Implement a check to validate local filesystem available space before attempting file download (#1971)

* Implement a check to validate local filesystem available space before attempting file download
* Implement 'space_reservation' config option with a default value of 50 MB
* Prevent the original run-away logging error message 'Failed initialization on handle XXXX' from occurring if the system is out of space
* Update documentation and man page

config

@@ -50,3 +50,4 @@
 # webhook_listening_port = "8888"
 # webhook_expiration_interval = "86400"
 # webhook_renewal_interval = "43200"
+# space_reservation = "50"

docs/USAGE.md

@@ -35,6 +35,7 @@
   * [Configuring the client to use older 'skilion' application identifier](#configuring-the-client-to-use-older-skilion-application-identifier)
   * [How to 'skip' directories from syncing?](#how-to-skip-directories-from-syncing)
   * [How to 'rate limit' the application to control bandwidth consumed for upload & download operations](#how-to-rate-limit-the-application-to-control-bandwidth-consumed-for-upload--download-operations)
+  * [Preventing your local disk from filling up](#preventing-your-local-disk-from-filling-up)
   * [Shared folders (OneDrive Personal)](#shared-folders-onedrive-personal)
   * [Shared folders (OneDrive Business or Office 365)](#shared-folders-onedrive-business-or-office-365)
   * [SharePoint / Office 365 Shared Libraries](#sharepoint--office-365-shared-libraries)
@@ -382,6 +383,7 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con
 # webhook_listening_port = "8888"
 # webhook_expiration_interval = "86400"
 # webhook_renewal_interval = "43200"
+# space_reservation = "50"
 ```
 
 ### 'config' file configuration examples:
@@ -765,6 +767,29 @@ rate_limit = "131072"
 
 **Note:** A number greater than '131072' is a valid value, with '104857600' being tested as an upper limit.
 
+### Preventing your local disk from filling up
+By default, the application will reserve 50MB of disk space to prevent your filesystem to run out of disk space. This value can be modified by adding the following to your config file:
+
+Example:
+```text
+...
+# webhook_expiration_interval = "86400"
+# webhook_renewal_interval = "43200"
+space_reservation = "10"
+```
+
+The value entered is in MB (Mega Bytes). In this example, a value of 10MB is being used, and will be converted to bytes by the application. The value being used can be reviewed when using `--display-config`:
+```
+Config option 'sync_dir_permissions'         = 700
+Config option 'sync_file_permissions'        = 600
+Config option 'space_reservation'            = 10485760
+Config option 'application_id'               = 
+Config option 'azure_ad_endpoint'            = 
+Config option 'azure_tenant_id'              = common
+```
+
+Any value is valid here, however, if you use a value of '0' a value of '1' will actually be used, so that you actually do not run out of disk space.
+
 ### Shared folders (OneDrive Personal)
 Folders shared with you can be synced by adding them to your OneDrive. To do that open your Onedrive, go to the Shared files list, right click on the folder you want to sync and then click on "Add to my OneDrive".
 
@@ -1202,6 +1227,8 @@ Options:
       Skip syncing of symlinks
   --source-directory ARG
       Source directory to rename or move on OneDrive - no sync will be performed.
+  --space-reservation ARG
+      The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation
   --sync-root-files
       Sync all files in sync_dir root when using sync_list.
   --sync-shared-folders

onedrive.1.in

@@ -219,9 +219,11 @@ Configuration file key: \fBskip_symlinks\fP (default: \fBfalse\fP)
 \fB\-\-source\-directory\fP ARG
 Source directory to rename or move on OneDrive \- no sync will be performed.
 .TP
+\fB\-\-space\-reservation\fP ARG
+The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation
+.TP
 \fB\-\-sync\-root\-files\fP
 Sync all files in sync_dir root when using sync_list.
-
 .TP
 \fB\-\-sync\-shared\-folders\fP
 Sync OneDrive Business Shared Folders

src/config.d

@@ -125,7 +125,8 @@ final class Config
 		// maximum time an operation is allowed to take
 		// This includes dns resolution, connecting, data transfer, etc.
 		longValues["operation_timeout"] = 3600;
-
+		// To ensure we do not fill up the load disk, how much disk space should be reserved by default
+		longValues["space_reservation"] = 50 * 2^^20; // 50 MB as Bytes
 		// Webhook options
 		boolValues["webhook_enabled"] = false;
 		stringValues["webhook_public_url"] = "";
@@ -457,6 +458,9 @@ final class Config
 				"source-directory",
 					"Source directory to rename or move on OneDrive - no sync will be performed.",
 					&stringValues["source_directory"],
+				"space-reservation",
+					"The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation",
+					&longValues["space_reservation"],
 				"syncdir",
 					"Specify the local directory used for synchronization to OneDrive",
 					&stringValues["sync_dir"],
@@ -661,6 +665,16 @@ final class Config
 						if (ppp) {
 							c.popFront();
 							setValueLong(key, to!long(c.front.dup));
+							// if key is space_reservation we have to calculate MB -> bytes
+							if (key == "space_reservation") {
+								// temp value
+								ulong tempValue = to!long(c.front.dup);
+								// a value of 0 needs to be made at least 1MB .. 
+								if (tempValue == 0) {
+									tempValue = 1;
+								}
+								setValueLong("space_reservation", to!long(tempValue * 2^^20));
+							}
 						} else {
 							log.log("Unknown key in config file: ", key);
 							return false;
@@ -766,6 +780,7 @@ void outputLongHelp(Option[] opt)
 		"--skip-file",
 		"--skip-size",
 		"--source-directory",
+		"--space-reservation",
 		"--syncdir",
 		"--user-agent" ];
 	writeln(`OneDrive - a client for OneDrive Cloud Services

src/main.d

@@ -672,6 +672,7 @@ int main(string[] args)
 		writeln("Config option 'remove_source_files'          = ", cfg.getValueBool("remove_source_files"));
 		writeln("Config option 'sync_dir_permissions'         = ", cfg.getValueLong("sync_dir_permissions"));
 		writeln("Config option 'sync_file_permissions'        = ", cfg.getValueLong("sync_file_permissions"));
+		writeln("Config option 'space_reservation'            = ", cfg.getValueLong("space_reservation"));
 		
 		// curl operations
 		writeln("Config option 'application_id'               = ", cfg.getValueString("application_id"));

src/onedrive.d

@@ -1472,6 +1472,16 @@ final class OneDriveApi
 				// Some other error was returned
 				log.error("  Error Message: ", errorMessage);
 				log.error("  Calling Function: ", getFunctionName!({}));
+				
+				// Was this a curl initialization error?
+				if (canFind(errorMessage, "Failed initialization on handle")) {
+					// initialization error ... prevent a run-away process if we have zero disk space
+					ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace("."));
+					if (localActualFreeSpace == 0) {
+						// force exit
+						exit(-1);
+					}
+				}
 			}
 			// return an empty JSON for handling
 			return json;

src/sync.d

@@ -2822,6 +2822,28 @@ final class SyncEngine
 				log.vdebug("WARNING: fileDetails['file']['hashes'] is missing - unable to compare file hash after download");
 			}
 			
+			// Is there enough free space locally to download the file
+			// - We can use '.' here as we change the current working directory to the configured 'sync_dir'
+			ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace("."));
+			// So that we are not responsible in making the disk 100% full if we can download the file, compare the current available space against the reservation set and file size
+			// The reservation value is user configurable in the config file, 50MB by default
+			ulong freeSpaceReservation = cfg.getValueLong("space_reservation");
+			// debug output
+			log.vdebug("Local Disk Space Actual: ", localActualFreeSpace);
+			log.vdebug("Free Space Reservation:  ", freeSpaceReservation);
+			log.vdebug("File Size to Download:   ", fileSize);
+			
+			// calculate if we can download file
+			if ((localActualFreeSpace < freeSpaceReservation) || (fileSize > localActualFreeSpace)) {
+				// localActualFreeSpace is less than freeSpaceReservation .. insufficient free space
+				// fileSize is greater than localActualFreeSpace .. insufficient free space
+				writeln("failed!");
+				log.log("Insufficient local disk space to download file");
+				downloadFailed = true;
+				return;
+			}
+			
+			// Attempt to download the file
 			try {
 				onedrive.downloadById(item.driveId, item.id, path, fileSize);
 			} catch (OneDriveException e) {

src/util.d

@@ -14,6 +14,7 @@ import std.uri;
 import std.json;
 import std.traits;
 import qxor;
+import core.stdc.stdlib;
 static import log;
 
 shared string deviceName;
@@ -332,7 +333,7 @@ void displayOneDriveErrorMessage(string message, string callingFunction)
 	}
 	
 	// Where in the code was this error generated
-	log.error("  Calling Function: ", callingFunction);
+	log.vlog("  Calling Function: ", callingFunction);
 }
 
 // Parse and display error message received from the local file system
@@ -343,7 +344,13 @@ void displayFileSystemErrorMessage(string message, string callingFunction)
 	// What was the error message
 	log.error("  Error Message:    ", errorArray[0]);
 	// Where in the code was this error generated
-	log.error("  Calling Function: ", callingFunction);
+	log.vlog("  Calling Function: ", callingFunction);
+	// If we are out of disk space (despite download reservations) we need to exit the application
+	ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace("."));
+	if (localActualFreeSpace == 0) {
+		// force exit
+		exit(-1);
+	}
 }
 
 // Get the function name that is being called to assist with identifying where an error is being generated