diff --git a/config b/config index 98d61cfb..0514a86d 100644 --- a/config +++ b/config @@ -50,3 +50,4 @@ # webhook_listening_port = "8888" # webhook_expiration_interval = "86400" # webhook_renewal_interval = "43200" +# space_reservation = "50" \ No newline at end of file diff --git a/docs/USAGE.md b/docs/USAGE.md index 86373094..2d498c06 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -35,6 +35,7 @@ * [Configuring the client to use older 'skilion' application identifier](#configuring-the-client-to-use-older-skilion-application-identifier) * [How to 'skip' directories from syncing?](#how-to-skip-directories-from-syncing) * [How to 'rate limit' the application to control bandwidth consumed for upload & download operations](#how-to-rate-limit-the-application-to-control-bandwidth-consumed-for-upload--download-operations) + * [Preventing your local disk from filling up](#preventing-your-local-disk-from-filling-up) * [Shared folders (OneDrive Personal)](#shared-folders-onedrive-personal) * [Shared folders (OneDrive Business or Office 365)](#shared-folders-onedrive-business-or-office-365) * [SharePoint / Office 365 Shared Libraries](#sharepoint--office-365-shared-libraries) @@ -382,6 +383,7 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con # webhook_listening_port = "8888" # webhook_expiration_interval = "86400" # webhook_renewal_interval = "43200" +# space_reservation = "50" ``` ### 'config' file configuration examples: @@ -765,6 +767,29 @@ rate_limit = "131072" **Note:** A number greater than '131072' is a valid value, with '104857600' being tested as an upper limit. +### Preventing your local disk from filling up +By default, the application will reserve 50MB of disk space to prevent your filesystem to run out of disk space. This value can be modified by adding the following to your config file: + +Example: +```text +... +# webhook_expiration_interval = "86400" +# webhook_renewal_interval = "43200" +space_reservation = "10" +``` + +The value entered is in MB (Mega Bytes). In this example, a value of 10MB is being used, and will be converted to bytes by the application. The value being used can be reviewed when using `--display-config`: +``` +Config option 'sync_dir_permissions' = 700 +Config option 'sync_file_permissions' = 600 +Config option 'space_reservation' = 10485760 +Config option 'application_id' = +Config option 'azure_ad_endpoint' = +Config option 'azure_tenant_id' = common +``` + +Any value is valid here, however, if you use a value of '0' a value of '1' will actually be used, so that you actually do not run out of disk space. + ### Shared folders (OneDrive Personal) Folders shared with you can be synced by adding them to your OneDrive. To do that open your Onedrive, go to the Shared files list, right click on the folder you want to sync and then click on "Add to my OneDrive". @@ -1202,6 +1227,8 @@ Options: Skip syncing of symlinks --source-directory ARG Source directory to rename or move on OneDrive - no sync will be performed. + --space-reservation ARG + The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation --sync-root-files Sync all files in sync_dir root when using sync_list. --sync-shared-folders diff --git a/onedrive.1.in b/onedrive.1.in index 6c5fd3ea..e06568fe 100644 --- a/onedrive.1.in +++ b/onedrive.1.in @@ -219,9 +219,11 @@ Configuration file key: \fBskip_symlinks\fP (default: \fBfalse\fP) \fB\-\-source\-directory\fP ARG Source directory to rename or move on OneDrive \- no sync will be performed. .TP +\fB\-\-space\-reservation\fP ARG +The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation +.TP \fB\-\-sync\-root\-files\fP Sync all files in sync_dir root when using sync_list. - .TP \fB\-\-sync\-shared\-folders\fP Sync OneDrive Business Shared Folders diff --git a/src/config.d b/src/config.d index 14301614..4c23538e 100644 --- a/src/config.d +++ b/src/config.d @@ -124,8 +124,9 @@ final class Config longValues["rate_limit"] = 0; // maximum time an operation is allowed to take // This includes dns resolution, connecting, data transfer, etc. - longValues["operation_timeout"] = 3600; - + longValues["operation_timeout"] = 3600; + // To ensure we do not fill up the load disk, how much disk space should be reserved by default + longValues["space_reservation"] = 50 * 2^^20; // 50 MB as Bytes // Webhook options boolValues["webhook_enabled"] = false; stringValues["webhook_public_url"] = ""; @@ -457,6 +458,9 @@ final class Config "source-directory", "Source directory to rename or move on OneDrive - no sync will be performed.", &stringValues["source_directory"], + "space-reservation", + "The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation", + &longValues["space_reservation"], "syncdir", "Specify the local directory used for synchronization to OneDrive", &stringValues["sync_dir"], @@ -661,6 +665,16 @@ final class Config if (ppp) { c.popFront(); setValueLong(key, to!long(c.front.dup)); + // if key is space_reservation we have to calculate MB -> bytes + if (key == "space_reservation") { + // temp value + ulong tempValue = to!long(c.front.dup); + // a value of 0 needs to be made at least 1MB .. + if (tempValue == 0) { + tempValue = 1; + } + setValueLong("space_reservation", to!long(tempValue * 2^^20)); + } } else { log.log("Unknown key in config file: ", key); return false; @@ -766,6 +780,7 @@ void outputLongHelp(Option[] opt) "--skip-file", "--skip-size", "--source-directory", + "--space-reservation", "--syncdir", "--user-agent" ]; writeln(`OneDrive - a client for OneDrive Cloud Services diff --git a/src/main.d b/src/main.d index f2d9620d..cdc6ce01 100644 --- a/src/main.d +++ b/src/main.d @@ -672,6 +672,7 @@ int main(string[] args) writeln("Config option 'remove_source_files' = ", cfg.getValueBool("remove_source_files")); writeln("Config option 'sync_dir_permissions' = ", cfg.getValueLong("sync_dir_permissions")); writeln("Config option 'sync_file_permissions' = ", cfg.getValueLong("sync_file_permissions")); + writeln("Config option 'space_reservation' = ", cfg.getValueLong("space_reservation")); // curl operations writeln("Config option 'application_id' = ", cfg.getValueString("application_id")); diff --git a/src/onedrive.d b/src/onedrive.d index b4e29190..b22b5cda 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -1472,6 +1472,16 @@ final class OneDriveApi // Some other error was returned log.error(" Error Message: ", errorMessage); log.error(" Calling Function: ", getFunctionName!({})); + + // Was this a curl initialization error? + if (canFind(errorMessage, "Failed initialization on handle")) { + // initialization error ... prevent a run-away process if we have zero disk space + ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace(".")); + if (localActualFreeSpace == 0) { + // force exit + exit(-1); + } + } } // return an empty JSON for handling return json; diff --git a/src/sync.d b/src/sync.d index c5bcb4e7..ee563771 100644 --- a/src/sync.d +++ b/src/sync.d @@ -269,7 +269,7 @@ final class SyncEngine private bool nationalCloudDeployment = false; // array of all OneDrive driveId's for use with OneDrive Business Folders private string[] driveIDsArray; - + this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync) { assert(onedrive && itemdb && selectiveSync); @@ -2822,6 +2822,28 @@ final class SyncEngine log.vdebug("WARNING: fileDetails['file']['hashes'] is missing - unable to compare file hash after download"); } + // Is there enough free space locally to download the file + // - We can use '.' here as we change the current working directory to the configured 'sync_dir' + ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace(".")); + // So that we are not responsible in making the disk 100% full if we can download the file, compare the current available space against the reservation set and file size + // The reservation value is user configurable in the config file, 50MB by default + ulong freeSpaceReservation = cfg.getValueLong("space_reservation"); + // debug output + log.vdebug("Local Disk Space Actual: ", localActualFreeSpace); + log.vdebug("Free Space Reservation: ", freeSpaceReservation); + log.vdebug("File Size to Download: ", fileSize); + + // calculate if we can download file + if ((localActualFreeSpace < freeSpaceReservation) || (fileSize > localActualFreeSpace)) { + // localActualFreeSpace is less than freeSpaceReservation .. insufficient free space + // fileSize is greater than localActualFreeSpace .. insufficient free space + writeln("failed!"); + log.log("Insufficient local disk space to download file"); + downloadFailed = true; + return; + } + + // Attempt to download the file try { onedrive.downloadById(item.driveId, item.id, path, fileSize); } catch (OneDriveException e) { diff --git a/src/util.d b/src/util.d index 8e666fbc..efcbd5ae 100644 --- a/src/util.d +++ b/src/util.d @@ -14,6 +14,7 @@ import std.uri; import std.json; import std.traits; import qxor; +import core.stdc.stdlib; static import log; shared string deviceName; @@ -332,7 +333,7 @@ void displayOneDriveErrorMessage(string message, string callingFunction) } // Where in the code was this error generated - log.error(" Calling Function: ", callingFunction); + log.vlog(" Calling Function: ", callingFunction); } // Parse and display error message received from the local file system @@ -343,7 +344,13 @@ void displayFileSystemErrorMessage(string message, string callingFunction) // What was the error message log.error(" Error Message: ", errorArray[0]); // Where in the code was this error generated - log.error(" Calling Function: ", callingFunction); + log.vlog(" Calling Function: ", callingFunction); + // If we are out of disk space (despite download reservations) we need to exit the application + ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace(".")); + if (localActualFreeSpace == 0) { + // force exit + exit(-1); + } } // Get the function name that is being called to assist with identifying where an error is being generated