Browse Source

Implement a check to validate local filesystem available space before attempting file download (#1971)

* Implement a check to validate local filesystem available space before attempting file download
* Implement 'space_reservation' config option with a default value of 50 MB
* Prevent the original run-away logging error message 'Failed initialization on handle XXXX' from occurring if the system is out of space
* Update documentation and man page
pull/1976/head
abraunegg 3 weeks ago committed by GitHub
parent
commit
0fffb8fbc0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      config
  2. 27
      docs/USAGE.md
  3. 4
      onedrive.1.in
  4. 19
      src/config.d
  5. 1
      src/main.d
  6. 10
      src/onedrive.d
  7. 24
      src/sync.d
  8. 11
      src/util.d

1
config

@ -50,3 +50,4 @@
# webhook_listening_port = "8888"
# webhook_expiration_interval = "86400"
# webhook_renewal_interval = "43200"
# space_reservation = "50"

27
docs/USAGE.md

@ -35,6 +35,7 @@
* [Configuring the client to use older 'skilion' application identifier](#configuring-the-client-to-use-older-skilion-application-identifier)
* [How to 'skip' directories from syncing?](#how-to-skip-directories-from-syncing)
* [How to 'rate limit' the application to control bandwidth consumed for upload & download operations](#how-to-rate-limit-the-application-to-control-bandwidth-consumed-for-upload--download-operations)
* [Preventing your local disk from filling up](#preventing-your-local-disk-from-filling-up)
* [Shared folders (OneDrive Personal)](#shared-folders-onedrive-personal)
* [Shared folders (OneDrive Business or Office 365)](#shared-folders-onedrive-business-or-office-365)
* [SharePoint / Office 365 Shared Libraries](#sharepoint--office-365-shared-libraries)
@ -382,6 +383,7 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con
# webhook_listening_port = "8888"
# webhook_expiration_interval = "86400"
# webhook_renewal_interval = "43200"
# space_reservation = "50"
```
### 'config' file configuration examples:
@ -765,6 +767,29 @@ rate_limit = "131072"
**Note:** A number greater than '131072' is a valid value, with '104857600' being tested as an upper limit.
### Preventing your local disk from filling up
By default, the application will reserve 50MB of disk space to prevent your filesystem to run out of disk space. This value can be modified by adding the following to your config file:
Example:
```text
...
# webhook_expiration_interval = "86400"
# webhook_renewal_interval = "43200"
space_reservation = "10"
```
The value entered is in MB (Mega Bytes). In this example, a value of 10MB is being used, and will be converted to bytes by the application. The value being used can be reviewed when using `--display-config`:
```
Config option 'sync_dir_permissions' = 700
Config option 'sync_file_permissions' = 600
Config option 'space_reservation' = 10485760
Config option 'application_id' =
Config option 'azure_ad_endpoint' =
Config option 'azure_tenant_id' = common
```
Any value is valid here, however, if you use a value of '0' a value of '1' will actually be used, so that you actually do not run out of disk space.
### Shared folders (OneDrive Personal)
Folders shared with you can be synced by adding them to your OneDrive. To do that open your Onedrive, go to the Shared files list, right click on the folder you want to sync and then click on "Add to my OneDrive".
@ -1202,6 +1227,8 @@ Options:
Skip syncing of symlinks
--source-directory ARG
Source directory to rename or move on OneDrive - no sync will be performed.
--space-reservation ARG
The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation
--sync-root-files
Sync all files in sync_dir root when using sync_list.
--sync-shared-folders

4
onedrive.1.in

@ -219,9 +219,11 @@ Configuration file key: \fBskip_symlinks\fP (default: \fBfalse\fP)
\fB\-\-source\-directory\fP ARG
Source directory to rename or move on OneDrive \- no sync will be performed.
.TP
\fB\-\-space\-reservation\fP ARG
The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation
.TP
\fB\-\-sync\-root\-files\fP
Sync all files in sync_dir root when using sync_list.
.TP
\fB\-\-sync\-shared\-folders\fP
Sync OneDrive Business Shared Folders

19
src/config.d

@ -124,8 +124,9 @@ final class Config
longValues["rate_limit"] = 0;
// maximum time an operation is allowed to take
// This includes dns resolution, connecting, data transfer, etc.
longValues["operation_timeout"] = 3600;
longValues["operation_timeout"] = 3600;
// To ensure we do not fill up the load disk, how much disk space should be reserved by default
longValues["space_reservation"] = 50 * 2^^20; // 50 MB as Bytes
// Webhook options
boolValues["webhook_enabled"] = false;
stringValues["webhook_public_url"] = "";
@ -457,6 +458,9 @@ final class Config
"source-directory",
"Source directory to rename or move on OneDrive - no sync will be performed.",
&stringValues["source_directory"],
"space-reservation",
"The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation",
&longValues["space_reservation"],
"syncdir",
"Specify the local directory used for synchronization to OneDrive",
&stringValues["sync_dir"],
@ -661,6 +665,16 @@ final class Config
if (ppp) {
c.popFront();
setValueLong(key, to!long(c.front.dup));
// if key is space_reservation we have to calculate MB -> bytes
if (key == "space_reservation") {
// temp value
ulong tempValue = to!long(c.front.dup);
// a value of 0 needs to be made at least 1MB ..
if (tempValue == 0) {
tempValue = 1;
}
setValueLong("space_reservation", to!long(tempValue * 2^^20));
}
} else {
log.log("Unknown key in config file: ", key);
return false;
@ -766,6 +780,7 @@ void outputLongHelp(Option[] opt)
"--skip-file",
"--skip-size",
"--source-directory",
"--space-reservation",
"--syncdir",
"--user-agent" ];
writeln(`OneDrive - a client for OneDrive Cloud Services

1
src/main.d

@ -672,6 +672,7 @@ int main(string[] args)
writeln("Config option 'remove_source_files' = ", cfg.getValueBool("remove_source_files"));
writeln("Config option 'sync_dir_permissions' = ", cfg.getValueLong("sync_dir_permissions"));
writeln("Config option 'sync_file_permissions' = ", cfg.getValueLong("sync_file_permissions"));
writeln("Config option 'space_reservation' = ", cfg.getValueLong("space_reservation"));
// curl operations
writeln("Config option 'application_id' = ", cfg.getValueString("application_id"));

10
src/onedrive.d

@ -1472,6 +1472,16 @@ final class OneDriveApi
// Some other error was returned
log.error(" Error Message: ", errorMessage);
log.error(" Calling Function: ", getFunctionName!({}));
// Was this a curl initialization error?
if (canFind(errorMessage, "Failed initialization on handle")) {
// initialization error ... prevent a run-away process if we have zero disk space
ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace("."));
if (localActualFreeSpace == 0) {
// force exit
exit(-1);
}
}
}
// return an empty JSON for handling
return json;

24
src/sync.d

@ -269,7 +269,7 @@ final class SyncEngine
private bool nationalCloudDeployment = false;
// array of all OneDrive driveId's for use with OneDrive Business Folders
private string[] driveIDsArray;
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
{
assert(onedrive && itemdb && selectiveSync);
@ -2822,6 +2822,28 @@ final class SyncEngine
log.vdebug("WARNING: fileDetails['file']['hashes'] is missing - unable to compare file hash after download");
}
// Is there enough free space locally to download the file
// - We can use '.' here as we change the current working directory to the configured 'sync_dir'
ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace("."));
// So that we are not responsible in making the disk 100% full if we can download the file, compare the current available space against the reservation set and file size
// The reservation value is user configurable in the config file, 50MB by default
ulong freeSpaceReservation = cfg.getValueLong("space_reservation");
// debug output
log.vdebug("Local Disk Space Actual: ", localActualFreeSpace);
log.vdebug("Free Space Reservation: ", freeSpaceReservation);
log.vdebug("File Size to Download: ", fileSize);
// calculate if we can download file
if ((localActualFreeSpace < freeSpaceReservation) || (fileSize > localActualFreeSpace)) {
// localActualFreeSpace is less than freeSpaceReservation .. insufficient free space
// fileSize is greater than localActualFreeSpace .. insufficient free space
writeln("failed!");
log.log("Insufficient local disk space to download file");
downloadFailed = true;
return;
}
// Attempt to download the file
try {
onedrive.downloadById(item.driveId, item.id, path, fileSize);
} catch (OneDriveException e) {

11
src/util.d

@ -14,6 +14,7 @@ import std.uri;
import std.json;
import std.traits;
import qxor;
import core.stdc.stdlib;
static import log;
shared string deviceName;
@ -332,7 +333,7 @@ void displayOneDriveErrorMessage(string message, string callingFunction)
}
// Where in the code was this error generated
log.error(" Calling Function: ", callingFunction);
log.vlog(" Calling Function: ", callingFunction);
}
// Parse and display error message received from the local file system
@ -343,7 +344,13 @@ void displayFileSystemErrorMessage(string message, string callingFunction)
// What was the error message
log.error(" Error Message: ", errorArray[0]);
// Where in the code was this error generated
log.error(" Calling Function: ", callingFunction);
log.vlog(" Calling Function: ", callingFunction);
// If we are out of disk space (despite download reservations) we need to exit the application
ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace("."));
if (localActualFreeSpace == 0) {
// force exit
exit(-1);
}
}
// Get the function name that is being called to assist with identifying where an error is being generated

Loading…
Cancel
Save