mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-10 09:52:13 +02:00
Implement a check to validate local filesystem available space before attempting file download (#1971)
* Implement a check to validate local filesystem available space before attempting file download * Implement 'space_reservation' config option with a default value of 50 MB * Prevent the original run-away logging error message 'Failed initialization on handle XXXX' from occurring if the system is out of space * Update documentation and man page
This commit is contained in:
parent
767352686f
commit
0fffb8fbc0
1
config
1
config
|
@ -50,3 +50,4 @@
|
||||||
# webhook_listening_port = "8888"
|
# webhook_listening_port = "8888"
|
||||||
# webhook_expiration_interval = "86400"
|
# webhook_expiration_interval = "86400"
|
||||||
# webhook_renewal_interval = "43200"
|
# webhook_renewal_interval = "43200"
|
||||||
|
# space_reservation = "50"
|
|
@ -35,6 +35,7 @@
|
||||||
* [Configuring the client to use older 'skilion' application identifier](#configuring-the-client-to-use-older-skilion-application-identifier)
|
* [Configuring the client to use older 'skilion' application identifier](#configuring-the-client-to-use-older-skilion-application-identifier)
|
||||||
* [How to 'skip' directories from syncing?](#how-to-skip-directories-from-syncing)
|
* [How to 'skip' directories from syncing?](#how-to-skip-directories-from-syncing)
|
||||||
* [How to 'rate limit' the application to control bandwidth consumed for upload & download operations](#how-to-rate-limit-the-application-to-control-bandwidth-consumed-for-upload--download-operations)
|
* [How to 'rate limit' the application to control bandwidth consumed for upload & download operations](#how-to-rate-limit-the-application-to-control-bandwidth-consumed-for-upload--download-operations)
|
||||||
|
* [Preventing your local disk from filling up](#preventing-your-local-disk-from-filling-up)
|
||||||
* [Shared folders (OneDrive Personal)](#shared-folders-onedrive-personal)
|
* [Shared folders (OneDrive Personal)](#shared-folders-onedrive-personal)
|
||||||
* [Shared folders (OneDrive Business or Office 365)](#shared-folders-onedrive-business-or-office-365)
|
* [Shared folders (OneDrive Business or Office 365)](#shared-folders-onedrive-business-or-office-365)
|
||||||
* [SharePoint / Office 365 Shared Libraries](#sharepoint--office-365-shared-libraries)
|
* [SharePoint / Office 365 Shared Libraries](#sharepoint--office-365-shared-libraries)
|
||||||
|
@ -382,6 +383,7 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con
|
||||||
# webhook_listening_port = "8888"
|
# webhook_listening_port = "8888"
|
||||||
# webhook_expiration_interval = "86400"
|
# webhook_expiration_interval = "86400"
|
||||||
# webhook_renewal_interval = "43200"
|
# webhook_renewal_interval = "43200"
|
||||||
|
# space_reservation = "50"
|
||||||
```
|
```
|
||||||
|
|
||||||
### 'config' file configuration examples:
|
### 'config' file configuration examples:
|
||||||
|
@ -765,6 +767,29 @@ rate_limit = "131072"
|
||||||
|
|
||||||
**Note:** A number greater than '131072' is a valid value, with '104857600' being tested as an upper limit.
|
**Note:** A number greater than '131072' is a valid value, with '104857600' being tested as an upper limit.
|
||||||
|
|
||||||
|
### Preventing your local disk from filling up
|
||||||
|
By default, the application will reserve 50MB of disk space to prevent your filesystem to run out of disk space. This value can be modified by adding the following to your config file:
|
||||||
|
|
||||||
|
Example:
|
||||||
|
```text
|
||||||
|
...
|
||||||
|
# webhook_expiration_interval = "86400"
|
||||||
|
# webhook_renewal_interval = "43200"
|
||||||
|
space_reservation = "10"
|
||||||
|
```
|
||||||
|
|
||||||
|
The value entered is in MB (Mega Bytes). In this example, a value of 10MB is being used, and will be converted to bytes by the application. The value being used can be reviewed when using `--display-config`:
|
||||||
|
```
|
||||||
|
Config option 'sync_dir_permissions' = 700
|
||||||
|
Config option 'sync_file_permissions' = 600
|
||||||
|
Config option 'space_reservation' = 10485760
|
||||||
|
Config option 'application_id' =
|
||||||
|
Config option 'azure_ad_endpoint' =
|
||||||
|
Config option 'azure_tenant_id' = common
|
||||||
|
```
|
||||||
|
|
||||||
|
Any value is valid here, however, if you use a value of '0' a value of '1' will actually be used, so that you actually do not run out of disk space.
|
||||||
|
|
||||||
### Shared folders (OneDrive Personal)
|
### Shared folders (OneDrive Personal)
|
||||||
Folders shared with you can be synced by adding them to your OneDrive. To do that open your Onedrive, go to the Shared files list, right click on the folder you want to sync and then click on "Add to my OneDrive".
|
Folders shared with you can be synced by adding them to your OneDrive. To do that open your Onedrive, go to the Shared files list, right click on the folder you want to sync and then click on "Add to my OneDrive".
|
||||||
|
|
||||||
|
@ -1202,6 +1227,8 @@ Options:
|
||||||
Skip syncing of symlinks
|
Skip syncing of symlinks
|
||||||
--source-directory ARG
|
--source-directory ARG
|
||||||
Source directory to rename or move on OneDrive - no sync will be performed.
|
Source directory to rename or move on OneDrive - no sync will be performed.
|
||||||
|
--space-reservation ARG
|
||||||
|
The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation
|
||||||
--sync-root-files
|
--sync-root-files
|
||||||
Sync all files in sync_dir root when using sync_list.
|
Sync all files in sync_dir root when using sync_list.
|
||||||
--sync-shared-folders
|
--sync-shared-folders
|
||||||
|
|
|
@ -219,9 +219,11 @@ Configuration file key: \fBskip_symlinks\fP (default: \fBfalse\fP)
|
||||||
\fB\-\-source\-directory\fP ARG
|
\fB\-\-source\-directory\fP ARG
|
||||||
Source directory to rename or move on OneDrive \- no sync will be performed.
|
Source directory to rename or move on OneDrive \- no sync will be performed.
|
||||||
.TP
|
.TP
|
||||||
|
\fB\-\-space\-reservation\fP ARG
|
||||||
|
The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation
|
||||||
|
.TP
|
||||||
\fB\-\-sync\-root\-files\fP
|
\fB\-\-sync\-root\-files\fP
|
||||||
Sync all files in sync_dir root when using sync_list.
|
Sync all files in sync_dir root when using sync_list.
|
||||||
|
|
||||||
.TP
|
.TP
|
||||||
\fB\-\-sync\-shared\-folders\fP
|
\fB\-\-sync\-shared\-folders\fP
|
||||||
Sync OneDrive Business Shared Folders
|
Sync OneDrive Business Shared Folders
|
||||||
|
|
17
src/config.d
17
src/config.d
|
@ -125,7 +125,8 @@ final class Config
|
||||||
// maximum time an operation is allowed to take
|
// maximum time an operation is allowed to take
|
||||||
// This includes dns resolution, connecting, data transfer, etc.
|
// This includes dns resolution, connecting, data transfer, etc.
|
||||||
longValues["operation_timeout"] = 3600;
|
longValues["operation_timeout"] = 3600;
|
||||||
|
// To ensure we do not fill up the load disk, how much disk space should be reserved by default
|
||||||
|
longValues["space_reservation"] = 50 * 2^^20; // 50 MB as Bytes
|
||||||
// Webhook options
|
// Webhook options
|
||||||
boolValues["webhook_enabled"] = false;
|
boolValues["webhook_enabled"] = false;
|
||||||
stringValues["webhook_public_url"] = "";
|
stringValues["webhook_public_url"] = "";
|
||||||
|
@ -457,6 +458,9 @@ final class Config
|
||||||
"source-directory",
|
"source-directory",
|
||||||
"Source directory to rename or move on OneDrive - no sync will be performed.",
|
"Source directory to rename or move on OneDrive - no sync will be performed.",
|
||||||
&stringValues["source_directory"],
|
&stringValues["source_directory"],
|
||||||
|
"space-reservation",
|
||||||
|
"The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation",
|
||||||
|
&longValues["space_reservation"],
|
||||||
"syncdir",
|
"syncdir",
|
||||||
"Specify the local directory used for synchronization to OneDrive",
|
"Specify the local directory used for synchronization to OneDrive",
|
||||||
&stringValues["sync_dir"],
|
&stringValues["sync_dir"],
|
||||||
|
@ -661,6 +665,16 @@ final class Config
|
||||||
if (ppp) {
|
if (ppp) {
|
||||||
c.popFront();
|
c.popFront();
|
||||||
setValueLong(key, to!long(c.front.dup));
|
setValueLong(key, to!long(c.front.dup));
|
||||||
|
// if key is space_reservation we have to calculate MB -> bytes
|
||||||
|
if (key == "space_reservation") {
|
||||||
|
// temp value
|
||||||
|
ulong tempValue = to!long(c.front.dup);
|
||||||
|
// a value of 0 needs to be made at least 1MB ..
|
||||||
|
if (tempValue == 0) {
|
||||||
|
tempValue = 1;
|
||||||
|
}
|
||||||
|
setValueLong("space_reservation", to!long(tempValue * 2^^20));
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
log.log("Unknown key in config file: ", key);
|
log.log("Unknown key in config file: ", key);
|
||||||
return false;
|
return false;
|
||||||
|
@ -766,6 +780,7 @@ void outputLongHelp(Option[] opt)
|
||||||
"--skip-file",
|
"--skip-file",
|
||||||
"--skip-size",
|
"--skip-size",
|
||||||
"--source-directory",
|
"--source-directory",
|
||||||
|
"--space-reservation",
|
||||||
"--syncdir",
|
"--syncdir",
|
||||||
"--user-agent" ];
|
"--user-agent" ];
|
||||||
writeln(`OneDrive - a client for OneDrive Cloud Services
|
writeln(`OneDrive - a client for OneDrive Cloud Services
|
||||||
|
|
|
@ -672,6 +672,7 @@ int main(string[] args)
|
||||||
writeln("Config option 'remove_source_files' = ", cfg.getValueBool("remove_source_files"));
|
writeln("Config option 'remove_source_files' = ", cfg.getValueBool("remove_source_files"));
|
||||||
writeln("Config option 'sync_dir_permissions' = ", cfg.getValueLong("sync_dir_permissions"));
|
writeln("Config option 'sync_dir_permissions' = ", cfg.getValueLong("sync_dir_permissions"));
|
||||||
writeln("Config option 'sync_file_permissions' = ", cfg.getValueLong("sync_file_permissions"));
|
writeln("Config option 'sync_file_permissions' = ", cfg.getValueLong("sync_file_permissions"));
|
||||||
|
writeln("Config option 'space_reservation' = ", cfg.getValueLong("space_reservation"));
|
||||||
|
|
||||||
// curl operations
|
// curl operations
|
||||||
writeln("Config option 'application_id' = ", cfg.getValueString("application_id"));
|
writeln("Config option 'application_id' = ", cfg.getValueString("application_id"));
|
||||||
|
|
|
@ -1472,6 +1472,16 @@ final class OneDriveApi
|
||||||
// Some other error was returned
|
// Some other error was returned
|
||||||
log.error(" Error Message: ", errorMessage);
|
log.error(" Error Message: ", errorMessage);
|
||||||
log.error(" Calling Function: ", getFunctionName!({}));
|
log.error(" Calling Function: ", getFunctionName!({}));
|
||||||
|
|
||||||
|
// Was this a curl initialization error?
|
||||||
|
if (canFind(errorMessage, "Failed initialization on handle")) {
|
||||||
|
// initialization error ... prevent a run-away process if we have zero disk space
|
||||||
|
ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace("."));
|
||||||
|
if (localActualFreeSpace == 0) {
|
||||||
|
// force exit
|
||||||
|
exit(-1);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// return an empty JSON for handling
|
// return an empty JSON for handling
|
||||||
return json;
|
return json;
|
||||||
|
|
22
src/sync.d
22
src/sync.d
|
@ -2822,6 +2822,28 @@ final class SyncEngine
|
||||||
log.vdebug("WARNING: fileDetails['file']['hashes'] is missing - unable to compare file hash after download");
|
log.vdebug("WARNING: fileDetails['file']['hashes'] is missing - unable to compare file hash after download");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Is there enough free space locally to download the file
|
||||||
|
// - We can use '.' here as we change the current working directory to the configured 'sync_dir'
|
||||||
|
ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace("."));
|
||||||
|
// So that we are not responsible in making the disk 100% full if we can download the file, compare the current available space against the reservation set and file size
|
||||||
|
// The reservation value is user configurable in the config file, 50MB by default
|
||||||
|
ulong freeSpaceReservation = cfg.getValueLong("space_reservation");
|
||||||
|
// debug output
|
||||||
|
log.vdebug("Local Disk Space Actual: ", localActualFreeSpace);
|
||||||
|
log.vdebug("Free Space Reservation: ", freeSpaceReservation);
|
||||||
|
log.vdebug("File Size to Download: ", fileSize);
|
||||||
|
|
||||||
|
// calculate if we can download file
|
||||||
|
if ((localActualFreeSpace < freeSpaceReservation) || (fileSize > localActualFreeSpace)) {
|
||||||
|
// localActualFreeSpace is less than freeSpaceReservation .. insufficient free space
|
||||||
|
// fileSize is greater than localActualFreeSpace .. insufficient free space
|
||||||
|
writeln("failed!");
|
||||||
|
log.log("Insufficient local disk space to download file");
|
||||||
|
downloadFailed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempt to download the file
|
||||||
try {
|
try {
|
||||||
onedrive.downloadById(item.driveId, item.id, path, fileSize);
|
onedrive.downloadById(item.driveId, item.id, path, fileSize);
|
||||||
} catch (OneDriveException e) {
|
} catch (OneDriveException e) {
|
||||||
|
|
11
src/util.d
11
src/util.d
|
@ -14,6 +14,7 @@ import std.uri;
|
||||||
import std.json;
|
import std.json;
|
||||||
import std.traits;
|
import std.traits;
|
||||||
import qxor;
|
import qxor;
|
||||||
|
import core.stdc.stdlib;
|
||||||
static import log;
|
static import log;
|
||||||
|
|
||||||
shared string deviceName;
|
shared string deviceName;
|
||||||
|
@ -332,7 +333,7 @@ void displayOneDriveErrorMessage(string message, string callingFunction)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Where in the code was this error generated
|
// Where in the code was this error generated
|
||||||
log.error(" Calling Function: ", callingFunction);
|
log.vlog(" Calling Function: ", callingFunction);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse and display error message received from the local file system
|
// Parse and display error message received from the local file system
|
||||||
|
@ -343,7 +344,13 @@ void displayFileSystemErrorMessage(string message, string callingFunction)
|
||||||
// What was the error message
|
// What was the error message
|
||||||
log.error(" Error Message: ", errorArray[0]);
|
log.error(" Error Message: ", errorArray[0]);
|
||||||
// Where in the code was this error generated
|
// Where in the code was this error generated
|
||||||
log.error(" Calling Function: ", callingFunction);
|
log.vlog(" Calling Function: ", callingFunction);
|
||||||
|
// If we are out of disk space (despite download reservations) we need to exit the application
|
||||||
|
ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace("."));
|
||||||
|
if (localActualFreeSpace == 0) {
|
||||||
|
// force exit
|
||||||
|
exit(-1);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the function name that is being called to assist with identifying where an error is being generated
|
// Get the function name that is being called to assist with identifying where an error is being generated
|
||||||
|
|
Loading…
Reference in a new issue