deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-22 07:36:36 +02:00
Code Issues Projects Releases Wiki Activity

Make webhooks more resilient

Browse source 
* Do not quit on subscription API errors. Instead, wait a configurable interval before retrying.
* Shorten default webhook expiration and renewal intervals to reduce the chance of http 409 errors.
* Handle http 409 on subscription creation by taking over the existing subscription.
* Handle http 404 on subscription renewal by creating a new subscription(current behavior, listed for completeness).
* Log other known http errors include 400 (bad webhook endpoint), 401 (auth failed), and 403 (too many subscriptions).
* Log detailed messages when encoutering unknown http errors to assist with future debugging.
This commit is contained in:
Lyncredible 2023-10-15 08:14:02 -07:00
parent 4a60654e3f
commit 30ee83a2ae
6 changed files with 362 additions and 193 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
config
View file

@ -48,8 +48,9 @@
# webhook_public_url = ""
# webhook_listening_host = ""
# webhook_listening_port = "8888"
# webhook_expiration_interval = "86400"
# webhook_renewal_interval = "43200"
# webhook_expiration_interval = "600"
# webhook_renewal_interval = "300"
# webhook_retry_interval = "60"
# space_reservation = "50"
# display_running_config = "false"
# read_only_auth_scope = "false"

31
docs/USAGE.md
View file

@ -57,8 +57,8 @@ Before reading this document, please ensure you are running application version
- [Running 'onedrive' in 'monitor' mode](#running-onedrive-in-monitor-mode)
* [Use webhook to subscribe to remote updates in 'monitor' mode](#use-webhook-to-subscribe-to-remote-updates-in-monitor-mode)
* [More webhook configuration options](#more-webhook-configuration-options)
+ [webhook_listening_host and webhook_listening_port](#webhook_listening_host-and-webhook_listening_port)
+ [webhook_expiration_interval and webhook_renewal_interval](#webhook_expiration_interval-and-webhook_renewal_interval)
+ [Webhook listening host and port](#webhook-listening-host-and-port)
+ [Webhook expiration, renewal and retry intervals](#webhook-expiration-renewal-and-retry-intervals)
- [Running 'onedrive' as a system service](#running-onedrive-as-a-system-service)
* [OneDrive service running as root user via init.d](#onedrive-service-running-as-root-user-via-initd)
* [OneDrive service running as root user via systemd (Arch, Ubuntu, Debian, OpenSuSE, Fedora)](#onedrive-service-running-as-root-user-via-systemd-arch-ubuntu-debian-opensuse-fedora)
@ -524,8 +524,8 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con
# webhook_public_url = ""
# webhook_listening_host = ""
# webhook_listening_port = "8888"
# webhook_expiration_interval = "86400"
# webhook_renewal_interval = "43200"
# webhook_expiration_interval = "600"
# webhook_renewal_interval = "300"
# space_reservation = "50"
# display_running_config = "false"
# read_only_auth_scope = "false"
@ -891,8 +891,8 @@ By default, the application will reserve 50MB of disk space to prevent your file
Example:
```text
...
# webhook_expiration_interval = "86400"
# webhook_renewal_interval = "43200"
# webhook_expiration_interval = "600"
# webhook_renewal_interval = "300"
space_reservation = "10"
```
@ -1059,7 +1059,7 @@ For any further nginx configuration assistance, please refer to: https://docs.ng
Below options can be optionally configured. The default is usually good enough.
#### webhook_listening_host and webhook_listening_port
#### Webhook listening host and port
Set `webhook_listening_host` and `webhook_listening_port` to change the webhook listening endpoint. If `webhook_listening_host` is left empty, which is the default, the webhook will bind to `0.0.0.0`. The default `webhook_listening_port` is `8888`.
@ -1068,16 +1068,21 @@ webhook_listening_host = ""
webhook_listening_port = "8888"
```
#### webhook_expiration_interval and webhook_renewal_interval
#### Webhook expiration, renewal and retry intervals
Set `webhook_expiration_interval` and `webhook_renewal_interval` to change the frequency of subscription renewal. By default, the webhook asks Microsoft to keep subscriptions alive for 24 hours, and it renews subscriptions when it is less than 12 hours before their expiration.
Set `webhook_expiration_interval` and `webhook_renewal_interval` to change the frequency of subscription renewal. Set `webhook_retry_interval` to change the delay before retrying after an error happens while communicating with Microsoft's subscription endpoints.
By default, the webhook asks Microsoft to keep subscriptions alive for 10 minutes, and it renews subscriptions when it is less than 5 minutes before their expiration. When errors happen, it waits 1 minute before retrying.
```
# Default expiration interval is 24 hours
webhook_expiration_interval = "86400"
# Default expiration interval is 10 minutes
webhook_expiration_interval = "600"
# Default renewal interval is 12 hours
webhook_renewal_interval = "43200"
# Default renewal interval is 5 minutes
webhook_renewal_interval = "300"
# Default retry interval is 1 minute
webhook_retry_interval = "60"
```
## Running 'onedrive' as a system service

33
src/config.d
View file

@ -43,9 +43,9 @@ final class Config
// Default file permission mode
public long defaultFilePermissionMode = 600;
public int configuredFilePermissionMode;
// Bring in v2.5.0 config items
// HTTP Struct items, used for configuring HTTP()
// Curl Timeout Handling
// libcurl dns_cache_timeout timeout
@ -70,8 +70,8 @@ final class Config
immutable int defaultMaxRedirects = 5;
// Specify what IP protocol version should be used when communicating with OneDrive
immutable int defaultIpProtocol = 0; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
this(string confdirOption)
{
@ -106,7 +106,7 @@ final class Config
longValues["min_notify_changes"] = 5;
longValues["monitor_log_frequency"] = 6;
// Number of N sync runs before performing a full local scan of sync_dir
// By default 12 which means every ~60 minutes a full disk scan of sync_dir will occur
// By default 12 which means every ~60 minutes a full disk scan of sync_dir will occur
// 'monitor_interval' * 'monitor_fullscan_frequency' = 3600 = 1 hour
longValues["monitor_fullscan_frequency"] = 12;
// Number of children in a path that is locally removed which will be classified as a 'big data delete'
@ -158,8 +158,9 @@ final class Config
stringValues["webhook_public_url"] = "";
stringValues["webhook_listening_host"] = "";
longValues["webhook_listening_port"] = 8888;
longValues["webhook_expiration_interval"] = 3600 * 24;
longValues["webhook_renewal_interval"] = 3600 * 12;
longValues["webhook_expiration_interval"] = 60 * 10;
longValues["webhook_renewal_interval"] = 60 * 5;
longValues["webhook_retry_interval"] = 60;
// Log to application output running configuration values
boolValues["display_running_config"] = false;
// Configure read-only authentication scope
@ -187,7 +188,7 @@ final class Config
// - Enabling this option will add function processing times to the console output
// - This then enables tracking of where the application is spending most amount of time when processing data when users have questions re performance
boolValues["display_processing_time"] = false;
// HTTPS & CURL Operation Settings
// - Maximum time an operation is allowed to take
// This includes dns resolution, connecting, data transfer, etc.
@ -200,7 +201,7 @@ final class Config
longValues["data_timeout"] = defaultDataTimeout;
// What IP protocol version should be used when communicating with OneDrive
longValues["ip_protocol_version"] = defaultIpProtocol; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
// EXPAND USERS HOME DIRECTORY
// Determine the users home directory.
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
@ -282,7 +283,7 @@ final class Config
writeln("ERROR: ~/.config/onedrive is a file rather than a directory");
}
// Must exit
exit(EXIT_FAILURE);
exit(EXIT_FAILURE);
}
}
@ -405,7 +406,7 @@ final class Config
&longValues["classify_as_big_delete"],
"cleanup-local-files",
"Cleanup additional local files when using --download-only. This will remove local data.",
&boolValues["cleanup_local_files"],
&boolValues["cleanup_local_files"],
"create-directory",
"Create a directory on OneDrive - no sync will be performed.",
&stringValues["create_directory"],
@ -642,11 +643,11 @@ final class Config
// Use exit scopes to shutdown API
return false;
}
// We were able to readText the config file - so, we should be able to open and read it
auto file = File(filename, "r");
string lineBuffer;
// configure scopes
// - failure
scope(failure) {
@ -711,7 +712,7 @@ final class Config
setValueString("skip_dir", configFileSkipDir);
}
}
// --single-directory Strip quotation marks from path
// --single-directory Strip quotation marks from path
// This is an issue when using ONEDRIVE_SINGLE_DIRECTORY with Docker
if (key == "single_directory") {
// Strip quotation marks from provided path
@ -751,7 +752,7 @@ final class Config
if (key == "space_reservation") {
// temp value
ulong tempValue = to!long(c.front.dup);
// a value of 0 needs to be made at least 1MB ..
// a value of 0 needs to be made at least 1MB ..
if (tempValue == 0) {
tempValue = 1;
}
@ -823,7 +824,7 @@ final class Config
}
return configuredFilePermissionMode;
}
void resetSkipToDefaults() {
// reset skip_file and skip_dir to application defaults
// skip_file

139
src/main.d
View file

@ -58,12 +58,12 @@ int main(string[] args)
bool cleanupLocalFilesGlobal = false;
bool synchronizeConfigured = false;
bool invalidSyncExit = false;
// start and finish messages
string startMessage = "Starting a sync with OneDrive";
string finishMessage = "Sync with OneDrive is complete";
string helpMessage = "Please use 'onedrive --help' for further assistance in regards to running this application.";
// hash file permission values
string hashPermissionValue = "600";
auto convertedPermissionValue = parse!long(hashPermissionValue, 8);
@ -150,7 +150,7 @@ int main(string[] args)
"verbose|v+", "Print more details, useful for debugging (repeat for extra debugging)", &log.verbose,
"version", "Print the version and exit", &printVersion
);
// print help and exit
if (opt.helpWanted) {
args ~= "--help";
@ -182,7 +182,7 @@ int main(string[] args)
// Error message already printed
return EXIT_FAILURE;
}
// How was this application started - what options were passed in
log.vdebug("passed in options: ", args);
log.vdebug("note --confdir and --verbose not listed in args");
@ -195,12 +195,12 @@ int main(string[] args)
// update configuration from command line args
cfg.update_from_args(args);
// --resync should be a 'last resort item' .. the user needs to 'accept' to proceed
// --resync should be a 'last resort item' .. the user needs to 'accept' to proceed
if ((cfg.getValueBool("resync")) && (!cfg.getValueBool("display_config"))) {
// what is the risk acceptance?
bool resyncRiskAcceptance = false;
if (!cfg.getValueBool("resync_auth")) {
// need to prompt user
char response;
@ -209,7 +209,7 @@ int main(string[] args)
writeln("This has the potential to overwrite local versions of files with potentially older versions downloaded from OneDrive which can lead to data loss");
writeln("If in-doubt, backup your local data first before proceeding with --resync");
write("\nAre you sure you wish to proceed with --resync? [Y/N] ");
try {
// Attempt to read user response
readf(" %c\n", &response);
@ -217,7 +217,7 @@ int main(string[] args)
// Caught an error
return EXIT_FAILURE;
}
// Evaluate user repsonse
if ((to!string(response) == "y") || (to!string(response) == "Y")) {
// User has accepted --resync risk to proceed
@ -229,7 +229,7 @@ int main(string[] args)
// resync_auth is true
resyncRiskAcceptance = true;
}
// Action based on response
if (!resyncRiskAcceptance){
// --resync risk not accepted
@ -320,7 +320,7 @@ int main(string[] args)
if (exists(configFilePath)) currentConfigHash = computeQuickXorHash(configFilePath);
if (exists(syncListFilePath)) currentSyncListHash = computeQuickXorHash(syncListFilePath);
if (exists(businessSharedFolderFilePath)) currentBusinessSharedFoldersHash = computeQuickXorHash(businessSharedFolderFilePath);
// read the existing hashes for each of the relevant configuration files if they exist
if (exists(configHashFile)) {
try {
@ -507,7 +507,7 @@ int main(string[] args)
if (cfg.getValueBool("list_business_shared_folders")) ignoreResyncRequirement = true;
if ((!cfg.getValueString("get_o365_drive_id").empty)) ignoreResyncRequirement = true;
if ((!cfg.getValueString("get_file_link").empty)) ignoreResyncRequirement = true;
// Do we need to ignore a --resync requirement?
if (!ignoreResyncRequirement) {
// We are not ignoring --requirement
@ -564,7 +564,7 @@ int main(string[] args)
}
// configure databaseFilePathDryRunGlobal
databaseFilePathDryRunGlobal = cfg.databaseFilePathDryRun;
string dryRunShmFile = databaseFilePathDryRunGlobal ~ "-shm";
string dryRunWalFile = databaseFilePathDryRunGlobal ~ "-wal";
// If the dry run database exists, clean this up
@ -681,7 +681,7 @@ int main(string[] args)
// Exit
return EXIT_SUCCESS;
}
// Handle --reauth to re-authenticate the client
if (cfg.getValueBool("reauth")) {
log.vdebug("--reauth requested");
@ -690,20 +690,20 @@ int main(string[] args)
safeRemove(cfg.refreshTokenFilePath);
}
}
// Display current application configuration
if ((cfg.getValueBool("display_config")) || (cfg.getValueBool("display_running_config"))) {
if (cfg.getValueBool("display_running_config")) {
writeln("--------------- Application Runtime Configuration ---------------");
}
// Display application version
writeln("onedrive version = ", strip(import("version")));
// Display all of the pertinent configuration options
writeln("Config path = ", cfg.configDirName);
// Does a config file exist or are we using application defaults
writeln("Config file found in config path = ", exists(configFilePath));
// Is config option drive_id configured?
if (cfg.getValueString("drive_id") != ""){
writeln("Config option 'drive_id' = ", cfg.getValueString("drive_id"));
@ -711,25 +711,25 @@ int main(string[] args)
// Config Options as per 'config' file
writeln("Config option 'sync_dir' = ", syncDir);
// logging and notifications
writeln("Config option 'enable_logging' = ", cfg.getValueBool("enable_logging"));
writeln("Config option 'log_dir' = ", cfg.getValueString("log_dir"));
writeln("Config option 'disable_notifications' = ", cfg.getValueBool("disable_notifications"));
writeln("Config option 'min_notify_changes' = ", cfg.getValueLong("min_notify_changes"));
// skip files and directory and 'matching' policy
writeln("Config option 'skip_dir' = ", cfg.getValueString("skip_dir"));
writeln("Config option 'skip_dir_strict_match' = ", cfg.getValueBool("skip_dir_strict_match"));
writeln("Config option 'skip_file' = ", cfg.getValueString("skip_file"));
writeln("Config option 'skip_dotfiles' = ", cfg.getValueBool("skip_dotfiles"));
writeln("Config option 'skip_symlinks' = ", cfg.getValueBool("skip_symlinks"));
// --monitor sync process options
writeln("Config option 'monitor_interval' = ", cfg.getValueLong("monitor_interval"));
writeln("Config option 'monitor_log_frequency' = ", cfg.getValueLong("monitor_log_frequency"));
writeln("Config option 'monitor_fullscan_frequency' = ", cfg.getValueLong("monitor_fullscan_frequency"));
// sync process and method
writeln("Config option 'read_only_auth_scope' = ", cfg.getValueBool("read_only_auth_scope"));
writeln("Config option 'dry_run' = ", cfg.getValueBool("dry_run"));
@ -751,7 +751,7 @@ int main(string[] args)
writeln("Config option 'sync_dir_permissions' = ", cfg.getValueLong("sync_dir_permissions"));
writeln("Config option 'sync_file_permissions' = ", cfg.getValueLong("sync_file_permissions"));
writeln("Config option 'space_reservation' = ", cfg.getValueLong("space_reservation"));
// curl operations
writeln("Config option 'application_id' = ", cfg.getValueString("application_id"));
writeln("Config option 'azure_ad_endpoint' = ", cfg.getValueString("azure_ad_endpoint"));
@ -765,11 +765,11 @@ int main(string[] args)
writeln("Config option 'connect_timeout' = ", cfg.getValueLong("connect_timeout"));
writeln("Config option 'data_timeout' = ", cfg.getValueLong("data_timeout"));
writeln("Config option 'ip_protocol_version' = ", cfg.getValueLong("ip_protocol_version"));
// Is sync_list configured ?
writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files"));
if (exists(syncListFilePath)){
writeln("Selective sync 'sync_list' configured = true");
writeln("sync_list contents:");
// Output the sync_list contents
@ -781,7 +781,7 @@ int main(string[] args)
}
} else {
writeln("Selective sync 'sync_list' configured = false");
}
// Is business_shared_folders enabled and configured ?
@ -799,7 +799,7 @@ int main(string[] args)
} else {
writeln("Business Shared Folders configured = false");
}
// Are webhooks enabled?
writeln("Config option 'webhook_enabled' = ", cfg.getValueBool("webhook_enabled"));
if (cfg.getValueBool("webhook_enabled")) {
@ -808,12 +808,13 @@ int main(string[] args)
writeln("Config option 'webhook_listening_port' = ", cfg.getValueLong("webhook_listening_port"));
writeln("Config option 'webhook_expiration_interval' = ", cfg.getValueLong("webhook_expiration_interval"));
writeln("Config option 'webhook_renewal_interval' = ", cfg.getValueLong("webhook_renewal_interval"));
writeln("Config option 'webhook_retry_interval' = ", cfg.getValueLong("webhook_retry_interval"));
}
if (cfg.getValueBool("display_running_config")) {
writeln("-----------------------------------------------------------------");
}
// Do we exit? We only exit if --display-config has been used
if (cfg.getValueBool("display_config")) {
return EXIT_SUCCESS;
@ -833,7 +834,7 @@ int main(string[] args)
log.vdebug("Testing if we have exclusive access to local database file");
// Are we the only running instance? Test that we can open the database file path
itemDb = new ItemDatabase(cfg.databaseFilePath);
// did we successfully initialise the database class?
if (!itemDb.isDatabaseInitialised()) {
// no .. destroy class
@ -841,7 +842,7 @@ int main(string[] args)
// exit application
return EXIT_FAILURE;
}
// If we have exclusive access we will not have exited
// destroy access test
destroy(itemDb);
@ -853,7 +854,7 @@ int main(string[] args)
safeRemove(cfg.uploadStateFilePath);
}
}
// Test if OneDrive service can be reached, exit if it cant be reached
log.vdebug("Testing network to ensure network connectivity to Microsoft OneDrive Service");
online = testNetwork(cfg);
@ -911,13 +912,13 @@ int main(string[] args)
}
}
}
// Check application version and Initialize OneDrive API, check for authorization
if (online) {
// Check Application Version
log.vlog("Checking Application Version ...");
checkApplicationVersion();
// we can only initialise if we are online
log.vlog("Initializing the OneDrive API ...");
oneDrive = new OneDriveApi(cfg);
@ -956,7 +957,7 @@ int main(string[] args)
emptyParameter = "--destination-directory";
dataParameter = "--source-directory";
}
log.error("ERROR: " ~ dataParameter ~ " was passed in without also using " ~ emptyParameter);
log.error("ERROR: " ~ dataParameter ~ " was passed in without also using " ~ emptyParameter);
}
// Use exit scopes to shutdown API
writeln();
@ -1023,7 +1024,7 @@ int main(string[] args)
log.vdebug("Using database file: ", asNormalizedPath(cfg.databaseFilePath));
itemDb = new ItemDatabase(cfg.databaseFilePath);
}
// did we successfully initialise the database class?
if (!itemDb.isDatabaseInitialised()) {
// no .. destroy class
@ -1154,14 +1155,14 @@ int main(string[] args)
// performing this action could have undesirable effects .. the user must accept this risk
// what is the risk acceptance?
bool resyncRiskAcceptance = false;
// need to prompt user
char response;
// warning message
writeln("\nThe use of --force-sync will reconfigure the application to use defaults. This may have untold and unknown future impacts.");
writeln("By proceeding in using this option you accept any impacts including any data loss that may occur as a result of using --force-sync.");
write("\nAre you sure you wish to proceed with --force-sync [Y/N] ");
try {
// Attempt to read user response
readf(" %c\n", &response);
@ -1169,7 +1170,7 @@ int main(string[] args)
// Caught an error
return EXIT_FAILURE;
}
// Evaluate user repsonse
if ((to!string(response) == "y") || (to!string(response) == "Y")) {
// User has accepted --force-sync risk to proceed
@ -1177,7 +1178,7 @@ int main(string[] args)
// Are you sure you wish .. does not use writeln();
write("\n");
}
// Action based on response
if (!resyncRiskAcceptance){
// --force-sync not accepted
@ -1188,7 +1189,7 @@ int main(string[] args)
cfg.resetSkipToDefaults();
// update sync engine regex with reset defaults
selectiveSync.setDirMask(cfg.getValueString("skip_dir"));
selectiveSync.setFileMask(cfg.getValueString("skip_file"));
selectiveSync.setFileMask(cfg.getValueString("skip_file"));
}
}
@ -1248,7 +1249,7 @@ int main(string[] args)
log.log("WARNING: Local data loss MAY occur in this scenario.");
sync.setBypassDataPreservation();
}
// Do we configure to clean up local files if using --download-only ?
if ((cfg.getValueBool("download_only")) && (cfg.getValueBool("cleanup_local_files"))) {
// --download-only and --cleanup-local-files were passed in
@ -1270,13 +1271,13 @@ int main(string[] args)
sync.setNationalCloudDeployment();
}
}
// Are we forcing to use /children scan instead of /delta to simulate National Cloud Deployment use of /children?
if (cfg.getValueBool("force_children_scan")) {
log.log("Forcing client to use /children scan rather than /delta to simulate National Cloud Deployment use of /children");
sync.setNationalCloudDeployment();
}
// Do we need to display the function processing timing
if (cfg.getValueBool("display_processing_time")) {
log.log("Forcing client to display function processing times");
@ -1324,14 +1325,14 @@ int main(string[] args)
// --create-share-link - Are we createing a shareable link for an existing file on OneDrive?
if (cfg.getValueString("create_share_link") != "") {
// Query OneDrive for the file, and if valid, create a shareable link for the file
// By default, the shareable link will be read-only.
// If the user adds:
// --with-editing-perms
// By default, the shareable link will be read-only.
// If the user adds:
// --with-editing-perms
// this will create a writeable link
bool writeablePermissions = cfg.getValueBool("with_editing_perms");
sync.createShareableLinkForFile(cfg.getValueString("create_share_link"), writeablePermissions);
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS;
@ -1345,7 +1346,7 @@ int main(string[] args)
// Use exit scopes to shutdown API
return EXIT_SUCCESS;
}
// --modified-by - Are we listing the modified-by details of a provided path?
if (cfg.getValueString("modified_by") != "") {
// Query OneDrive for the file link
@ -1379,7 +1380,7 @@ int main(string[] args)
log.error("ERROR: Unsupported account type for syncing OneDrive Business Shared Folders");
}
}
// Ensure that the value stored for cfg.getValueString("single_directory") does not contain any extra quotation marks
if (cfg.getValueString("single_directory") != ""){
string originalSingleDirectoryValue = cfg.getValueString("single_directory");
@ -1409,7 +1410,7 @@ int main(string[] args)
if (online) {
// set flag for exit scope
synchronizeConfigured = true;
// Check user entry for local path - the above chdir means we are already in ~/OneDrive/ thus singleDirectory is local to this path
if (cfg.getValueString("single_directory") != "") {
// Does the directory we want to sync actually exist?
@ -1521,7 +1522,7 @@ int main(string[] args)
immutable long fullScanFrequency = cfg.getValueLong("monitor_fullscan_frequency");
MonoTime lastCheckTime = MonoTime.currTime();
MonoTime lastGitHubCheckTime = MonoTime.currTime();
long logMonitorCounter = 0;
long fullScanCounter = 0;
// set fullScanRequired to true so that at application startup we perform a full walk
@ -1589,7 +1590,7 @@ int main(string[] args)
// update when we have performed this check
lastGitHubCheckTime = MonoTime.currTime();
}
// monitor sync loop
logOutputMessage = "################################################## NEW LOOP ##################################################";
if (displaySyncOptions) {
@ -1647,7 +1648,7 @@ int main(string[] args)
try {
// performance timing
SysTime startSyncProcessingTime = Clock.currTime();
// perform a --monitor sync
if ((cfg.getValueLong("verbose") > 0) || (logMonitorCounter == logInterval) || (fullScanRequired) ) {
// log to console and log file if enabled
@ -1695,25 +1696,25 @@ int main(string[] args)
}
// performSync complete, set lastCheckTime to current time
lastCheckTime = MonoTime.currTime();
// Display memory details before cleanup
if (displayMemoryUsage) log.displayMemoryUsagePreGC();
// Perform Garbage Cleanup
GC.collect();
// Display memory details after cleanup
if (displayMemoryUsage) log.displayMemoryUsagePostGC();
// If we did a full scan, make sure we merge the conents of the WAL and SHM to disk
if (fullScanRequired) {
// Write WAL and SHM data to file for this loop
log.vdebug("Merge contents of WAL and SHM files into main database file");
itemDb.performVacuum();
}
// reset fullScanRequired and syncListConfiguredFullScanOverride
fullScanRequired = false;
if (syncListConfigured) syncListConfiguredFullScanOverride = false;
// monitor loop complete
logOutputMessage = "################################################ LOOP COMPLETE ###############################################";
@ -1840,13 +1841,13 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// OneDrive First
if (logLevel < MONITOR_LOG_QUIET) log.log("Syncing changes from selected OneDrive path ...");
sync.applyDifferencesSingleDirectory(remotePath);
// Is this a --download-only --cleanup-local-files request?
// If yes, scan for local changes - but --cleanup-local-files is being used, a further flag will trigger local file deletes rather than attempt to upload files to OneDrive
if (cleanupLocalFiles) {
// --download-only and --cleanup-local-files were passed in
log.log("Searching local filesystem for extra files and folders which need to be removed");
sync.scanForDifferencesFilesystemScan(localPath);
sync.scanForDifferencesFilesystemScan(localPath);
} else {
// is this a --download-only request?
if (!downloadOnly) {
@ -1894,13 +1895,13 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
log.vdebug(syncCallLogOutput);
}
sync.applyDifferences(false);
// Is this a --download-only --cleanup-local-files request?
// If yes, scan for local changes - but --cleanup-local-files is being used, a further flag will trigger local file deletes rather than attempt to upload files to OneDrive
if (cleanupLocalFiles) {
// --download-only and --cleanup-local-files were passed in
log.log("Searching local filesystem for extra files and folders which need to be removed");
sync.scanForDifferencesFilesystemScan(localPath);
sync.scanForDifferencesFilesystemScan(localPath);
} else {
// is this a --download-only request?
if (!downloadOnly) {
@ -1916,14 +1917,14 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
SysTime startIntegrityCheckProcessingTime = Clock.currTime();
if (sync.getPerformanceProcessingOutput()) {
// performance timing for DB and file system integrity check - start
writeln("============================================================");
writeln("Start Integrity Check Processing Time: ", startIntegrityCheckProcessingTime);
}
// What sort of local scan do we want to do?
// In --monitor mode, when performing the DB scan, a race condition occurs where by if a file or folder is moved during this process
// the inotify event is discarded once performSync() is finished (see m.update(false) above), so these events need to be handled
@ -1936,12 +1937,12 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
} else {
// --monitor in use
// Use individual calls with inotify checks between to avoid a race condition between these 2 functions
// Database scan integrity check to compare DB data vs actual content on disk to ensure what we think is local, is local
// Database scan integrity check to compare DB data vs actual content on disk to ensure what we think is local, is local
// and that the data 'hash' as recorded in the DB equals the hash of the actual content
// This process can be extremely expensive time and CPU processing wise
//
// fullScanRequired is set to TRUE when the application starts up, or the config option 'monitor_fullscan_frequency' count is reached
// By default, 'monitor_fullscan_frequency' = 12, and 'monitor_interval' = 300, meaning that by default, a full database consistency check
// By default, 'monitor_fullscan_frequency' = 12, and 'monitor_interval' = 300, meaning that by default, a full database consistency check
// is done once an hour.
//
// To change this behaviour adjust 'monitor_interval' and 'monitor_fullscan_frequency' to desired values in the application config file
@ -1954,14 +1955,14 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
log.vdebug("NOT performing Database Integrity Check .. fullScanRequired = FALSE");
m.update(true);
}
// Filesystem walk to find new files not uploaded
log.vdebug("Searching local filesystem for new data");
sync.scanForDifferencesFilesystemScan(localPath);
// handle any inotify events that occured 'whilst' we were scanning the local filesystem
m.update(true);
}
SysTime endIntegrityCheckProcessingTime = Clock.currTime();
if (sync.getPerformanceProcessingOutput()) {
// performance timing for DB and file system integrity check - finish
@ -1969,7 +1970,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
writeln("Elapsed Function Processing Time: ", (endIntegrityCheckProcessingTime - startIntegrityCheckProcessingTime));
writeln("============================================================");
}
// At this point, all OneDrive changes / local changes should be uploaded and in sync
// This MAY not be the case when using sync_list, thus a full walk of OneDrive ojects is required

251
src/onedrive.d
View file

@ -201,8 +201,8 @@ final class OneDriveApi
private SysTime accessTokenExpiration;
private HTTP http;
private OneDriveWebhook webhook;
private SysTime subscriptionExpiration;
private Duration subscriptionExpirationInterval, subscriptionRenewalInterval;
private SysTime subscriptionExpiration, subscriptionLastErrorAt;
private Duration subscriptionExpirationInterval, subscriptionRenewalInterval, subscriptionRetryInternal;
private string notificationUrl;
// if true, every new access token is printed
@ -240,7 +240,7 @@ final class OneDriveApi
if (cfg.getValueBool("debug_https")) {
http.verbose = true;
.debugResponse = true;
// Output what options we are using so that in the debug log this can be tracked
log.vdebug("http.dnsTimeout = ", cfg.getValueLong("dns_timeout"));
log.vdebug("http.connectTimeout = ", cfg.getValueLong("connect_timeout"));
@ -475,7 +475,7 @@ final class OneDriveApi
// https://curl.se/libcurl/c/CURLOPT_FORBID_REUSE.html
// Ensure that we ARE reusing connections - setting to 0 ensures that we are reusing connections
http.handle.set(CurlOption.forbid_reuse,0);
// Do we set the dryRun handlers?
if (cfg.getValueBool("dry_run")) {
.dryRun = true;
@ -485,8 +485,10 @@ final class OneDriveApi
}
subscriptionExpiration = Clock.currTime(UTC());
subscriptionLastErrorAt = SysTime.fromUnixTime(0);
subscriptionExpirationInterval = dur!"seconds"(cfg.getValueLong("webhook_expiration_interval"));
subscriptionRenewalInterval = dur!"seconds"(cfg.getValueLong("webhook_renewal_interval"));
subscriptionRetryInternal = dur!"seconds"(cfg.getValueLong("webhook_retry_interval"));
notificationUrl = cfg.getValueString("webhook_public_url");
}
@ -576,7 +578,7 @@ final class OneDriveApi
// read-write authentication scopes will be used (default)
authScope = "&scope=Files.ReadWrite%20Files.ReadWrite.All%20Sites.ReadWrite.All%20offline_access&response_type=code&prompt=login&redirect_uri=";
}
string url = authUrl ~ "?client_id=" ~ clientId ~ authScope ~ redirectUrl;
string authFilesString = cfg.getValueString("auth_files");
string authResponseString = cfg.getValueString("auth_response");
@ -586,7 +588,7 @@ final class OneDriveApi
string[] authFiles = authFilesString.split(":");
string authUrl = authFiles[0];
string responseUrl = authFiles[1];
try {
// Try and write out the auth URL to the nominated file
auto authUrlFile = File(authUrl, "w");
@ -598,7 +600,7 @@ final class OneDriveApi
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
return false;
}
while (!exists(responseUrl)) {
Thread.sleep(dur!("msecs")(100));
}
@ -636,7 +638,7 @@ final class OneDriveApi
redeemToken(c.front);
return true;
}
string getSiteSearchUrl()
{
// Return the actual siteSearchUrl being used and/or requested when performing 'siteQuery = onedrive.o365SiteSearch(nextLink);' call
@ -1004,17 +1006,25 @@ final class OneDriveApi
spawn(&OneDriveWebhook.serve);
}
if (!hasValidSubscription()) {
createSubscription();
} else if (isSubscriptionUpForRenewal()) {
try {
auto elapsed = Clock.currTime(UTC()) - subscriptionLastErrorAt;
if (elapsed < subscriptionRetryInternal) {
return;
}
try {
if (!hasValidSubscription()) {
createSubscription();
} else if (isSubscriptionUpForRenewal()) {
renewSubscription();
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
log.log("The subscription is not found on the server. Recreating subscription ...");
createSubscription();
}
}
} catch (OneDriveException e) {
logSubscriptionError(e);
subscriptionLastErrorAt = Clock.currTime(UTC());
log.log("Will retry creating or renewing subscription in ", subscriptionRetryInternal);
} catch (JSONException e) {
log.error("ERROR: Unexpected JSON error: ", e.msg);
subscriptionLastErrorAt = Clock.currTime(UTC());
log.log("Will retry creating or renewing subscription in ", subscriptionRetryInternal);
}
}
@ -1039,7 +1049,7 @@ final class OneDriveApi
} else {
resourceItem = "/me/drive/root";
}
// create JSON request to create webhook subscription
const JSONValue request = [
"changeType": "updated",
@ -1049,22 +1059,56 @@ final class OneDriveApi
"clientState": randomUUID().toString()
];
http.addRequestHeader("Content-Type", "application/json");
JSONValue response;
try {
response = post(url, request.toString());
} catch (OneDriveException e) {
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
// We need to exit here, user needs to fix issue
log.error("ERROR: Unable to initialize subscriptions for updates. Please fix this issue.");
shutdown();
exit(-1);
}
JSONValue response = post(url, request.toString());
// Save important subscription metadata including id and expiration
subscriptionId = response["id"].str;
subscriptionExpiration = SysTime.fromISOExtString(response["expirationDateTime"].str);
// Save important subscription metadata including id and expiration
subscriptionId = response["id"].str;
subscriptionExpiration = SysTime.fromISOExtString(response["expirationDateTime"].str);
log.log("Created new subscription ", subscriptionId, " with expiration: ", subscriptionExpiration.toISOExtString());
} catch (OneDriveException e) {
if (e.httpStatusCode == 409) {
// Take over an existing subscription on HTTP 409.
//
// Sample 409 error:
// {
// "error": {
// "code": "ObjectIdentifierInUse",
// "innerError": {
// "client-request-id": "615af209-467a-4ab7-8eff-27c1d1efbc2d",
// "date": "2023-09-26T09:27:45",
// "request-id": "615af209-467a-4ab7-8eff-27c1d1efbc2d"
// },
// "message": "Subscription Id c0bba80e-57a3-43a7-bac2-e6f525a76e7c already exists for the requested combination"
// }
// }
// Make sure the error code is "ObjectIdentifierInUse"
try {
if (e.error["error"]["code"].str != "ObjectIdentifierInUse") {
throw e;
}
} catch (JSONException jsonEx) {
throw e;
}
// Extract the existing subscription id from the error message
import std.regex;
auto idReg = ctRegex!(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", "i");
auto m = matchFirst(e.error["error"]["message"].str, idReg);
if (!m) {
throw e;
}
// Save the subscription id and renew it immediately since we don't know the expiration timestamp
subscriptionId = m[0];
log.log("Found existing subscription ", subscriptionId);
renewSubscription();
} else {
throw e;
}
}
}
private void renewSubscription() {
@ -1077,10 +1121,23 @@ final class OneDriveApi
"expirationDateTime": expirationDateTime.toISOExtString()
];
http.addRequestHeader("Content-Type", "application/json");
JSONValue response = patch(url, request.toString());
// Update subscription expiration from the response
subscriptionExpiration = SysTime.fromISOExtString(response["expirationDateTime"].str);
try {
JSONValue response = patch(url, request.toString());
// Update subscription expiration from the response
subscriptionExpiration = SysTime.fromISOExtString(response["expirationDateTime"].str);
log.log("Renewed subscription ", subscriptionId, " with expiration: ", subscriptionExpiration.toISOExtString());
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
log.log("The subscription is not found on the server. Recreating subscription ...");
subscriptionId = null;
subscriptionExpiration = Clock.currTime(UTC());
createSubscription();
} else {
throw e;
}
}
}
private void deleteSubscription() {
@ -1094,6 +1151,100 @@ final class OneDriveApi
log.log("Deleted subscription");
}
private void logSubscriptionError(OneDriveException e) {
if (e.httpStatusCode == 400) {
// Log known 400 error where Microsoft cannot get a 200 OK from the webhook endpoint
//
// Sample 400 error:
// {
// "error": {
// "code": "InvalidRequest",
// "innerError": {
// "client-request-id": "<uuid>",
// "date": "<timestamp>",
// "request-id": "<uuid>"
// },
// "message": "Subscription validation request failed. Notification endpoint must respond with 200 OK to validation request."
// }
// }
try {
if (e.error["error"]["code"].str == "InvalidRequest") {
import std.regex;
auto msgReg = ctRegex!(r"Subscription validation request failed", "i");
auto m = matchFirst(e.error["error"]["message"].str, msgReg);
if (m) {
log.error("ERROR: Cannot create or renew subscription: Microsoft did not get 200 OK from the webhook endpoint.");
return;
}
}
} catch (JSONException) {
// fallthrough
}
} else if (e.httpStatusCode == 401) {
// Log known 401 error where authentication failed
//
// Sample 401 error:
// {
// "error": {
// "code": "ExtensionError",
// "innerError": {
// "client-request-id": "<uuid>",
// "date": "<timestamp>",
// "request-id": "<uuid>"
// },
// "message": "Operation: Create; Exception: [Status Code: Unauthorized; Reason: Authentication failed]"
// }
// }
try {
if (e.error["error"]["code"].str == "ExtensionError") {
import std.regex;
auto msgReg = ctRegex!(r"Authentication failed", "i");
auto m = matchFirst(e.error["error"]["message"].str, msgReg);
if (m) {
log.error("ERROR: Cannot create or renew subscription: Authentication failed.");
return;
}
}
} catch (JSONException) {
// fallthrough
}
} else if (e.httpStatusCode == 403) {
// Log known 403 error where the number of subscriptions on item has exceeded limit
//
// Sample 403 error:
// {
// "error": {
// "code": "ExtensionError",
// "innerError": {
// "client-request-id": "<uuid>",
// "date": "<timestamp>",
// "request-id": "<uuid>"
// },
// "message": "Operation: Create; Exception: [Status Code: Forbidden; Reason: Number of subscriptions on item has exceeded limit]"
// }
// }
try {
if (e.error["error"]["code"].str == "ExtensionError") {
import std.regex;
auto msgReg = ctRegex!(r"Number of subscriptions on item has exceeded limit", "i");
auto m = matchFirst(e.error["error"]["message"].str, msgReg);
if (m) {
log.error("ERROR: Cannot create or renew subscription: Number of subscriptions has exceeded limit.");
return;
}
}
} catch (JSONException) {
// fallthrough
}
}
// Log detailed message for unknown errors
log.error("ERROR: Cannot create or renew subscription.");
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
}
private void redeemToken(const(char)[] authCode)
{
const(char)[] postData =
@ -1148,7 +1299,7 @@ final class OneDriveApi
}
}
}
if ("access_token" in response){
accessToken = "bearer " ~ response["access_token"].str();
refreshToken = response["refresh_token"].str();
@ -1229,11 +1380,11 @@ final class OneDriveApi
{
// Threshold for displaying download bar
long thresholdFileSize = 4 * 2^^20; // 4 MiB
// To support marking of partially-downloaded files,
// To support marking of partially-downloaded files,
string originalFilename = filename;
string downloadFilename = filename ~ ".partial";
// open downloadFilename as write in binary mode
auto file = File(downloadFilename, "wb");
@ -1302,7 +1453,7 @@ final class OneDriveApi
// Data Received = 13685777
// Expected Total = 52428800
// Percent Complete = 26
if (cfg.getValueLong("rate_limit") > 0) {
// User configured rate limit
// How much data should be in each segment to qualify for 5%
@ -1314,7 +1465,7 @@ final class OneDriveApi
if ((dlnow > thisSegmentData) && (dlnow < nextSegmentData) && (previousProgressPercent != currentDLPercent) || (dlnow == dltotal)) {
// Downloaded data equals approx 5%
log.vdebug("Incrementing Progress Bar using calculated 5% of data received");
// Downloading 50% |oooooooooooooooooooo | ETA 00:01:40
// Downloading 50% |oooooooooooooooooooo | ETA 00:01:40
// increment progress bar
p.next();
// update values
@ -1328,7 +1479,7 @@ final class OneDriveApi
if ((isIdentical(fmod(currentDLPercent, percentCheck), 0.0)) && (previousProgressPercent != currentDLPercent)) {
// currentDLPercent matches a new increment
log.vdebug("Incrementing Progress Bar using fmod match");
// Downloading 50% |oooooooooooooooooooo | ETA 00:01:40
// Downloading 50% |oooooooooooooooooooo | ETA 00:01:40
// increment progress bar
p.next();
// update values
@ -1491,7 +1642,7 @@ final class OneDriveApi
log.vdebug("onedrive.perform() Generated a OneDrive CurlException");
auto errorArray = splitLines(e.msg);
string errorMessage = errorArray[0];
// what is contained in the curl error message?
if (canFind(errorMessage, "Couldn't connect to server on handle") || canFind(errorMessage, "Couldn't resolve host name on handle") || canFind(errorMessage, "Timeout was reached on handle")) {
// This is a curl timeout
@ -1505,12 +1656,12 @@ final class OneDriveApi
int timestampAlign = 0;
bool retrySuccess = false;
SysTime currentTime;
// what caused the initial curl exception?
if (canFind(errorMessage, "Couldn't connect to server on handle")) log.vdebug("Unable to connect to server - HTTPS access blocked?");
if (canFind(errorMessage, "Couldn't resolve host name on handle")) log.vdebug("Unable to resolve server - DNS access blocked?");
if (canFind(errorMessage, "Timeout was reached on handle")) log.vdebug("A timeout was triggered - data too slow, no response ... use --debug-https to diagnose further");
while (!retrySuccess){
try {
// configure libcurl to perform a fresh connection
@ -1540,16 +1691,16 @@ final class OneDriveApi
if (canFind(e.msg, "Couldn't connect to server on handle")) {
log.log(" - Check HTTPS access or Firewall Rules");
timestampAlign = 9;
}
}
if (canFind(e.msg, "Couldn't resolve host name on handle")) {
log.log(" - Check DNS resolution or Firewall Rules");
timestampAlign = 0;
}
// increment backoff interval
backoffInterval++;
int thisBackOffInterval = retryAttempts*backoffInterval;
// display retry information
currentTime.fracSecs = Duration.zero;
auto timeString = currentTime.toString();
@ -1558,13 +1709,13 @@ final class OneDriveApi
if (thisBackOffInterval > maxBackoffInterval) {
thisBackOffInterval = maxBackoffInterval;
}
// detail when the next attempt will be tried
// factor in the delay for curl to generate the exception - otherwise the next timestamp appears to be 'out' even though technically correct
auto nextRetry = currentTime + dur!"seconds"(thisBackOffInterval) + dur!"seconds"(timestampAlign);
log.vlog(" Next retry in approx: ", (thisBackOffInterval + timestampAlign), " seconds");
log.vlog(" Next retry approx: ", nextRetry);
// thread sleep
Thread.sleep(dur!"seconds"(thisBackOffInterval));
}
@ -1585,7 +1736,7 @@ final class OneDriveApi
// Some other error was returned
log.error(" Error Message: ", errorMessage);
log.error(" Calling Function: ", getFunctionName!({}));
// Was this a curl initialization error?
if (canFind(errorMessage, "Failed initialization on handle")) {
// initialization error ... prevent a run-away process if we have zero disk space

96
src/util.d
View file

@ -96,7 +96,7 @@ Regex!char wild2regex(const(char)[] pattern)
break;
case ' ':
str ~= "\\s+";
break;
break;
case '/':
str ~= "\\/";
break;
@ -129,10 +129,10 @@ bool testNetwork(Config cfg)
http.dataTimeout = (dur!"seconds"(cfg.getValueLong("data_timeout")));
// maximum time any operation is allowed to take
// This includes dns resolution, connecting, data transfer, etc.
http.operationTimeout = (dur!"seconds"(cfg.getValueLong("operation_timeout")));
http.operationTimeout = (dur!"seconds"(cfg.getValueLong("operation_timeout")));
// What IP protocol version should be used when using Curl - IPv4 & IPv6, IPv4 or IPv6
http.handle.set(CurlOption.ipresolve,cfg.getValueLong("ip_protocol_version")); // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
// HTTP connection test method
http.method = HTTP.Method.head;
// Attempt to contact the Microsoft Online Service
@ -190,7 +190,7 @@ bool isValidName(string path)
// Restriction and limitations about windows naming files
// https://msdn.microsoft.com/en-us/library/aa365247
// https://support.microsoft.com/en-us/help/3125202/restrictions-and-limitations-when-you-sync-files-and-folders
// allow root item
if (path == ".") {
return true;
@ -210,7 +210,7 @@ bool isValidName(string path)
);
auto m = match(itemName, invalidNameReg);
matched = m.empty;
// Additional explicit validation checks
if (itemName == ".lock") {matched = false;}
if (itemName == "desktop.ini") {matched = false;}
@ -218,7 +218,7 @@ bool isValidName(string path)
if(canFind(itemName, "_vti_")){matched = false;}
// Item name cannot equal '~'
if (itemName == "~") {matched = false;}
// return response
return matched;
}
@ -229,7 +229,7 @@ bool containsBadWhiteSpace(string path)
if (path == ".") {
return true;
}
// https://github.com/abraunegg/onedrive/issues/35
// Issue #35 presented an interesting issue where the filename contained a newline item
// 'State-of-the-art, challenges, and open issues in the integration of Internet of'$'\n''Things and Cloud Computing.pdf'
@ -237,7 +237,7 @@ bool containsBadWhiteSpace(string path)
// /v1.0/me/drive/root:/.%2FState-of-the-art%2C%20challenges%2C%20and%20open%20issues%20in%20the%20integration%20of%20Internet%20of%0AThings%20and%20Cloud%20Computing.pdf
// The '$'\n'' is translated to %0A which causes the OneDrive query to fail
// Check for the presence of '%0A' via regex
string itemName = encodeComponent(baseName(path));
auto invalidWhitespaceReg =
ctRegex!(
@ -254,12 +254,12 @@ bool containsASCIIHTMLCodes(string path)
// If a filename contains ASCII HTML codes, regardless of if it gets encoded, it generates an error
// Check if the filename contains an ASCII HTML code sequence
auto invalidASCIICode =
auto invalidASCIICode =
ctRegex!(
// Check to see if &#XXXX is in the filename
`(?:&#|&#[0-9][0-9]|&#[0-9][0-9][0-9]|&#[0-9][0-9][0-9][0-9])`
);
auto m = match(path, invalidASCIICode);
return m.empty;
}
@ -276,14 +276,15 @@ void displayOneDriveErrorMessage(string message, string callingFunction)
// extra debug
log.vdebug("Raw Error Data: ", message);
log.vdebug("JSON Message: ", errorMessage);
// What is the reason for the error
if (errorMessage.type() == JSONType.object) {
// configure the error reason
string errorReason;
string errorCode;
string requestDate;
string requestId;
// set the reason for the error
try {
// Use error_description as reason
@ -291,15 +292,15 @@ void displayOneDriveErrorMessage(string message, string callingFunction)
} catch (JSONException e) {
// we dont want to do anything here
}
// set the reason for the error
try {
// Use ["error"]["message"] as reason
errorReason = errorMessage["error"]["message"].str;
errorReason = errorMessage["error"]["message"].str;
} catch (JSONException e) {
// we dont want to do anything here
}
// Display the error reason
if (errorReason.startsWith("<!DOCTYPE")) {
// a HTML Error Reason was given
@ -309,34 +310,43 @@ void displayOneDriveErrorMessage(string message, string callingFunction)
// a non HTML Error Reason was given
log.error(" Error Reason: ", errorReason);
}
// Get the error code if available
try {
// Use ["error"]["code"] as code
errorCode = errorMessage["error"]["code"].str;
} catch (JSONException e) {
// we dont want to do anything here
}
// Get the date of request if available
try {
// Use ["error"]["innerError"]["date"] as date
requestDate = errorMessage["error"]["innerError"]["date"].str;
requestDate = errorMessage["error"]["innerError"]["date"].str;
} catch (JSONException e) {
// we dont want to do anything here
}
// Get the request-id if available
try {
// Use ["error"]["innerError"]["request-id"] as request-id
requestId = errorMessage["error"]["innerError"]["request-id"].str;
requestId = errorMessage["error"]["innerError"]["request-id"].str;
} catch (JSONException e) {
// we dont want to do anything here
}
// Display the date and request id if available
// Display the error code, date and request id if available
if (errorCode != "") log.error(" Error Code: ", errorCode);
if (requestDate != "") log.error(" Error Timestamp: ", requestDate);
if (requestId != "") log.error(" API Request ID: ", requestId);
}
// Where in the code was this error generated
log.vlog(" Calling Function: ", callingFunction);
}
// Parse and display error message received from the local file system
void displayFileSystemErrorMessage(string message, string callingFunction)
void displayFileSystemErrorMessage(string message, string callingFunction)
{
writeln();
log.error("ERROR: The local file system returned an error with the following message:");
@ -367,14 +377,14 @@ JSONValue getLatestReleaseDetails() {
JSONValue versionDetails;
string latestTag;
string publishedDate;
try {
content = get("https://api.github.com/repos/abraunegg/onedrive/releases/latest");
} catch (CurlException e) {
// curl generated an error - meaning we could not query GitHub
log.vdebug("Unable to query GitHub for latest release");
}
try {
githubLatest = content.parseJSON();
} catch (JSONException e) {
@ -382,7 +392,7 @@ JSONValue getLatestReleaseDetails() {
log.vdebug("Unable to parse GitHub JSON response");
githubLatest = parseJSON("{}");
}
// githubLatest has to be a valid JSON object
if (githubLatest.type() == JSONType.object){
// use the returned tag_name
@ -409,15 +419,15 @@ JSONValue getLatestReleaseDetails() {
log.vdebug("Invalid JSON Object. Setting GitHub 'tag_name' release version to 0.0.0");
latestTag = "0.0.0";
log.vdebug("Invalid JSON Object. Setting GitHub 'published_at' date to 2018-07-18T18:00:00Z");
publishedDate = "2018-07-18T18:00:00Z";
publishedDate = "2018-07-18T18:00:00Z";
}
// return the latest github version and published date as our own JSON
versionDetails = [
"latestTag": JSONValue(latestTag),
"publishedDate": JSONValue(publishedDate)
];
// return JSON
return versionDetails;
}
@ -431,14 +441,14 @@ JSONValue getCurrentVersionDetails(string thisVersion) {
JSONValue versionDetails;
string versionTag = "v" ~ thisVersion;
string publishedDate;
try {
content = get("https://api.github.com/repos/abraunegg/onedrive/releases");
} catch (CurlException e) {
// curl generated an error - meaning we could not query GitHub
log.vdebug("Unable to query GitHub for release details");
}
try {
githubDetails = content.parseJSON();
} catch (JSONException e) {
@ -446,7 +456,7 @@ JSONValue getCurrentVersionDetails(string thisVersion) {
log.vdebug("Unable to parse GitHub JSON response");
githubDetails = parseJSON("{}");
}
// githubDetails has to be a valid JSON array
if (githubDetails.type() == JSONType.array){
foreach (searchResult; githubDetails.array) {
@ -458,7 +468,7 @@ JSONValue getCurrentVersionDetails(string thisVersion) {
publishedDate = searchResult["published_at"].str;
}
}
if (publishedDate.empty) {
// empty .. no version match ?
// set to v2.0.0 release date
@ -468,15 +478,15 @@ JSONValue getCurrentVersionDetails(string thisVersion) {
} else {
// JSONValue is not an Array
log.vdebug("Invalid JSON Array. Setting GitHub 'published_at' date to 2018-07-18T18:00:00Z");
publishedDate = "2018-07-18T18:00:00Z";
publishedDate = "2018-07-18T18:00:00Z";
}
// return the latest github version and published date as our own JSON
versionDetails = [
"versionTag": JSONValue(thisVersion),
"publishedDate": JSONValue(publishedDate)
];
// return JSON
return versionDetails;
}
@ -489,7 +499,7 @@ void checkApplicationVersion() {
SysTime publishedDate = SysTime.fromISOExtString(latestVersionDetails["publishedDate"].str).toUTC();
SysTime releaseGracePeriod = publishedDate;
SysTime currentTime = Clock.currTime().toUTC();
// drop fraction seconds
publishedDate.fracSecs = Duration.zero;
currentTime.fracSecs = Duration.zero;
@ -500,20 +510,20 @@ void checkApplicationVersion() {
// what is this clients version?
auto currentVersionArray = strip(strip(import("version"), "v")).split("-");
string applicationVersion = currentVersionArray[0];
// debug output
log.vdebug("applicationVersion: ", applicationVersion);
log.vdebug("latestVersion: ", latestVersion);
log.vdebug("publishedDate: ", publishedDate);
log.vdebug("currentTime: ", currentTime);
log.vdebug("releaseGracePeriod: ", releaseGracePeriod);
// display details if not current
// is application version is older than available on GitHub
if (applicationVersion != latestVersion) {
// application version is different
bool displayObsolete = false;
// what warning do we present?
if (applicationVersion < latestVersion) {
// go get this running version details
@ -521,12 +531,12 @@ void checkApplicationVersion() {
SysTime thisVersionPublishedDate = SysTime.fromISOExtString(thisVersionDetails["publishedDate"].str).toUTC();
thisVersionPublishedDate.fracSecs = Duration.zero;
log.vdebug("thisVersionPublishedDate: ", thisVersionPublishedDate);
// the running version grace period is its release date + 1 month
SysTime thisVersionReleaseGracePeriod = thisVersionPublishedDate;
thisVersionReleaseGracePeriod = thisVersionReleaseGracePeriod.add!"months"(1);
log.vdebug("thisVersionReleaseGracePeriod: ", thisVersionReleaseGracePeriod);
// is this running version obsolete ?
if (!displayObsolete) {
// if releaseGracePeriod > currentTime
@ -539,7 +549,7 @@ void checkApplicationVersion() {
displayObsolete = true;
}
}
// display version response
writeln();
if (!displayObsolete) {