From af86b55b35857dc18ec24f19a3d906671c905b3c Mon Sep 17 00:00:00 2001 From: abraunegg Date: Tue, 21 Jun 2022 11:59:02 +1000 Subject: [PATCH 1/4] Release 2.4.20 Dev Prep (#2013) * Release 2.4.20 Dev Prep * Clarify RHEL 7.x requirements * Add RHEL 9.x after confirming build on platform --- configure | 18 +++++++++--------- configure.ac | 2 +- docs/INSTALL.md | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/configure b/configure index addc294b..e4cc54d2 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for onedrive v2.4.19. +# Generated by GNU Autoconf 2.69 for onedrive v2.4.20-dev. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='onedrive' PACKAGE_TARNAME='onedrive' -PACKAGE_VERSION='v2.4.19' -PACKAGE_STRING='onedrive v2.4.19' +PACKAGE_VERSION='v2.4.20-dev' +PACKAGE_STRING='onedrive v2.4.20-dev' PACKAGE_BUGREPORT='https://github.com/abraunegg/onedrive' PACKAGE_URL='' @@ -1219,7 +1219,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures onedrive v2.4.19 to adapt to many kinds of systems. +\`configure' configures onedrive v2.4.20-dev to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1280,7 +1280,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of onedrive v2.4.19:";; + short | recursive ) echo "Configuration of onedrive v2.4.20-dev:";; esac cat <<\_ACEOF @@ -1393,7 +1393,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -onedrive configure v2.4.19 +onedrive configure v2.4.20-dev generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1410,7 +1410,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by onedrive $as_me v2.4.19, which was +It was created by onedrive $as_me v2.4.20-dev, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3159,7 +3159,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by onedrive $as_me v2.4.19, which was +This file was extended by onedrive $as_me v2.4.20-dev, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -3212,7 +3212,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -onedrive config.status v2.4.19 +onedrive config.status v2.4.20-dev configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 02b8b386..180be96f 100644 --- a/configure.ac +++ b/configure.ac @@ -9,7 +9,7 @@ dnl - commit the changed files (configure.ac, configure) dnl - tag the release AC_PREREQ([2.69]) -AC_INIT([onedrive],[v2.4.19], [https://github.com/abraunegg/onedrive], [onedrive]) +AC_INIT([onedrive],[v2.4.20-dev], [https://github.com/abraunegg/onedrive], [onedrive]) AC_CONFIG_SRCDIR([src/main.d]) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 775f91ff..78b81c37 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -144,14 +144,14 @@ CentOS 6.x and RHEL 6.x reached End of Life status on November 30th 2020 and is sudo yum groupinstall 'Development Tools' sudo yum install libcurl-devel sudo yum install sqlite-devel -curl -fsS https://dlang.org/install.sh | bash -s dmd +curl -fsS https://dlang.org/install.sh | bash -s dmd-2.099.0 ``` For notifications the following is also necessary: ```text sudo yum install libnotify-devel ``` -### Dependencies: Fedora > Version 18 / CentOS 8.x / RHEL 8.x +### Dependencies: Fedora > Version 18 / CentOS 8.x / RHEL 8.x / RHEL 9.x ```text sudo dnf groupinstall 'Development Tools' sudo dnf install libcurl-devel @@ -296,7 +296,7 @@ sudo make install ## Upgrading the client If you have installed the client from a distribution package, the client will be updated when the distribution package is updated by the package maintainer and will be updated to the new application version when you perform your package update. -If you have built the client from source, to upgrade your client, you must first uninstall your existing 'onedrive' binary (see above), then re-install the client by re-cloning, re-compiling and re-installing the client again to install the new version. +If you have built the client from source, to upgrade your client, you must first uninstall your existing 'onedrive' binary (see below), then re-install the client by re-cloning, re-compiling and re-installing the client again to install the new version. To confirm you have the new version installed, use `onedrive --version` to determine the version that is now installed. @@ -314,7 +314,7 @@ If you are not upgrading your client, to remove your application state and confi ``` rm -rf ~/.config/onedrive ``` -**Note:** If you are using the `--confdir option`, substitute `~/.config/onedrive` above for that directory. +**Note:** If you are using the `--confdir option`, substitute `~/.config/onedrive` for the correct directory storing your client configuration. If you want to just delete the application key, but keep the items database: ```text From 2bab99b62ccf83a8a61b4ba98673d545b7bd4652 Mon Sep 17 00:00:00 2001 From: paulsharpeY <61544260+paulsharpeY@users.noreply.github.com> Date: Wed, 22 Jun 2022 11:37:34 +0100 Subject: [PATCH 2/4] Update known-issues.md (#2016) Fix spelling. --- docs/known-issues.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/known-issues.md b/docs/known-issues.md index e40a9532..aff32173 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -6,7 +6,7 @@ The below are known issues with this client: **Description:** -When running the client in standalone mode (`--synchronize`) moving folders that are sucessfully synced around between subseqant standalone syncs causes a deletion & re-upload of data to occur. +When running the client in standalone mode (`--synchronize`) moving folders that are sucessfully synced around between subsequent standalone syncs causes a deletion & re-upload of data to occur. **Explanation:** From 042949f1c135ee6049f73f75bb149fe30ad8d9a3 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Fri, 24 Jun 2022 14:54:32 +1000 Subject: [PATCH 3/4] Fix 'foreign key constraint failed' when using OneDrive Business Shared Folders (#2017) In some OneDrive Business scenarios, the shared folder /delta response lacks the 'root' drive details. When this occurs, this creates the following error: A database statement execution error occurred: foreign key constraint failed. Ensure we query independently the root details for this shared folder and ensure that it is added before we process the /delta response --- src/sync.d | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/sync.d b/src/sync.d index 06c78047..103f367b 100644 --- a/src/sync.d +++ b/src/sync.d @@ -1776,6 +1776,44 @@ final class SyncEngine } } + // In some OneDrive Business scenarios, the shared folder /delta response lacks the 'root' drive details + // When this occurs, this creates the following error: A database statement execution error occurred: foreign key constraint failed + // Ensure we query independently the root details for this shared folder and ensure that it is added before we process the /delta response + if ((!nationalCloudDeployment) && (driveId!= defaultDriveId) && (syncBusinessFolders)) { + // fetch this driveId root details to ensure we add this to the database for this remote drive + JSONValue rootData; + + try { + rootData = onedrive.getDriveIdRoot(driveId); + } catch (OneDriveException e) { + log.vdebug("rootData = onedrive.getDriveIdRoot(driveId) generated a OneDriveException"); + // HTTP request returned status code 504 (Gateway Timeout) or 429 retry + if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + if (e.httpStatusCode == 429) { + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - retrying applicable request"); + handleOneDriveThrottleRequest(); + } + if (e.httpStatusCode == 504) { + log.vdebug("Retrying original request that generated the HTTP 504 (Gateway Timeout) - retrying applicable request"); + Thread.sleep(dur!"seconds"(30)); + } + // Retry original request by calling function again to avoid replicating any further error handling + rootData = onedrive.getDriveIdRoot(driveId); + + } else { + // There was a HTTP 5xx Server Side Error + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + // Must exit here + exit(-1); + } + } + + // apply this root drive data + applyDifference(rootData, driveId, true); + } + + // Process /delta response from OneDrive // is changes a valid JSON response if (changes.type() == JSONType.object) { // Are there any changes to process? From ca984eba70ae21536e0bf250a3e9137eb8857846 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Thu, 30 Jun 2022 07:08:29 +1000 Subject: [PATCH 4/4] Update OneDrive API response handling for National Cloud Deployments (#2023) * Update OneDrive API response handling for National Cloud Deployments * Add developer option to allow easy switch between /children and /delta to query OneDrive for changes --- src/config.d | 7 +++- src/main.d | 9 +++++ src/sync.d | 94 +++++++++++++++++++++++++--------------------------- 3 files changed, 61 insertions(+), 49 deletions(-) diff --git a/src/config.d b/src/config.d index 4c23538e..c0d0968e 100644 --- a/src/config.d +++ b/src/config.d @@ -147,7 +147,12 @@ final class Config // display_sync_options = true | false // - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging boolValues["display_sync_options"] = false; - + // force_children_scan = true | false + // - Force client to use /children rather than /delta to query changes on OneDrive + // - This option flags nationalCloudDeployment as true, forcing the client to act like it is using a National Cloud Deployment + boolValues["force_children_scan"] = false; + + // EXPAND USERS HOME DIRECTORY // Determine the users home directory. // Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts // Check for HOME environment variable diff --git a/src/main.d b/src/main.d index 17a7e08d..768a6eb9 100644 --- a/src/main.d +++ b/src/main.d @@ -1100,9 +1100,18 @@ int main(string[] args) // value is configured, is it a valid value? if ((cfg.getValueString("azure_ad_endpoint") == "USL4") || (cfg.getValueString("azure_ad_endpoint") == "USL5") || (cfg.getValueString("azure_ad_endpoint") == "DE") || (cfg.getValueString("azure_ad_endpoint") == "CN")) { // valid entries to flag we are using a National Cloud Deployment + // National Cloud Deployments do not support /delta as a query + // https://docs.microsoft.com/en-us/graph/deployments#supported-features + // Flag that we have a valid National Cloud Deployment that cannot use /delta queries sync.setNationalCloudDeployment(); } } + + // Are we forcing to use /children scan instead of /delta to simulate National Cloud Deployment use of /children? + if (cfg.getValueBool("force_children_scan")) { + log.vdebug("Forcing client to use /children scan rather than /delta to simulate National Cloud Deployment use of /children"); + sync.setNationalCloudDeployment(); + } // Do we need to validate the syncDir to check for the presence of a '.nosync' file if (cfg.getValueBool("check_nomount")) { diff --git a/src/sync.d b/src/sync.d index 103f367b..ed4e4528 100644 --- a/src/sync.d +++ b/src/sync.d @@ -186,7 +186,7 @@ private Item makeItem(const ref JSONValue driveItem) item.remoteId = driveItem["remoteItem"]["id"].str; } - // National Cloud Deployments (US and DE) do not support /delta as a query + // National Cloud Deployments do not support /delta as a query // Thus we need to track in the database that this item is in sync // As we are making an item, set the syncStatus to Y // ONLY when using a National Cloud Deployment, all the existing DB entries will get set to N @@ -1471,36 +1471,39 @@ final class SyncEngine // To view changes correctly, we need to use the correct path id for the request if (driveId == defaultDriveId) { // The drive id matches our users default drive id - idToQuery = defaultRootId.dup; log.vdebug("Configuring 'idToQuery' as defaultRootId duplicate"); + idToQuery = defaultRootId.dup; } else { // The drive id does not match our users default drive id // Potentially the 'path id' we are requesting the details of is a Shared Folder (remote item) // Use the 'id' that was passed in (folderId) - idToQuery = id.dup; log.vdebug("Configuring 'idToQuery' as 'id' duplicate"); + idToQuery = id.dup; } // what path id are we going to query? log.vdebug("Path object to query configured as 'idToQuery' = ", idToQuery); long deltaChanges = 0; // What query do we use? - // National Cloud Deployments (US and DE) do not support /delta as a query + // National Cloud Deployments do not support /delta as a query // https://docs.microsoft.com/en-us/graph/deployments#supported-features // Are we running against a National Cloud Deployments that does not support /delta if (nationalCloudDeployment) { - // Have to query /children rather than /delta + // National Cloud Deployment that does not support /delta query + // Have to query /children and build our own /delta response nationalCloudChildrenScan = true; log.vdebug("Using /children call to query drive for items to populate 'changes' and 'changesAvailable'"); // In a OneDrive Business Shared Folder scenario + nationalCloudDeployment, if ALL items are downgraded, then this leads to local file deletion // Downgrade ONLY files associated with this driveId and idToQuery log.vdebug("Downgrading all children for this driveId (" ~ driveId ~ ") and idToQuery (" ~ idToQuery ~ ") to an out-of-sync state"); + // Before we get any data, flag any object in the database as out-of-sync for this driveID & ID auto drivePathChildren = itemdb.selectChildren(driveId, idToQuery); if (count(drivePathChildren) > 0) { // Children to process and flag as out-of-sync foreach (drivePathChild; drivePathChildren) { // Flag any object in the database as out-of-sync for this driveID & ID + log.vdebug("Downgrading item as out-of-sync: ", drivePathChild.id); itemdb.downgradeSyncStatusFlag(drivePathChild.driveId, drivePathChild.id); } } @@ -1779,6 +1782,9 @@ final class SyncEngine // In some OneDrive Business scenarios, the shared folder /delta response lacks the 'root' drive details // When this occurs, this creates the following error: A database statement execution error occurred: foreign key constraint failed // Ensure we query independently the root details for this shared folder and ensure that it is added before we process the /delta response + + // However, if we are using a National Cloud Deployment, these deployments do not support /delta, so we generate a /delta response via generateDeltaResponse() + // This specifically adds the root drive details to the self generated /delta response if ((!nationalCloudDeployment) && (driveId!= defaultDriveId) && (syncBusinessFolders)) { // fetch this driveId root details to ensure we add this to the database for this remote drive JSONValue rootData; @@ -2506,9 +2512,13 @@ final class SyncEngine SysTime remoteModifiedTime = item.mtime; remoteModifiedTime.fracSecs = Duration.zero; - if (localModifiedTime != remoteModifiedTime) { - // Database update needed for this item because our record is out-of-date - log.vdebug("Updating local database with item details as timestamps of items are different"); + // If the timestamp is different, or we are running on a National Cloud Deployment that does not support /delta queries - we have to update the DB with the details from OneDrive + // Unfortunatly because of the consequence of Nataional Cloud Deployments not supporting /delta queries, the application uses the local database to flag what is out-of-date / track changes + // This means that the constant disk writing to the database fix implemented with https://github.com/abraunegg/onedrive/pull/2004 cannot be utilised when using Nataional Cloud Deployments + // as all records are touched / updated when performing the OneDrive sync operations. The only way to change this, is for Microsoft to support /delta queries for Nataional Cloud Deployments + if ((localModifiedTime != remoteModifiedTime) || (nationalCloudDeployment)) { + // Database update needed for this item because our local record is out-of-date + log.vdebug("Updating local database with item details from OneDrive as local record needs to be updated"); itemdb.update(item); } } else { @@ -3198,27 +3208,9 @@ final class SyncEngine } // Are we configured to use a National Cloud Deployment - // Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB - // Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query - if ((nationalCloudDeployment) || (syncBusinessFolders)) { + if (nationalCloudDeployment) { // Select items that have a out-of-sync flag set - foreach (driveId; driveIDsArray) { - // For each unique OneDrive driveID we know about - Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId); - foreach (item; outOfSyncItems) { - if (!dryRun) { - // clean up idsToDelete - idsToDelete.length = 0; - assumeSafeAppend(idsToDelete); - // flag to delete local file as it now is no longer in sync with OneDrive - log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive"); - log.vdebug("item: ", item); - idsToDelete ~= [item.driveId, item.id]; - // delete items in idsToDelete - if (idsToDelete.length > 0) deleteItems(); - } - } - } + flagNationalCloudDeploymentOutOfSyncItems(); } // scan for changes in the path provided @@ -3299,27 +3291,9 @@ final class SyncEngine } // Are we configured to use a National Cloud Deployment - // Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB - // Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query - if ((nationalCloudDeployment) || (syncBusinessFolders)) { + if (nationalCloudDeployment) { // Select items that have a out-of-sync flag set - foreach (driveId; driveIDsArray) { - // For each unique OneDrive driveID we know about - Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId); - foreach (item; outOfSyncItems) { - if (!dryRun) { - // clean up idsToDelete - idsToDelete.length = 0; - assumeSafeAppend(idsToDelete); - // flag to delete local file as it now is no longer in sync with OneDrive - log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive"); - log.vdebug("item: ", item); - idsToDelete ~= [item.driveId, item.id]; - // delete items in idsToDelete - if (idsToDelete.length > 0) deleteItems(); - } - } - } + flagNationalCloudDeploymentOutOfSyncItems(); } // scan for changes in the path provided @@ -3360,6 +3334,30 @@ final class SyncEngine } } + void flagNationalCloudDeploymentOutOfSyncItems() { + // Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB + // Normally, this is done at the end of processing all /delta queries, however National Cloud Deployments do not support /delta as a query + // https://docs.microsoft.com/en-us/graph/deployments#supported-features + // Select items that have a out-of-sync flag set + foreach (driveId; driveIDsArray) { + // For each unique OneDrive driveID we know about + Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId); + foreach (item; outOfSyncItems) { + if (!dryRun) { + // clean up idsToDelete + idsToDelete.length = 0; + assumeSafeAppend(idsToDelete); + // flag to delete local file as it now is no longer in sync with OneDrive + log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive"); + log.vdebug("item: ", item); + idsToDelete ~= [item.driveId, item.id]; + // delete items in idsToDelete + if (idsToDelete.length > 0) deleteItems(); + } + } + } + } + void handleUploadOnlyBusinessSharedFoldersEdgeCase() { // read in the business_shared_folders file contents string[] businessSharedFoldersList;