From ca984eba70ae21536e0bf250a3e9137eb8857846 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Thu, 30 Jun 2022 07:08:29 +1000 Subject: [PATCH] Update OneDrive API response handling for National Cloud Deployments (#2023) * Update OneDrive API response handling for National Cloud Deployments * Add developer option to allow easy switch between /children and /delta to query OneDrive for changes --- src/config.d | 7 +++- src/main.d | 9 +++++ src/sync.d | 94 +++++++++++++++++++++++++--------------------------- 3 files changed, 61 insertions(+), 49 deletions(-) diff --git a/src/config.d b/src/config.d index 4c23538e..c0d0968e 100644 --- a/src/config.d +++ b/src/config.d @@ -147,7 +147,12 @@ final class Config // display_sync_options = true | false // - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging boolValues["display_sync_options"] = false; - + // force_children_scan = true | false + // - Force client to use /children rather than /delta to query changes on OneDrive + // - This option flags nationalCloudDeployment as true, forcing the client to act like it is using a National Cloud Deployment + boolValues["force_children_scan"] = false; + + // EXPAND USERS HOME DIRECTORY // Determine the users home directory. // Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts // Check for HOME environment variable diff --git a/src/main.d b/src/main.d index 17a7e08d..768a6eb9 100644 --- a/src/main.d +++ b/src/main.d @@ -1100,9 +1100,18 @@ int main(string[] args) // value is configured, is it a valid value? if ((cfg.getValueString("azure_ad_endpoint") == "USL4") || (cfg.getValueString("azure_ad_endpoint") == "USL5") || (cfg.getValueString("azure_ad_endpoint") == "DE") || (cfg.getValueString("azure_ad_endpoint") == "CN")) { // valid entries to flag we are using a National Cloud Deployment + // National Cloud Deployments do not support /delta as a query + // https://docs.microsoft.com/en-us/graph/deployments#supported-features + // Flag that we have a valid National Cloud Deployment that cannot use /delta queries sync.setNationalCloudDeployment(); } } + + // Are we forcing to use /children scan instead of /delta to simulate National Cloud Deployment use of /children? + if (cfg.getValueBool("force_children_scan")) { + log.vdebug("Forcing client to use /children scan rather than /delta to simulate National Cloud Deployment use of /children"); + sync.setNationalCloudDeployment(); + } // Do we need to validate the syncDir to check for the presence of a '.nosync' file if (cfg.getValueBool("check_nomount")) { diff --git a/src/sync.d b/src/sync.d index 103f367b..ed4e4528 100644 --- a/src/sync.d +++ b/src/sync.d @@ -186,7 +186,7 @@ private Item makeItem(const ref JSONValue driveItem) item.remoteId = driveItem["remoteItem"]["id"].str; } - // National Cloud Deployments (US and DE) do not support /delta as a query + // National Cloud Deployments do not support /delta as a query // Thus we need to track in the database that this item is in sync // As we are making an item, set the syncStatus to Y // ONLY when using a National Cloud Deployment, all the existing DB entries will get set to N @@ -1471,36 +1471,39 @@ final class SyncEngine // To view changes correctly, we need to use the correct path id for the request if (driveId == defaultDriveId) { // The drive id matches our users default drive id - idToQuery = defaultRootId.dup; log.vdebug("Configuring 'idToQuery' as defaultRootId duplicate"); + idToQuery = defaultRootId.dup; } else { // The drive id does not match our users default drive id // Potentially the 'path id' we are requesting the details of is a Shared Folder (remote item) // Use the 'id' that was passed in (folderId) - idToQuery = id.dup; log.vdebug("Configuring 'idToQuery' as 'id' duplicate"); + idToQuery = id.dup; } // what path id are we going to query? log.vdebug("Path object to query configured as 'idToQuery' = ", idToQuery); long deltaChanges = 0; // What query do we use? - // National Cloud Deployments (US and DE) do not support /delta as a query + // National Cloud Deployments do not support /delta as a query // https://docs.microsoft.com/en-us/graph/deployments#supported-features // Are we running against a National Cloud Deployments that does not support /delta if (nationalCloudDeployment) { - // Have to query /children rather than /delta + // National Cloud Deployment that does not support /delta query + // Have to query /children and build our own /delta response nationalCloudChildrenScan = true; log.vdebug("Using /children call to query drive for items to populate 'changes' and 'changesAvailable'"); // In a OneDrive Business Shared Folder scenario + nationalCloudDeployment, if ALL items are downgraded, then this leads to local file deletion // Downgrade ONLY files associated with this driveId and idToQuery log.vdebug("Downgrading all children for this driveId (" ~ driveId ~ ") and idToQuery (" ~ idToQuery ~ ") to an out-of-sync state"); + // Before we get any data, flag any object in the database as out-of-sync for this driveID & ID auto drivePathChildren = itemdb.selectChildren(driveId, idToQuery); if (count(drivePathChildren) > 0) { // Children to process and flag as out-of-sync foreach (drivePathChild; drivePathChildren) { // Flag any object in the database as out-of-sync for this driveID & ID + log.vdebug("Downgrading item as out-of-sync: ", drivePathChild.id); itemdb.downgradeSyncStatusFlag(drivePathChild.driveId, drivePathChild.id); } } @@ -1779,6 +1782,9 @@ final class SyncEngine // In some OneDrive Business scenarios, the shared folder /delta response lacks the 'root' drive details // When this occurs, this creates the following error: A database statement execution error occurred: foreign key constraint failed // Ensure we query independently the root details for this shared folder and ensure that it is added before we process the /delta response + + // However, if we are using a National Cloud Deployment, these deployments do not support /delta, so we generate a /delta response via generateDeltaResponse() + // This specifically adds the root drive details to the self generated /delta response if ((!nationalCloudDeployment) && (driveId!= defaultDriveId) && (syncBusinessFolders)) { // fetch this driveId root details to ensure we add this to the database for this remote drive JSONValue rootData; @@ -2506,9 +2512,13 @@ final class SyncEngine SysTime remoteModifiedTime = item.mtime; remoteModifiedTime.fracSecs = Duration.zero; - if (localModifiedTime != remoteModifiedTime) { - // Database update needed for this item because our record is out-of-date - log.vdebug("Updating local database with item details as timestamps of items are different"); + // If the timestamp is different, or we are running on a National Cloud Deployment that does not support /delta queries - we have to update the DB with the details from OneDrive + // Unfortunatly because of the consequence of Nataional Cloud Deployments not supporting /delta queries, the application uses the local database to flag what is out-of-date / track changes + // This means that the constant disk writing to the database fix implemented with https://github.com/abraunegg/onedrive/pull/2004 cannot be utilised when using Nataional Cloud Deployments + // as all records are touched / updated when performing the OneDrive sync operations. The only way to change this, is for Microsoft to support /delta queries for Nataional Cloud Deployments + if ((localModifiedTime != remoteModifiedTime) || (nationalCloudDeployment)) { + // Database update needed for this item because our local record is out-of-date + log.vdebug("Updating local database with item details from OneDrive as local record needs to be updated"); itemdb.update(item); } } else { @@ -3198,27 +3208,9 @@ final class SyncEngine } // Are we configured to use a National Cloud Deployment - // Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB - // Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query - if ((nationalCloudDeployment) || (syncBusinessFolders)) { + if (nationalCloudDeployment) { // Select items that have a out-of-sync flag set - foreach (driveId; driveIDsArray) { - // For each unique OneDrive driveID we know about - Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId); - foreach (item; outOfSyncItems) { - if (!dryRun) { - // clean up idsToDelete - idsToDelete.length = 0; - assumeSafeAppend(idsToDelete); - // flag to delete local file as it now is no longer in sync with OneDrive - log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive"); - log.vdebug("item: ", item); - idsToDelete ~= [item.driveId, item.id]; - // delete items in idsToDelete - if (idsToDelete.length > 0) deleteItems(); - } - } - } + flagNationalCloudDeploymentOutOfSyncItems(); } // scan for changes in the path provided @@ -3299,27 +3291,9 @@ final class SyncEngine } // Are we configured to use a National Cloud Deployment - // Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB - // Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query - if ((nationalCloudDeployment) || (syncBusinessFolders)) { + if (nationalCloudDeployment) { // Select items that have a out-of-sync flag set - foreach (driveId; driveIDsArray) { - // For each unique OneDrive driveID we know about - Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId); - foreach (item; outOfSyncItems) { - if (!dryRun) { - // clean up idsToDelete - idsToDelete.length = 0; - assumeSafeAppend(idsToDelete); - // flag to delete local file as it now is no longer in sync with OneDrive - log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive"); - log.vdebug("item: ", item); - idsToDelete ~= [item.driveId, item.id]; - // delete items in idsToDelete - if (idsToDelete.length > 0) deleteItems(); - } - } - } + flagNationalCloudDeploymentOutOfSyncItems(); } // scan for changes in the path provided @@ -3360,6 +3334,30 @@ final class SyncEngine } } + void flagNationalCloudDeploymentOutOfSyncItems() { + // Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB + // Normally, this is done at the end of processing all /delta queries, however National Cloud Deployments do not support /delta as a query + // https://docs.microsoft.com/en-us/graph/deployments#supported-features + // Select items that have a out-of-sync flag set + foreach (driveId; driveIDsArray) { + // For each unique OneDrive driveID we know about + Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId); + foreach (item; outOfSyncItems) { + if (!dryRun) { + // clean up idsToDelete + idsToDelete.length = 0; + assumeSafeAppend(idsToDelete); + // flag to delete local file as it now is no longer in sync with OneDrive + log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive"); + log.vdebug("item: ", item); + idsToDelete ~= [item.driveId, item.id]; + // delete items in idsToDelete + if (idsToDelete.length > 0) deleteItems(); + } + } + } + } + void handleUploadOnlyBusinessSharedFoldersEdgeCase() { // read in the business_shared_folders file contents string[] businessSharedFoldersList;