From db5ceccffbd3d135425092d2fea27c66b41d942f Mon Sep 17 00:00:00 2001 From: abraunegg Date: Thu, 21 Nov 2019 19:58:34 +1100 Subject: [PATCH] Check database for excluded sync_list items previously in scope (Issue #723) (#724) * When using sync_list and files are moved out of sync scope online via OneDrive website, check if the item was previously in-sync in database. if true, flag for local delete. * Rework fix for #568 so that if a directory is moved / created in sync_list scope, on the 'true-up' sync, any files that were moved with the folder are now downloaded * Update when sync_list override is done * Add debugging for each loop for sync_list scan handling --- src/main.d | 17 +++++++++++------ src/sync.d | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/src/main.d b/src/main.d index 28e4bd02..af2097a3 100644 --- a/src/main.d +++ b/src/main.d @@ -690,7 +690,7 @@ int main(string[] args) auto logMonitorCounter = 0; auto fullScanCounter = 0; bool fullScanRequired = true; - bool syncListConfiguredOverride = false; + bool syncListConfiguredOverride = true; while (true) { if (!cfg.getValueBool("download_only")) m.update(online); auto currTime = MonoTime.currTime(); @@ -709,6 +709,11 @@ int main(string[] args) syncListConfiguredOverride = true; } } + + // sync option handling per sync loop + log.vdebug("syncListConfigured = ", syncListConfigured); + log.vdebug("fullScanRequired = ", fullScanRequired); + log.vdebug("syncListConfiguredOverride = ", syncListConfiguredOverride); // log.logAndNotify("DEBUG trying to create checkpoint"); // auto res = itemdb.db_checkpoint(); @@ -852,8 +857,8 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo } else { // sync from OneDrive first before uploading files to OneDrive if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ..."); - // if sync_list is configured, syncListConfigured = true, thus a FULL walk of all OneDrive objects will be performed - sync.applyDifferences(syncListConfigured); + // For the initial sync, always use the delta link so that we capture all the right delta changes including adds, moves & deletes + sync.applyDifferences(false); // Is a full scan of the entire sync_dir required? if (fullScanRequired) { // is this a download only request? @@ -862,9 +867,9 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo // in monitor mode all local changes are captured via inotify // thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required sync.scanForDifferences(localPath); - // ensure that the current remote state is updated locally - // for the 'true-up' set this to false so that we use the delta link for second operation - sync.applyDifferences(false); + // ensure that the current remote state is updated locally to ensure everything is consistent + // for the 'true-up' sync, if sync_list is configured, syncListConfigured = true, thus a FULL walk of all OneDrive objects will be requested and used if required + sync.applyDifferences(syncListConfigured); } } } diff --git a/src/sync.d b/src/sync.d index c0fa36e0..2d8e08df 100644 --- a/src/sync.d +++ b/src/sync.d @@ -233,6 +233,8 @@ final class SyncEngine private bool syncBusinessFolders = false; // single directory scope flag private bool singleDirectoryScope = false; + // sync_list new folder added, trigger delta scan override + private bool syncListFullScanTrigger = false; this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync) { @@ -410,6 +412,22 @@ final class SyncEngine log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments"); } + // Issue #658 Handling + // If an existing folder is moved into a sync_list valid path (where it previously was out of scope due to sync_list), + // then set this flag to true, so that on the second 'true-up' sync, we force a rescan of the OneDrive path to capture any 'files' + void setSyncListFullScanTrigger() + { + syncListFullScanTrigger = true; + log.vdebug("Setting syncListFullScanTrigger = true due to new folder creation request in a location that is in-scope via sync_list"); + } + + // unset method + void unsetSyncListFullScanTrigger() + { + syncListFullScanTrigger = false; + log.vdebug("Setting syncListFullScanTrigger = false"); + } + // download all new changes from OneDrive void applyDifferences(bool performFullItemScan) { @@ -733,9 +751,10 @@ final class SyncEngine log.vdebug("onedrive.getPathDetailsById call returned an invalid JSON Object"); } + // Issue #658 // If we are using a sync_list file, using deltaLink will actually 'miss' changes (moves & deletes) on OneDrive as using sync_list discards changes // Use the performFullItemScan boolean to control whether we perform a full object scan of use the delta link for the root folder - // When using --synchronize the process order is: + // When using --synchronize the normal process order is: // 1. Scan OneDrive for changes // 2. Scan local folder for changes // 3. Scan OneDrive for changes @@ -827,7 +846,7 @@ final class SyncEngine // is changes a valid JSON response if (changes.type() == JSONType.object) { // Are there any changes to process? - if ((("value" in changes) != null) && (deltaChanges > 0)) { + if ((("value" in changes) != null) && ((deltaChanges > 0) || (syncListFullScanTrigger))) { auto nrChanges = count(changes["value"].array); auto changeCount = 0; @@ -848,6 +867,11 @@ final class SyncEngine // There are valid changes log.vdebug("Number of items from OneDrive to process: ", nrChanges); } + + // unset now the full scan trigger if set + if (syncListFullScanTrigger) { + unsetSyncListFullScanTrigger(); + } } foreach (item; changes["value"].array) { @@ -1187,7 +1211,7 @@ final class SyncEngine // check for selective sync string path; if (!unwanted) { - // Is the item parent in the local database + // Is the item parent in the local database? if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){ // compute the item path to see if the path is excluded path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name; @@ -1203,6 +1227,12 @@ final class SyncEngine // path is unwanted unwanted = true; log.vlog("Skipping item - excluded by sync_list config: ", path); + // flagging to skip this file now, but does this exist in the DB thus needs to be removed / deleted? + if (itemdb.idInLocalDatabase(item.driveId, item.id)){ + log.vlog("Flagging item for local delete as item exists in database: ", path); + // flag to delete + idsToDelete ~= [item.driveId, item.id]; + } } } } else { @@ -1249,9 +1279,11 @@ final class SyncEngine // Item name we will attempt to delete will be printed out later if (cached) { // flag to delete + log.vdebug("Flagging item for deletion: ", item); idsToDelete ~= [item.driveId, item.id]; } else { // flag to ignore + log.vdebug("Flagging item to skip: ", item); skippedItems ~= item.id; } return; @@ -1379,6 +1411,15 @@ final class SyncEngine case ItemType.dir: case ItemType.remote: log.log("Creating directory: ", path); + + // Issue #658 handling + auto syncListExcluded = selectiveSync.isPathExcludedViaSyncList(path); + log.vdebug("sync_list excluded: ", syncListExcluded); + if (!syncListExcluded) { + // path we are creating is not excluded via sync_list + setSyncListFullScanTrigger(); + } + if (!dryRun) { mkdirRecurse(path); } else {