diff --git a/src/main.d b/src/main.d index e9bacdd0..2a40237c 100644 --- a/src/main.d +++ b/src/main.d @@ -78,6 +78,7 @@ int main(string[] args) string syncListHashFile = cfg.configDirName ~ "/.sync_list.hash"; string configBackupFile = cfg.configDirName ~ "/.config.backup"; bool configOptionsDifferent = false; + bool syncListConfigured = false; bool syncListDifferent = false; bool syncDirDifferent = false; bool skipFileDifferent = false; @@ -487,6 +488,7 @@ int main(string[] args) auto selectiveSync = new SelectiveSync(); if (exists(cfg.syncListFilePath)){ log.vdebug("Loading user configured sync_list file ..."); + syncListConfigured = true; // list what will be synced auto syncListFile = File(cfg.syncListFilePath); auto range = syncListFile.byLine(); @@ -611,8 +613,8 @@ int main(string[] args) return EXIT_FAILURE; } } - - performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, true); + // perform a --synchronize sync + performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, true, syncListConfigured); } } @@ -679,6 +681,7 @@ int main(string[] args) auto logMonitorCounter = 0; auto fullScanCounter = 0; bool fullScanRequired = true; + bool syncListConfiguredOverride = false; while (true) { if (!cfg.getValueBool("download_only")) m.update(online); auto currTime = MonoTime.currTime(); @@ -693,6 +696,9 @@ int main(string[] args) if (fullScanCounter > fullScanFrequency){ fullScanCounter = 1; fullScanRequired = true; + if (syncListConfigured) { + syncListConfiguredOverride = true; + } } // log.logAndNotify("DEBUG trying to create checkpoint"); @@ -706,7 +712,8 @@ int main(string[] args) return EXIT_FAILURE; } try { - performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired); + // perform a --monitor sync + performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired, syncListConfiguredOverride); if (!cfg.getValueBool("download_only")) { // discard all events that may have been generated by the sync m.update(false); @@ -723,6 +730,9 @@ int main(string[] args) } // performSync complete, set lastCheckTime to current time fullScanRequired = false; + if (syncListConfigured) { + syncListConfiguredOverride = false; + } lastCheckTime = MonoTime.currTime(); GC.collect(); } @@ -768,7 +778,7 @@ bool initSyncEngine(SyncEngine sync) } // try to synchronize the folder three times -void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired) +void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired, bool syncListConfigured) { int count; string remotePath = "/"; @@ -829,11 +839,11 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo // sync local files first before downloading from OneDrive if (logLevel < MONITOR_LOG_QUIET) log.log("Syncing changes from local path first before downloading changes from OneDrive ..."); sync.scanForDifferences(localPath); - sync.applyDifferences(); + sync.applyDifferences(syncListConfigured); } else { // sync from OneDrive first before uploading files to OneDrive if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ..."); - sync.applyDifferences(); + sync.applyDifferences(syncListConfigured); // Is a full scan of the entire sync_dir required? if (fullScanRequired) { // is this a download only request? @@ -843,7 +853,8 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo // thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required sync.scanForDifferences(localPath); // ensure that the current remote state is updated locally - sync.applyDifferences(); + // for the 'true-up' set this to false so that we use the delta link for second operation + sync.applyDifferences(false); } } } diff --git a/src/sync.d b/src/sync.d index c0f47ba6..75920795 100644 --- a/src/sync.d +++ b/src/sync.d @@ -411,13 +411,13 @@ final class SyncEngine } // download all new changes from OneDrive - void applyDifferences() + void applyDifferences(bool performFullItemScan) { // Set defaults for the root folder // Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls string driveId = defaultDriveId; string rootId = defaultRootId; - applyDifferences(driveId, rootId); + applyDifferences(driveId, rootId, performFullItemScan); // Check OneDrive Personal Shared Folders // https://github.com/OneDrive/onedrive-api-docs/issues/764 @@ -425,7 +425,7 @@ final class SyncEngine foreach (item; items) { log.vdebug("------------------------------------------------------------------"); log.vlog("Syncing OneDrive Shared Folder: ", item.name); - applyDifferences(item.remoteDriveId, item.remoteId); + applyDifferences(item.remoteDriveId, item.remoteId, performFullItemScan); } } @@ -462,19 +462,19 @@ final class SyncEngine // 2. Download changes specific to the remote path // root remote - applyDifferences(defaultDriveId, onedrivePathDetails["id"].str); + applyDifferences(defaultDriveId, onedrivePathDetails["id"].str, false); // remote changes driveId = onedrivePathDetails["remoteItem"]["parentReference"]["driveId"].str; // Should give something like 66d53be8a5056eca folderId = onedrivePathDetails["remoteItem"]["id"].str; // Should give something like BC7D88EC1F539DCF!107 // Apply any differences found on OneDrive for this path (download data) - applyDifferences(driveId, folderId); + applyDifferences(driveId, folderId, false); } else { // use the item id as folderId folderId = onedrivePathDetails["id"].str; // Should give something like 12345ABCDE1234A1!101 // Apply any differences found on OneDrive for this path (download data) - applyDifferences(defaultDriveId, folderId); + applyDifferences(defaultDriveId, folderId, false); } } else { // Log that an invalid JSON object was returned @@ -586,16 +586,24 @@ final class SyncEngine // download the new changes of a specific item // id is the root of the drive or a shared folder - private void applyDifferences(string driveId, const(char)[] id) + private void applyDifferences(string driveId, const(char)[] id, bool performFullItemScan) { log.vlog("Applying changes of Path ID: " ~ id); JSONValue changes; // If we are using a sync_list file, using deltaLink will actually 'miss' changes (moves & deletes) on OneDrive as using sync_list discards changes + // Use the performFullItemScan boolean to control whether we perform a full object scan of use the delta link for the root folder + // When using --synchronize the process order is: + // 1. Scan OneDrive for changes + // 2. Scan local folder for changes + // 3. Scan OneDrive for changes + // When using sync_list and performing a full scan, what this means is a full scan is performed twice, which leads to massive processing & time overheads + // Control this via performFullItemScan + string deltaLink = ""; - string userSyncList = cfg.configDirName ~ "/sync_list"; - if (!exists(userSyncList)){ - // not using sync_list file, use the delta link + if (!performFullItemScan){ + // performFullItemScan == false + // use delta link deltaLink = itemdb.getDeltaLink(driveId, id); } @@ -792,7 +800,7 @@ final class SyncEngine // HTTP request returned status code 504 (Gateway Timeout) // Retry by calling applyDifferences() again log.vlog("OneDrive returned a 'HTTP 504 - Gateway Timeout' - gracefully handling error"); - applyDifferences(driveId, idToQuery); + applyDifferences(driveId, idToQuery, performFullItemScan); } else { // Default operation if not 404, 410, 500, 504 errors // display what the error is @@ -808,12 +816,24 @@ final class SyncEngine if (("value" in changes) != null) { auto nrChanges = count(changes["value"].array); auto changeCount = 0; - - if (nrChanges >= cfg.getValueLong("min_notify_changes")) { - log.logAndNotify("Processing ", nrChanges, " changes"); + + if (!performFullItemScan){ + // Display the number of changes we are processing + if (nrChanges >= cfg.getValueLong("min_notify_changes")) { + log.logAndNotify("Processing ", nrChanges, " changes"); + } else { + // There are valid changes + log.vdebug("Number of changes from OneDrive to process: ", nrChanges); + } } else { - // There are valid changes - log.vdebug("Number of changes from OneDrive to process: ", nrChanges); + // Do not display anything unless we are doing a verbose debug as due to #658 we are essentially doing a --resync each time when using sync_list + // Display the number of items we are processing + if (nrChanges >= cfg.getValueLong("min_notify_changes")) { + log.logAndNotify("Processing ", nrChanges, " OneDrive items to ensure consistent state due to sync_list being used"); + } else { + // There are valid changes + log.vdebug("Number of items from OneDrive to process: ", nrChanges); + } } foreach (item; changes["value"].array) {