deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-22 07:36:36 +02:00
Code Issues Projects Releases Wiki Activity

Modify 'Processing X changes' output to be more reflective of actual activity (Issue #679) (#680)

Browse source 
* When using 'sync_list' do not output that we are processing 'changes' as this is misleading.
* When using sync_list perform a full scan only the first time when using --synchronize & --monitor to avoid double processing of all OneDrive items each and every time
* When using sync_list and --monitor only perform a full object scan when we are also configured to perform a full scan
This commit is contained in:
abraunegg 2019-10-09 12:02:17 +11:00 committed by GitHub
parent 808a2f331b
commit 6a8d575941
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 54 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
src/main.d
View file

@ -78,6 +78,7 @@ int main(string[] args)
string syncListHashFile = cfg.configDirName ~ "/.sync_list.hash";
string configBackupFile = cfg.configDirName ~ "/.config.backup";
bool configOptionsDifferent = false;
bool syncListConfigured = false;
bool syncListDifferent = false;
bool syncDirDifferent = false;
bool skipFileDifferent = false;
@ -487,6 +488,7 @@ int main(string[] args)
auto selectiveSync = new SelectiveSync();
if (exists(cfg.syncListFilePath)){
log.vdebug("Loading user configured sync_list file ...");
syncListConfigured = true;
// list what will be synced
auto syncListFile = File(cfg.syncListFilePath);
auto range = syncListFile.byLine();
@ -611,8 +613,8 @@ int main(string[] args)
return EXIT_FAILURE;
}
}
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, true);
// perform a --synchronize sync
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, true, syncListConfigured);
}
}
@ -679,6 +681,7 @@ int main(string[] args)
auto logMonitorCounter = 0;
auto fullScanCounter = 0;
bool fullScanRequired = true;
bool syncListConfiguredOverride = false;
while (true) {
if (!cfg.getValueBool("download_only")) m.update(online);
auto currTime = MonoTime.currTime();
@ -693,6 +696,9 @@ int main(string[] args)
if (fullScanCounter > fullScanFrequency){
fullScanCounter = 1;
fullScanRequired = true;
if (syncListConfigured) {
syncListConfiguredOverride = true;
}
}
// log.logAndNotify("DEBUG trying to create checkpoint");
@ -706,7 +712,8 @@ int main(string[] args)
return EXIT_FAILURE;
}
try {
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired);
// perform a --monitor sync
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired, syncListConfiguredOverride);
if (!cfg.getValueBool("download_only")) {
// discard all events that may have been generated by the sync
m.update(false);
@ -723,6 +730,9 @@ int main(string[] args)
}
// performSync complete, set lastCheckTime to current time
fullScanRequired = false;
if (syncListConfigured) {
syncListConfiguredOverride = false;
}
lastCheckTime = MonoTime.currTime();
GC.collect();
}
@ -768,7 +778,7 @@ bool initSyncEngine(SyncEngine sync)
}
// try to synchronize the folder three times
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired)
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired, bool syncListConfigured)
{
int count;
string remotePath = "/";
@ -829,11 +839,11 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// sync local files first before downloading from OneDrive
if (logLevel < MONITOR_LOG_QUIET) log.log("Syncing changes from local path first before downloading changes from OneDrive ...");
sync.scanForDifferences(localPath);
sync.applyDifferences();
sync.applyDifferences(syncListConfigured);
} else {
// sync from OneDrive first before uploading files to OneDrive
if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ...");
sync.applyDifferences();
sync.applyDifferences(syncListConfigured);
// Is a full scan of the entire sync_dir required?
if (fullScanRequired) {
// is this a download only request?
@ -843,7 +853,8 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required
sync.scanForDifferences(localPath);
// ensure that the current remote state is updated locally
sync.applyDifferences();
// for the 'true-up' set this to false so that we use the delta link for second operation
sync.applyDifferences(false);
}
}
}

52
src/sync.d
View file

@ -411,13 +411,13 @@ final class SyncEngine
}
// download all new changes from OneDrive
void applyDifferences()
void applyDifferences(bool performFullItemScan)
{
// Set defaults for the root folder
// Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls
string driveId = defaultDriveId;
string rootId = defaultRootId;
applyDifferences(driveId, rootId);
applyDifferences(driveId, rootId, performFullItemScan);
// Check OneDrive Personal Shared Folders
// https://github.com/OneDrive/onedrive-api-docs/issues/764
@ -425,7 +425,7 @@ final class SyncEngine
foreach (item; items) {
log.vdebug("------------------------------------------------------------------");
log.vlog("Syncing OneDrive Shared Folder: ", item.name);
applyDifferences(item.remoteDriveId, item.remoteId);
applyDifferences(item.remoteDriveId, item.remoteId, performFullItemScan);
}
}
@ -462,19 +462,19 @@ final class SyncEngine
// 2. Download changes specific to the remote path
// root remote
applyDifferences(defaultDriveId, onedrivePathDetails["id"].str);
applyDifferences(defaultDriveId, onedrivePathDetails["id"].str, false);
// remote changes
driveId = onedrivePathDetails["remoteItem"]["parentReference"]["driveId"].str; // Should give something like 66d53be8a5056eca
folderId = onedrivePathDetails["remoteItem"]["id"].str; // Should give something like BC7D88EC1F539DCF!107
// Apply any differences found on OneDrive for this path (download data)
applyDifferences(driveId, folderId);
applyDifferences(driveId, folderId, false);
} else {
// use the item id as folderId
folderId = onedrivePathDetails["id"].str; // Should give something like 12345ABCDE1234A1!101
// Apply any differences found on OneDrive for this path (download data)
applyDifferences(defaultDriveId, folderId);
applyDifferences(defaultDriveId, folderId, false);
}
} else {
// Log that an invalid JSON object was returned
@ -586,16 +586,24 @@ final class SyncEngine
// download the new changes of a specific item
// id is the root of the drive or a shared folder
private void applyDifferences(string driveId, const(char)[] id)
private void applyDifferences(string driveId, const(char)[] id, bool performFullItemScan)
{
log.vlog("Applying changes of Path ID: " ~ id);
JSONValue changes;
// If we are using a sync_list file, using deltaLink will actually 'miss' changes (moves & deletes) on OneDrive as using sync_list discards changes
// Use the performFullItemScan boolean to control whether we perform a full object scan of use the delta link for the root folder
// When using --synchronize the process order is:
// 1. Scan OneDrive for changes
// 2. Scan local folder for changes
// 3. Scan OneDrive for changes
// When using sync_list and performing a full scan, what this means is a full scan is performed twice, which leads to massive processing & time overheads
// Control this via performFullItemScan
string deltaLink = "";
string userSyncList = cfg.configDirName ~ "/sync_list";
if (!exists(userSyncList)){
// not using sync_list file, use the delta link
if (!performFullItemScan){
// performFullItemScan == false
// use delta link
deltaLink = itemdb.getDeltaLink(driveId, id);
}
@ -792,7 +800,7 @@ final class SyncEngine
// HTTP request returned status code 504 (Gateway Timeout)
// Retry by calling applyDifferences() again
log.vlog("OneDrive returned a 'HTTP 504 - Gateway Timeout' - gracefully handling error");
applyDifferences(driveId, idToQuery);
applyDifferences(driveId, idToQuery, performFullItemScan);
} else {
// Default operation if not 404, 410, 500, 504 errors
// display what the error is
@ -808,12 +816,24 @@ final class SyncEngine
if (("value" in changes) != null) {
auto nrChanges = count(changes["value"].array);
auto changeCount = 0;
if (nrChanges >= cfg.getValueLong("min_notify_changes")) {
log.logAndNotify("Processing ", nrChanges, " changes");
if (!performFullItemScan){
// Display the number of changes we are processing
if (nrChanges >= cfg.getValueLong("min_notify_changes")) {
log.logAndNotify("Processing ", nrChanges, " changes");
} else {
// There are valid changes
log.vdebug("Number of changes from OneDrive to process: ", nrChanges);
}
} else {
// There are valid changes
log.vdebug("Number of changes from OneDrive to process: ", nrChanges);
// Do not display anything unless we are doing a verbose debug as due to #658 we are essentially doing a --resync each time when using sync_list
// Display the number of items we are processing
if (nrChanges >= cfg.getValueLong("min_notify_changes")) {
log.logAndNotify("Processing ", nrChanges, " OneDrive items to ensure consistent state due to sync_list being used");
} else {
// There are valid changes
log.vdebug("Number of items from OneDrive to process: ", nrChanges);
}
}
foreach (item; changes["value"].array) {