deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-10 09:46:40 +02:00
Code Issues Projects Releases Wiki Activity

Perform full scan of OneDrive when required to ensure consistent local state (Issue #865) (#867)

Browse source 
* Use the configured 'fullScanRequired' every 10th loop to ensure that the local repository actually is in sync with OneDrive when sync_list is not used
* Use 'oneDriveFullScanTrigger' if it is set to trigger a full scan earlier than wait for next full sync if in monitor mode
This commit is contained in:
abraunegg 2020-04-15 14:29:50 +10:00 committed by GitHub
parent 7ca1ff95a8
commit d3aa0032bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 45 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/main.d
View file

@ -790,6 +790,8 @@ int main(string[] args)
}
}
// Monitor Loop Counter
log.vdebug("fullScanCounter = ", fullScanCounter);
// sync option handling per sync loop
log.vdebug("syncListConfigured = ", syncListConfigured);
log.vdebug("fullScanRequired = ", fullScanRequired);
@ -950,8 +952,13 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required
sync.scanForDifferences(localPath);
// ensure that the current remote state is updated locally to ensure everything is consistent
// for the 'true-up' sync, if sync_list is configured, syncListConfigured = true, thus a FULL walk of all OneDrive objects will be requested and used if required
sync.applyDifferences(syncListConfigured);
if (syncListConfigured) {
// for the 'true-up' sync, if sync_list is configured, syncListConfigured = true, thus a FULL walk of all OneDrive objects will be requested and used if required
sync.applyDifferences(syncListConfigured);
} else {
// for the 'true-up' sync, if fullScanRequired = true, thus a FULL walk of all OneDrive objects will be requested and used if required
sync.applyDifferences(fullScanRequired);
}
}
}
}

53
src/sync.d
View file

@ -239,7 +239,7 @@ final class SyncEngine
// is sync_list configured
private bool syncListConfigured = false;
// sync_list new folder added, trigger delta scan override
private bool syncListFullScanTrigger = false;
private bool oneDriveFullScanTrigger = false;
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
{
@ -452,17 +452,17 @@ final class SyncEngine
// Issue #658 Handling
// If an existing folder is moved into a sync_list valid path (where it previously was out of scope due to sync_list),
// then set this flag to true, so that on the second 'true-up' sync, we force a rescan of the OneDrive path to capture any 'files'
void setSyncListFullScanTrigger()
void setOneDriveFullScanTrigger()
{
syncListFullScanTrigger = true;
log.vdebug("Setting syncListFullScanTrigger = true due to new folder creation request in a location that is in-scope via sync_list");
oneDriveFullScanTrigger = true;
log.vdebug("Setting oneDriveFullScanTrigger = true due to new folder creation request in a location that is now in-scope which was previously out of scope");
}
// unset method
void unsetSyncListFullScanTrigger()
void unsetOneDriveFullScanTrigger()
{
syncListFullScanTrigger = false;
log.vdebug("Setting syncListFullScanTrigger = false");
oneDriveFullScanTrigger = false;
log.vdebug("Setting oneDriveFullScanTrigger = false");
}
// set syncListConfigured to true
@ -853,9 +853,18 @@ final class SyncEngine
string deltaLink = "";
string deltaLinkAvailable = itemdb.getDeltaLink(driveId, id);
log.vdebug("syncListConfigured = ", syncListConfigured);
log.vdebug("syncListFullScanTrigger = ", syncListFullScanTrigger);
log.vdebug("oneDriveFullScanTrigger = ", oneDriveFullScanTrigger);
log.vdebug("performFullItemScan = ", performFullItemScan);
// if sync_list is not configured, syncListConfigured should be false
// do we override performFullItemScan if it is currently false and oneDriveFullScanTrigger is true?
if ((!performFullItemScan) && (oneDriveFullScanTrigger)) {
// forcing a full scan earlier than potentially normal
// oneDriveFullScanTrigger = true due to new folder creation request in a location that is now in-scope which was previously out of scope
performFullItemScan = true;
log.vdebug("overriding performFullItemScan as oneDriveFullScanTrigger was set");
}
// depending on the scan type (--monitor or --synchronize) performFullItemScan is set depending on the number of sync passes performed (--monitor) or ALWAYS if just --synchronize is used
if (!performFullItemScan){
// performFullItemScan == false
@ -867,6 +876,8 @@ final class SyncEngine
} else {
log.vdebug("deltaLink contains valid data - resulting API query will be treated as a delta scan of OneDrive");
}
} else {
log.vdebug("performFullItemScan is true, not using deltaLink or deltaLinkAvailable to force query of all OneDrive items");
}
for (;;) {
@ -956,7 +967,7 @@ final class SyncEngine
// is changes a valid JSON response
if (changes.type() == JSONType.object) {
// Are there any changes to process?
if ((("value" in changes) != null) && ((deltaChanges > 0) || (syncListFullScanTrigger))) {
if ((("value" in changes) != null) && ((deltaChanges > 0) || (oneDriveFullScanTrigger))) {
auto nrChanges = count(changes["value"].array);
auto changeCount = 0;
@ -970,25 +981,33 @@ final class SyncEngine
// verbose log, no 'notify' .. it is over the top
if (!syncListConfigured) {
// sync_list is not being used - lets use the right messaging here
log.vlog("Processing ", nrChanges, " changes");
if (oneDriveFullScanTrigger) {
// full scan was triggered out of cycle
log.vlog("Processing ", nrChanges, " OneDrive items to ensure consistent local state due to a full scan being triggered by actions on OneDrive");
} else {
// no sync_list, no full scan was triggered
log.vlog("Processing ", nrChanges, " changes");
}
} else {
// sync_list is being used - why are we going through the entire OneDrive contents?
log.vlog("Processing ", nrChanges, " OneDrive items to ensure consistent state due to sync_list being used");
log.vlog("Processing ", nrChanges, " OneDrive items to ensure consistent local state due to sync_list being used");
}
} else {
// There are valid changes but less than the min_notify_changes configured threshold
// We will only output the number of changes being processed to debug log if this is set to assist with debugging
// As this is debug logging, messaging can be the same, regardless of sync_list being used or not
log.vdebug("Number of changes from OneDrive to process: ", nrChanges);
// is performFullItemScan set due to a full scan required?
if (performFullItemScan){
// full scan was triggered due to using sync_list
log.vdebug("Number of items from OneDrive to process: ", nrChanges);
// full scan was triggered due to sync_list or skip_dir being used
log.vdebug("Number of items from OneDrive to process due to a full scan being triggered: ", nrChanges);
// unset now the full scan trigger if set
if (syncListFullScanTrigger) {
unsetSyncListFullScanTrigger();
if (oneDriveFullScanTrigger) {
unsetOneDriveFullScanTrigger();
}
} else {
// standard message
log.vdebug("Number of changes from OneDrive to process: ", nrChanges);
}
}
@ -1598,7 +1617,7 @@ final class SyncEngine
log.vdebug("sync_list excluded: ", syncListExcluded);
if (!syncListExcluded) {
// path we are creating is not excluded via sync_list
setSyncListFullScanTrigger();
setOneDriveFullScanTrigger();
}
if (!dryRun) {