deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-15 12:16:49 +02:00
Code Issues Projects Releases Wiki Activity

Check database for excluded sync_list items previously in scope (Issue #723) (#724)

Browse source 
* When using sync_list and files are moved out of sync scope online via OneDrive website, check if the item was previously in-sync in database. if true, flag for local delete.
* Rework fix for #568 so that if a directory is moved / created in sync_list scope, on the 'true-up' sync, any files that were moved with the folder are now downloaded
* Update when sync_list override is done
* Add debugging for each loop for sync_list scan handling
This commit is contained in:
abraunegg 2019-11-21 19:58:34 +11:00 committed by GitHub
parent 387bd53438
commit db5ceccffb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 55 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/main.d
View file

@ -690,7 +690,7 @@ int main(string[] args)
auto logMonitorCounter = 0;
auto fullScanCounter = 0;
bool fullScanRequired = true;
bool syncListConfiguredOverride = false;
bool syncListConfiguredOverride = true;
while (true) {
if (!cfg.getValueBool("download_only")) m.update(online);
auto currTime = MonoTime.currTime();
@ -709,6 +709,11 @@ int main(string[] args)
syncListConfiguredOverride = true;
}
}
// sync option handling per sync loop
log.vdebug("syncListConfigured = ", syncListConfigured);
log.vdebug("fullScanRequired = ", fullScanRequired);
log.vdebug("syncListConfiguredOverride = ", syncListConfiguredOverride);
// log.logAndNotify("DEBUG trying to create checkpoint");
// auto res = itemdb.db_checkpoint();
@ -852,8 +857,8 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
} else {
// sync from OneDrive first before uploading files to OneDrive
if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ...");
// if sync_list is configured, syncListConfigured = true, thus a FULL walk of all OneDrive objects will be performed
sync.applyDifferences(syncListConfigured);
// For the initial sync, always use the delta link so that we capture all the right delta changes including adds, moves & deletes
sync.applyDifferences(false);
// Is a full scan of the entire sync_dir required?
if (fullScanRequired) {
// is this a download only request?
@ -862,9 +867,9 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// in monitor mode all local changes are captured via inotify
// thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required
sync.scanForDifferences(localPath);
// ensure that the current remote state is updated locally
// for the 'true-up' set this to false so that we use the delta link for second operation
sync.applyDifferences(false);
// ensure that the current remote state is updated locally to ensure everything is consistent
// for the 'true-up' sync, if sync_list is configured, syncListConfigured = true, thus a FULL walk of all OneDrive objects will be requested and used if required
sync.applyDifferences(syncListConfigured);
}
}
}

47
src/sync.d
View file

@ -233,6 +233,8 @@ final class SyncEngine
private bool syncBusinessFolders = false;
// single directory scope flag
private bool singleDirectoryScope = false;
// sync_list new folder added, trigger delta scan override
private bool syncListFullScanTrigger = false;
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
{
@ -410,6 +412,22 @@ final class SyncEngine
log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments");
}
// Issue #658 Handling
// If an existing folder is moved into a sync_list valid path (where it previously was out of scope due to sync_list),
// then set this flag to true, so that on the second 'true-up' sync, we force a rescan of the OneDrive path to capture any 'files'
void setSyncListFullScanTrigger()
{
syncListFullScanTrigger = true;
log.vdebug("Setting syncListFullScanTrigger = true due to new folder creation request in a location that is in-scope via sync_list");
}
// unset method
void unsetSyncListFullScanTrigger()
{
syncListFullScanTrigger = false;
log.vdebug("Setting syncListFullScanTrigger = false");
}
// download all new changes from OneDrive
void applyDifferences(bool performFullItemScan)
{
@ -733,9 +751,10 @@ final class SyncEngine
log.vdebug("onedrive.getPathDetailsById call returned an invalid JSON Object");
}
// Issue #658
// If we are using a sync_list file, using deltaLink will actually 'miss' changes (moves & deletes) on OneDrive as using sync_list discards changes
// Use the performFullItemScan boolean to control whether we perform a full object scan of use the delta link for the root folder
// When using --synchronize the process order is:
// When using --synchronize the normal process order is:
// 1. Scan OneDrive for changes
// 2. Scan local folder for changes
// 3. Scan OneDrive for changes
@ -827,7 +846,7 @@ final class SyncEngine
// is changes a valid JSON response
if (changes.type() == JSONType.object) {
// Are there any changes to process?
if ((("value" in changes) != null) && (deltaChanges > 0)) {
if ((("value" in changes) != null) && ((deltaChanges > 0) || (syncListFullScanTrigger))) {
auto nrChanges = count(changes["value"].array);
auto changeCount = 0;
@ -848,6 +867,11 @@ final class SyncEngine
// There are valid changes
log.vdebug("Number of items from OneDrive to process: ", nrChanges);
}
// unset now the full scan trigger if set
if (syncListFullScanTrigger) {
unsetSyncListFullScanTrigger();
}
}
foreach (item; changes["value"].array) {
@ -1187,7 +1211,7 @@ final class SyncEngine
// check for selective sync
string path;
if (!unwanted) {
// Is the item parent in the local database
// Is the item parent in the local database?
if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){
// compute the item path to see if the path is excluded
path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name;
@ -1203,6 +1227,12 @@ final class SyncEngine
// path is unwanted
unwanted = true;
log.vlog("Skipping item - excluded by sync_list config: ", path);
// flagging to skip this file now, but does this exist in the DB thus needs to be removed / deleted?
if (itemdb.idInLocalDatabase(item.driveId, item.id)){
log.vlog("Flagging item for local delete as item exists in database: ", path);
// flag to delete
idsToDelete ~= [item.driveId, item.id];
}
}
}
} else {
@ -1249,9 +1279,11 @@ final class SyncEngine
// Item name we will attempt to delete will be printed out later
if (cached) {
// flag to delete
log.vdebug("Flagging item for deletion: ", item);
idsToDelete ~= [item.driveId, item.id];
} else {
// flag to ignore
log.vdebug("Flagging item to skip: ", item);
skippedItems ~= item.id;
}
return;
@ -1379,6 +1411,15 @@ final class SyncEngine
case ItemType.dir:
case ItemType.remote:
log.log("Creating directory: ", path);
// Issue #658 handling
auto syncListExcluded = selectiveSync.isPathExcludedViaSyncList(path);
log.vdebug("sync_list excluded: ", syncListExcluded);
if (!syncListExcluded) {
// path we are creating is not excluded via sync_list
setSyncListFullScanTrigger();
}
if (!dryRun) {
mkdirRecurse(path);
} else {