mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-04 06:52:18 +02:00
* When using sync_list and files are moved out of sync scope online via OneDrive website, check if the item was previously in-sync in database. if true, flag for local delete. * Rework fix for #568 so that if a directory is moved / created in sync_list scope, on the 'true-up' sync, any files that were moved with the folder are now downloaded * Update when sync_list override is done * Add debugging for each loop for sync_list scan handling
This commit is contained in:
parent
387bd53438
commit
db5ceccffb
17
src/main.d
17
src/main.d
|
@ -690,7 +690,7 @@ int main(string[] args)
|
||||||
auto logMonitorCounter = 0;
|
auto logMonitorCounter = 0;
|
||||||
auto fullScanCounter = 0;
|
auto fullScanCounter = 0;
|
||||||
bool fullScanRequired = true;
|
bool fullScanRequired = true;
|
||||||
bool syncListConfiguredOverride = false;
|
bool syncListConfiguredOverride = true;
|
||||||
while (true) {
|
while (true) {
|
||||||
if (!cfg.getValueBool("download_only")) m.update(online);
|
if (!cfg.getValueBool("download_only")) m.update(online);
|
||||||
auto currTime = MonoTime.currTime();
|
auto currTime = MonoTime.currTime();
|
||||||
|
@ -709,6 +709,11 @@ int main(string[] args)
|
||||||
syncListConfiguredOverride = true;
|
syncListConfiguredOverride = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sync option handling per sync loop
|
||||||
|
log.vdebug("syncListConfigured = ", syncListConfigured);
|
||||||
|
log.vdebug("fullScanRequired = ", fullScanRequired);
|
||||||
|
log.vdebug("syncListConfiguredOverride = ", syncListConfiguredOverride);
|
||||||
|
|
||||||
// log.logAndNotify("DEBUG trying to create checkpoint");
|
// log.logAndNotify("DEBUG trying to create checkpoint");
|
||||||
// auto res = itemdb.db_checkpoint();
|
// auto res = itemdb.db_checkpoint();
|
||||||
|
@ -852,8 +857,8 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
||||||
} else {
|
} else {
|
||||||
// sync from OneDrive first before uploading files to OneDrive
|
// sync from OneDrive first before uploading files to OneDrive
|
||||||
if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ...");
|
if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ...");
|
||||||
// if sync_list is configured, syncListConfigured = true, thus a FULL walk of all OneDrive objects will be performed
|
// For the initial sync, always use the delta link so that we capture all the right delta changes including adds, moves & deletes
|
||||||
sync.applyDifferences(syncListConfigured);
|
sync.applyDifferences(false);
|
||||||
// Is a full scan of the entire sync_dir required?
|
// Is a full scan of the entire sync_dir required?
|
||||||
if (fullScanRequired) {
|
if (fullScanRequired) {
|
||||||
// is this a download only request?
|
// is this a download only request?
|
||||||
|
@ -862,9 +867,9 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
||||||
// in monitor mode all local changes are captured via inotify
|
// in monitor mode all local changes are captured via inotify
|
||||||
// thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required
|
// thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required
|
||||||
sync.scanForDifferences(localPath);
|
sync.scanForDifferences(localPath);
|
||||||
// ensure that the current remote state is updated locally
|
// ensure that the current remote state is updated locally to ensure everything is consistent
|
||||||
// for the 'true-up' set this to false so that we use the delta link for second operation
|
// for the 'true-up' sync, if sync_list is configured, syncListConfigured = true, thus a FULL walk of all OneDrive objects will be requested and used if required
|
||||||
sync.applyDifferences(false);
|
sync.applyDifferences(syncListConfigured);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
47
src/sync.d
47
src/sync.d
|
@ -233,6 +233,8 @@ final class SyncEngine
|
||||||
private bool syncBusinessFolders = false;
|
private bool syncBusinessFolders = false;
|
||||||
// single directory scope flag
|
// single directory scope flag
|
||||||
private bool singleDirectoryScope = false;
|
private bool singleDirectoryScope = false;
|
||||||
|
// sync_list new folder added, trigger delta scan override
|
||||||
|
private bool syncListFullScanTrigger = false;
|
||||||
|
|
||||||
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
|
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
|
||||||
{
|
{
|
||||||
|
@ -410,6 +412,22 @@ final class SyncEngine
|
||||||
log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments");
|
log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue #658 Handling
|
||||||
|
// If an existing folder is moved into a sync_list valid path (where it previously was out of scope due to sync_list),
|
||||||
|
// then set this flag to true, so that on the second 'true-up' sync, we force a rescan of the OneDrive path to capture any 'files'
|
||||||
|
void setSyncListFullScanTrigger()
|
||||||
|
{
|
||||||
|
syncListFullScanTrigger = true;
|
||||||
|
log.vdebug("Setting syncListFullScanTrigger = true due to new folder creation request in a location that is in-scope via sync_list");
|
||||||
|
}
|
||||||
|
|
||||||
|
// unset method
|
||||||
|
void unsetSyncListFullScanTrigger()
|
||||||
|
{
|
||||||
|
syncListFullScanTrigger = false;
|
||||||
|
log.vdebug("Setting syncListFullScanTrigger = false");
|
||||||
|
}
|
||||||
|
|
||||||
// download all new changes from OneDrive
|
// download all new changes from OneDrive
|
||||||
void applyDifferences(bool performFullItemScan)
|
void applyDifferences(bool performFullItemScan)
|
||||||
{
|
{
|
||||||
|
@ -733,9 +751,10 @@ final class SyncEngine
|
||||||
log.vdebug("onedrive.getPathDetailsById call returned an invalid JSON Object");
|
log.vdebug("onedrive.getPathDetailsById call returned an invalid JSON Object");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue #658
|
||||||
// If we are using a sync_list file, using deltaLink will actually 'miss' changes (moves & deletes) on OneDrive as using sync_list discards changes
|
// If we are using a sync_list file, using deltaLink will actually 'miss' changes (moves & deletes) on OneDrive as using sync_list discards changes
|
||||||
// Use the performFullItemScan boolean to control whether we perform a full object scan of use the delta link for the root folder
|
// Use the performFullItemScan boolean to control whether we perform a full object scan of use the delta link for the root folder
|
||||||
// When using --synchronize the process order is:
|
// When using --synchronize the normal process order is:
|
||||||
// 1. Scan OneDrive for changes
|
// 1. Scan OneDrive for changes
|
||||||
// 2. Scan local folder for changes
|
// 2. Scan local folder for changes
|
||||||
// 3. Scan OneDrive for changes
|
// 3. Scan OneDrive for changes
|
||||||
|
@ -827,7 +846,7 @@ final class SyncEngine
|
||||||
// is changes a valid JSON response
|
// is changes a valid JSON response
|
||||||
if (changes.type() == JSONType.object) {
|
if (changes.type() == JSONType.object) {
|
||||||
// Are there any changes to process?
|
// Are there any changes to process?
|
||||||
if ((("value" in changes) != null) && (deltaChanges > 0)) {
|
if ((("value" in changes) != null) && ((deltaChanges > 0) || (syncListFullScanTrigger))) {
|
||||||
auto nrChanges = count(changes["value"].array);
|
auto nrChanges = count(changes["value"].array);
|
||||||
auto changeCount = 0;
|
auto changeCount = 0;
|
||||||
|
|
||||||
|
@ -848,6 +867,11 @@ final class SyncEngine
|
||||||
// There are valid changes
|
// There are valid changes
|
||||||
log.vdebug("Number of items from OneDrive to process: ", nrChanges);
|
log.vdebug("Number of items from OneDrive to process: ", nrChanges);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unset now the full scan trigger if set
|
||||||
|
if (syncListFullScanTrigger) {
|
||||||
|
unsetSyncListFullScanTrigger();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (item; changes["value"].array) {
|
foreach (item; changes["value"].array) {
|
||||||
|
@ -1187,7 +1211,7 @@ final class SyncEngine
|
||||||
// check for selective sync
|
// check for selective sync
|
||||||
string path;
|
string path;
|
||||||
if (!unwanted) {
|
if (!unwanted) {
|
||||||
// Is the item parent in the local database
|
// Is the item parent in the local database?
|
||||||
if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){
|
if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){
|
||||||
// compute the item path to see if the path is excluded
|
// compute the item path to see if the path is excluded
|
||||||
path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name;
|
path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name;
|
||||||
|
@ -1203,6 +1227,12 @@ final class SyncEngine
|
||||||
// path is unwanted
|
// path is unwanted
|
||||||
unwanted = true;
|
unwanted = true;
|
||||||
log.vlog("Skipping item - excluded by sync_list config: ", path);
|
log.vlog("Skipping item - excluded by sync_list config: ", path);
|
||||||
|
// flagging to skip this file now, but does this exist in the DB thus needs to be removed / deleted?
|
||||||
|
if (itemdb.idInLocalDatabase(item.driveId, item.id)){
|
||||||
|
log.vlog("Flagging item for local delete as item exists in database: ", path);
|
||||||
|
// flag to delete
|
||||||
|
idsToDelete ~= [item.driveId, item.id];
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
@ -1249,9 +1279,11 @@ final class SyncEngine
|
||||||
// Item name we will attempt to delete will be printed out later
|
// Item name we will attempt to delete will be printed out later
|
||||||
if (cached) {
|
if (cached) {
|
||||||
// flag to delete
|
// flag to delete
|
||||||
|
log.vdebug("Flagging item for deletion: ", item);
|
||||||
idsToDelete ~= [item.driveId, item.id];
|
idsToDelete ~= [item.driveId, item.id];
|
||||||
} else {
|
} else {
|
||||||
// flag to ignore
|
// flag to ignore
|
||||||
|
log.vdebug("Flagging item to skip: ", item);
|
||||||
skippedItems ~= item.id;
|
skippedItems ~= item.id;
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
|
@ -1379,6 +1411,15 @@ final class SyncEngine
|
||||||
case ItemType.dir:
|
case ItemType.dir:
|
||||||
case ItemType.remote:
|
case ItemType.remote:
|
||||||
log.log("Creating directory: ", path);
|
log.log("Creating directory: ", path);
|
||||||
|
|
||||||
|
// Issue #658 handling
|
||||||
|
auto syncListExcluded = selectiveSync.isPathExcludedViaSyncList(path);
|
||||||
|
log.vdebug("sync_list excluded: ", syncListExcluded);
|
||||||
|
if (!syncListExcluded) {
|
||||||
|
// path we are creating is not excluded via sync_list
|
||||||
|
setSyncListFullScanTrigger();
|
||||||
|
}
|
||||||
|
|
||||||
if (!dryRun) {
|
if (!dryRun) {
|
||||||
mkdirRecurse(path);
|
mkdirRecurse(path);
|
||||||
} else {
|
} else {
|
||||||
|
|
Loading…
Reference in a new issue