mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-15 20:25:18 +02:00
* When using 'sync_list' do not output that we are processing 'changes' as this is misleading. * When using sync_list perform a full scan only the first time when using --synchronize & --monitor to avoid double processing of all OneDrive items each and every time * When using sync_list and --monitor only perform a full object scan when we are also configured to perform a full scan
This commit is contained in:
parent
808a2f331b
commit
6a8d575941
25
src/main.d
25
src/main.d
|
@ -78,6 +78,7 @@ int main(string[] args)
|
|||
string syncListHashFile = cfg.configDirName ~ "/.sync_list.hash";
|
||||
string configBackupFile = cfg.configDirName ~ "/.config.backup";
|
||||
bool configOptionsDifferent = false;
|
||||
bool syncListConfigured = false;
|
||||
bool syncListDifferent = false;
|
||||
bool syncDirDifferent = false;
|
||||
bool skipFileDifferent = false;
|
||||
|
@ -487,6 +488,7 @@ int main(string[] args)
|
|||
auto selectiveSync = new SelectiveSync();
|
||||
if (exists(cfg.syncListFilePath)){
|
||||
log.vdebug("Loading user configured sync_list file ...");
|
||||
syncListConfigured = true;
|
||||
// list what will be synced
|
||||
auto syncListFile = File(cfg.syncListFilePath);
|
||||
auto range = syncListFile.byLine();
|
||||
|
@ -611,8 +613,8 @@ int main(string[] args)
|
|||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, true);
|
||||
// perform a --synchronize sync
|
||||
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, true, syncListConfigured);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -679,6 +681,7 @@ int main(string[] args)
|
|||
auto logMonitorCounter = 0;
|
||||
auto fullScanCounter = 0;
|
||||
bool fullScanRequired = true;
|
||||
bool syncListConfiguredOverride = false;
|
||||
while (true) {
|
||||
if (!cfg.getValueBool("download_only")) m.update(online);
|
||||
auto currTime = MonoTime.currTime();
|
||||
|
@ -693,6 +696,9 @@ int main(string[] args)
|
|||
if (fullScanCounter > fullScanFrequency){
|
||||
fullScanCounter = 1;
|
||||
fullScanRequired = true;
|
||||
if (syncListConfigured) {
|
||||
syncListConfiguredOverride = true;
|
||||
}
|
||||
}
|
||||
|
||||
// log.logAndNotify("DEBUG trying to create checkpoint");
|
||||
|
@ -706,7 +712,8 @@ int main(string[] args)
|
|||
return EXIT_FAILURE;
|
||||
}
|
||||
try {
|
||||
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired);
|
||||
// perform a --monitor sync
|
||||
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired, syncListConfiguredOverride);
|
||||
if (!cfg.getValueBool("download_only")) {
|
||||
// discard all events that may have been generated by the sync
|
||||
m.update(false);
|
||||
|
@ -723,6 +730,9 @@ int main(string[] args)
|
|||
}
|
||||
// performSync complete, set lastCheckTime to current time
|
||||
fullScanRequired = false;
|
||||
if (syncListConfigured) {
|
||||
syncListConfiguredOverride = false;
|
||||
}
|
||||
lastCheckTime = MonoTime.currTime();
|
||||
GC.collect();
|
||||
}
|
||||
|
@ -768,7 +778,7 @@ bool initSyncEngine(SyncEngine sync)
|
|||
}
|
||||
|
||||
// try to synchronize the folder three times
|
||||
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired)
|
||||
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired, bool syncListConfigured)
|
||||
{
|
||||
int count;
|
||||
string remotePath = "/";
|
||||
|
@ -829,11 +839,11 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
// sync local files first before downloading from OneDrive
|
||||
if (logLevel < MONITOR_LOG_QUIET) log.log("Syncing changes from local path first before downloading changes from OneDrive ...");
|
||||
sync.scanForDifferences(localPath);
|
||||
sync.applyDifferences();
|
||||
sync.applyDifferences(syncListConfigured);
|
||||
} else {
|
||||
// sync from OneDrive first before uploading files to OneDrive
|
||||
if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ...");
|
||||
sync.applyDifferences();
|
||||
sync.applyDifferences(syncListConfigured);
|
||||
// Is a full scan of the entire sync_dir required?
|
||||
if (fullScanRequired) {
|
||||
// is this a download only request?
|
||||
|
@ -843,7 +853,8 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
// thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required
|
||||
sync.scanForDifferences(localPath);
|
||||
// ensure that the current remote state is updated locally
|
||||
sync.applyDifferences();
|
||||
// for the 'true-up' set this to false so that we use the delta link for second operation
|
||||
sync.applyDifferences(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
42
src/sync.d
42
src/sync.d
|
@ -411,13 +411,13 @@ final class SyncEngine
|
|||
}
|
||||
|
||||
// download all new changes from OneDrive
|
||||
void applyDifferences()
|
||||
void applyDifferences(bool performFullItemScan)
|
||||
{
|
||||
// Set defaults for the root folder
|
||||
// Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls
|
||||
string driveId = defaultDriveId;
|
||||
string rootId = defaultRootId;
|
||||
applyDifferences(driveId, rootId);
|
||||
applyDifferences(driveId, rootId, performFullItemScan);
|
||||
|
||||
// Check OneDrive Personal Shared Folders
|
||||
// https://github.com/OneDrive/onedrive-api-docs/issues/764
|
||||
|
@ -425,7 +425,7 @@ final class SyncEngine
|
|||
foreach (item; items) {
|
||||
log.vdebug("------------------------------------------------------------------");
|
||||
log.vlog("Syncing OneDrive Shared Folder: ", item.name);
|
||||
applyDifferences(item.remoteDriveId, item.remoteId);
|
||||
applyDifferences(item.remoteDriveId, item.remoteId, performFullItemScan);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -462,19 +462,19 @@ final class SyncEngine
|
|||
// 2. Download changes specific to the remote path
|
||||
|
||||
// root remote
|
||||
applyDifferences(defaultDriveId, onedrivePathDetails["id"].str);
|
||||
applyDifferences(defaultDriveId, onedrivePathDetails["id"].str, false);
|
||||
|
||||
// remote changes
|
||||
driveId = onedrivePathDetails["remoteItem"]["parentReference"]["driveId"].str; // Should give something like 66d53be8a5056eca
|
||||
folderId = onedrivePathDetails["remoteItem"]["id"].str; // Should give something like BC7D88EC1F539DCF!107
|
||||
|
||||
// Apply any differences found on OneDrive for this path (download data)
|
||||
applyDifferences(driveId, folderId);
|
||||
applyDifferences(driveId, folderId, false);
|
||||
} else {
|
||||
// use the item id as folderId
|
||||
folderId = onedrivePathDetails["id"].str; // Should give something like 12345ABCDE1234A1!101
|
||||
// Apply any differences found on OneDrive for this path (download data)
|
||||
applyDifferences(defaultDriveId, folderId);
|
||||
applyDifferences(defaultDriveId, folderId, false);
|
||||
}
|
||||
} else {
|
||||
// Log that an invalid JSON object was returned
|
||||
|
@ -586,16 +586,24 @@ final class SyncEngine
|
|||
|
||||
// download the new changes of a specific item
|
||||
// id is the root of the drive or a shared folder
|
||||
private void applyDifferences(string driveId, const(char)[] id)
|
||||
private void applyDifferences(string driveId, const(char)[] id, bool performFullItemScan)
|
||||
{
|
||||
log.vlog("Applying changes of Path ID: " ~ id);
|
||||
JSONValue changes;
|
||||
|
||||
// If we are using a sync_list file, using deltaLink will actually 'miss' changes (moves & deletes) on OneDrive as using sync_list discards changes
|
||||
// Use the performFullItemScan boolean to control whether we perform a full object scan of use the delta link for the root folder
|
||||
// When using --synchronize the process order is:
|
||||
// 1. Scan OneDrive for changes
|
||||
// 2. Scan local folder for changes
|
||||
// 3. Scan OneDrive for changes
|
||||
// When using sync_list and performing a full scan, what this means is a full scan is performed twice, which leads to massive processing & time overheads
|
||||
// Control this via performFullItemScan
|
||||
|
||||
string deltaLink = "";
|
||||
string userSyncList = cfg.configDirName ~ "/sync_list";
|
||||
if (!exists(userSyncList)){
|
||||
// not using sync_list file, use the delta link
|
||||
if (!performFullItemScan){
|
||||
// performFullItemScan == false
|
||||
// use delta link
|
||||
deltaLink = itemdb.getDeltaLink(driveId, id);
|
||||
}
|
||||
|
||||
|
@ -792,7 +800,7 @@ final class SyncEngine
|
|||
// HTTP request returned status code 504 (Gateway Timeout)
|
||||
// Retry by calling applyDifferences() again
|
||||
log.vlog("OneDrive returned a 'HTTP 504 - Gateway Timeout' - gracefully handling error");
|
||||
applyDifferences(driveId, idToQuery);
|
||||
applyDifferences(driveId, idToQuery, performFullItemScan);
|
||||
} else {
|
||||
// Default operation if not 404, 410, 500, 504 errors
|
||||
// display what the error is
|
||||
|
@ -809,12 +817,24 @@ final class SyncEngine
|
|||
auto nrChanges = count(changes["value"].array);
|
||||
auto changeCount = 0;
|
||||
|
||||
if (!performFullItemScan){
|
||||
// Display the number of changes we are processing
|
||||
if (nrChanges >= cfg.getValueLong("min_notify_changes")) {
|
||||
log.logAndNotify("Processing ", nrChanges, " changes");
|
||||
} else {
|
||||
// There are valid changes
|
||||
log.vdebug("Number of changes from OneDrive to process: ", nrChanges);
|
||||
}
|
||||
} else {
|
||||
// Do not display anything unless we are doing a verbose debug as due to #658 we are essentially doing a --resync each time when using sync_list
|
||||
// Display the number of items we are processing
|
||||
if (nrChanges >= cfg.getValueLong("min_notify_changes")) {
|
||||
log.logAndNotify("Processing ", nrChanges, " OneDrive items to ensure consistent state due to sync_list being used");
|
||||
} else {
|
||||
// There are valid changes
|
||||
log.vdebug("Number of items from OneDrive to process: ", nrChanges);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (item; changes["value"].array) {
|
||||
bool isRoot = false;
|
||||
|
|
Loading…
Reference in a new issue