deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-29 20:12:36 +02:00
Code Issues Projects Releases Wiki Activity

Reduce scanning of local filesystem needlessly every sync in monitor mode (#433)

Browse source 
* Reduce scanning the entire local system in monitor mode for local changes
* Add 'monitor_fullscan_frequency' to set the frequency of performing a full disk scan when in monitor mode
This commit is contained in:
abraunegg 2019-04-02 03:58:56 +11:00 committed by GitHub
parent 770456fe13
commit 4444b9cd8a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 31 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/config.d
View file

@ -52,6 +52,9 @@ final class Config
longValues["min_notif_changes"] = 5; longValues["min_notif_changes"] = 5;
longValues["min_notif_changes"] = 5; longValues["min_notif_changes"] = 5;
longValues["monitor_log_frequency"] = 5; longValues["monitor_log_frequency"] = 5;
// Number of n sync runs before performing a full local scan of sync_dir
// By default 10 which means every ~7.5 minutes a full disk scan of sync_dir will occur
longValues["monitor_fullscan_frequency"] = 10;
// Determine the users home directory. // Determine the users home directory.
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts // Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts

37
src/main.d
View file

@ -374,7 +374,7 @@ int main(string[] args)
} }
// Perform the sync // Perform the sync
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL); performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, true);
} }
} }
@ -436,26 +436,39 @@ int main(string[] args)
// monitor loop // monitor loop
immutable auto checkInterval = dur!"seconds"(cfg.getValueLong("monitor_interval")); immutable auto checkInterval = dur!"seconds"(cfg.getValueLong("monitor_interval"));
immutable auto logInterval = cfg.getValueLong("monitor_log_frequency"); immutable auto logInterval = cfg.getValueLong("monitor_log_frequency");
immutable auto fullScanFrequency = cfg.getValueLong("monitor_fullscan_frequency");
auto lastCheckTime = MonoTime.currTime(); auto lastCheckTime = MonoTime.currTime();
auto logMonitorCounter = 0; auto logMonitorCounter = 0;
auto fullScanCounter = 0;
bool fullScanRequired = true;
while (true) { while (true) {
if (!cfg.getValueBool("download_only")) m.update(online); if (!cfg.getValueBool("download_only")) m.update(online);
auto currTime = MonoTime.currTime(); auto currTime = MonoTime.currTime();
if (currTime - lastCheckTime > checkInterval) { if (currTime - lastCheckTime > checkInterval) {
// log monitor output suppression
logMonitorCounter += 1; logMonitorCounter += 1;
if (logMonitorCounter > logInterval) if (logMonitorCounter > logInterval)
logMonitorCounter = 1; logMonitorCounter = 1;
// full scan of sync_dir
fullScanCounter += 1;
if (fullScanCounter > fullScanFrequency){
fullScanCounter = 1;
fullScanRequired = true;
}
// log.logAndNotify("DEBUG trying to create checkpoint"); // log.logAndNotify("DEBUG trying to create checkpoint");
// auto res = itemdb.db_checkpoint(); // auto res = itemdb.db_checkpoint();
// log.logAndNotify("Checkpoint return: ", res); // log.logAndNotify("Checkpoint return: ", res);
// itemdb.dump_open_statements(); // itemdb.dump_open_statements();
try { try {
if (!initSyncEngine(sync)) { if (!initSyncEngine(sync)) {
oneDrive.http.shutdown(); oneDrive.http.shutdown();
return EXIT_FAILURE; return EXIT_FAILURE;
} }
try { try {
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT)); performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired);
if (!cfg.getValueBool("download_only")) { if (!cfg.getValueBool("download_only")) {
// discard all events that may have been generated by the sync // discard all events that may have been generated by the sync
m.update(false); m.update(false);
@ -471,6 +484,7 @@ int main(string[] args)
log.log("Cannot initialize connection to OneDrive"); log.log("Cannot initialize connection to OneDrive");
} }
// performSync complete, set lastCheckTime to current time // performSync complete, set lastCheckTime to current time
fullScanRequired = false;
lastCheckTime = MonoTime.currTime(); lastCheckTime = MonoTime.currTime();
GC.collect(); GC.collect();
} }
@ -516,7 +530,7 @@ bool initSyncEngine(SyncEngine sync)
} }
// try to synchronize the folder three times // try to synchronize the folder three times
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel) void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired)
{ {
int count; int count;
string remotePath = "/"; string remotePath = "/";
@ -575,12 +589,17 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// sync from OneDrive first before uploading files to OneDrive // sync from OneDrive first before uploading files to OneDrive
if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ..."); if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ...");
sync.applyDifferences(); sync.applyDifferences();
// is this a download only request? // Is a full scan of the entire sync_dir required?
if (!downloadOnly) { if (fullScanRequired) {
// process local changes // is this a download only request?
sync.scanForDifferences(localPath); if (!downloadOnly) {
// ensure that the current remote state is updated locally // process local changes walking the entire path checking for changes
sync.applyDifferences(); // in monitor mode all local changes are captured via inotify
// thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required
sync.scanForDifferences(localPath);
// ensure that the current remote state is updated locally
sync.applyDifferences();
}
} }
} }
} }