deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-06-08 00:42:13 +02:00
Code Issues Projects Releases Wiki Activity

Add developer option to display sync options (#924)

Browse source 
* It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging. This has been useful when tracking down 'sync_list' / sync issue & other performance related items without having to enable full verbose debugging to see what is going on
This commit is contained in:
abraunegg 2020-05-25 11:30:04 +10:00 committed by GitHub
parent 0096e7efce
commit f9c3ccc311
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 111 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/config.d
View file

@ -92,6 +92,9 @@ final class Config
// - It may be desirable to, when running in monitor mode, force monitor mode to 'quit' after X number of loops // - It may be desirable to, when running in monitor mode, force monitor mode to 'quit' after X number of loops
// - This is especially beneficial when debugging or performing memory tests with Valgrind // - This is especially beneficial when debugging or performing memory tests with Valgrind
longValues["monitor_max_loop"] = 0; longValues["monitor_max_loop"] = 0;
// display_sync_options = true | false
// - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging
boolValues["display_sync_options"] = false;
// Determine the users home directory. // Determine the users home directory.
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts // Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts

140
src/main.d
View file

@ -31,6 +31,7 @@ int main(string[] args)
string syncListHashFile; string syncListHashFile;
string configBackupFile; string configBackupFile;
string syncDir; string syncDir;
string logOutputMessage;
bool configOptionsDifferent = false; bool configOptionsDifferent = false;
bool syncListConfigured = false; bool syncListConfigured = false;
bool syncListDifferent = false; bool syncListDifferent = false;
@ -41,6 +42,7 @@ int main(string[] args)
bool performSyncOK = false; bool performSyncOK = false;
bool onedriveInitialised = false; bool onedriveInitialised = false;
bool displayMemoryUsage = false; bool displayMemoryUsage = false;
bool displaySyncOptions = false;
// Define scopes // Define scopes
scope(exit) { scope(exit) {
@ -129,6 +131,9 @@ int main(string[] args)
// set memory display // set memory display
displayMemoryUsage = cfg.getValueBool("display_memory"); displayMemoryUsage = cfg.getValueBool("display_memory");
// set display sync options
displaySyncOptions = cfg.getValueBool("display_sync_options");
// update configuration from command line args // update configuration from command line args
cfg.update_from_args(args); cfg.update_from_args(args);
@ -332,13 +337,9 @@ int main(string[] args)
} }
} }
// dry-run notification // dry-run notification and database setup
if (cfg.getValueBool("dry_run")) { if (cfg.getValueBool("dry_run")) {
log.log("DRY-RUN Configured. Output below shows what 'would' have occurred."); log.log("DRY-RUN Configured. Output below shows what 'would' have occurred.");
}
// dry-run database setup
if (cfg.getValueBool("dry_run")) {
// If the dry run database exists, clean this up // If the dry run database exists, clean this up
if (exists(cfg.databaseFilePathDryRun)) { if (exists(cfg.databaseFilePathDryRun)) {
// remove the existing file // remove the existing file
@ -748,7 +749,7 @@ int main(string[] args)
// perform a --synchronize sync // perform a --synchronize sync
// fullScanRequired = false, for final true-up // fullScanRequired = false, for final true-up
// but if we have sync_list configured, use syncListConfigured which = true // but if we have sync_list configured, use syncListConfigured which = true
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, false, syncListConfigured); performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, false, syncListConfigured, displaySyncOptions);
} }
} }
@ -858,7 +859,12 @@ int main(string[] args)
auto currTime = MonoTime.currTime(); auto currTime = MonoTime.currTime();
if (currTime - lastCheckTime > checkInterval) { if (currTime - lastCheckTime > checkInterval) {
// monitor sync loop // monitor sync loop
log.vdebug("################################################## NEW LOOP ##################################################"); logOutputMessage = "################################################## NEW LOOP ##################################################";
if (displaySyncOptions) {
log.log(logOutputMessage);
} else {
log.vdebug(logOutputMessage);
}
// Increment monitorLoopFullCount // Increment monitorLoopFullCount
monitorLoopFullCount++; monitorLoopFullCount++;
// Display memory details at start of loop // Display memory details at start of loop
@ -888,12 +894,19 @@ int main(string[] args)
} }
} }
// Monitor Loop Counter if (displaySyncOptions) {
log.vdebug("fullScanCounter = ", fullScanCounter); // sync option handling per sync loop
// sync option handling per sync loop log.log("fullScanCounter = ", fullScanCounter);
log.vdebug("syncListConfigured = ", syncListConfigured); log.log("syncListConfigured = ", syncListConfigured);
log.vdebug("fullScanRequired = ", fullScanRequired); log.log("fullScanRequired = ", fullScanRequired);
log.vdebug("syncListConfiguredFullScanOverride = ", syncListConfiguredFullScanOverride); log.log("syncListConfiguredFullScanOverride = ", syncListConfiguredFullScanOverride);
} else {
// sync option handling per sync loop via debug
log.vdebug("fullScanCounter = ", fullScanCounter);
log.vdebug("syncListConfigured = ", syncListConfigured);
log.vdebug("fullScanRequired = ", fullScanRequired);
log.vdebug("syncListConfiguredFullScanOverride = ", syncListConfiguredFullScanOverride);
}
try { try {
if (!initSyncEngine(sync)) { if (!initSyncEngine(sync)) {
@ -903,7 +916,7 @@ int main(string[] args)
try { try {
// perform a --monitor sync // perform a --monitor sync
log.vlog("Starting a sync with OneDrive"); log.vlog("Starting a sync with OneDrive");
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired, syncListConfiguredFullScanOverride); performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired, syncListConfiguredFullScanOverride, displaySyncOptions);
if (!cfg.getValueBool("download_only")) { if (!cfg.getValueBool("download_only")) {
// discard all events that may have been generated by the sync // discard all events that may have been generated by the sync
try { try {
@ -940,7 +953,13 @@ int main(string[] args)
if (displayMemoryUsage) { if (displayMemoryUsage) {
log.displayMemoryUsagePostGC(); log.displayMemoryUsagePostGC();
} }
// monitor loop complete
logOutputMessage = "################################################ LOOP COMPLETE ###############################################";
if (displaySyncOptions) {
log.log(logOutputMessage);
} else {
log.vdebug(logOutputMessage);
}
// Developer break via config option // Developer break via config option
if (cfg.getValueLong("monitor_max_loop") > 0) { if (cfg.getValueLong("monitor_max_loop") > 0) {
// developer set option to limit --monitor loops // developer set option to limit --monitor loops
@ -988,11 +1007,12 @@ bool initSyncEngine(SyncEngine sync)
} }
// try to synchronize the folder three times // try to synchronize the folder three times
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired, bool syncListConfiguredFullScanOverride) void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired, bool syncListConfiguredFullScanOverride, bool displaySyncOptions)
{ {
int count; int count;
string remotePath = "/"; string remotePath = "/";
string localPath = "."; string localPath = ".";
string logOutputMessage;
// performSync API scan triggers // performSync API scan triggers
log.vdebug("performSync API scan triggers"); log.vdebug("performSync API scan triggers");
@ -1017,7 +1037,13 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
do { do {
try { try {
log.vdebug("################################################## NEW SYNC ##################################################"); // starting a sync
logOutputMessage = "################################################## NEW SYNC ##################################################";
if (displaySyncOptions) {
log.log(logOutputMessage);
} else {
log.vdebug(logOutputMessage);
}
if (singleDirectory != ""){ if (singleDirectory != ""){
// we were requested to sync a single directory // we were requested to sync a single directory
log.vlog("Syncing changes from this selected path: ", singleDirectory); log.vlog("Syncing changes from this selected path: ", singleDirectory);
@ -1053,6 +1079,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
sync.scanForDifferences(localPath); sync.scanForDifferences(localPath);
} else { } else {
// No upload only // No upload only
string syncCallLogOutput;
if (localFirst) { if (localFirst) {
// sync local files first before downloading from OneDrive // sync local files first before downloading from OneDrive
if (logLevel < MONITOR_LOG_QUIET) log.log("Syncing changes from local path first before downloading changes from OneDrive ..."); if (logLevel < MONITOR_LOG_QUIET) log.log("Syncing changes from local path first before downloading changes from OneDrive ...");
@ -1070,15 +1097,31 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ..."); if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ...");
// For the initial sync, always use the delta link so that we capture all the right delta changes including adds, moves & deletes // For the initial sync, always use the delta link so that we capture all the right delta changes including adds, moves & deletes
log.vdebug("Calling sync.applyDifferences(false);"); logOutputMessage = "Initial Scan: Call OneDrive Delta API for delta changes as compared to last successful sync.";
syncCallLogOutput = "Calling sync.applyDifferences(false);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
sync.applyDifferences(false); sync.applyDifferences(false);
// is this a download only request? // is this a download only request?
if (!downloadOnly) { if (!downloadOnly) {
// process local changes walking the entire path checking for changes // process local changes walking the entire path checking for changes
// in monitor mode all local changes are captured via inotify // in monitor mode all local changes are captured via inotify
// thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required // thus scanning every 'monitor_interval' (default 300 seconds) for local changes is excessive and not required
log.vdebug("Calling sync.scanForDifferences(localPath);"); logOutputMessage = "Process local filesystem (sync_dir) for file changes as compared to database entries";
syncCallLogOutput = "Calling sync.scanForDifferences(localPath);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
sync.scanForDifferences(localPath); sync.scanForDifferences(localPath);
// At this point, all OneDrive changes / local changes should be uploaded and in sync // At this point, all OneDrive changes / local changes should be uploaded and in sync
@ -1096,39 +1139,72 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// Do not perform a full walk of the OneDrive objects // Do not perform a full walk of the OneDrive objects
if ((!fullScanRequired) && (!syncListConfiguredFullScanOverride)){ if ((!fullScanRequired) && (!syncListConfiguredFullScanOverride)){
log.vdebug("Final True-Up: Do not perform a full walk of the OneDrive objects - not required"); logOutputMessage = "Final True-Up: Do not perform a full walk of the OneDrive objects - not required";
log.vdebug("Calling sync.applyDifferences(false);"); syncCallLogOutput = "Calling sync.applyDifferences(false);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
sync.applyDifferences(false); sync.applyDifferences(false);
return;
} }
// Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop // Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop
if ((!fullScanRequired) && (syncListConfiguredFullScanOverride)){ if ((!fullScanRequired) && (syncListConfiguredFullScanOverride)){
log.vdebug("Final True-Up: Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop"); logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop";
log.vdebug("Calling sync.applyDifferences(true);"); syncCallLogOutput = "Calling sync.applyDifferences(true);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
sync.applyDifferences(true); sync.applyDifferences(true);
return;
} }
// Perform a full walk of OneDrive objects because a full scan was required // Perform a full walk of OneDrive objects because a full scan was required
if ((fullScanRequired) && (!syncListConfiguredFullScanOverride)){ if ((fullScanRequired) && (!syncListConfiguredFullScanOverride)){
log.vdebug("Final True-Up: Perform a full walk of OneDrive objects because a full scan was required"); logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required";
log.vdebug("Calling sync.applyDifferences(true);"); syncCallLogOutput = "Calling sync.applyDifferences(true);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
sync.applyDifferences(true); sync.applyDifferences(true);
return;
} }
// Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop // Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop
if ((fullScanRequired) && (syncListConfiguredFullScanOverride)){ if ((fullScanRequired) && (syncListConfiguredFullScanOverride)){
log.vdebug("Final True-Up: Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop"); logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop";
log.vdebug("Calling sync.applyDifferences(true);"); syncCallLogOutput = "Calling sync.applyDifferences(true);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
sync.applyDifferences(true); sync.applyDifferences(true);
return;
} }
} }
} }
} }
} }
// sync is complete
logOutputMessage = "################################################ SYNC COMPLETE ###############################################";
if (displaySyncOptions) {
log.log(logOutputMessage);
} else {
log.vdebug(logOutputMessage);
}
count = -1; count = -1;
} catch (Exception e) { } catch (Exception e) {
if (++count == 3) { if (++count == 3) {