From ad0daf2df500bf8f44916be1e108fee1cbfd4ad5 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Sat, 24 Aug 2019 17:18:58 +1000 Subject: [PATCH] Detect the need for --resync when config changes (#617) * Detect the need for --resync when config changes either via config file or cli override --- docs/USAGE.md | 2 + src/config.d | 34 +++++--- src/main.d | 217 ++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 233 insertions(+), 20 deletions(-) diff --git a/docs/USAGE.md b/docs/USAGE.md index 04e1bbbd..5111eca4 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -236,6 +236,8 @@ Proceed with caution here when changing the default sync dir from ~/OneDrive to The issue here is around how the client stores the sync_dir path in the database. If the config file is missing, or you don't use the `--syncdir` parameter - what will happen is the client will default back to `~/OneDrive` and 'think' that either all your data has been deleted - thus delete the content on OneDrive, or will start downloading all data from OneDrive into the default location. +**Note:** After changing `sync_dir`, you must perform a full re-synchronization by adding `--resync` to your existing command line - for example: `onedrive --synchronize --resync` + ### skip_dir Example: `skip_dir = "Desktop|Documents/IISExpress|Documents/SQL Server Management Studio|Documents/Visual Studio*|Documents/WindowsPowerShell"` diff --git a/src/config.d b/src/config.d index 2ae7ac4d..515cecaa 100644 --- a/src/config.d +++ b/src/config.d @@ -14,7 +14,13 @@ final class Config public string syncListFilePath; public string homePath; public string configDirName; - + public string defaultSyncDir = "~/OneDrive"; + public string defaultSkipFile = "~*|.~*|*.tmp"; + public string defaultSkipDir = ""; + public string configFileSyncDir; + public string configFileSkipFile; + public string configFileSkipDir; + private string userConfigFilePath; // hashmap for the values found in the user config file // ARGGGG D is stupid and cannot make hashmap initializations!!! @@ -23,13 +29,12 @@ final class Config private bool[string] boolValues; private long[string] longValues; - this(string confdirOption) { // default configuration - stringValues["sync_dir"] = "~/OneDrive"; - stringValues["skip_file"] = "~*|.~*|*.tmp"; - stringValues["skip_dir"] = ""; + stringValues["sync_dir"] = defaultSyncDir; + stringValues["skip_file"] = defaultSkipFile; + stringValues["skip_dir"] = defaultSkipDir; stringValues["log_dir"] = "/var/log/onedrive/"; stringValues["drive_id"] = ""; boolValues["upload_only"] = false; @@ -80,8 +85,7 @@ final class Config // Output homePath calculation log.vdebug("homePath: ", homePath); - - + // Determine the correct configuration directory to use string configDirBase; if (confdirOption != "") { @@ -113,7 +117,6 @@ final class Config configDirName = configDirBase ~ "/onedrive"; } - log.vlog("Using Config Dir: ", configDirName); if (!exists(configDirName)) mkdirRecurse(configDirName); @@ -141,10 +144,8 @@ final class Config return true; } - void update_from_args(string[] args) { - // Add additional options that are NOT configurable via config file stringValues["create_directory"] = ""; stringValues["destination_directory"] = ""; @@ -162,7 +163,6 @@ final class Config boolValues["monitor"] = false; boolValues["synchronize"] = false; - // Application Startup option validation try { string tmpStr; @@ -268,6 +268,9 @@ final class Config "skip-file", "Skip any files that match this pattern from syncing", &stringValues["skip_file"], + "skip-dir", + "Skip any directories that match this pattern from syncing", + &stringValues["skip_dir"], "skip-size", "Skip new files larger than this size (in MB)", &longValues["skip_size"], @@ -299,7 +302,6 @@ final class Config "version", "Print the version and exit", &tmpBol - ); if (opt.helpWanted) { outputLongHelp(opt.options); @@ -317,7 +319,6 @@ final class Config } } - string getValueString(string key) { auto p = key in stringValues; @@ -385,6 +386,13 @@ final class Config if (pp) { c.popFront(); setValueString(key, c.front.dup); + // detect need for --resync for these: + // --syncdir ARG + // --skip-file ARG + // --skip-dir ARG + if (key == "sync_dir") configFileSyncDir = c.front.dup; + if (key == "skip_file") configFileSkipFile = c.front.dup; + if (key == "skip_dir") configFileSkipDir = c.front.dup; } else { auto ppp = key in longValues; if (ppp) { diff --git a/src/main.d b/src/main.d index 5d613cf2..2f655117 100644 --- a/src/main.d +++ b/src/main.d @@ -1,6 +1,6 @@ import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit; import core.memory, core.time, core.thread; -import std.getopt, std.file, std.path, std.process, std.stdio, std.conv, std.algorithm.searching, std.string; +import std.getopt, std.file, std.path, std.process, std.stdio, std.conv, std.algorithm.searching, std.string, std.regex; import config, itemdb, monitor, onedrive, selective, sync, util; import std.net.curl: CurlException; import core.stdc.signal; @@ -53,8 +53,7 @@ int main(string[] args) log.error("Try 'onedrive -h' for more information"); return EXIT_FAILURE; } - - + // load configuration file if available auto cfg = new config.Config(confdirOption); if (!cfg.initialize()) { @@ -62,15 +61,216 @@ int main(string[] args) // Error message already printed return EXIT_FAILURE; } + // update configuration from command line args cfg.update_from_args(args); - + + // Has any of our configuration that would require a --resync been changed? + // 1. sync_list file modification + // 2. config file modification - but only if sync_dir, skip_dir, skip_file or drive_id was modified + // 3. CLI input overriding configured config file option + + string currentConfigHash; + string currentSyncListHash; + string previousConfigHash; + string previousSyncListHash; + string configHashFile = cfg.configDirName ~ "/.config.hash"; + string syncListHashFile = cfg.configDirName ~ "/.sync_list.hash"; + string configBackupFile = cfg.configDirName ~ "/.config.backup"; + bool configOptionsDifferent = false; + bool syncListDifferent = false; + bool syncDirDifferent = false; + bool skipFileDifferent = false; + bool skipDirDifferent = false; + + if ((exists(cfg.configDirName ~ "/config")) && (!exists(configHashFile))) { + // Hash of config file needs to be created + std.file.write(configHashFile, computeQuickXorHash(cfg.configDirName ~ "/config")); + } + + if ((exists(cfg.configDirName ~ "/sync_list")) && (!exists(syncListHashFile))) { + // Hash of sync_list file needs to be created + std.file.write(syncListHashFile, computeQuickXorHash(cfg.configDirName ~ "/sync_list")); + } + + // If hash files exist, but config files do not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different + if ((!exists(cfg.configDirName ~ "/config")) && (exists(configHashFile))) { + // if --resync safe remove config.hash and config.backup + if (cfg.getValueBool("resync")) { + safeRemove(configHashFile); + safeRemove(configBackupFile); + } + } + + if ((!exists(cfg.configDirName ~ "/sync_list")) && (exists(syncListHashFile))) { + // if --resync safe remove sync_list.hash + if (cfg.getValueBool("resync")) safeRemove(syncListHashFile); + } + + // Read config hashes if they exist + if (exists(cfg.configDirName ~ "/config")) currentConfigHash = computeQuickXorHash(cfg.configDirName ~ "/config"); + if (exists(cfg.configDirName ~ "/sync_list")) currentSyncListHash = computeQuickXorHash(cfg.configDirName ~ "/sync_list"); + if (exists(configHashFile)) previousConfigHash = readText(configHashFile); + if (exists(syncListHashFile)) previousSyncListHash = readText(syncListHashFile); + + // Was sync_list updated? + if (currentSyncListHash != previousSyncListHash) { + // Debugging output to assist what changed + log.vdebug("sync_list file has been updated, --resync needed"); + syncListDifferent = true; + } + + // Was config updated? + if (currentConfigHash != previousConfigHash) { + // config file was updated, however we only want to trigger a --resync requirement if sync_dir, skip_dir, skip_file or drive_id was modified + log.vdebug("config file has been updated, checking if --resync needed"); + if (exists(configBackupFile)) { + // check backup config what has changed for these configuration options if anything + // # sync_dir = "~/OneDrive" + // # skip_file = "~*|.~*|*.tmp" + // # skip_dir = "" + // # drive_id = "" + string[string] stringValues; + stringValues["sync_dir"] = ""; + stringValues["skip_file"] = ""; + stringValues["skip_dir"] = ""; + stringValues["drive_id"] = ""; + + auto file = File(configBackupFile, "r"); + auto r = regex(`^(\w+)\s*=\s*"(.*)"\s*$`); + foreach (line; file.byLine()) { + line = stripLeft(line); + if (line.length == 0 || line[0] == ';' || line[0] == '#') continue; + auto c = line.matchFirst(r); + if (!c.empty) { + c.popFront(); // skip the whole match + string key = c.front.dup; + auto p = key in stringValues; + if (p) { + c.popFront(); + // compare this key + if ((key == "sync_dir") && (c.front.dup != cfg.getValueString("sync_dir"))) { + log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed"); + configOptionsDifferent = true; + } + + if ((key == "skip_file") && (c.front.dup != cfg.getValueString("skip_file"))){ + log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed"); + configOptionsDifferent = true; + } + if ((key == "skip_dir") && (c.front.dup != cfg.getValueString("skip_dir"))){ + log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed"); + configOptionsDifferent = true; + } + if ((key == "drive_id") && (c.front.dup != cfg.getValueString("drive_id"))){ + log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed"); + configOptionsDifferent = true; + } + } + } + } + } else { + // no backup to check + log.vdebug("WARNING: no backup config file was found, unable to validate if any changes made"); + } + + // If there was a backup, any modified values we need to worry about would been detected + if (!cfg.getValueBool("display_config")) { + // we are not testing the configuration + if (!configOptionsDifferent) { + // no options are different + if (!cfg.getValueBool("dry_run")) { + // we are not in a dry-run scenario + // update config hash + log.vdebug("updating config hash as it is out of date"); + std.file.write(configHashFile, computeQuickXorHash(cfg.configDirName ~ "/config")); + // create backup copy of current config file + log.vdebug("making backup of config file as it is out of date"); + std.file.copy(cfg.configDirName ~ "/config", configBackupFile); + } + } + } + } + + // Is there a backup of the config file if the config file exists? + if ((exists(cfg.configDirName ~ "/config")) && (!exists(configBackupFile))) { + // create backup copy of current config file + std.file.copy(cfg.configDirName ~ "/config", configBackupFile); + } + + // config file set options can be changed via CLI input, specifically these will impact sync and --resync will be needed: + // --syncdir ARG + // --skip-file ARG + // --skip-dir ARG + if (exists(cfg.configDirName ~ "/config")) { + // config file exists + // was the sync_dir updated by CLI? + if (cfg.configFileSyncDir != "") { + // sync_dir was set in config file + if (cfg.configFileSyncDir != cfg.getValueString("sync_dir")) { + // config file was set and CLI input changed this + log.vdebug("sync_dir: CLI override of config file option, --resync needed"); + syncDirDifferent = true; + } + } + + // was the skip_file updated by CLI? + if (cfg.configFileSkipFile != "") { + // skip_file was set in config file + if (cfg.configFileSkipFile != cfg.getValueString("skip_file")) { + // config file was set and CLI input changed this + log.vdebug("skip_file: CLI override of config file option, --resync needed"); + skipFileDifferent = true; + } + } + + // was the skip_dir updated by CLI? + if (cfg.configFileSkipDir != "") { + // skip_dir was set in config file + if (cfg.configFileSkipDir != cfg.getValueString("skip_dir")) { + // config file was set and CLI input changed this + log.vdebug("skip_dir: CLI override of config file option, --resync needed"); + skipDirDifferent = true; + } + } + } + + // Has anything triggered a --resync requirement? + if (configOptionsDifferent || syncListDifferent || syncDirDifferent || skipFileDifferent || skipDirDifferent) { + // --resync needed, is the user just testing configuration changes? + if (!cfg.getValueBool("display_config")){ + // not testing configuration changes + if (!cfg.getValueBool("resync")) { + // --resync not issued, fail fast + log.error("An application configuration change has been detected where a --resync is required"); + return EXIT_FAILURE; + } else { + // --resync issued, update hashes of config files if they exist + if (!cfg.getValueBool("dry_run")) { + // not doing a dry run, update hash files if config & sync_list exist + if (exists(cfg.configDirName ~ "/config")) { + // update hash + log.vdebug("updating config hash as --resync issued"); + std.file.write(configHashFile, computeQuickXorHash(cfg.configDirName ~ "/config")); + // create backup copy of current config file + log.vdebug("making backup of config file as --resync issued"); + std.file.copy(cfg.configDirName ~ "/config", configBackupFile); + } + if (exists(cfg.configDirName ~ "/sync_list")) { + // update sync_list hash + log.vdebug("updating sync_list hash as --resync issued"); + std.file.write(syncListHashFile, computeQuickXorHash(cfg.configDirName ~ "/sync_list")); + } + } + } + } + } + // dry-run notification if (cfg.getValueBool("dry_run")) { log.log("DRY-RUN Configured. Output below shows what 'would' have occurred."); } - // Are we able to reach the OneDrive Service bool online = false; @@ -134,6 +334,7 @@ int main(string[] args) } if (cfg.getValueBool("resync") || cfg.getValueBool("logout")) { + if (cfg.getValueBool("resync")) log.vdebug("--resync requested"); log.vlog("Deleting the saved status ..."); if (!cfg.getValueBool("dry_run")) { safeRemove(cfg.databaseFilePath); @@ -141,6 +342,7 @@ int main(string[] args) safeRemove(cfg.uploadStateFilePath); } if (cfg.getValueBool("logout")) { + log.vdebug("--logout requested"); if (!cfg.getValueBool("dry_run")) { safeRemove(cfg.refreshTokenFilePath); } @@ -196,6 +398,7 @@ int main(string[] args) writeln("Selective sync configured = false"); } + // exit return EXIT_SUCCESS; } @@ -257,11 +460,11 @@ int main(string[] args) log.vlog("Opening the item database ..."); if (!cfg.getValueBool("dry_run")) { // Load the items.sqlite3 file as the database - log.vdebug("Using database file: ", cfg.databaseFilePath); + log.vdebug("Using database file: ", asNormalizedPath(cfg.databaseFilePath)); itemDb = new ItemDatabase(cfg.databaseFilePath); } else { // Load the items-dryrun.sqlite3 file as the database - log.vdebug("Using database file: ", cfg.databaseFilePathDryRun); + log.vdebug("Using database file: ", asNormalizedPath(cfg.databaseFilePathDryRun)); itemDb = new ItemDatabase(cfg.databaseFilePathDryRun); }