diff --git a/README.md b/README.md index a714cce3..6c81e34c 100644 --- a/README.md +++ b/README.md @@ -249,11 +249,6 @@ skip_file = "= .*|~*" ```text skip_file = "~*" ``` -**Default valid configuration:** -```text -skip_file = "~*|.~*|*.tmp" -``` - Do not use a skip_file entry of `.*` as this will prevent correct searching of local changes to process. ### Important - curl compatibility @@ -318,7 +313,7 @@ Config path = /home/alex/.config/onedrive Config file found in config path = false Config option 'sync_dir' = /home/alex/OneDrive Config option 'skip_dir' = -Config option 'skip_file' = ~*|.~*|*.tmp +Config option 'skip_file' = ~* Config option 'skip_dotfiles' = false Config option 'skip_symlinks' = false Config option 'monitor_interval' = 45 @@ -492,11 +487,6 @@ Files can be skipped in the following fashion: * Explicitly specify the filename and it's full path relative to your sync_dir, eg: 'path/to/file/filename.ext' * Explicitly specify the filename only and skip every instance of this filename, eg: 'filename.ext' -By default, the following files will be skipped: -* Files that start with ~ -* Files that start with .~ (like .~lock.* files generated by LibreOffice) -* Files that end in .tmp - **Note:** after changing `skip_file`, you must perform a full re-synchronization by adding `--resync` to your existing command line - for example: `onedrive --synchronize --resync` **Note:** Do not use a skip_file entry of `.*` as this will prevent correct searching of local changes to process. diff --git a/config b/config index 3095f4f2..79cc666b 100644 --- a/config +++ b/config @@ -1,6 +1,6 @@ # Directory where the files will be synced sync_dir = "~/OneDrive" # Skip files and directories that match this pattern -skip_file = "~*|.~*|*.tmp" +skip_file = "~*" # Wait time (seconds) between sync operations in monitor mode monitor_interval = "45" diff --git a/src/config.d b/src/config.d index f0921850..5d07a86a 100644 --- a/src/config.d +++ b/src/config.d @@ -1,6 +1,4 @@ -import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit; -import std.file, std.string, std.regex, std.stdio, std.process, std.algorithm.searching, std.getopt, std.conv; -import std.algorithm.sorting: sort; +import std.file, std.string, std.regex, std.stdio; import selective; static import log; @@ -12,110 +10,13 @@ final class Config public string databaseFilePathDryRun; public string uploadStateFilePath; public string syncListFilePath; - public string homePath; - public string configDirName; private string userConfigFilePath; // hashmap for the values found in the user config file - // ARGGGG D is stupid and cannot make hashmap initializations!!! - // private string[string] foobar = [ "aa": "bb" ] does NOT work!!! - private string[string] stringValues; - private bool[string] boolValues; - private long[string] longValues; + private string[string] values; - - this(string confdirOption) + this(string configDirName) { - // default configuration - stringValues["single_directory"] = ""; - stringValues["sync_dir"] = "~/OneDrive"; - stringValues["skip_file"] = "~*|.~*|*.tmp"; - stringValues["skip_dir"] = ""; - stringValues["log_dir"] = "/var/log/onedrive/"; - stringValues["drive_id"] = ""; - boolValues["upload_only"] = false; - boolValues["check_nomount"] = false; - boolValues["check_nosync"] = false; - boolValues["download_only"] = false; - boolValues["disable_notifications"] = false; - boolValues["disable_upload_validation"] = false; - boolValues["enable_logging"] = false; - boolValues["force_http_11"] = false; - boolValues["local_first"] = false; - boolValues["no_remote_delete"] = false; - boolValues["skip_symlinks"] = false; - boolValues["debug_https"] = false; - boolValues["skip_dotfiles"] = false; - boolValues["dry_run"] = false; - longValues["verbose"] = log.verbose; // might be initialized by the first getopt call! - longValues["monitor_interval"] = 45, - longValues["min_notif_changes"] = 5; - longValues["min_notif_changes"] = 5; - longValues["monitor_log_frequency"] = 5; - // Number of n sync runs before performing a full local scan of sync_dir - // By default 10 which means every ~7.5 minutes a full disk scan of sync_dir will occur - longValues["monitor_fullscan_frequency"] = 10; - - // Determine the users home directory. - // Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts - // Check for HOME environment variable - if (environment.get("HOME") != ""){ - // Use HOME environment variable - log.vdebug("homePath: HOME environment variable set"); - homePath = environment.get("HOME"); - } else { - if ((environment.get("SHELL") == "") && (environment.get("USER") == "")){ - // No shell is set or username - observed case when running as systemd service under CentOS 7.x - log.vdebug("homePath: WARNING - no HOME environment variable set"); - log.vdebug("homePath: WARNING - no SHELL environment variable set"); - log.vdebug("homePath: WARNING - no USER environment variable set"); - homePath = "/root"; - } else { - // A shell & valid user is set, but no HOME is set, use ~ which can be expanded - log.vdebug("homePath: WARNING - no HOME environment variable set"); - homePath = "~"; - } - } - - // Output homePath calculation - log.vdebug("homePath: ", homePath); - - - // Determine the correct configuration directory to use - string configDirBase; - if (confdirOption != "") { - // A CLI 'confdir' was passed in - log.vdebug("configDirName: CLI override to set configDirName to: ", confdirOption); - if (canFind(configDirName,"~")) { - // A ~ was found - log.vdebug("configDirName: A '~' was found in configDirName, using the calculated 'homePath' to replace '~'"); - configDirName = homePath ~ strip(confdirOption,"~","~"); - } else { - configDirName = confdirOption; - } - } else { - // Determine the base directory relative to which user specific configuration files should be stored. - if (environment.get("XDG_CONFIG_HOME") != ""){ - log.vdebug("configDirBase: XDG_CONFIG_HOME environment variable set"); - configDirBase = environment.get("XDG_CONFIG_HOME"); - } else { - // XDG_CONFIG_HOME does not exist on systems where X11 is not present - ie - headless systems / servers - log.vdebug("configDirBase: WARNING - no XDG_CONFIG_HOME environment variable set"); - configDirBase = homePath ~ "/.config"; - } - - // Output configDirBase calculation - log.vdebug("configDirBase: ", configDirBase); - // Set the default application configuration directory - log.vdebug("configDirName: Configuring application to use default config path"); - // configDirBase contains the correct path so we do not need to check for presence of '~' - configDirName = configDirBase ~ "/onedrive"; - } - - - log.vlog("Using Config Dir: ", configDirName); - if (!exists(configDirName)) mkdirRecurse(configDirName); - refreshTokenFilePath = configDirName ~ "/refresh_token"; deltaLinkFilePath = configDirName ~ "/delta_link"; databaseFilePath = configDirName ~ "/items.sqlite3"; @@ -125,8 +26,34 @@ final class Config syncListFilePath = configDirName ~ "/sync_list"; } - bool initialize() + bool init() { + // Default configuration directory + setValue("sync_dir", "~/OneDrive"); + // Skip Directories - no directories are skipped by default + setValue("skip_dir", ""); + // Configure to skip ONLY temp files (~*.doc etc) by default + // Prior configuration was: .*|~* + setValue("skip_file", "~*"); + // By default skip dot files & folders are not skipped + setValue("skip_dotfiles", "false"); + // By default symlinks are not skipped (using string type + // instead of boolean because hashmap only stores string types) + setValue("skip_symlinks", "false"); + // Configure the monitor mode loop - the number of seconds by which + // each sync operation is undertaken when idle under monitor mode + setValue("monitor_interval", "45"); + // Configure the default logging directory to be /var/log/onedrive/ + setValue("log_dir", "/var/log/onedrive/"); + // Configure a default empty value for drive_id + setValue("drive_id", ""); + // Minimal changes that trigger a log and notification on sync + setValue("min_notif_changes", "5"); + // Frequency of log messages in monitor, ie after n sync runs ship out a log message + setValue("monitor_log_frequency", "5"); + // Check if we should ignore a directory if a special file (.nosync) is present + setValue("check_nosync", "false"); + if (!load(userConfigFilePath)) { // What was the reason for failure? if (!exists(userConfigFilePath)) { @@ -140,154 +67,9 @@ final class Config return true; } - - void update_from_args(string[] args) + string getValue(string key) { - - // Add additional options that are NOT configurable via config file - stringValues["create_directory"] = ""; - stringValues["destination_directory"] = ""; - stringValues["get_o365_drive_id"] = ""; - stringValues["remove_directory"] = ""; - stringValues["single_directory"] = ""; - stringValues["source_directory"] = ""; - boolValues["display_config"] = false; - boolValues["display_sync_status"] = false; - boolValues["resync"] = false; - boolValues["print_token"] = false; - boolValues["logout"] = false; - boolValues["monitor"] = false; - boolValues["synchronize"] = false; - - - // Application Startup option validation - try { - string tmpStr; - bool tmpBol; - long tmpVerb; - auto opt = getopt( - args, - std.getopt.config.bundling, - std.getopt.config.caseSensitive, - "check-for-nomount", - "Check for the presence of .nosync in the syncdir root. If found, do not perform sync.", - &boolValues["check_nomount"], - "check-for-nosync", - "Check for the presence of .nosync in each directory. If found, skip directory from sync.", - &boolValues["check_nosync"], - "create-directory", - "Create a directory on OneDrive - no sync will be performed.", - &stringValues["create_directory"], - "debug-https", - "Debug OneDrive HTTPS communication.", - &boolValues["debug_https"], - "destination-directory", - "Destination directory for renamed or move on OneDrive - no sync will be performed.", - &stringValues["destination_directory"], - "disable-notifications", - "Do not use desktop notifications in monitor mode.", - &boolValues["disable_notifications"], - "disable-upload-validation", - "Disable upload validation when uploading to OneDrive", - &boolValues["disable_upload_validation"], - "display-config", - "Display what options the client will use as currently configured - no sync will be performed.", - &boolValues["display_config"], - "display-sync-status", - "Display the sync status of the client - no sync will be performed.", - &boolValues["display_sync_status"], - "download-only|d", - "Only download remote changes", - &boolValues["download_only"], - "dry-run", - "Perform a trial sync with no changes made", - &boolValues["dry_run"], - "enable-logging", - "Enable client activity to a separate log file", - &boolValues["enable_logging"], - "force-http-1.1", - "Force the use of HTTP 1.1 for all operations", - &boolValues["force_http_11"], - "get-O365-drive-id", - "Query and return the Office 365 Drive ID for a given Office 365 SharePoint Shared Library", - &stringValues["get_o365_drive_id"], - "local-first", - "Synchronize from the local directory source first, before downloading changes from OneDrive.", - &boolValues["local_first"], - "logout", - "Logout the current user", - &boolValues["logout"], - "monitor|m", - "Keep monitoring for local and remote changes", - &boolValues["monitor"], - "monitor-log-frequency", - "Frequency of logging in monitor mode", - &longValues["monitor_log_frequency"], - "no-remote-delete", - "Do not delete local file 'deletes' from OneDrive when using --upload-only", - &boolValues["no_remote_delete"], - "print-token", - "Print the access token, useful for debugging", - &boolValues["print_token"], - "resync", - "Forget the last saved state, perform a full sync", - &boolValues["resync"], - "remove-directory", - "Remove a directory on OneDrive - no sync will be performed.", - &stringValues["remove_directory"], - "skip-dot-files", - "Skip dot files and folders from syncing", - &boolValues["skip_dotfiles"], - "single-directory", - "Specify a single local directory within the OneDrive root to sync.", - &stringValues["single_directory"], - "skip-symlinks", - "Skip syncing of symlinks", - &boolValues["skip_symlinks"], - "source-directory", - "Source directory to rename or move on OneDrive - no sync will be performed.", - &stringValues["source_directory"], - "syncdir", - "Specify the local directory used for synchronization to OneDrive", - &stringValues["sync_dir"], - "synchronize", - "Perform a synchronization", - &boolValues["synchronize"], - "upload-only", - "Only upload to OneDrive, do not sync changes from OneDrive locally", - &boolValues["upload_only"], - // duplicated from main.d to get full help output! - "confdir", - "Set the directory used to store the configuration files", - &tmpStr, - "verbose|v+", - "Print more details, useful for debugging (repeat for extra debugging)", - &tmpVerb, - "version", - "Print the version and exit", - &tmpBol - - ); - if (opt.helpWanted) { - outputLongHelp(opt.options); - exit(EXIT_SUCCESS); - } - } catch (GetOptException e) { - log.error(e.msg); - log.error("Try 'onedrive -h' for more information"); - exit(EXIT_FAILURE); - } catch (Exception e) { - // error - log.error(e.msg); - log.error("Try 'onedrive -h' for more information"); - exit(EXIT_FAILURE); - } - } - - - string getValueString(string key) - { - auto p = key in stringValues; + auto p = key in values; if (p) { return *p; } else { @@ -295,39 +77,19 @@ final class Config } } - long getValueLong(string key) + string getValue(string key, string value) { - auto p = key in longValues; + auto p = key in values; if (p) { return *p; } else { - throw new Exception("Missing config value: " ~ key); + return value; } } - bool getValueBool(string key) + void setValue(string key, string value) { - auto p = key in boolValues; - if (p) { - return *p; - } else { - throw new Exception("Missing config value: " ~ key); - } - } - - void setValueBool(string key, bool value) - { - boolValues[key] = value; - } - - void setValueString(string key, string value) - { - stringValues[key] = value; - } - - void setValueLong(string key, long value) - { - longValues[key] = value; + values[key] = value; } private bool load(string filename) @@ -342,26 +104,13 @@ final class Config if (!c.empty) { c.popFront(); // skip the whole match string key = c.front.dup; - auto p = key in boolValues; + auto p = key in values; if (p) { c.popFront(); - // only accept "true" as true value. TODO Should we support other formats? - setValueBool(key, c.front.dup == "true" ? true : false); + setValue(key, c.front.dup); } else { - auto pp = key in stringValues; - if (pp) { - c.popFront(); - setValueString(key, c.front.dup); - } else { - auto ppp = key in longValues; - if (ppp) { - c.popFront(); - setValueLong(key, to!long(c.front.dup)); - } else { - log.log("Unknown key in config file: ", key); - return false; - } - } + log.log("Unknown key in config file: ", key); + return false; } } else { log.log("Malformed config line: ", line); @@ -372,49 +121,10 @@ final class Config } } - -void outputLongHelp(Option[] opt) -{ - auto argsNeedingOptions = [ - "--confdir", - "--create-directory", - "--destination-directory", - "--get-O365-drive-id", - "--remove-directory", - "--single-directory", - "--source-directory", - "--syncdir" ]; - writeln(`OneDrive - a client for OneDrive Cloud Services - -Usage: - onedrive [options] --synchronize - Do a one time synchronization - onedrive [options] --monitor - Monitor filesystem and sync regularly - onedrive [options] --display-config - Display the currently used configuration - onedrive [options] --display-sync-status - Query OneDrive service and report on pending changes - onedrive -h | --help - Show this help screen - onedrive --version - Show version - -Options: -`); - foreach (it; opt.sort!("a.optLong < b.optLong")) { - writefln(" %s%s%s%s\n %s", - it.optLong, - it.optShort == "" ? "" : " " ~ it.optShort, - argsNeedingOptions.canFind(it.optLong) ? " ARG" : "", - it.required ? " (required)" : "", it.help); - } -} - unittest { auto cfg = new Config(""); cfg.load("config"); - assert(cfg.getValueString("sync_dir") == "~/OneDrive"); + assert(cfg.getValue("sync_dir") == "~/OneDrive"); + assert(cfg.getValue("empty", "default") == "default"); } - diff --git a/src/itemdb.d b/src/itemdb.d index 8b39f3d5..fd8132eb 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -31,7 +31,7 @@ struct Item { final class ItemDatabase { // increment this for every change in the db schema - immutable int itemDatabaseVersion = 9; + immutable int itemDatabaseVersion = 8; Database db; string insertItemStmt; diff --git a/src/log.d b/src/log.d index 9b5930d4..5aa754f7 100644 --- a/src/log.d +++ b/src/log.d @@ -10,7 +10,7 @@ version(Notifications) { } // enable verbose logging -long verbose; +int verbose; bool writeLogFile = false; private bool doNotifications; diff --git a/src/main.d b/src/main.d index 9682bf47..bb9627e7 100644 --- a/src/main.d +++ b/src/main.d @@ -19,26 +19,114 @@ int main(string[] args) // Disable buffering on stdout stdout.setvbuf(0, _IONBF); + // Application Option Variables + // Add a check mounts option to resolve https://github.com/abraunegg/onedrive/issues/8 + bool checkMount = false; + // Check if we should ignore a directory if a special file (.nosync) is present - https://github.com/abraunegg/onedrive/issues/163 + bool checkNoSync = false; // configuration directory - string confdirOption; - + string configDirName; + // Create a single root directory on OneDrive + string createDirectory; + // The destination directory if we are using the OneDrive client to rename a directory + string destinationDirectory; + // Debug the HTTPS submit operations if required + bool debugHttp = false; + // Do not use notifications in monitor mode + bool disableNotifications = false; + // Display application configuration but do not sync + bool displayConfiguration = false; + // Display sync status + bool displaySyncStatus = false; + // only download remote changes + bool downloadOnly = false; + // Does the user want to disable upload validation - https://github.com/abraunegg/onedrive/issues/205 + // SharePoint will associate some metadata from the library the file is uploaded to directly in the file - thus change file size & checksums + bool disableUploadValidation = false; + // Perform only a dry run - not applicable for --monitor mode + bool dryRun = false; + // Do we enable a log file + bool enableLogFile = false; + // Force the use of HTTP 1.1 to overcome curl => 7.62.0 where some operations are now sent via HTTP/2 + // Whilst HTTP/2 operations are handled, in some cases the handling of this outside of the client is not being done correctly (router, other) thus the client breaks + // This flag then allows the user to downgrade all HTTP operations to HTTP 1.1 for maximum network path compatibility + bool forceHTTP11 = false; + // SharePoint / Office 365 Shared Library name to query + string o365SharedLibraryName; + // Local sync - Upload local changes first before downloading changes from OneDrive + bool localFirst = false; + // remove the current user and sync state + bool logout = false; + // enable monitor mode + bool monitor = false; + // Add option for no remote delete + bool noRemoteDelete = false; + // print the access token + bool printAccessToken = false; + // force a full resync + bool resync = false; + // Remove a single directory on OneDrive + string removeDirectory; + // This allows for selective directory syncing instead of everything under ~/OneDrive/ + string singleDirectory; + // Skip dot files & folders - eg .file or /.folder/ + bool skipDotFiles = false; + // Add option to skip symlinks + bool skipSymlinks = false; + // The source directory if we are using the OneDrive client to rename a directory + string sourceDirectory; + // override the sync directory + string syncDirName; + // Configure a flag to perform a sync + // This is beneficial so that if just running the client itself - without any options, or sync check, the client does not perform a sync + bool synchronize = false; + // Upload Only + bool uploadOnly = false; + // enable verbose logging + bool verbose = false; + // print the version and exit + bool printVersion = false; + + // Application Startup option validation try { - // print the version and exit - bool printVersion = false; auto opt = getopt( args, - std.getopt.config.passThrough, std.getopt.config.bundling, std.getopt.config.caseSensitive, - "confdir", "Set the directory used to store the configuration files", &confdirOption, + "check-for-nomount", "Check for the presence of .nosync in the syncdir root. If found, do not perform sync.", &checkMount, + "check-for-nosync", "Check for the presence of .nosync in each directory. If found, skip directory from sync.", &checkNoSync, + "confdir", "Set the directory used to store the configuration files", &configDirName, + "create-directory", "Create a directory on OneDrive - no sync will be performed.", &createDirectory, + "destination-directory", "Destination directory for renamed or move on OneDrive - no sync will be performed.", &destinationDirectory, + "debug-https", "Debug OneDrive HTTPS communication.", &debugHttp, + "disable-notifications", "Do not use desktop notifications in monitor mode.", &disableNotifications, + "display-config", "Display what options the client will use as currently configured - no sync will be performed.", &displayConfiguration, + "display-sync-status", "Display the sync status of the client - no sync will be performed.", &displaySyncStatus, + "download-only|d", "Only download remote changes", &downloadOnly, + "disable-upload-validation", "Disable upload validation when uploading to OneDrive", &disableUploadValidation, + "dry-run", "Perform a trial sync with no changes made", &dryRun, + "enable-logging", "Enable client activity to a separate log file", &enableLogFile, + "force-http-1.1", "Force the use of HTTP 1.1 for all operations", &forceHTTP11, + "get-O365-drive-id", "Query and return the Office 365 Drive ID for a given Office 365 SharePoint Shared Library", &o365SharedLibraryName, + "local-first", "Synchronize from the local directory source first, before downloading changes from OneDrive.", &localFirst, + "logout", "Logout the current user", &logout, + "monitor|m", "Keep monitoring for local and remote changes", &monitor, + "no-remote-delete", "Do not delete local file 'deletes' from OneDrive when using --upload-only", &noRemoteDelete, + "print-token", "Print the access token, useful for debugging", &printAccessToken, + "resync", "Forget the last saved state, perform a full sync", &resync, + "remove-directory", "Remove a directory on OneDrive - no sync will be performed.", &removeDirectory, + "single-directory", "Specify a single local directory within the OneDrive root to sync.", &singleDirectory, + "skip-dot-files", "Skip dot files and folders from syncing", &skipDotFiles, + "skip-symlinks", "Skip syncing of symlinks", &skipSymlinks, + "source-directory", "Source directory to rename or move on OneDrive - no sync will be performed.", &sourceDirectory, + "syncdir", "Specify the local directory used for synchronization to OneDrive", &syncDirName, + "synchronize", "Perform a synchronization", &synchronize, + "upload-only", "Only upload to OneDrive, do not sync changes from OneDrive locally", &uploadOnly, "verbose|v+", "Print more details, useful for debugging (repeat for extra debugging)", &log.verbose, "version", "Print the version and exit", &printVersion ); if (opt.helpWanted) { - args ~= "--help"; - } - if (printVersion) { - std.stdio.write("onedrive ", import("version")); + outputLongHelp(opt.options); return EXIT_SUCCESS; } } catch (GetOptException e) { @@ -52,28 +140,92 @@ int main(string[] args) return EXIT_FAILURE; } + // Main function variables + string homePath = ""; + string configDirBase = ""; + // Debug the HTTPS response operations if required + bool debugHttpSubmit; + // Are we able to reach the OneDrive Service + bool online = false; + // simulateNoRefreshTokenFile in case of --dry-run & --logout + bool simulateNoRefreshTokenFile = false; + + // Determine the users home directory. + // Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts + // Check for HOME environment variable + if (environment.get("HOME") != ""){ + // Use HOME environment variable + log.vdebug("homePath: HOME environment variable set"); + homePath = environment.get("HOME"); + } else { + if ((environment.get("SHELL") == "") && (environment.get("USER") == "")){ + // No shell is set or username - observed case when running as systemd service under CentOS 7.x + log.vdebug("homePath: WARNING - no HOME environment variable set"); + log.vdebug("homePath: WARNING - no SHELL environment variable set"); + log.vdebug("homePath: WARNING - no USER environment variable set"); + homePath = "/root"; + } else { + // A shell & valid user is set, but no HOME is set, use ~ which can be expanded + log.vdebug("homePath: WARNING - no HOME environment variable set"); + homePath = "~"; + } + } + + // Output homePath calculation + log.vdebug("homePath: ", homePath); - // load configuration file if available - auto cfg = new config.Config(confdirOption); - if (!cfg.initialize()) { + // Determine the base directory relative to which user specific configuration files should be stored. + if (environment.get("XDG_CONFIG_HOME") != ""){ + log.vdebug("configDirBase: XDG_CONFIG_HOME environment variable set"); + configDirBase = environment.get("XDG_CONFIG_HOME"); + } else { + // XDG_CONFIG_HOME does not exist on systems where X11 is not present - ie - headless systems / servers + log.vdebug("configDirBase: WARNING - no XDG_CONFIG_HOME environment variable set"); + configDirBase = homePath ~ "/.config"; + } + + // Output configDirBase calculation + log.vdebug("configDirBase: ", configDirBase); + + // Determine the correct configuration directory to use + if (configDirName != "") { + // A CLI 'confdir' was passed in + log.vdebug("configDirName: CLI override to set configDirName to: ", configDirName); + if (canFind(configDirName,"~")) { + // A ~ was found + log.vdebug("configDirName: A '~' was found in configDirName, using the calculated 'homePath' to replace '~'"); + configDirName = homePath ~ strip(configDirName,"~","~"); + } + } else { + // Set the default application configuration directory + log.vdebug("configDirName: Configuring application to use default config path"); + // configDirBase contains the correct path so we do not need to check for presence of '~' + configDirName = configDirBase ~ "/onedrive"; + } + + if (printVersion) { + std.stdio.write("onedrive ", import("version")); + return EXIT_SUCCESS; + } + + // dry-run notification + if (dryRun) { + log.log("DRY-RUN Configured. Output below shows what 'would' have occurred."); + } + + // load application configuration + log.vlog("Loading config ..."); + log.vlog("Using Config Dir: ", configDirName); + if (!exists(configDirName)) mkdirRecurse(configDirName); + auto cfg = new config.Config(configDirName); + if(!cfg.init()){ // There was an error loading the configuration // Error message already printed return EXIT_FAILURE; } - // update configuration from command line args - cfg.update_from_args(args); - - // dry-run notification - if (cfg.getValueBool("dry_run")) { - log.log("DRY-RUN Configured. Output below shows what 'would' have occurred."); - } - - // Are we able to reach the OneDrive Service - bool online = false; - // dry-run database setup - if (cfg.getValueBool("dry_run")) { + if (dryRun) { // Make a copy of the original items.sqlite3 for use as the dry run copy if it exists if (exists(cfg.databaseFilePath)) { // copy the file @@ -82,29 +234,58 @@ int main(string[] args) } } + // command line parameters to override default 'config' & take precedence + // Set the client to skip specific directories if .nosync is found AND ONLY if --check-for-nosync was passed in + if (checkNoSync) { + log.vdebug("CLI override to set check_nosync to: true"); + cfg.setValue("check_nosync", "true"); + } + + // Set the client to skip dot files & folders if --skip-dot-files was passed in + if (skipDotFiles) { + // The user passed in an alternate skip_dotfiles as to what was either in 'config' file or application default + log.vdebug("CLI override to set skip_dotfiles to: true"); + cfg.setValue("skip_dotfiles", "true"); + } + + // Set the client to skip symbolic links if --skip-symlinks was passed in + if (skipSymlinks) { + // The user passed in an alternate skip_symlinks as to what was either in 'config' file or application default + log.vdebug("CLI override to set skip_symlinks to: true"); + cfg.setValue("skip_symlinks", "true"); + } + + // Set the OneDrive Local Sync Directory if was passed in via --syncdir + if (syncDirName) { + // The user passed in an alternate sync_dir as to what was either in 'config' file or application default + // Do not expandTilde here as we do not know if we reliably can + log.vdebug("CLI override to set sync_dir to: ", syncDirName); + cfg.setValue("sync_dir", syncDirName); + } + // sync_dir environment handling to handle ~ expansion properly string syncDir; if ((environment.get("SHELL") == "") && (environment.get("USER") == "")){ log.vdebug("sync_dir: No SHELL or USER environment variable configuration detected"); // No shell or user set, so expandTilde() will fail - usually headless system running under init.d / systemd or potentially Docker // Does the 'currently configured' sync_dir include a ~ - if (canFind(cfg.getValueString("sync_dir"), "~")) { + if (canFind(cfg.getValue("sync_dir"),"~")) { // A ~ was found log.vdebug("sync_dir: A '~' was found in sync_dir, using the calculated 'homePath' to replace '~'"); - syncDir = cfg.homePath ~ strip(cfg.getValueString("sync_dir"), "~"); + syncDir = homePath ~ strip(cfg.getValue("sync_dir"),"~","~"); } else { // No ~ found in sync_dir, use as is log.vdebug("sync_dir: Getting syncDir from config value sync_dir"); - syncDir = cfg.getValueString("sync_dir"); + syncDir = cfg.getValue("sync_dir"); } } else { // A shell and user is set, expand any ~ as this will be expanded correctly if present log.vdebug("sync_dir: Getting syncDir from config value sync_dir"); - if (canFind(cfg.getValueString("sync_dir"), "~")) { + if (canFind(cfg.getValue("sync_dir"),"~")) { log.vdebug("sync_dir: A '~' was found in configured sync_dir, automatically expanding as SHELL and USER environment variable is set"); - syncDir = expandTilde(cfg.getValueString("sync_dir")); + syncDir = expandTilde(cfg.getValue("sync_dir")); } else { - syncDir = cfg.getValueString("sync_dir"); + syncDir = cfg.getValue("sync_dir"); } } @@ -112,47 +293,50 @@ int main(string[] args) log.vdebug("syncDir: ", syncDir); // Configure logging if enabled - if (cfg.getValueBool("enable_logging")){ + if (enableLogFile){ // Read in a user defined log directory or use the default - string logDir = cfg.getValueString("log_dir"); + string logDir = cfg.getValue("log_dir"); log.vlog("Using logfile dir: ", logDir); log.init(logDir); } // Configure whether notifications are used - log.setNotifications(cfg.getValueBool("monitor") && !cfg.getValueBool("disable_notifications")); + log.setNotifications(monitor && !disableNotifications); // upgrades - if (exists(cfg.configDirName ~ "/items.db")) { - if (!cfg.getValueBool("dry_run")) { - safeRemove(cfg.configDirName ~ "/items.db"); + if (exists(configDirName ~ "/items.db")) { + if (!dryRun) { + safeRemove(configDirName ~ "/items.db"); } log.logAndNotify("Database schema changed, resync needed"); - cfg.setValueBool("resync", true); + resync = true; } - if (cfg.getValueBool("resync") || cfg.getValueBool("logout")) { + if (resync || logout) { log.vlog("Deleting the saved status ..."); - if (!cfg.getValueBool("dry_run")) { + if (!dryRun) { safeRemove(cfg.databaseFilePath); safeRemove(cfg.deltaLinkFilePath); safeRemove(cfg.uploadStateFilePath); } - if (cfg.getValueBool("logout")) { - if (!cfg.getValueBool("dry_run")) { + if (logout) { + if (!dryRun) { safeRemove(cfg.refreshTokenFilePath); + } else { + // simulate file being removed / unavailable + simulateNoRefreshTokenFile = true; } } } // Display current application configuration, no application initialisation - if (cfg.getValueBool("display_config")){ - string userConfigFilePath = cfg.configDirName ~ "/config"; - string userSyncList = cfg.configDirName ~ "/sync_list"; + if (displayConfiguration){ + string userConfigFilePath = configDirName ~ "/config"; + string userSyncList = configDirName ~ "/sync_list"; // Display application version std.stdio.write("onedrive version = ", import("version")); // Display all of the pertinent configuration options - writeln("Config path = ", cfg.configDirName); + writeln("Config path = ", configDirName); // Does a config file exist or are we using application defaults if (exists(userConfigFilePath)){ @@ -162,19 +346,19 @@ int main(string[] args) } // Config Options - writeln("Config option 'check_nosync' = ", cfg.getValueBool("check_nosync")); + writeln("Config option 'check_nosync' = ", cfg.getValue("check_nosync")); writeln("Config option 'sync_dir' = ", syncDir); - writeln("Config option 'skip_dir' = ", cfg.getValueString("skip_dir")); - writeln("Config option 'skip_file' = ", cfg.getValueString("skip_file")); - writeln("Config option 'skip_dotfiles' = ", cfg.getValueBool("skip_dotfiles")); - writeln("Config option 'skip_symlinks' = ", cfg.getValueBool("skip_symlinks")); - writeln("Config option 'monitor_interval' = ", cfg.getValueLong("monitor_interval")); - writeln("Config option 'min_notif_changes' = ", cfg.getValueLong("min_notif_changes")); - writeln("Config option 'log_dir' = ", cfg.getValueString("log_dir")); + writeln("Config option 'skip_dir' = ", cfg.getValue("skip_dir")); + writeln("Config option 'skip_file' = ", cfg.getValue("skip_file")); + writeln("Config option 'skip_dotfiles' = ", cfg.getValue("skip_dotfiles")); + writeln("Config option 'skip_symlinks' = ", cfg.getValue("skip_symlinks")); + writeln("Config option 'monitor_interval' = ", cfg.getValue("monitor_interval")); + writeln("Config option 'min_notif_changes' = ", cfg.getValue("min_notif_changes")); + writeln("Config option 'log_dir' = ", cfg.getValue("log_dir")); // Is config option drive_id configured? - if (cfg.getValueString("drive_id") != ""){ - writeln("Config option 'drive_id' = ", cfg.getValueString("drive_id")); + if (cfg.getValue("drive_id", "") != ""){ + writeln("Config option 'drive_id' = ", cfg.getValue("drive_id")); } // Is sync_list configured? @@ -201,14 +385,14 @@ int main(string[] args) } catch (CurlException e) { // No network connection to OneDrive Service log.error("No network connection to Microsoft OneDrive Service"); - if (!cfg.getValueBool("monitor")) { + if (!monitor) { return EXIT_FAILURE; } } // Initialize OneDrive, check for authorization - oneDrive = new OneDriveApi(cfg); - oneDrive.printAccessToken = cfg.getValueBool("print_token"); + oneDrive = new OneDriveApi(cfg, debugHttp, forceHTTP11, dryRun, simulateNoRefreshTokenFile); + oneDrive.printAccessToken = printAccessToken; if (!oneDrive.init()) { log.error("Could not initialize the OneDrive API"); // workaround for segfault in std.net.curl.Curl.shutdown() on exit @@ -218,14 +402,13 @@ int main(string[] args) // if --synchronize or --monitor not passed in, exit & display help auto performSyncOK = false; - - if (cfg.getValueBool("synchronize") || cfg.getValueBool("monitor")) { + if (synchronize || monitor) { performSyncOK = true; } // create-directory, remove-directory, source-directory, destination-directory // are activities that dont perform a sync no error message for these items either - if (((cfg.getValueString("create_directory") != "") || (cfg.getValueString("remove_directory") != "")) || ((cfg.getValueString("source_directory") != "") && (cfg.getValueString("destination_directory") != "")) || (cfg.getValueString("get_o365_drive_id") != "") || cfg.getValueBool("display_sync_status")) { + if (((createDirectory != "") || (removeDirectory != "")) || ((sourceDirectory != "") && (destinationDirectory != "")) || (o365SharedLibraryName != "") || (displaySyncStatus == true)) { performSyncOK = true; } @@ -237,7 +420,7 @@ int main(string[] args) } // if --synchronize && --monitor passed in, exit & display help as these conflict with each other - if (cfg.getValueBool("synchronize") && cfg.getValueBool("monitor")) { + if (synchronize && monitor) { writeln("\nERROR: --synchronize and --monitor cannot be used together\n"); writeln("Refer to --help to determine which command option you should use.\n"); oneDrive.http.shutdown(); @@ -246,7 +429,7 @@ int main(string[] args) // Initialize the item database log.vlog("Opening the item database ..."); - if (!cfg.getValueBool("dry_run")) { + if (!dryRun) { // Load the items.sqlite3 file as the database log.vdebug("Using database file: ", cfg.databaseFilePath); itemDb = new ItemDatabase(cfg.databaseFilePath); @@ -279,15 +462,15 @@ int main(string[] args) // Configure skip_dir & skip_file from config entries log.vdebug("Configuring skip_dir ..."); - log.vdebug("skip_dir: ", cfg.getValueString("skip_dir")); - selectiveSync.setDirMask(cfg.getValueString("skip_dir")); + log.vdebug("skip_dir: ", cfg.getValue("skip_dir")); + selectiveSync.setDirMask(cfg.getValue("skip_dir")); log.vdebug("Configuring skip_file ..."); - log.vdebug("skip_file: ", cfg.getValueString("skip_file")); - selectiveSync.setFileMask(cfg.getValueString("skip_file")); + log.vdebug("skip_file: ", cfg.getValue("skip_file")); + selectiveSync.setFileMask(cfg.getValue("skip_file")); // Initialize the sync engine log.logAndNotify("Initializing the Synchronization Engine ..."); - auto sync = new SyncEngine(cfg, oneDrive, itemDb, selectiveSync); + auto sync = new SyncEngine(cfg, oneDrive, itemDb, selectiveSync, dryRun); try { if (!initSyncEngine(sync)) { @@ -295,21 +478,21 @@ int main(string[] args) return EXIT_FAILURE; } } catch (CurlException e) { - if (!cfg.getValueBool("monitor")) { - log.log("\nNo internet connection."); + if (!monitor) { + log.log("\nNo Internet connection."); oneDrive.http.shutdown(); return EXIT_FAILURE; } } // We should only set noRemoteDelete in an upload-only scenario - if ((cfg.getValueBool("upload_only"))&&(cfg.getValueBool("no_remote_delete"))) sync.setNoRemoteDelete(); + if ((uploadOnly)&&(noRemoteDelete)) sync.setNoRemoteDelete(); // Do we configure to disable the upload validation routine - if (cfg.getValueBool("disable_upload_validation")) sync.setDisableUploadValidation(); + if(disableUploadValidation) sync.setDisableUploadValidation(); // Do we need to validate the syncDir to check for the presence of a '.nosync' file - if (cfg.getValueBool("check_nomount")) { + if (checkMount) { // we were asked to check the mounts if (exists(syncDir ~ "/.nosync")) { log.logAndNotify("ERROR: .nosync file found. Aborting synchronization process to safeguard data."); @@ -319,53 +502,53 @@ int main(string[] args) } // Do we need to create or remove a directory? - if ((cfg.getValueString("create_directory") != "") || (cfg.getValueString("remove_directory") != "")) { + if ((createDirectory != "") || (removeDirectory != "")) { - if (cfg.getValueString("create_directory") != "") { + if (createDirectory != "") { // create a directory on OneDrive - sync.createDirectoryNoSync(cfg.getValueString("create_directory")); + sync.createDirectoryNoSync(createDirectory); } - if (cfg.getValueString("remove_directory") != "") { + if (removeDirectory != "") { // remove a directory on OneDrive - sync.deleteDirectoryNoSync(cfg.getValueString("remove_directory")); + sync.deleteDirectoryNoSync(removeDirectory); } } // Are we renaming or moving a directory? - if ((cfg.getValueString("source_directory") != "") && (cfg.getValueString("destination_directory") != "")) { + if ((sourceDirectory != "") && (destinationDirectory != "")) { // We are renaming or moving a directory - sync.renameDirectoryNoSync(cfg.getValueString("source_directory"), cfg.getValueString("destination_directory")); + sync.renameDirectoryNoSync(sourceDirectory, destinationDirectory); } // Are we obtaining the Office 365 Drive ID for a given Office 365 SharePoint Shared Library? - if (cfg.getValueString("get_o365_drive_id") != ""){ - sync.querySiteCollectionForDriveID(cfg.getValueString("get_o365_drive_id")); + if (o365SharedLibraryName != ""){ + sync.querySiteCollectionForDriveID(o365SharedLibraryName); } // Are we displaying the sync status of the client? - if (cfg.getValueBool("display_sync_status")) { + if (displaySyncStatus) { string remotePath = "/"; string localPath = "."; // Are we doing a single directory check? - if (cfg.getValueString("single_directory") != ""){ + if (singleDirectory != ""){ // Need two different path strings here - remotePath = cfg.getValueString("single_directory"); - localPath = cfg.getValueString("single_directory"); + remotePath = singleDirectory; + localPath = singleDirectory; } sync.queryDriveForChanges(remotePath); } // Are we performing a sync, resync or monitor operation? - if ((cfg.getValueBool("synchronize")) || (cfg.getValueBool("resync")) || (cfg.getValueBool("monitor"))) { + if ((synchronize) || (resync) || (monitor)) { - if ((cfg.getValueBool("synchronize")) || (cfg.getValueBool("resync"))) { + if ((synchronize) || (resync)) { if (online) { // Check user entry for local path - the above chdir means we are already in ~/OneDrive/ thus singleDirectory is local to this path - if (cfg.getValueString("single_directory") != ""){ + if (singleDirectory != ""){ // Does the directory we want to sync actually exist? - if (!exists(cfg.getValueString("single_directory"))){ + if (!exists(singleDirectory)){ // the requested directory does not exist .. log.logAndNotify("ERROR: The requested local directory does not exist. Please check ~/OneDrive/ for requested path"); oneDrive.http.shutdown(); @@ -374,13 +557,13 @@ int main(string[] args) } // Perform the sync - performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, true); + performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, LOG_NORMAL); } } - if (cfg.getValueBool("monitor")) { + if (monitor) { log.logAndNotify("Initializing monitor ..."); - log.log("OneDrive monitor interval (seconds): ", cfg.getValueLong("monitor_interval")); + log.log("OneDrive monitor interval (seconds): ", to!long(cfg.getValue("monitor_interval"))); Monitor m = new Monitor(selectiveSync); m.onDirCreated = delegate(string path) { log.vlog("[M] Directory created: ", path); @@ -432,44 +615,33 @@ int main(string[] args) signal(SIGTERM, &exitHandler); // initialise the monitor class - if (!cfg.getValueBool("download_only")) m.init(cfg, cfg.getValueLong("verbose") > 0, cfg.getValueBool("skip_symlinks"), cfg.getValueBool("check_nosync")); + if (cfg.getValue("skip_symlinks") == "true") skipSymlinks = true; + if (cfg.getValue("check_nosync") == "true") checkNoSync = true; + if (!downloadOnly) m.init(cfg, verbose, skipSymlinks, checkNoSync); // monitor loop - immutable auto checkInterval = dur!"seconds"(cfg.getValueLong("monitor_interval")); - immutable auto logInterval = cfg.getValueLong("monitor_log_frequency"); - immutable auto fullScanFrequency = cfg.getValueLong("monitor_fullscan_frequency"); + immutable auto checkInterval = dur!"seconds"(to!long(cfg.getValue("monitor_interval"))); + immutable auto logInterval = to!long(cfg.getValue("monitor_log_frequency")); auto lastCheckTime = MonoTime.currTime(); auto logMonitorCounter = 0; - auto fullScanCounter = 0; - bool fullScanRequired = true; while (true) { - if (!cfg.getValueBool("download_only")) m.update(online); + if (!downloadOnly) m.update(online); auto currTime = MonoTime.currTime(); if (currTime - lastCheckTime > checkInterval) { - // log monitor output suppression logMonitorCounter += 1; if (logMonitorCounter > logInterval) logMonitorCounter = 1; - - // full scan of sync_dir - fullScanCounter += 1; - if (fullScanCounter > fullScanFrequency){ - fullScanCounter = 1; - fullScanRequired = true; - } - // log.logAndNotify("DEBUG trying to create checkpoint"); // auto res = itemdb.db_checkpoint(); // log.logAndNotify("Checkpoint return: ", res); // itemdb.dump_open_statements(); - try { if (!initSyncEngine(sync)) { oneDrive.http.shutdown(); return EXIT_FAILURE; } try { - performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired); - if (!cfg.getValueBool("download_only")) { + performSync(sync, singleDirectory, downloadOnly, localFirst, uploadOnly, (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT)); + if (!downloadOnly) { // discard all events that may have been generated by the sync m.update(false); } @@ -484,7 +656,6 @@ int main(string[] args) log.log("Cannot initialize connection to OneDrive"); } // performSync complete, set lastCheckTime to current time - fullScanRequired = false; lastCheckTime = MonoTime.currTime(); GC.collect(); } @@ -500,7 +671,7 @@ int main(string[] args) destroy(itemDb); // --dry-run temp database cleanup - if (cfg.getValueBool("dry_run")) { + if (dryRun) { if (exists(cfg.databaseFilePathDryRun)) { // remove the file log.vdebug("Removing items-dryrun.sqlite3 as dry run operations complete"); @@ -530,7 +701,7 @@ bool initSyncEngine(SyncEngine sync) } // try to synchronize the folder three times -void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired) +void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel) { int count; string remotePath = "/"; @@ -589,17 +760,12 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo // sync from OneDrive first before uploading files to OneDrive if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ..."); sync.applyDifferences(); - // Is a full scan of the entire sync_dir required? - if (fullScanRequired) { - // is this a download only request? - if (!downloadOnly) { - // process local changes walking the entire path checking for changes - // in monitor mode all local changes are captured via inotify - // thus scanning every 'monitor_interval' (default 45 seconds) for local changes is excessive and not required - sync.scanForDifferences(localPath); - // ensure that the current remote state is updated locally - sync.applyDifferences(); - } + // is this a download only request? + if (!downloadOnly) { + // process local changes + sync.scanForDifferences(localPath); + // ensure that the current remote state is updated locally + sync.applyDifferences(); } } } @@ -635,4 +801,41 @@ extern(C) nothrow @nogc @system void exitHandler(int value) { } catch(Exception e) {} exit(0); } +void outputLongHelp(Option[] opt) +{ + auto argsNeedingOptions = [ + "--confdir", + "--create-directory", + "--destination-directory", + "--get-O365-drive-id", + "--remove-directory", + "--single-directory", + "--source-directory", + "--syncdir" ]; + writeln(`OneDrive - a client for OneDrive Cloud Services + +Usage: + onedrive [options] --synchronize + Do a one time synchronization + onedrive [options] --monitor + Monitor filesystem and sync regularly + onedrive [options] --display-config + Display the currently used configuration + onedrive [options] --display-sync-status + Query OneDrive service and report on pending changes + onedrive -h | --help + Show this help screen + onedrive --version + Show version + +Options: +`); + foreach (it; opt) { + writefln(" %s%s%s%s\n %s", + it.optShort == "" ? "" : it.optShort ~ " ", + it.optLong, + argsNeedingOptions.canFind(it.optLong) ? " ARG" : "", + it.required ? " (required)" : "", it.help); + } +} diff --git a/src/onedrive.d b/src/onedrive.d index 97b0f39c..847be0ec 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -8,8 +8,8 @@ import progress; import config; static import log; shared bool debugResponse = false; -private bool dryRun = false; -private bool simulateNoRefreshTokenFile = false; +shared bool dryRun = false; +shared bool simulateNoRefreshTokenFile = false; private immutable { // Client Identifier @@ -66,7 +66,7 @@ final class OneDriveApi // if true, every new access token is printed bool printAccessToken; - this(Config cfg) + this(Config cfg, bool debugHttp, bool forceHTTP11, bool dryRun, bool simulateNoRefreshTokenFile) { this.cfg = cfg; http = HTTP(); @@ -94,36 +94,36 @@ final class OneDriveApi http.maxRedirects(5); // Do we enable curl debugging? - if (cfg.getValueBool("debug_https")) { + if (debugHttp) { http.verbose = true; .debugResponse = true; } // What version of HTTP protocol do we use? // Curl >= 7.62.0 defaults to http2 for a significant number of operations - if (cfg.getValueBool("force_http_11")) { + if (forceHTTP11) { log.vdebug("Downgrading all HTTP operations to HTTP 1.1"); // Downgrade to HTTP 1.1 - yes version = 2 is HTTP 1.1 http.handle.set(CurlOption.http_version,2); } // Do we set the dryRun handlers? - if (cfg.getValueBool("dry_run")) { + if (dryRun) { .dryRun = true; - if (cfg.getValueBool("logout")) { - .simulateNoRefreshTokenFile = true; - } + } + if (simulateNoRefreshTokenFile) { + .simulateNoRefreshTokenFile = true; } } bool init() { try { - driveId = cfg.getValueString("drive_id"); + driveId = cfg.getValue("drive_id"); if (driveId.length) { driveUrl = driveByIdUrl ~ driveId; - itemByIdUrl = driveUrl ~ "/items"; - itemByPathUrl = driveUrl ~ "/root:/"; + itemByIdUrl = driveUrl ~ "/items"; + itemByPathUrl = driveUrl ~ "/root:/"; } } catch (Exception e) {} @@ -265,7 +265,7 @@ final class OneDriveApi // string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; if ((path == ".")||(path == "/")) url = driveUrl ~ "/root/"; else url = itemByPathUrl ~ encodeComponent(path) ~ ":/"; - url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference"; return get(url); } @@ -277,7 +277,7 @@ final class OneDriveApi const(char)[] url; // string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/"; url = driveByIdUrl ~ driveId ~ "/items/" ~ id; - url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference"; return get(url); } @@ -730,8 +730,8 @@ final class OneDriveApi case 400: // Bad Request .. how should we act? log.vlog("OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error"); - break; - + break; + // 412 - Precondition Failed case 412: log.vlog("OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error"); diff --git a/src/sync.d b/src/sync.d index 2465cf46..9ff2f383 100644 --- a/src/sync.d +++ b/src/sync.d @@ -205,7 +205,7 @@ final class SyncEngine // sync engine dryRun flag private bool dryRun = false; - this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync) + this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync, bool dryRun) { assert(onedrive && itemdb && selectiveSync); this.cfg = cfg; @@ -213,7 +213,7 @@ final class SyncEngine this.itemdb = itemdb; this.selectiveSync = selectiveSync; // session = UploadSession(onedrive, cfg.uploadStateFilePath); - this.dryRun = cfg.getValueBool("dry_run"); + this.dryRun = dryRun; } void reset() @@ -240,7 +240,7 @@ final class SyncEngine // OneDrive responded with 400 error: Bad Request log.error("\nERROR: OneDrive returned a 'HTTP 400 Bad Request' - Cannot Initialize Sync Engine"); // Check this - if (cfg.getValueString("drive_id").length) { + if (cfg.getValue("drive_id").length) { log.error("ERROR: Check your 'drive_id' entry in your configuration file as it may be incorrect\n"); } // Must exit here @@ -308,7 +308,6 @@ final class SyncEngine void setDisableUploadValidation() { disableUploadValidation = true; - log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments"); } @@ -604,7 +603,7 @@ final class SyncEngine if (("value" in changes) != null) { auto nrChanges = count(changes["value"].array); - if (nrChanges >= cfg.getValueLong("min_notif_changes")) { + if (nrChanges >= to!long(cfg.getValue("min_notif_changes"))) { log.logAndNotify("Processing ", nrChanges, " changes"); } else { // There are valid changes @@ -757,11 +756,8 @@ final class SyncEngine if (unwanted) log.vdebug("Flagging as unwanted: find(item.parentId).length != 0"); // Check if this is a directory to skip if (!unwanted) { - // Only check path if config is != "" - if (cfg.getValueString("skip_dir") != "") { - unwanted = selectiveSync.isDirNameExcluded(item.name); - if (unwanted) log.vlog("Skipping item - excluded by skip_dir config: ", item.name); - } + unwanted = selectiveSync.isDirNameExcluded(item.name); + if (unwanted) log.vlog("Skipping item - excluded by skip_dir config: ", item.name); } // Check if this is a file to skip if (!unwanted) { @@ -802,7 +798,7 @@ final class SyncEngine } // skip downloading dot files if configured - if (cfg.getValueBool("skip_dotfiles")) { + if (cfg.getValue("skip_dotfiles") == "true") { if (isDotFile(path)) { log.vlog("Skipping item - .file or .folder: ", path); unwanted = true; @@ -1364,45 +1360,28 @@ final class SyncEngine writeln("done."); } } else { - // OneDrive Business Account - // We need to always use a session to upload, but handle the changed file correctly - if (accountType == "business"){ - // For logging consistency - writeln(""); - try { - response = session.upload(path, item.driveId, item.parentId, baseName(path), item.eTag); - } catch (OneDriveException e) { - // Resolve https://github.com/abraunegg/onedrive/issues/36 - if ((e.httpStatusCode == 409) || (e.httpStatusCode == 423)) { - // The file is currently checked out or locked for editing by another user - // We cant upload this file at this time - writeln("skipped."); - log.fileOnly("Uploading modified file ", path, " ... skipped."); - writeln("", path, " is currently checked out or locked for editing by another user."); - log.fileOnly(path, " is currently checked out or locked for editing by another user."); - return; - } - // what is this error????? - else throw e; + // OneDrive Business Account - always use a session to upload + writeln(""); + + try { + response = session.upload(path, item.driveId, item.parentId, baseName(path)); + } catch (OneDriveException e) { + + // Resolve https://github.com/abraunegg/onedrive/issues/36 + if ((e.httpStatusCode == 409) || (e.httpStatusCode == 423)) { + // The file is currently checked out or locked for editing by another user + // We cant upload this file at this time + writeln(" skipped."); + log.fileOnly("Uploading modified file ", path, " ... skipped."); + writeln("", path, " is currently checked out or locked for editing by another user."); + log.fileOnly(path, " is currently checked out or locked for editing by another user."); + return; } - // As the session.upload includes the last modified time, save the response - saveItem(response); } - // OneDrive documentLibrary - if (accountType == "documentLibrary"){ - // Due to https://github.com/OneDrive/onedrive-api-docs/issues/935 Microsoft modifies all PDF, MS Office & HTML files with added XML content. It is a 'feature' of SharePoint. - // This means, as a session upload, on 'completion' the file is 'moved' and generates a 404 ...... - // Delete record from the local database - file will be uploaded as a new file - writeln("skipped."); - log.fileOnly("Uploading modified file ", path, " ... skipped."); - log.vlog("Skip Reason: Microsoft Sharepoint 'enrichment' after upload issue"); - log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details"); - itemdb.deleteById(item.driveId, item.id); - return; - } - - // log line completion + writeln("done."); + // As the session.upload includes the last modified time, save the response + saveItem(response); } log.fileOnly("Uploading modified file ", path, " ... done."); // use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded via simple upload @@ -1470,7 +1449,7 @@ final class SyncEngine // path is less than maxPathLength // skip dot files if configured - if (cfg.getValueBool("skip_dotfiles")) { + if (cfg.getValue("skip_dotfiles") == "true") { if (isDotFile(path)) { log.vlog("Skipping item - .file or .folder: ", path); return; @@ -1478,7 +1457,7 @@ final class SyncEngine } // Do we need to check for .nosync? Only if --check-for-nosync was passed in - if (cfg.getValueBool("check_nosync")) { + if (cfg.getValue("check_nosync") == "true") { if (exists(path ~ "/.nosync")) { log.vlog("Skipping item - .nosync found & --check-for-nosync enabled: ", path); return; @@ -1487,7 +1466,7 @@ final class SyncEngine if (isSymlink(path)) { // if config says so we skip all symlinked items - if (cfg.getValueBool("skip_symlinks")) { + if (cfg.getValue("skip_symlinks") == "true") { log.vlog("Skipping item - skip symbolic links configured: ", path); return; @@ -1521,12 +1500,9 @@ final class SyncEngine if (path != ".") { if (isDir(path)) { log.vdebug("Checking path: ", path); - // Only check path if config is != "" - if (cfg.getValueString("skip_dir") != "") { - if (selectiveSync.isDirNameExcluded(strip(path,"./"))) { - log.vlog("Skipping item - excluded by skip_dir config: ", path); - return; - } + if (selectiveSync.isDirNameExcluded(strip(path,"./"))) { + log.vlog("Skipping item - excluded by skip_dir config: ", path); + return; } } if (isFile(path)) { @@ -1910,32 +1886,11 @@ final class SyncEngine // use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded uploadLastModifiedTime(parent.driveId, id, cTag, mtime); } else { - // OneDrive Business account modified file upload handling - if (accountType == "business"){ - writeln(""); - // session upload - response = session.upload(path, parent.driveId, parent.id, baseName(path), fileDetailsFromOneDrive["eTag"].str); - writeln(" done."); - saveItem(response); - } - - // OneDrive SharePoint account modified file upload handling - if (accountType == "documentLibrary"){ - // If this is a Microsoft SharePoint site, we need to remove the existing file before upload - onedrive.deleteById(fileDetailsFromOneDrive["parentReference"]["driveId"].str, fileDetailsFromOneDrive["id"].str, fileDetailsFromOneDrive["eTag"].str); - // Due to https://github.com/OneDrive/onedrive-api-docs/issues/935 Microsoft modifies all PDF, MS Office & HTML files with added XML content. It is a 'feature' of SharePoint. - // This means, as a session upload, on 'completion' the file is 'moved' and generates a 404 ...... - // Upload modified file via simpleUpload to avoid the session 404 problem - response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path)); - writeln(" done."); - saveItem(response); - // So - now the 'local' and 'remote' file is technically DIFFERENT ... thanks Microsoft .. NO way to disable this stupidity - // Download the Microsoft 'modified' file so 'local' is now in sync - log.vlog("Due to Microsoft Sharepoint 'enrichment' of files, downloading 'enriched' file to ensure local file is in-sync"); - log.vlog("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details"); - auto fileSize = response["size"].integer; - onedrive.downloadById(response["parentReference"]["driveId"].str, response["id"].str, path, fileSize); - } + // OneDrive Business account upload handling + writeln(""); + response = session.upload(path, parent.driveId, parent.id, baseName(path)); + writeln(" done."); + saveItem(response); } } else { // we are --dry-run - simulate the file upload @@ -2066,11 +2021,10 @@ final class SyncEngine // Takes a JSON input and formats to an item which can be used by the database Item item = makeItem(jsonItem); // Add to the local database - log.vdebug("Adding to database: ", item); itemdb.upsert(item); } else { // log error - log.error("ERROR: OneDrive response missing required 'id' element"); + log.error("ERROR: OneDrive response missing required 'id' element:"); log.error("ERROR: ", jsonItem); } } else {