diff --git a/src/config.d b/src/config.d index a17f823e..fb8eaa22 100644 --- a/src/config.d +++ b/src/config.d @@ -1,66 +1,68 @@ import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit; -import std.file, std.string, std.regex, std.stdio, std.process, std.algorithm.searching, std.getopt, std.conv; +import std.file, std.string, std.regex, std.stdio, std.process, std.algorithm.searching, std.getopt, std.conv, std.path; import std.algorithm.sorting: sort; import selective; static import log; final class Config { - public string refreshTokenFilePath; - public string deltaLinkFilePath; - public string databaseFilePath; - public string databaseFilePathDryRun; - public string uploadStateFilePath; - public string syncListFilePath; - public string homePath; - public string configDirName; + // application defaults public string defaultSyncDir = "~/OneDrive"; public string defaultSkipFile = "~*|.~*|*.tmp"; public string defaultSkipDir = ""; - public string configFileSyncDir; - public string configFileSkipFile; - public string configFileSkipDir; + // application set items + public string refreshTokenFilePath = ""; + public string deltaLinkFilePath = ""; + public string databaseFilePath = ""; + public string databaseFilePathDryRun = ""; + public string uploadStateFilePath = ""; + public string syncListFilePath = ""; + public string homePath = ""; + public string configDirName = ""; + public string configFileSyncDir = ""; + public string configFileSkipFile = ""; + public string configFileSkipDir = ""; + private string userConfigFilePath = ""; // was the application just authorised - paste of response uri public bool applicationAuthorizeResponseUri = false; - - private string userConfigFilePath; // hashmap for the values found in the user config file // ARGGGG D is stupid and cannot make hashmap initializations!!! // private string[string] foobar = [ "aa": "bb" ] does NOT work!!! private string[string] stringValues; private bool[string] boolValues; private long[string] longValues; + public auto configRegex = ctRegex!(`^(\w+)\s*=\s*"(.*)"\s*$`); this(string confdirOption) { // default configuration - entries in config file ~/.config/onedrive/config // an entry here means it can be set via the config file if there is a coresponding read and set in update_from_args() - stringValues["sync_dir"] = defaultSyncDir; - stringValues["skip_file"] = defaultSkipFile; - stringValues["skip_dir"] = defaultSkipDir; - stringValues["log_dir"] = "/var/log/onedrive/"; - stringValues["drive_id"] = ""; - stringValues["user_agent"] = ""; - boolValues["upload_only"] = false; - boolValues["check_nomount"] = false; - boolValues["check_nosync"] = false; - boolValues["download_only"] = false; + stringValues["sync_dir"] = defaultSyncDir; + stringValues["skip_file"] = defaultSkipFile; + stringValues["skip_dir"] = defaultSkipDir; + stringValues["log_dir"] = "/var/log/onedrive/"; + stringValues["drive_id"] = ""; + stringValues["user_agent"] = ""; + boolValues["upload_only"] = false; + boolValues["check_nomount"] = false; + boolValues["check_nosync"] = false; + boolValues["download_only"] = false; boolValues["disable_notifications"] = false; boolValues["disable_upload_validation"] = false; - boolValues["enable_logging"] = false; - boolValues["force_http_11"] = false; - boolValues["force_http_2"] = false; - boolValues["local_first"] = false; - boolValues["no_remote_delete"] = false; - boolValues["skip_symlinks"] = false; - boolValues["debug_https"] = false; - boolValues["skip_dotfiles"] = false; - boolValues["dry_run"] = false; - boolValues["sync_root_files"] = false; - longValues["verbose"] = log.verbose; // might be initialized by the first getopt call! - longValues["monitor_interval"] = 45, - longValues["skip_size"] = 0, - longValues["min_notify_changes"] = 5; + boolValues["enable_logging"] = false; + boolValues["force_http_11"] = false; + boolValues["force_http_2"] = false; + boolValues["local_first"] = false; + boolValues["no_remote_delete"] = false; + boolValues["skip_symlinks"] = false; + boolValues["debug_https"] = false; + boolValues["skip_dotfiles"] = false; + boolValues["dry_run"] = false; + boolValues["sync_root_files"] = false; + longValues["verbose"] = log.verbose; // might be initialized by the first getopt call! + longValues["monitor_interval"] = 45; + longValues["skip_size"] = 0; + longValues["min_notify_changes"] = 5; longValues["monitor_log_frequency"] = 5; // Number of n sync runs before performing a full local scan of sync_dir // By default 10 which means every ~7.5 minutes a full disk scan of sync_dir will occur @@ -73,9 +75,19 @@ final class Config boolValues["skip_dir_strict_match"] = false; // Allow for a custom Client ID / Application ID to be used to replace the inbuilt default // This is a config file option ONLY - stringValues["application_id"] = ""; + stringValues["application_id"] = ""; // allow for resync to be set via config file - boolValues["resync"] = false; + boolValues["resync"] = false; + + // DEVELOPER OPTIONS + // display_memory = true | false + // - It may be desirable to display the memory usage of the application to assist with diagnosing memory issues with the application + // - This is especially beneficial when debugging or performing memory tests with Valgrind + boolValues["display_memory"] = false; + // monitor_max_loop = long value + // - It may be desirable to, when running in monitor mode, force monitor mode to 'quit' after X number of loops + // - This is especially beneficial when debugging or performing memory tests with Valgrind + longValues["monitor_max_loop"] = 0; // Determine the users home directory. // Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts @@ -131,17 +143,29 @@ final class Config // configDirBase contains the correct path so we do not need to check for presence of '~' configDirName = configDirBase ~ "/onedrive"; } - + + // Config directory options all determined + // configDirName has a trailing / log.vlog("Using Config Dir: ", configDirName); if (!exists(configDirName)) mkdirRecurse(configDirName); - - refreshTokenFilePath = configDirName ~ "/refresh_token"; - deltaLinkFilePath = configDirName ~ "/delta_link"; - databaseFilePath = configDirName ~ "/items.sqlite3"; - databaseFilePathDryRun = configDirName ~ "/items-dryrun.sqlite3"; - uploadStateFilePath = configDirName ~ "/resume_upload"; - userConfigFilePath = configDirName ~ "/config"; - syncListFilePath = configDirName ~ "/sync_list"; + + // Update application set variables based on configDirName + refreshTokenFilePath = buildNormalizedPath(configDirName ~ "/refresh_token"); + deltaLinkFilePath = buildNormalizedPath(configDirName ~ "/delta_link"); + databaseFilePath = buildNormalizedPath(configDirName ~ "/items.sqlite3"); + databaseFilePathDryRun = buildNormalizedPath(configDirName ~ "/items-dryrun.sqlite3"); + uploadStateFilePath = buildNormalizedPath(configDirName ~ "/resume_upload"); + userConfigFilePath = buildNormalizedPath(configDirName ~ "/config"); + syncListFilePath = buildNormalizedPath(configDirName ~ "/sync_list"); + + // Debug Output for application set variables based on configDirName + log.vdebug("refreshTokenFilePath = ", refreshTokenFilePath); + log.vdebug("deltaLinkFilePath = ", deltaLinkFilePath); + log.vdebug("databaseFilePath = ", databaseFilePath); + log.vdebug("databaseFilePathDryRun = ", databaseFilePathDryRun); + log.vdebug("uploadStateFilePath = ", uploadStateFilePath); + log.vdebug("userConfigFilePath = ", userConfigFilePath); + log.vdebug("syncListFilePath = ", syncListFilePath); } bool initialize() @@ -403,15 +427,38 @@ final class Config longValues[key] = value; } + // load a configuration file private bool load(string filename) { - scope(failure) return false; + // configure function variables auto file = File(filename, "r"); - auto r = regex(`^(\w+)\s*=\s*"(.*)"\s*$`); - foreach (line; file.byLine()) { - line = stripLeft(line); - if (line.length == 0 || line[0] == ';' || line[0] == '#') continue; - auto c = line.matchFirst(r); + string lineBuffer; + + // configure scopes + // - failure + scope(failure) { + // close file if open + if (file.isOpen()){ + // close open file + file.close(); + } + return false; + } + // - exit + scope(exit) { + // close file if open + if (file.isOpen()){ + // close open file + file.close(); + } + } + + // read file line by line + auto range = file.byLine(); + foreach (line; range) { + lineBuffer = stripLeft(line).to!string; + if (lineBuffer.length == 0 || lineBuffer[0] == ';' || lineBuffer[0] == '#') continue; + auto c = lineBuffer.matchFirst(configRegex); if (!c.empty) { c.popFront(); // skip the whole match string key = c.front.dup; @@ -444,7 +491,7 @@ final class Config } } } else { - log.log("Malformed config line: ", line); + log.log("Malformed config line: ", lineBuffer); return false; } } @@ -452,7 +499,6 @@ final class Config } } - void outputLongHelp(Option[] opt) { auto argsNeedingOptions = [ diff --git a/src/log.d b/src/log.d index c868fd50..e555486b 100644 --- a/src/log.d +++ b/src/log.d @@ -3,6 +3,7 @@ import std.file; import std.datetime; import std.process; import std.conv; +import core.memory; import core.sys.posix.pwd, core.sys.posix.unistd, core.stdc.string : strlen; import std.algorithm : splitter; version(Notifications) { @@ -201,3 +202,25 @@ private string getUserName() return "unknown"; } } + +void displayMemoryUsagePreGC() +{ +// Display memory usage +writeln("\nMemory Usage pre GC (bytes)"); +writeln("--------------------"); +writeln("memory usedSize = ", GC.stats.usedSize); +writeln("memory freeSize = ", GC.stats.freeSize); +// uncomment this if required, if not using LDC 1.16 as this does not exist in that version +//writeln("memory allocatedInCurrentThread = ", GC.stats.allocatedInCurrentThread, "\n"); +} + +void displayMemoryUsagePostGC() +{ +// Display memory usage +writeln("\nMemory Usage post GC (bytes)"); +writeln("--------------------"); +writeln("memory usedSize = ", GC.stats.usedSize); +writeln("memory freeSize = ", GC.stats.freeSize); +// uncomment this if required, if not using LDC 1.16 as this does not exist in that version +//writeln("memory allocatedInCurrentThread = ", GC.stats.allocatedInCurrentThread, "\n"); +} diff --git a/src/main.d b/src/main.d index db2be57b..b1b81815 100644 --- a/src/main.d +++ b/src/main.d @@ -21,11 +21,72 @@ int main(string[] args) // Disable buffering on stdout stdout.setvbuf(0, _IONBF); - // configuration directory + // main function variables string confdirOption; + string currentConfigHash; + string currentSyncListHash; + string previousConfigHash; + string previousSyncListHash; + string configHashFile; + string syncListHashFile; + string configBackupFile; + string syncDir; + bool configOptionsDifferent = false; + bool syncListConfigured = false; + bool syncListDifferent = false; + bool syncDirDifferent = false; + bool skipFileDifferent = false; + bool skipDirDifferent = false; + bool online = false; + bool performSyncOK = false; + bool onedriveInitialised = false; + bool displayMemoryUsage = false; + + // Define scopes + scope(exit) { + // Display memory details + if (displayMemoryUsage) { + log.displayMemoryUsagePreGC(); + } + // if initialised, shut down the HTTP instance + if (onedriveInitialised) { + oneDrive.shutdown(); + } + // Make sure the .wal file is incorporated into the main db before we exit + destroy(itemDb); + // free API instance + oneDrive = null; + // Perform Garbage Cleanup + GC.collect(); + // Display memory details + if (displayMemoryUsage) { + log.displayMemoryUsagePostGC(); + } + } + + scope(failure) { + // Display memory details + if (displayMemoryUsage) { + log.displayMemoryUsagePreGC(); + } + // if initialised, shut down the HTTP instance + if (onedriveInitialised) { + oneDrive.shutdown(); + } + // Make sure the .wal file is incorporated into the main db before we exit + destroy(itemDb); + // free API instance + oneDrive = null; + // Perform Garbage Cleanup + GC.collect(); + // Display memory details + if (displayMemoryUsage) { + log.displayMemoryUsagePostGC(); + } + } + // read in application options as passed in try { - // print the version and exit bool printVersion = false; auto opt = getopt( args, @@ -36,19 +97,22 @@ int main(string[] args) "verbose|v+", "Print more details, useful for debugging (repeat for extra debugging)", &log.verbose, "version", "Print the version and exit", &printVersion ); + // print help and exit if (opt.helpWanted) { args ~= "--help"; } + // print the version and exit if (printVersion) { - std.stdio.write("onedrive ", import("version")); + writeln("onedrive ", strip(import("version"))); return EXIT_SUCCESS; } } catch (GetOptException e) { + // option errors log.error(e.msg); log.error("Try 'onedrive -h' for more information"); return EXIT_FAILURE; } catch (Exception e) { - // error + // generic error log.error(e.msg); log.error("Try 'onedrive -h' for more information"); return EXIT_FAILURE; @@ -62,6 +126,9 @@ int main(string[] args) return EXIT_FAILURE; } + // set memory display + displayMemoryUsage = cfg.getValueBool("display_memory"); + // update configuration from command line args cfg.update_from_args(args); @@ -69,33 +136,24 @@ int main(string[] args) // 1. sync_list file modification // 2. config file modification - but only if sync_dir, skip_dir, skip_file or drive_id was modified // 3. CLI input overriding configured config file option + configHashFile = buildNormalizedPath(cfg.configDirName ~ "/.config.hash"); + syncListHashFile = buildNormalizedPath(cfg.configDirName ~ "/.sync_list.hash"); + configBackupFile = buildNormalizedPath(cfg.configDirName ~ "/.config.backup"); - string currentConfigHash; - string currentSyncListHash; - string previousConfigHash; - string previousSyncListHash; - string configHashFile = cfg.configDirName ~ "/.config.hash"; - string syncListHashFile = cfg.configDirName ~ "/.sync_list.hash"; - string configBackupFile = cfg.configDirName ~ "/.config.backup"; - bool configOptionsDifferent = false; - bool syncListConfigured = false; - bool syncListDifferent = false; - bool syncDirDifferent = false; - bool skipFileDifferent = false; - bool skipDirDifferent = false; - - if ((exists(cfg.configDirName ~ "/config")) && (!exists(configHashFile))) { + // Does a config file exist with a valid hash file + if ((exists(buildNormalizedPath(cfg.configDirName ~ "/config"))) && (!exists(configHashFile))) { // Hash of config file needs to be created - std.file.write(configHashFile, computeQuickXorHash(cfg.configDirName ~ "/config")); + std.file.write(configHashFile, computeQuickXorHash(buildNormalizedPath(cfg.configDirName ~ "/config"))); } - if ((exists(cfg.configDirName ~ "/sync_list")) && (!exists(syncListHashFile))) { + // Does a sync_list file exist with a valid hash file + if ((exists(buildNormalizedPath(cfg.configDirName ~ "/sync_list"))) && (!exists(syncListHashFile))) { // Hash of sync_list file needs to be created - std.file.write(syncListHashFile, computeQuickXorHash(cfg.configDirName ~ "/sync_list")); + std.file.write(syncListHashFile, computeQuickXorHash(buildNormalizedPath(cfg.configDirName ~ "/sync_list"))); } // If hash files exist, but config files do not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different - if ((!exists(cfg.configDirName ~ "/config")) && (exists(configHashFile))) { + if ((!exists(buildNormalizedPath(cfg.configDirName ~ "/config"))) && (exists(configHashFile))) { // if --resync safe remove config.hash and config.backup if (cfg.getValueBool("resync")) { safeRemove(configHashFile); @@ -103,28 +161,29 @@ int main(string[] args) } } - if ((!exists(cfg.configDirName ~ "/sync_list")) && (exists(syncListHashFile))) { + // If sync_list hash file exists, but sync_list file does not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different + if ((!exists(buildNormalizedPath(cfg.configDirName ~ "/sync_list"))) && (exists(syncListHashFile))) { // if --resync safe remove sync_list.hash if (cfg.getValueBool("resync")) safeRemove(syncListHashFile); } // Read config hashes if they exist - if (exists(cfg.configDirName ~ "/config")) currentConfigHash = computeQuickXorHash(cfg.configDirName ~ "/config"); - if (exists(cfg.configDirName ~ "/sync_list")) currentSyncListHash = computeQuickXorHash(cfg.configDirName ~ "/sync_list"); + if (exists(buildNormalizedPath(cfg.configDirName ~ "/config"))) currentConfigHash = computeQuickXorHash(buildNormalizedPath(cfg.configDirName ~ "/config")); + if (exists(buildNormalizedPath(cfg.configDirName ~ "/sync_list"))) currentSyncListHash = computeQuickXorHash(buildNormalizedPath(cfg.configDirName ~ "/sync_list")); if (exists(configHashFile)) previousConfigHash = readText(configHashFile); if (exists(syncListHashFile)) previousSyncListHash = readText(syncListHashFile); - // Was sync_list updated? + // Was sync_list file updated? if (currentSyncListHash != previousSyncListHash) { // Debugging output to assist what changed log.vdebug("sync_list file has been updated, --resync needed"); syncListDifferent = true; } - // Was config updated? + // Was config file updated between last execution ang this execution? if (currentConfigHash != previousConfigHash) { // config file was updated, however we only want to trigger a --resync requirement if sync_dir, skip_dir, skip_file or drive_id was modified - log.vdebug("config file has been updated, checking if --resync needed"); + log.log("config file has been updated, checking if --resync needed"); if (exists(configBackupFile)) { // check backup config what has changed for these configuration options if anything // # sync_dir = "~/OneDrive" @@ -136,13 +195,14 @@ int main(string[] args) stringValues["skip_file"] = ""; stringValues["skip_dir"] = ""; stringValues["drive_id"] = ""; - - auto file = File(configBackupFile, "r"); - auto r = regex(`^(\w+)\s*=\s*"(.*)"\s*$`); - foreach (line; file.byLine()) { - line = stripLeft(line); - if (line.length == 0 || line[0] == ';' || line[0] == '#') continue; - auto c = line.matchFirst(r); + auto configBackupFileHandle = File(configBackupFile, "r"); + string lineBuffer; + auto range = configBackupFileHandle.byLine(); + // read configBackupFile line by line + foreach (line; range) { + lineBuffer = stripLeft(line).to!string; + if (lineBuffer.length == 0 || lineBuffer[0] == ';' || lineBuffer[0] == '#') continue; + auto c = lineBuffer.matchFirst(cfg.configRegex); if (!c.empty) { c.popFront(); // skip the whole match string key = c.front.dup; @@ -170,6 +230,11 @@ int main(string[] args) } } } + // close file if open + if (configBackupFileHandle.isOpen()){ + // close open file + configBackupFileHandle.close(); + } } else { // no backup to check log.vdebug("WARNING: no backup config file was found, unable to validate if any changes made"); @@ -194,16 +259,16 @@ int main(string[] args) } // Is there a backup of the config file if the config file exists? - if ((exists(cfg.configDirName ~ "/config")) && (!exists(configBackupFile))) { + if ((exists(buildNormalizedPath(cfg.configDirName ~ "/config"))) && (!exists(configBackupFile))) { // create backup copy of current config file - std.file.copy(cfg.configDirName ~ "/config", configBackupFile); + std.file.copy(buildNormalizedPath(cfg.configDirName ~ "/config"), configBackupFile); } // config file set options can be changed via CLI input, specifically these will impact sync and --resync will be needed: // --syncdir ARG // --skip-file ARG // --skip-dir ARG - if (exists(cfg.configDirName ~ "/config")) { + if (exists(buildNormalizedPath(cfg.configDirName ~ "/config"))) { // config file exists // was the sync_dir updated by CLI? if (cfg.configFileSyncDir != "") { @@ -249,18 +314,18 @@ int main(string[] args) // --resync issued, update hashes of config files if they exist if (!cfg.getValueBool("dry_run")) { // not doing a dry run, update hash files if config & sync_list exist - if (exists(cfg.configDirName ~ "/config")) { + if (exists(buildNormalizedPath(cfg.configDirName ~ "/config"))) { // update hash log.vdebug("updating config hash as --resync issued"); - std.file.write(configHashFile, computeQuickXorHash(cfg.configDirName ~ "/config")); + std.file.write(configHashFile, computeQuickXorHash(buildNormalizedPath(cfg.configDirName ~ "/config"))); // create backup copy of current config file log.vdebug("making backup of config file as --resync issued"); - std.file.copy(cfg.configDirName ~ "/config", configBackupFile); + std.file.copy(buildNormalizedPath(cfg.configDirName ~ "/config"), configBackupFile); } - if (exists(cfg.configDirName ~ "/sync_list")) { + if (exists(buildNormalizedPath(cfg.configDirName ~ "/sync_list"))) { // update sync_list hash log.vdebug("updating sync_list hash as --resync issued"); - std.file.write(syncListHashFile, computeQuickXorHash(cfg.configDirName ~ "/sync_list")); + std.file.write(syncListHashFile, computeQuickXorHash(buildNormalizedPath(cfg.configDirName ~ "/sync_list"))); } } } @@ -271,9 +336,6 @@ int main(string[] args) if (cfg.getValueBool("dry_run")) { log.log("DRY-RUN Configured. Output below shows what 'would' have occurred."); } - - // Are we able to reach the OneDrive Service - bool online = false; // dry-run database setup if (cfg.getValueBool("dry_run")) { @@ -299,7 +361,6 @@ int main(string[] args) } // sync_dir environment handling to handle ~ expansion properly - string syncDir; if ((environment.get("SHELL") == "") && (environment.get("USER") == "")){ log.vdebug("sync_dir: No SHELL or USER environment variable configuration detected"); // No shell or user set, so expandTilde() will fail - usually headless system running under init.d / systemd or potentially Docker @@ -338,15 +399,16 @@ int main(string[] args) // Configure whether notifications are used log.setNotifications(cfg.getValueBool("monitor") && !cfg.getValueBool("disable_notifications")); - // upgrades - if (exists(cfg.configDirName ~ "/items.db")) { + // Application upgrades - skilion version etc + if (exists(buildNormalizedPath(cfg.configDirName ~ "/items.db"))) { if (!cfg.getValueBool("dry_run")) { - safeRemove(cfg.configDirName ~ "/items.db"); + safeRemove(buildNormalizedPath(cfg.configDirName ~ "/items.db")); } log.logAndNotify("Database schema changed, resync needed"); cfg.setValueBool("resync", true); } - + + // Handle --resync and --logout to remove local files if (cfg.getValueBool("resync") || cfg.getValueBool("logout")) { if (cfg.getValueBool("resync")) log.vdebug("--resync requested"); log.vlog("Deleting the saved status ..."); @@ -362,11 +424,11 @@ int main(string[] args) } } } - + // Display current application configuration, no application initialisation if (cfg.getValueBool("display_config")){ - string userConfigFilePath = cfg.configDirName ~ "/config"; - string userSyncList = cfg.configDirName ~ "/sync_list"; + string userConfigFilePath = buildNormalizedPath(cfg.configDirName ~ "/config"); + string userSyncList = buildNormalizedPath(cfg.configDirName ~ "/sync_list"); // Display application version writeln("onedrive version = ", strip(import("version"))); @@ -381,8 +443,6 @@ int main(string[] args) } // Config Options - - writeln("Config option 'check_nosync' = ", cfg.getValueBool("check_nosync")); writeln("Config option 'sync_dir' = ", syncDir); writeln("Config option 'skip_dir' = ", cfg.getValueString("skip_dir")); @@ -394,8 +454,6 @@ int main(string[] args) writeln("Config option 'log_dir' = ", cfg.getValueString("log_dir")); writeln("Config option 'classify_as_big_delete' = ", cfg.getValueLong("classify_as_big_delete")); - - // Is config option drive_id configured? if (cfg.getValueString("drive_id") != ""){ writeln("Config option 'drive_id' = ", cfg.getValueString("drive_id")); @@ -422,6 +480,7 @@ int main(string[] args) return EXIT_SUCCESS; } + // If the user is still using --force-http-1.1 advise its no longer required if (cfg.getValueBool("force_http_11")) { log.log("NOTE: The use of --force-http-1.1 is depreciated"); } @@ -442,39 +501,40 @@ int main(string[] args) // Initialize OneDrive, check for authorization log.vlog("Initializing the OneDrive API ..."); oneDrive = new OneDriveApi(cfg); + onedriveInitialised = oneDrive.init(); oneDrive.printAccessToken = cfg.getValueBool("print_token"); - if (!oneDrive.init()) { + + if (!onedriveInitialised) { log.error("Could not initialize the OneDrive API"); - // workaround for segfault in std.net.curl.Curl.shutdown() on exit - oneDrive.http.shutdown(); + // Use exit scopes to shutdown API return EXIT_UNAUTHORIZED; } // if --synchronize or --monitor not passed in, configure the flag to display help & exit - auto performSyncOK = false; if (cfg.getValueBool("synchronize") || cfg.getValueBool("monitor")) { performSyncOK = true; } // create-directory, remove-directory, source-directory, destination-directory - // are activities that dont perform a sync no error message for these items either + // these are activities that dont perform a sync, so to not generate an error message for these items either if (((cfg.getValueString("create_directory") != "") || (cfg.getValueString("remove_directory") != "")) || ((cfg.getValueString("source_directory") != "") && (cfg.getValueString("destination_directory") != "")) || (cfg.getValueString("get_file_link") != "") || (cfg.getValueString("get_o365_drive_id") != "") || cfg.getValueBool("display_sync_status")) { performSyncOK = true; } + // Were acceptable sync operations provided? Was --synchronize or --monitor passed in if (!performSyncOK) { // was the application just authorised? if (cfg.applicationAuthorizeResponseUri) { // Application was just authorised log.log("\nApplication has been successfully authorised, however no additional command switches were provided.\n"); log.log("Please use --help for further assistance in regards to running this application.\n"); - oneDrive.http.shutdown(); + // Use exit scopes to shutdown API return EXIT_SUCCESS; } else { // Application was not just authorised log.log("\n--synchronize or --monitor switches missing from your command line input. Please add one (not both) of these switches to your command line or use --help for further assistance.\n"); log.log("No OneDrive sync will be performed without one of these two arguments being present.\n"); - oneDrive.http.shutdown(); + // Use exit scopes to shutdown API return EXIT_FAILURE; } } @@ -483,7 +543,7 @@ int main(string[] args) if (cfg.getValueBool("synchronize") && cfg.getValueBool("monitor")) { writeln("\nERROR: --synchronize and --monitor cannot be used together\n"); writeln("Refer to --help to determine which command option you should use.\n"); - oneDrive.http.shutdown(); + // Use exit scopes to shutdown API return EXIT_FAILURE; } @@ -499,6 +559,7 @@ int main(string[] args) itemDb = new ItemDatabase(cfg.databaseFilePathDryRun); } + // configure the sync direcory based on syncDir config option log.vlog("All operations will be performed in: ", syncDir); if (!exists(syncDir)) { log.vdebug("syncDir: Configured syncDir is missing. Creating: ", syncDir); @@ -508,7 +569,7 @@ int main(string[] args) } catch (std.file.FileException e) { // Creating the sync directory failed log.error("ERROR: Unable to create local OneDrive syncDir - ", e.msg); - oneDrive.http.shutdown(); + // Use exit scopes to shutdown API return EXIT_FAILURE; } } @@ -526,6 +587,11 @@ int main(string[] args) { log.vdebug("sync_list: ", line); } + // close syncListFile if open + if (syncListFile.isOpen()){ + // close open file + syncListFile.close(); + } } selectiveSync.load(cfg.syncListFilePath); @@ -563,12 +629,12 @@ int main(string[] args) // All skip_file entries are valid log.vdebug("skip_file: ", cfg.getValueString("skip_file")); selectiveSync.setFileMask(cfg.getValueString("skip_file")); - + // Initialize the sync engine auto sync = new SyncEngine(cfg, oneDrive, itemDb, selectiveSync); try { if (!initSyncEngine(sync)) { - oneDrive.http.shutdown(); + // Use exit scopes to shutdown API return EXIT_FAILURE; } else { if (cfg.getValueString("get_file_link") == "") { @@ -579,7 +645,7 @@ int main(string[] args) } catch (CurlException e) { if (!cfg.getValueBool("monitor")) { log.log("\nNo Internet connection."); - oneDrive.http.shutdown(); + // Use exit scopes to shutdown API return EXIT_FAILURE; } } @@ -612,7 +678,7 @@ int main(string[] args) // we were asked to check the mounts if (exists(syncDir ~ "/.nosync")) { log.logAndNotify("ERROR: .nosync file found. Aborting synchronization process to safeguard data."); - oneDrive.http.shutdown(); + // Use exit scopes to shutdown API return EXIT_FAILURE; } } @@ -650,13 +716,10 @@ int main(string[] args) // Are we displaying the sync status of the client? if (cfg.getValueBool("display_sync_status")) { string remotePath = "/"; - string localPath = "."; - // Are we doing a single directory check? if (cfg.getValueString("single_directory") != ""){ // Need two different path strings here remotePath = cfg.getValueString("single_directory"); - localPath = cfg.getValueString("single_directory"); } sync.queryDriveForChanges(remotePath); } @@ -672,7 +735,7 @@ int main(string[] args) if (!exists(cfg.getValueString("single_directory"))){ // the requested directory does not exist .. log.logAndNotify("ERROR: The requested local directory does not exist. Please check ~/OneDrive/ for requested path"); - oneDrive.http.shutdown(); + // Use exit scopes to shutdown API return EXIT_FAILURE; } } @@ -759,12 +822,14 @@ int main(string[] args) } // monitor loop + bool performMonitor = true; + ulong monitorLoopFullCount = 0; immutable auto checkInterval = dur!"seconds"(cfg.getValueLong("monitor_interval")); - immutable auto logInterval = cfg.getValueLong("monitor_log_frequency"); - immutable auto fullScanFrequency = cfg.getValueLong("monitor_fullscan_frequency"); - auto lastCheckTime = MonoTime.currTime(); - auto logMonitorCounter = 0; - auto fullScanCounter = 0; + immutable long logInterval = cfg.getValueLong("monitor_log_frequency"); + immutable long fullScanFrequency = cfg.getValueLong("monitor_fullscan_frequency"); + MonoTime lastCheckTime = MonoTime.currTime(); + long logMonitorCounter = 0; + long fullScanCounter = 0; bool fullScanRequired = false; bool syncListConfiguredFullScanOverride = false; // if sync list is configured, set to true @@ -773,7 +838,7 @@ int main(string[] args) syncListConfiguredFullScanOverride = true; } - while (true) { + while (performMonitor) { if (!cfg.getValueBool("download_only")) { try { m.update(online); @@ -785,6 +850,13 @@ int main(string[] args) auto currTime = MonoTime.currTime(); if (currTime - lastCheckTime > checkInterval) { + // Increment monitorLoopFullCount + monitorLoopFullCount++; + // Display memory details at start of loop + if (displayMemoryUsage) { + log.displayMemoryUsagePreGC(); + } + // log monitor output suppression logMonitorCounter += 1; if (logMonitorCounter > logInterval) { @@ -812,7 +884,7 @@ int main(string[] args) try { if (!initSyncEngine(sync)) { - oneDrive.http.shutdown(); + // Use exit scopes to shutdown API return EXIT_FAILURE; } try { @@ -845,19 +917,31 @@ int main(string[] args) syncListConfiguredFullScanOverride = false; } lastCheckTime = MonoTime.currTime(); + // Display memory details before cleanup + if (displayMemoryUsage) { + log.displayMemoryUsagePreGC(); + } + // Perform Garbage Cleanup GC.collect(); - } + // Display memory details after cleanup + if (displayMemoryUsage) { + log.displayMemoryUsagePostGC(); + } + + // Developer break via config option + if (cfg.getValueLong("monitor_max_loop") > 0) { + // developer set option to limit --monitor loops + if (monitorLoopFullCount == (cfg.getValueLong("monitor_max_loop"))) { + performMonitor = false; + log.log("Exiting after ", monitorLoopFullCount, " loops due to developer set option"); + } + } + } Thread.sleep(dur!"msecs"(500)); } } } - // Workaround for segfault in std.net.curl.Curl.shutdown() on exit - oneDrive.http.shutdown(); - - // Make sure the .wal file is incorporated into the main db before we exit - destroy(itemDb); - // --dry-run temp database cleanup if (cfg.getValueBool("dry_run")) { if (exists(cfg.databaseFilePathDryRun)) { @@ -867,6 +951,8 @@ int main(string[] args) } } + // Exit application + // Use exit scopes to shutdown API return EXIT_SUCCESS; } @@ -1054,8 +1140,7 @@ extern(C) nothrow @nogc @system void exitHandler(int value) { log.log("Got termination signal, shutting down db connection"); // make sure the .wal file is incorporated into the main db destroy(itemDb); - // workaround for segfault in std.net.curl.Curl.shutdown() on exit - oneDrive.http.shutdown(); + // Use exit scopes to shutdown OneDrive API })(); } catch(Exception e) {} exit(0); diff --git a/src/onedrive.d b/src/onedrive.d index 3da3d4c6..76451737 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -73,7 +73,7 @@ final class OneDriveApi private Config cfg; private string refreshToken, accessToken; private SysTime accessTokenExpiration; - /* private */ HTTP http; + private HTTP http; // if true, every new access token is printed bool printAccessToken; @@ -147,6 +147,20 @@ final class OneDriveApi } } + // Shutdown OneDrive HTTP construct + void shutdown() + { + // reset any values to defaults, freeing any set objects + http.clearRequestHeaders(); + http.onSend = null; + http.onReceive = null; + http.onReceiveHeader = null; + http.onReceiveStatusLine = null; + http.contentLength = 0; + // shut down the curl instance + http.shutdown(); + } + bool init() { // Update clientId if application_id is set in config file @@ -266,6 +280,8 @@ final class OneDriveApi JSONValue getDefaultDrive() { checkAccessTokenExpired(); + const(char)[] url; + url = driveUrl; return get(driveUrl); } @@ -273,17 +289,22 @@ final class OneDriveApi JSONValue getDefaultRoot() { checkAccessTokenExpired(); - return get(driveUrl ~ "/root"); + const(char)[] url; + url = driveUrl ~ "/root"; + return get(url); } // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta JSONValue viewChangesById(const(char)[] driveId, const(char)[] id, const(char)[] deltaLink) { checkAccessTokenExpired(); - const(char)[] url = deltaLink; - if (url == null) { + const(char)[] url; + // configure deltaLink to query + if (deltaLink.empty) { url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/delta"; url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; + } else { + url = deltaLink; } return get(url); } @@ -409,18 +430,29 @@ final class OneDriveApi JSONValue uploadFragment(const(char)[] uploadUrl, string filepath, long offset, long offsetSize, long fileSize) { checkAccessTokenExpired(); + // open file as read-only in binary mode + auto file = File(filepath, "rb"); + file.seek(offset); + string contentRange = "bytes " ~ to!string(offset) ~ "-" ~ to!string(offset + offsetSize - 1) ~ "/" ~ to!string(fileSize); + + // function scopes scope(exit) { http.clearRequestHeaders(); http.onSend = null; + http.onReceive = null; + http.onReceiveHeader = null; + http.onReceiveStatusLine = null; + http.contentLength = 0; + // close file if open + if (file.isOpen()){ + // close open file + file.close(); + } } + http.method = HTTP.Method.put; http.url = uploadUrl; - - import std.conv; - string contentRange = "bytes " ~ to!string(offset) ~ "-" ~ to!string(offset + offsetSize - 1) ~ "/" ~ to!string(fileSize); http.addRequestHeader("Content-Range", contentRange); - auto file = File(filepath, "rb"); - file.seek(offset); http.onSend = data => file.rawRead(data).length; http.contentLength = offsetSize; auto response = perform(); @@ -524,10 +556,12 @@ final class OneDriveApi private JSONValue get(const(char)[] url, bool skipToken = false) { scope(exit) http.clearRequestHeaders(); + log.vdebug("Request URL = ", url); http.method = HTTP.Method.get; http.url = url; if (!skipToken) addAccessTokenHeader(); // HACK: requestUploadStatus - auto response = perform(); + JSONValue response; + response = perform(); checkHttpCode(response); // OneDrive API Response Debugging if --https-debug is being used if (.debugResponse){ @@ -550,14 +584,35 @@ final class OneDriveApi { // Threshold for displaying download bar long thresholdFileSize = 4 * 2^^20; // 4 MiB + // open file as write in binary mode + auto file = File(filename, "wb"); + + // function scopes + scope(exit) { + http.clearRequestHeaders(); + http.onSend = null; + http.onReceive = null; + http.onReceiveHeader = null; + http.onReceiveStatusLine = null; + http.contentLength = 0; + // Reset onProgress to not display anything for next download + http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) + { + return 0; + }; + // close file if open + if (file.isOpen()){ + // close open file + file.close(); + } + } - scope(exit) http.clearRequestHeaders(); http.method = HTTP.Method.get; http.url = url; addAccessTokenHeader(); - auto f = File(filename, "wb"); + http.onReceive = (ubyte[] data) { - f.rawWrite(data); + file.rawWrite(data); return data.length; }; @@ -604,14 +659,12 @@ final class OneDriveApi // try and catch any curl error http.perform(); writeln(); - // Reset onProgress to not display anything for next download - http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) - { - return 0; - }; + // Reset onProgress to not display anything for next download done using exit scope } catch (CurlException e) { displayOneDriveErrorMessage(e.msg); } + // free progress bar memory + p = null; } else { // No progress bar try { @@ -661,16 +714,29 @@ final class OneDriveApi private JSONValue upload(string filepath, string url) { + checkAccessTokenExpired(); + // open file as read-only in binary mode + auto file = File(filepath, "rb"); + + // function scopes scope(exit) { http.clearRequestHeaders(); http.onSend = null; + http.onReceive = null; + http.onReceiveHeader = null; + http.onReceiveStatusLine = null; http.contentLength = 0; + // close file if open + if (file.isOpen()){ + // close open file + file.close(); + } } + http.method = HTTP.Method.put; http.url = url; addAccessTokenHeader(); http.addRequestHeader("Content-Type", "application/octet-stream"); - auto file = File(filepath, "rb"); http.onSend = data => file.rawRead(data).length; http.contentLength = file.size; auto response = perform(); @@ -704,6 +770,8 @@ final class OneDriveApi { scope(exit) http.onReceive = null; char[] content; + JSONValue json; + http.onReceive = (ubyte[] data) { content ~= data; // HTTP Server Response Code Debugging if --https-debug is being used @@ -713,8 +781,6 @@ final class OneDriveApi return data.length; }; - JSONValue json; - try { http.perform(); // Get the HTTP Response headers - needed for correct 429 handling @@ -723,18 +789,17 @@ final class OneDriveApi if (.debugResponse){ log.vdebug("onedrive.perform() => HTTP Response Headers: ", responseHeaders); } - + // is retry-after in the response headers if ("retry-after" in http.responseHeaders) { - // retry-after as in the response headers - // Set the value + // Set the retry-after value log.vdebug("onedrive.perform() => Received a 'Retry-After' Header Response with the following value: ", http.responseHeaders["retry-after"]); log.vdebug("onedrive.perform() => Setting retryAfterValue to: ", http.responseHeaders["retry-after"]); .retryAfterValue = to!ulong(http.responseHeaders["retry-after"]); } - } catch (CurlException e) { // Parse and display error message received from OneDrive log.error("ERROR: OneDrive returned an error with the following message:"); + auto errorArray = splitLines(e.msg); string errorMessage = errorArray[0]; diff --git a/src/sqlite.d b/src/sqlite.d index 120562cd..5e1839ec 100644 --- a/src/sqlite.d +++ b/src/sqlite.d @@ -176,7 +176,8 @@ struct Statement row.length = 0; } else if (rc == SQLITE_ROW) { // https://www.sqlite.org/c3ref/data_count.html - int count = sqlite3_data_count(pStmt); + int count = 0; + count = sqlite3_data_count(pStmt); row = new const(char)[][count]; foreach (size_t i, ref column; row) { // https://www.sqlite.org/c3ref/column_blob.html diff --git a/src/sync.d b/src/sync.d index ef77894b..80c4967a 100644 --- a/src/sync.d +++ b/src/sync.d @@ -97,13 +97,11 @@ private bool hasSha1Hash(const ref JSONValue item) return ("sha1Hash" in item["file"]["hashes"]) != null; } -private bool isDotFile(string path) +private bool isDotFile(const(string) path) { // always allow the root if (path == ".") return false; - - path = buildNormalizedPath(path); - auto paths = pathSplitter(path); + auto paths = pathSplitter(buildNormalizedPath(path)); foreach(base; paths) { if (startsWith(base, ".")){ return true; @@ -177,7 +175,7 @@ private Item makeItem(const ref JSONValue driveItem) return item; } -private bool testFileHash(string path, const ref Item item) +private bool testFileHash(const(string) path, const ref Item item) { if (item.crc32Hash) { if (item.crc32Hash == computeCrc32(path)) return true; @@ -504,7 +502,7 @@ final class SyncEngine } // download all new changes from a specified folder on OneDrive - void applyDifferencesSingleDirectory(string path) + void applyDifferencesSingleDirectory(const(string) path) { log.vlog("Getting path details from OneDrive ..."); JSONValue onedrivePathDetails; @@ -600,7 +598,7 @@ final class SyncEngine } // create a directory on OneDrive without syncing - auto createDirectoryNoSync(string path) + auto createDirectoryNoSync(const(string) path) { // Attempt to create the requested path within OneDrive without performing a sync log.vlog("Attempting to create the requested path within OneDrive"); @@ -610,7 +608,7 @@ final class SyncEngine } // delete a directory on OneDrive without syncing - auto deleteDirectoryNoSync(string path) + auto deleteDirectoryNoSync(const(string) path) { // Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls const(char)[] rootId = defaultRootId; @@ -697,14 +695,18 @@ final class SyncEngine private void applyDifferences(string driveId, const(char)[] id, bool performFullItemScan) { log.vlog("Applying changes of Path ID: " ~ id); + // function variables + const(char)[] idToQuery; JSONValue changes; JSONValue changesAvailable; - - // Query the name of this folder id + JSONValue idDetails; string syncFolderName; string syncFolderPath; string syncFolderChildPath; - JSONValue idDetails = parseJSON("{}"); + string deltaLink; + string deltaLinkAvailable; + + // Query the name of this folder id try { idDetails = onedrive.getPathDetailsById(driveId, id); } catch (OneDriveException e) { @@ -862,8 +864,7 @@ final class SyncEngine // Control this via performFullItemScan // Get the current delta link - string deltaLink = ""; - string deltaLinkAvailable = itemdb.getDeltaLink(driveId, id); + deltaLinkAvailable = itemdb.getDeltaLink(driveId, id); // if sync_list is not configured, syncListConfigured should be false log.vdebug("syncListConfigured = ", syncListConfigured); // oneDriveFullScanTrigger should be false unless set by actions on OneDrive and only if sync_list or skip_dir is used @@ -883,16 +884,18 @@ final class SyncEngine if (!performFullItemScan){ // performFullItemScan == false // use delta link - deltaLink = deltaLinkAvailable; log.vdebug("performFullItemScan is false, using the deltaLink as per database entry"); if (deltaLinkAvailable == ""){ + deltaLink = ""; log.vdebug("deltaLink was requested to be used, but contains no data - resulting API query will be treated as a full scan of OneDrive"); } else { + deltaLink = deltaLinkAvailable; log.vdebug("deltaLink contains valid data - resulting API query will be treated as a delta scan of OneDrive"); } } else { // performFullItemScan == true // do not use delta-link + deltaLink = ""; log.vdebug("performFullItemScan is true, not using the database deltaLink so that we query all objects on OneDrive to compare against all local objects"); } @@ -901,7 +904,6 @@ final class SyncEngine // If we used the 'id' passed in & when using --single-directory with a business account we get: // 'HTTP request returned status code 501 (Not Implemented): view.delta can only be called on the root.' // To view changes correctly, we need to use the correct path id for the request - const(char)[] idToQuery; if (driveId == defaultDriveId) { // The drive id matches our users default drive id idToQuery = defaultRootId.dup; @@ -911,6 +913,8 @@ final class SyncEngine // Use the 'id' that was passed in (folderId) idToQuery = id; } + // what path id are we going to query? + log.vdebug("path idToQuery = ", idToQuery); // query for changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink); try { @@ -1387,9 +1391,9 @@ final class SyncEngine } else { log.vdebug("onedrive.viewChangesByDriveId call returned an invalid JSON Object"); } - } + } } - + // delete items in idsToDelete if (idsToDelete.length > 0) deleteItems(); // empty the skipped items @@ -1525,7 +1529,7 @@ final class SyncEngine } // check the item type - string path; + string path = ""; if (!unwanted) { if (isItemFile(driveItem)) { log.vdebug("The item we are syncing is a file"); @@ -1705,7 +1709,7 @@ final class SyncEngine } // download an item that was not synced before - private void applyNewItem(Item item, string path) + private void applyNewItem(const ref Item item, const(string) path) { if (exists(path)) { // path exists locally @@ -1717,11 +1721,13 @@ final class SyncEngine // file is not in sync with the database // is the local file technically 'newer' based on UTC timestamp? SysTime localModifiedTime = timeLastModified(path).toUTC(); + SysTime itemModifiedTime = item.mtime; + // HACK: reduce time resolution to seconds before comparing + itemModifiedTime.fracSecs = Duration.zero; localModifiedTime.fracSecs = Duration.zero; - item.mtime.fracSecs = Duration.zero; // is the local modified time greater than that from OneDrive? - if (localModifiedTime > item.mtime) { + if (localModifiedTime > itemModifiedTime) { // local file is newer than item on OneDrive based on file modified time // Is this item id in the database? if (itemdb.idInLocalDatabase(item.driveId, item.id)){ @@ -1872,7 +1878,7 @@ final class SyncEngine } // downloads a File resource - private void downloadFileItem(Item item, string path) + private void downloadFileItem(const ref Item item, const(string) path) { assert(item.type == ItemType.file); write("Downloading file ", path, " ... "); @@ -2072,20 +2078,21 @@ final class SyncEngine } // returns true if the given item corresponds to the local one - private bool isItemSynced(Item item, string path) + private bool isItemSynced(const ref Item item, const(string) path) { if (!exists(path)) return false; final switch (item.type) { case ItemType.file: if (isFile(path)) { SysTime localModifiedTime = timeLastModified(path).toUTC(); + SysTime itemModifiedTime = item.mtime; // HACK: reduce time resolution to seconds before comparing - item.mtime.fracSecs = Duration.zero; + itemModifiedTime.fracSecs = Duration.zero; localModifiedTime.fracSecs = Duration.zero; - if (localModifiedTime == item.mtime) { + if (localModifiedTime == itemModifiedTime) { return true; } else { - log.vlog("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime); + log.vlog("The local item has a different modified time ", localModifiedTime, " remote is ", itemModifiedTime); } if (testFileHash(path, item)) { return true; @@ -2113,7 +2120,7 @@ final class SyncEngine foreach_reverse (i; idsToDelete) { Item item; if (!itemdb.selectById(i[0], i[1], item)) continue; // check if the item is in the db - string path = itemdb.computePath(i[0], i[1]); + const(string) path = itemdb.computePath(i[0], i[1]); log.log("Trying to delete item ", path); if (!dryRun) { // Actually process the database entry removal @@ -2170,7 +2177,7 @@ final class SyncEngine } // scan the given directory for differences and new items - void scanForDifferences(string path) + void scanForDifferences(const(string) path) { // scan for changes in the path provided log.vlog("Uploading differences of ", path); @@ -2178,6 +2185,7 @@ final class SyncEngine if (itemdb.selectByPath(path, defaultDriveId, item)) { uploadDifferences(item); } + log.vlog("Uploading new items of ", path); uploadNewItems(path); @@ -2188,7 +2196,7 @@ final class SyncEngine } } - private void uploadDifferences(Item item) + private void uploadDifferences(const ref Item item) { // see if this item.id we were supposed to have deleted // match early and return @@ -2255,7 +2263,7 @@ final class SyncEngine } } - private void uploadDirDifferences(Item item, string path) + private void uploadDirDifferences(const ref Item item, const(string) path) { assert(item.type == ItemType.dir); if (exists(path)) { @@ -2284,7 +2292,8 @@ final class SyncEngine } else { // we are in a --dry-run situation, directory appears to have deleted locally - this directory may never have existed as we never downloaded it .. // Check if path does not exist in database - if (!itemdb.selectByPath(path, defaultDriveId, item)) { + Item databaseItem; + if (!itemdb.selectByPath(path, defaultDriveId, databaseItem)) { // Path not found in database log.vlog("The directory has been deleted locally"); if (noRemoteDelete) { @@ -2311,7 +2320,7 @@ final class SyncEngine } } - private void uploadRemoteDirDifferences(Item item, string path) + private void uploadRemoteDirDifferences(const ref Item item, const(string) path) { assert(item.type == ItemType.remote); if (exists(path)) { @@ -2344,7 +2353,8 @@ final class SyncEngine } else { // we are in a --dry-run situation, directory appears to have deleted locally - this directory may never have existed as we never downloaded it .. // Check if path does not exist in database - if (!itemdb.selectByPathWithRemote(path, defaultDriveId, item)) { + Item databaseItem; + if (!itemdb.selectByPathWithRemote(path, defaultDriveId, databaseItem)) { // Path not found in database log.vlog("The directory has been deleted locally"); if (noRemoteDelete) { @@ -2372,7 +2382,7 @@ final class SyncEngine } // upload local file system differences to OneDrive - private void uploadFileDifferences(Item item, string path) + private void uploadFileDifferences(const ref Item item, const(string) path) { // Reset upload failure - OneDrive or filesystem issue (reading data) uploadFailed = false; @@ -2381,11 +2391,12 @@ final class SyncEngine if (exists(path)) { if (isFile(path)) { SysTime localModifiedTime = timeLastModified(path).toUTC(); + SysTime itemModifiedTime = item.mtime; // HACK: reduce time resolution to seconds before comparing - item.mtime.fracSecs = Duration.zero; + itemModifiedTime.fracSecs = Duration.zero; localModifiedTime.fracSecs = Duration.zero; - if (localModifiedTime != item.mtime) { + if (localModifiedTime != itemModifiedTime) { log.vlog("The file last modified time has changed"); string eTag = item.eTag; if (!testFileHash(path, item)) { @@ -2689,7 +2700,8 @@ final class SyncEngine } else { // We are in a --dry-run situation, file appears to have deleted locally - this file may never have existed as we never downloaded it .. // Check if path does not exist in database - if (!itemdb.selectByPath(path, defaultDriveId, item)) { + Item databaseItem; + if (!itemdb.selectByPath(path, defaultDriveId, databaseItem)) { // file not found in database log.vlog("The file has been deleted locally"); if (noRemoteDelete) { @@ -2722,24 +2734,27 @@ final class SyncEngine } // upload new items to OneDrive - private void uploadNewItems(string path) + private void uploadNewItems(const(string) path) { + import std.range : walkLength; + import std.uni : byGrapheme; // https://support.microsoft.com/en-us/help/3125202/restrictions-and-limitations-when-you-sync-files-and-folders // If the path is greater than allowed characters, then one drive will return a '400 - Bad Request' // Need to ensure that the URI is encoded before the check is made // 400 Character Limit for OneDrive Business / Office 365 // 430 Character Limit for OneDrive Personal - auto maxPathLength = 0; - import std.range : walkLength; - import std.uni : byGrapheme; - if (accountType == "business"){ - // Business Account - maxPathLength = 400; - } else { + long maxPathLength = 0; + long pathWalkLength = path.byGrapheme.walkLength; + + // Configure maxPathLength based on account type + if (accountType == "personal"){ // Personal Account maxPathLength = 430; + } else { + // Business Account / Office365 + maxPathLength = 400; } - + // A short lived file that has disappeared will cause an error - is the path valid? if (!exists(path)) { log.log("Skipping item - has disappeared: ", path); @@ -2755,8 +2770,8 @@ final class SyncEngine return; } - if(path.byGrapheme.walkLength < maxPathLength){ - // path is less than maxPathLength + if(pathWalkLength < maxPathLength){ + // path length is less than maxPathLength // skip dot files if configured if (cfg.getValueBool("skip_dotfiles")) { @@ -2846,6 +2861,7 @@ final class SyncEngine // This item passed all the unwanted checks // We want to upload this new item if (isDir(path)) { + Item item; if (!itemdb.selectByPath(path, defaultDriveId, item)) { uploadCreateDir(path); @@ -2862,7 +2878,8 @@ final class SyncEngine try { auto entries = dirEntries(path, SpanMode.shallow, false); foreach (DirEntry entry; entries) { - uploadNewItems(entry.name); + string thisPath = entry.name; + uploadNewItems(thisPath); } } catch (FileException e) { // display the error message @@ -2871,7 +2888,7 @@ final class SyncEngine } } else { // This item is a file - auto fileSize = getSize(path); + long fileSize = getSize(path); // Can we upload this file - is there enough free space? - https://github.com/skilion/onedrive/issues/73 // However if the OneDrive account does not provide the quota details, we have no idea how much free space is available if ((!quotaAvailable) || ((remainingFreeSpace - fileSize) > 0)){ @@ -3104,7 +3121,7 @@ final class SyncEngine } // upload a new file to OneDrive - private void uploadNewFile(string path) + private void uploadNewFile(const(string) path) { // Reset upload failure - OneDrive or filesystem issue (reading data) uploadFailed = false; @@ -3810,7 +3827,7 @@ final class SyncEngine } // delete an item on OneDrive - private void uploadDeleteItem(Item item, string path) + private void uploadDeleteItem(Item item, const(string) path) { log.log("Deleting item from OneDrive: ", path); bool flagAsBigDelete = false; @@ -3909,9 +3926,11 @@ final class SyncEngine // update the item's last modified time private void uploadLastModifiedTime(const(char)[] driveId, const(char)[] id, const(char)[] eTag, SysTime mtime) { + string itemModifiedTime; + itemModifiedTime = mtime.toISOExtString(); JSONValue data = [ "fileSystemInfo": JSONValue([ - "lastModifiedDateTime": mtime.toISOExtString() + "lastModifiedDateTime": itemModifiedTime ]) ]; @@ -4072,7 +4091,7 @@ final class SyncEngine } // delete an item by it's path - void deleteByPath(string path) + void deleteByPath(const(string) path) { Item item; if (!itemdb.selectByPath(path, defaultDriveId, item)) { @@ -4243,7 +4262,7 @@ final class SyncEngine } // Query the OneDrive 'drive' to determine if we are 'in sync' or if there are pending changes - void queryDriveForChanges(string path) { + void queryDriveForChanges(const(string) path) { // Function variables int validChanges = 0; @@ -4416,7 +4435,7 @@ final class SyncEngine } // Create a fake OneDrive response suitable for use with saveItem - JSONValue createFakeResponse(string path) { + JSONValue createFakeResponse(const(string) path) { import std.digest.sha; // Generate a simulated JSON response which can be used // At a minimum we need: diff --git a/src/util.d b/src/util.d index a4081816..3155d462 100644 --- a/src/util.d +++ b/src/util.d @@ -125,11 +125,15 @@ Regex!char wild2regex(const(char)[] pattern) // returns true if the network connection is available bool testNetwork() { - try { - HTTP http = HTTP("https://login.microsoftonline.com"); - http.dnsTimeout = (dur!"seconds"(5)); - http.method = HTTP.Method.head; + // Use low level HTTP struct + auto http = HTTP(); + http.url = "https://login.microsoftonline.com"; + http.dnsTimeout = (dur!"seconds"(5)); + http.method = HTTP.Method.head; + // Attempt to contact the Microsoft Online Service + try { http.perform(); + http.shutdown(); return true; } catch (SocketException) { return false;