// What is this module called? module main; // What does this module require to function? import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit; import core.stdc.signal; import core.memory; import core.time; import core.thread; import std.stdio; import std.getopt; import std.string; import std.file; import std.process; import std.algorithm; import std.path; import std.concurrency; import std.parallelism; import std.conv; import std.traits; import std.net.curl: CurlException; import std.datetime; // What other modules that we have created do we need to import? import config; import log; import curlEngine; import util; import onedrive; import progress; import syncEngine; import itemdb; import clientSideFiltering; import monitor; import webhook; // What other constant variables do we require? const int EXIT_RESYNC_REQUIRED = 126; // Class objects ApplicationConfig appConfig; OneDriveWebhook oneDriveWebhook; SyncEngine syncEngineInstance; ItemDatabase itemDB; ClientSideFiltering selectiveSync; Monitor filesystemMonitor; int main(string[] cliArgs) { // Application Start Time - used during monitor loop to detail how long it has been running for auto applicationStartTime = Clock.currTime(); // Disable buffering on stdout - this is needed so that when we are using plain write() it will go to the terminal without flushing stdout.setvbuf(0, _IONBF); // Required main function variables string genericHelpMessage = "Please use 'onedrive --help' for further assistance in regards to running this application."; // If the user passes in --confdir we need to store this as a variable string confdirOption = ""; // running as what user? string runtimeUserName = ""; // Are we online? bool online = false; // Does the operating environment have shell environment variables set bool shellEnvSet = false; // What is the runtime syncronisation directory that will be used // Typically this will be '~/OneDrive' .. however tilde expansion is unreliable string runtimeSyncDirectory = ""; // Configure the runtime database file path. Typically this will be the default, but in a --dry-run scenario, we use a separate database file string runtimeDatabaseFile = ""; // Verbosity Logging Count - this defines if verbose or debug logging is being used long verbosityCount = 0; // Application Logging Level bool verboseLogging = false; bool debugLogging = false; // Monitor loop failures bool monitorFailures = false; // Help requested bool helpRequested = false; // DEVELOPER OPTIONS OUTPUT VARIABLES bool displayMemoryUsage = false; bool displaySyncOptions = false; // Application Version //immutable string applicationVersion = "onedrive " ~ strip(import("version")); immutable string applicationVersion = "v2.5.0-alpha-5" ~ " GitHub version: " ~ strip(import("version")); // Define 'exit' and 'failure' scopes scope(exit) { // Detail what scope was called addLogEntry("Exit scope was called", ["debug"]); // Perform exit tasks performStandardExitProcess("exitScope"); } scope(failure) { // Detail what scope was called addLogEntry("Failure scope was called", ["debug"]); // Perform exit tasks performStandardExitProcess("failureScope"); } // Read in application options as passed in try { bool printVersion = false; auto cliOptions = getopt( cliArgs, std.getopt.config.passThrough, std.getopt.config.bundling, std.getopt.config.caseSensitive, "confdir", "Set the directory used to store the configuration files", &confdirOption, "verbose|v+", "Print more details, useful for debugging (repeat for extra debugging)", &verbosityCount, "version", "Print the version and exit", &printVersion ); // Print help and exit if (cliOptions.helpWanted) { cliArgs ~= "--help"; helpRequested = true; } // Print the version and exit if (printVersion) { writeln(applicationVersion); exit(EXIT_SUCCESS); } } catch (GetOptException e) { // Option errors writeln(e.msg); writeln(genericHelpMessage); return EXIT_FAILURE; } catch (Exception e) { // Generic error writeln(e.msg); writeln(genericHelpMessage); return EXIT_FAILURE; } // Determine the application logging verbosity if (verbosityCount == 1) { verboseLogging = true;} if (verbosityCount >= 2) { debugLogging = true;} // Initialize the application progress manager class initialiseProgressManager(verboseLogging, debugLogging); // Initialize the application logging class, as we know the application verbosity level // If we need to enable logging to a file, we can only do this once we know the application configuration which is done slightly later on initialiseLogging(verboseLogging, debugLogging); /** // most used addLogEntry("Basic 'info' message", ["info"]); .... or just use addLogEntry("Basic 'info' message"); addLogEntry("Basic 'verbose' message", ["verbose"]); addLogEntry("Basic 'debug' message", ["debug"]); // GUI notify only addLogEntry("Basic 'notify' ONLY message and displayed in GUI if notifications are enabled", ["notify"]); // info and notify addLogEntry("Basic 'info and notify' message and displayed in GUI if notifications are enabled", ["info", "notify"]); // log file only addLogEntry("Information sent to the log file only, and only if logging to a file is enabled", ["logFileOnly"]); // Console only (session based upload|download) addLogEntry("Basic 'Console only with new line' message", ["consoleOnly"]); // Console only with no new line addLogEntry("Basic 'Console only with no new line' message", ["consoleOnlyNoNewLine"]); **/ // Log application start time addLogEntry("Application started", ["debug"]); // Who are we running as? This will print the ProcessID, UID, GID and username the application is running as runtimeUserName = getUserName(); // Print in debug the application version as soon as possible addLogEntry("Application Version: " ~ applicationVersion, ["debug"]); // How was this application started - what options were passed in addLogEntry("Passed in 'cliArgs': " ~ to!string(cliArgs), ["debug"]); addLogEntry("Note: --confdir and --verbose are not listed in 'cliArgs' array", ["debug"]); addLogEntry("Passed in --confdir if present: " ~ confdirOption, ["debug"]); addLogEntry("Passed in --verbose count if present: " ~ to!string(verbosityCount), ["debug"]); // Create a new AppConfig object with default values, appConfig = new ApplicationConfig(); // Update the default application configuration with the logging level so these can be used as a config option throughout the application appConfig.setConfigLoggingLevels(verboseLogging, debugLogging, verbosityCount); // Initialise the application configuration, utilising --confdir if it was passed in // Otherwise application defaults will be used to configure the application if (!appConfig.initialise(confdirOption, helpRequested)) { // There was an error loading the user specified application configuration // Error message already printed return EXIT_FAILURE; } // Update the current runtime application configuration (default or 'config' fileread-in options) from any passed in command line arguments appConfig.updateFromArgs(cliArgs); // As early as possible, now re-configure the logging class, given that we have read in any applicable 'config' file and updated the application running config from CLI input: // - Enable logging to a file if this is required // - Disable GUI notifications if this has been configured // Configure application logging to a log file only if this has been enabled // This is the earliest point that this can be done, as the client configuration has been read in, and any CLI arguments have been processed. // Either of those ('confif' file, CPU arguments) could be enabling logging, thus this is the earliest point at which this can be validated and enabled. // The buffered logging also ensures that all 'output' to this point is also captured and written out to the log file if (appConfig.getValueBool("enable_logging")) { // Calculate the application logging directory string calculatedLogDirPath = appConfig.calculateLogDirectory(); string calculatedLogFilePath; // Initialise using the configured logging directory addLogEntry("Using the following path to store the runtime application log: " ~ calculatedLogDirPath, ["verbose"]); // Calculate the logfile name if (calculatedLogDirPath != appConfig.defaultHomePath) { // Log file is not going to the home directory string logfileName = runtimeUserName ~ ".onedrive.log"; calculatedLogFilePath = buildNormalizedPath(buildPath(calculatedLogDirPath, logfileName)); } else { // Log file is going to the users home directory calculatedLogFilePath = buildNormalizedPath(buildPath(calculatedLogDirPath, "onedrive.log")); } // Update the logging class to use 'calculatedLogFilePath' for the application log file now that this has been determined enableLogFileOutput(calculatedLogFilePath); } // Disable GUI Notifications if configured to do so // - This option is reverse action. If 'disable_notifications' is 'true', we need to send 'false' if (appConfig.getValueBool("disable_notifications")) { // disable_notifications is true, ensure GUI notifications is initialised with false so that NO GUI notification is sent disableGUINotifications(false); addLogEntry("Disabling GUI notifications as per user configuration"); } // Perform a depreciated options check now that the config file (if present) and CLI options have all been parsed to advise the user that their option usage might change appConfig.checkDepreciatedOptions(cliArgs); // Configure Client Side Filtering (selective sync) by parsing and getting a usable regex for skip_file, skip_dir and sync_list config components selectiveSync = new ClientSideFiltering(appConfig); if (!selectiveSync.initialise()) { // exit here as something triggered a selective sync configuration failure return EXIT_FAILURE; } // Set runtimeDatabaseFile, this will get updated if we are using --dry-run runtimeDatabaseFile = appConfig.databaseFilePath; // Read in 'sync_dir' from appConfig with '~' if present expanded runtimeSyncDirectory = appConfig.initialiseRuntimeSyncDirectory(); // DEVELOPER OPTIONS OUTPUT // Set to display memory details as early as possible displayMemoryUsage = appConfig.getValueBool("display_memory"); // set to display sync options displaySyncOptions = appConfig.getValueBool("display_sync_options"); // Display the current application configuration (based on all defaults, 'config' file parsing and/or options passed in via the CLI) and exit if --display-config has been used if ((appConfig.getValueBool("display_config")) || (appConfig.getValueBool("display_running_config"))) { // Display the application configuration appConfig.displayApplicationConfiguration(); // Do we exit? We exit only if '--display-config' has been used if (appConfig.getValueBool("display_config")) { return EXIT_SUCCESS; } } // Check for basic application option conflicts - flags that should not be used together and/or flag combinations that conflict with each other, values that should be present and are not if (appConfig.checkForBasicOptionConflicts) { // Any error will have been printed by the function itself, but we need a small delay here to allow the buffered logging to output any error return EXIT_FAILURE; } // Check for --dry-run operation // If this has been requested, we need to ensure that all actions are performed against the dry-run database copy, and, // no actual action takes place - such as deleting files if deleted online, moving files if moved online or local, downloading new & changed files, uploading new & changed files if (appConfig.getValueBool("dry_run")) { // this is a --dry-run operation addLogEntry("DRY-RUN Configured. Output below shows what 'would' have occurred."); // Cleanup any existing dry-run elements ... these should never be left hanging around cleanupDryRunDatabaseFiles(appConfig.databaseFilePathDryRun); // Make a copy of the original items.sqlite3 for use as the dry run copy if it exists if (exists(appConfig.databaseFilePath)) { // In a --dry-run --resync scenario, we should not copy the existing database file if (!appConfig.getValueBool("resync")) { // Copy the existing DB file to the dry-run copy addLogEntry("DRY-RUN: Copying items.sqlite3 to items-dryrun.sqlite3 to use for dry run operations"); copy(appConfig.databaseFilePath,appConfig.databaseFilePathDryRun); } else { // No database copy due to --resync addLogEntry("DRY-RUN: No database copy created for --dry-run due to --resync also being used"); } } // update runtimeDatabaseFile now that we are using the dry run path runtimeDatabaseFile = appConfig.databaseFilePathDryRun; } // Handle --logout as separate item, do not 'resync' on a --logout if (appConfig.getValueBool("logout")) { addLogEntry("--logout requested", ["debug"]); addLogEntry("Deleting the saved authentication status ..."); if (!appConfig.getValueBool("dry_run")) { safeRemove(appConfig.refreshTokenFilePath); } else { // --dry-run scenario ... technically we should not be making any local file changes ....... addLogEntry("DRY RUN: Not removing the saved authentication status"); } // Exit return EXIT_SUCCESS; } // Handle --reauth to re-authenticate the client if (appConfig.getValueBool("reauth")) { addLogEntry("--reauth requested", ["debug"]); addLogEntry("Deleting the saved authentication status ... re-authentication requested"); if (!appConfig.getValueBool("dry_run")) { safeRemove(appConfig.refreshTokenFilePath); } else { // --dry-run scenario ... technically we should not be making any local file changes ....... addLogEntry("DRY RUN: Not removing the saved authentication status"); } } // --resync should be considered a 'last resort item' or if the application configuration has changed, where a resync is needed .. the user needs to 'accept' this warning to proceed // If --resync has not been used (bool value is false), check the application configuration for 'changes' that require a --resync to ensure that the data locally reflects the users requested configuration if (appConfig.getValueBool("resync")) { // what is the risk acceptance for --resync? bool resyncRiskAcceptance = appConfig.displayResyncRiskForAcceptance(); addLogEntry("Returned --resync risk acceptance: " ~ to!string(resyncRiskAcceptance), ["debug"]); // Action based on user response if (!resyncRiskAcceptance){ // --resync risk not accepted return EXIT_FAILURE; } else { addLogEntry("--resync issued and risk accepted", ["debug"]); // --resync risk accepted, perform a cleanup of items that require a cleanup appConfig.cleanupHashFilesDueToResync(); // Make a backup of the applicable configuration file appConfig.createBackupConfigFile(); // Update hash files and generate a new config backup appConfig.updateHashContentsForConfigFiles(); // Remove the items database processResyncDatabaseRemoval(runtimeDatabaseFile); } } else { // Is the application currently authenticated? If not, it is pointless checking if a --resync is required until the application is authenticated if (exists(appConfig.refreshTokenFilePath)) { // Has any of our application configuration that would require a --resync been changed? if (appConfig.applicationChangeWhereResyncRequired()) { // Application configuration has changed however --resync not issued, fail fast addLogEntry(); addLogEntry("An application configuration change has been detected where a --resync is required"); addLogEntry(); return EXIT_RESYNC_REQUIRED; } else { // No configuration change that requires a --resync to be issued // Special cases need to be checked - if these options were enabled, it creates a false 'Resync Required' flag, so do not create a backup if ((!appConfig.getValueBool("list_business_shared_items"))) { // Make a backup of the applicable configuration file appConfig.createBackupConfigFile(); // Update hash files and generate a new config backup appConfig.updateHashContentsForConfigFiles(); } } } } // Implement https://github.com/abraunegg/onedrive/issues/1129 // Force a synchronization of a specific folder, only when using --synchronize --single-directory and ignoring all non-default skip_dir and skip_file rules if (appConfig.getValueBool("force_sync")) { // appConfig.checkForBasicOptionConflicts() has already checked for the basic requirements for --force-sync addLogEntry(); addLogEntry("WARNING: Overriding application configuration to use application defaults for skip_dir and skip_file due to --sync --single-directory --force-sync being used"); addLogEntry(); bool forceSyncRiskAcceptance = appConfig.displayForceSyncRiskForAcceptance(); addLogEntry("Returned --force-sync risk acceptance: " ~ forceSyncRiskAcceptance, ["debug"]); // Action based on user response if (!forceSyncRiskAcceptance){ // --force-sync risk not accepted return EXIT_FAILURE; } else { // --force-sync risk accepted // reset set config using function to use application defaults appConfig.resetSkipToDefaults(); // update sync engine regex with reset defaults selectiveSync.setDirMask(appConfig.getValueString("skip_dir")); selectiveSync.setFileMask(appConfig.getValueString("skip_file")); } } // What IP Protocol are we going to use to access the network with appConfig.displayIPProtocol(); // Test if OneDrive service can be reached, exit if it cant be reached addLogEntry("Testing network to ensure network connectivity to Microsoft OneDrive Service", ["debug"]); online = testInternetReachability(appConfig); // If we are not 'online' - how do we handle this situation? if (!online) { // We are unable to initialise the OneDrive API as we are not online if (!appConfig.getValueBool("monitor")) { // Running as --synchronize addLogEntry(); addLogEntry("ERROR: Unable to reach Microsoft OneDrive API service, unable to initialise application"); addLogEntry(); return EXIT_FAILURE; } else { // Running as --monitor addLogEntry(); addLogEntry("Unable to reach the Microsoft OneDrive API service at this point in time, re-trying network tests based on applicable intervals"); addLogEntry(); if (!retryInternetConnectivtyTest(appConfig)) { return EXIT_FAILURE; } } } // This needs to be a separate 'if' statement, as, if this was an 'if-else' from above, if we were originally offline and using --monitor, we would never get to this point if (online) { // Check Application Version addLogEntry("Checking Application Version ...", ["verbose"]); checkApplicationVersion(); // Initialise the OneDrive API addLogEntry("Attempting to initialise the OneDrive API ...", ["verbose"]); OneDriveApi oneDriveApiInstance = new OneDriveApi(appConfig); appConfig.apiWasInitialised = oneDriveApiInstance.initialise(); if (appConfig.apiWasInitialised) { addLogEntry("The OneDrive API was initialised successfully", ["verbose"]); // Flag that we were able to initalise the API in the application config oneDriveApiInstance.debugOutputConfiguredAPIItems(); oneDriveApiInstance.shutdown(); object.destroy(oneDriveApiInstance); // Need to configure the itemDB and syncEngineInstance for 'sync' and 'non-sync' operations addLogEntry("Opening the item database ...", ["verbose"]); // Configure the Item Database itemDB = new ItemDatabase(runtimeDatabaseFile); // Was the database successfully initialised? if (!itemDB.isDatabaseInitialised()) { // no .. destroy class itemDB = null; // exit application return EXIT_FAILURE; } // Initialise the syncEngine syncEngineInstance = new SyncEngine(appConfig, itemDB, selectiveSync); appConfig.syncEngineWasInitialised = syncEngineInstance.initialise(); // Are we not doing a --sync or a --monitor operation? Both of these will be false if they are not set if ((!appConfig.getValueBool("synchronize")) && (!appConfig.getValueBool("monitor"))) { // Are we performing some sort of 'no-sync' task? // - Are we obtaining the Office 365 Drive ID for a given Office 365 SharePoint Shared Library? // - Are we displaying the sync satus? // - Are we getting the URL for a file online? // - Are we listing who modified a file last online? // - Are we listing OneDrive Business Shared Items? // - Are we createing a shareable link for an existing file on OneDrive? // - Are we just creating a directory online, without any sync being performed? // - Are we just deleting a directory online, without any sync being performed? // - Are we renaming or moving a directory? // - Are we displaying the quota information? // - Did we just authorise the client? // --get-sharepoint-drive-id - Get the SharePoint Library drive_id if (appConfig.getValueString("sharepoint_library_name") != "") { // Get the SharePoint Library drive_id syncEngineInstance.querySiteCollectionForDriveID(appConfig.getValueString("sharepoint_library_name")); // Exit application // Use exit scopes to shutdown API and cleanup data return EXIT_SUCCESS; } // --display-sync-status - Query the sync status if (appConfig.getValueBool("display_sync_status")) { // path to query variable string pathToQueryStatusOn; // What path do we query? if (!appConfig.getValueString("single_directory").empty) { pathToQueryStatusOn = "/" ~ appConfig.getValueString("single_directory"); } else { pathToQueryStatusOn = "/"; } // Query the sync status syncEngineInstance.queryOneDriveForSyncStatus(pathToQueryStatusOn); // Exit application // Use exit scopes to shutdown API and cleanup data return EXIT_SUCCESS; } // --get-file-link - Get the URL path for a synced file? if (appConfig.getValueString("get_file_link") != "") { // Query the OneDrive API for the file link syncEngineInstance.queryOneDriveForFileDetails(appConfig.getValueString("get_file_link"), runtimeSyncDirectory, "URL"); // Exit application // Use exit scopes to shutdown API and cleanup data return EXIT_SUCCESS; } // --modified-by - Are we listing the modified-by details of a provided path? if (appConfig.getValueString("modified_by") != "") { // Query the OneDrive API for the last modified by details syncEngineInstance.queryOneDriveForFileDetails(appConfig.getValueString("modified_by"), runtimeSyncDirectory, "ModifiedBy"); // Exit application // Use exit scopes to shutdown API and cleanup data return EXIT_SUCCESS; } // --list-shared-items - Are we listing OneDrive Business Shared Items if (appConfig.getValueBool("list_business_shared_items")) { // Is this a business account type? if (appConfig.accountType == "business") { // List OneDrive Business Shared Items syncEngineInstance.listBusinessSharedObjects(); } else { addLogEntry("ERROR: Unsupported account type for listing OneDrive Business Shared Items"); } // Exit application // Use exit scopes to shutdown API return EXIT_SUCCESS; } // --create-share-link - Are we createing a shareable link for an existing file on OneDrive? if (appConfig.getValueString("create_share_link") != "") { // Query OneDrive for the file, and if valid, create a shareable link for the file // By default, the shareable link will be read-only. // If the user adds: // --with-editing-perms // this will create a writeable link syncEngineInstance.queryOneDriveForFileDetails(appConfig.getValueString("create_share_link"), runtimeSyncDirectory, "ShareableLink"); // Exit application // Use exit scopes to shutdown API return EXIT_SUCCESS; } // --create-directory - Are we just creating a directory online, without any sync being performed? if ((appConfig.getValueString("create_directory") != "")) { // Handle the remote path creation and updating of the local database without performing a sync syncEngineInstance.createDirectoryOnline(appConfig.getValueString("create_directory")); // Exit application // Use exit scopes to shutdown API return EXIT_SUCCESS; } // --remove-directory - Are we just deleting a directory online, without any sync being performed? if ((appConfig.getValueString("remove_directory") != "")) { // Handle the remote path deletion without performing a sync syncEngineInstance.deleteByPath(appConfig.getValueString("remove_directory")); // Exit application // Use exit scopes to shutdown API return EXIT_SUCCESS; } // Are we renaming or moving a directory? // onedrive --source-directory 'path/as/source/' --destination-directory 'path/as/destination' if ((appConfig.getValueString("source_directory") != "") && (appConfig.getValueString("destination_directory") != "")) { // We are renaming or moving a directory syncEngineInstance.uploadMoveItem(appConfig.getValueString("source_directory"), appConfig.getValueString("destination_directory")); // Exit application // Use exit scopes to shutdown API return EXIT_SUCCESS; } // Are we displaying the quota information? if (appConfig.getValueBool("display_quota")) { // Query and respond with the quota details syncEngineInstance.queryOneDriveForQuotaDetails(); // Exit application // Use exit scopes to shutdown API return EXIT_SUCCESS; } // If we get to this point, we have not performed a 'no-sync' task .. // Did we just authorise the client? if (appConfig.applicationAuthorizeResponseUri) { // Authorisation activity if (exists(appConfig.refreshTokenFilePath)) { // OneDrive refresh token exists addLogEntry(); addLogEntry("The application has been successfully authorised, but no extra command options have been specified."); addLogEntry(); addLogEntry(genericHelpMessage); addLogEntry(); // Use exit scopes to shutdown API return EXIT_SUCCESS; } else { // We just authorised, but refresh_token does not exist .. probably an auth error? addLogEntry(); addLogEntry("Your application's authorisation was unsuccessful. Please review your URI response entry, then attempt authorisation again with a new URI response."); addLogEntry(); // Use exit scopes to shutdown API return EXIT_FAILURE; } } else { // No authorisation activity addLogEntry(); addLogEntry("Your command line input is missing either the '--sync' or '--monitor' switches. Please include one (but not both) of these switches in your command line, or refer to 'onedrive --help' for additional guidance."); addLogEntry(); addLogEntry("It is important to note that you must include one of these two arguments in your command line for the application to perform a synchronisation with Microsoft OneDrive"); addLogEntry(); // Use exit scopes to shutdown API // invalidSyncExit = true; return EXIT_FAILURE; } } } else { // API could not be initialised addLogEntry("The OneDrive API could not be initialised"); return EXIT_FAILURE; } } // Configure the sync direcory based on the runtimeSyncDirectory configured directory addLogEntry("All application operations will be performed in the configured local 'sync_dir' directory: " ~ runtimeSyncDirectory, ["verbose"]); try { if (!exists(runtimeSyncDirectory)) { addLogEntry("runtimeSyncDirectory: Configured 'sync_dir' is missing locally. Creating: " ~ runtimeSyncDirectory, ["debug"]); try { // Attempt to create the sync dir we have been configured with mkdirRecurse(runtimeSyncDirectory); // Configure the applicable permissions for the folder addLogEntry("Setting directory permissions for: " ~ runtimeSyncDirectory, ["debug"]); runtimeSyncDirectory.setAttributes(appConfig.returnRequiredDirectoryPermisions()); } catch (std.file.FileException e) { // Creating the sync directory failed addLogEntry("ERROR: Unable to create the configured local 'sync_dir' directory: " ~ e.msg); // Use exit scopes to shutdown API return EXIT_FAILURE; } } } catch (std.file.FileException e) { // Creating the sync directory failed addLogEntry("ERROR: Unable to test for the existence of the configured local 'sync_dir' directory: " ~ e.msg); // Use exit scopes to shutdown API return EXIT_FAILURE; } // Change the working directory to the 'sync_dir' as configured chdir(runtimeSyncDirectory); // Do we need to validate the runtimeSyncDirectory to check for the presence of a '.nosync' file checkForNoMountScenario(); // Set the default thread pool value defaultPoolThreads(to!int(appConfig.getValueLong("threads"))); // Is the sync engine initiallised correctly? if (appConfig.syncEngineWasInitialised) { // Configure some initial variables string singleDirectoryPath; string localPath = "."; string remotePath = "/"; if (!appConfig.getValueBool("resync")) { // Check if there are interrupted upload session(s) if (syncEngineInstance.checkForInterruptedSessionUploads) { // Need to re-process the session upload files to resume the failed session uploads addLogEntry("There are interrupted session uploads that need to be resumed ..."); // Process the session upload files syncEngineInstance.processForInterruptedSessionUploads(); } } else { // Clean up any upload session files due to --resync being used syncEngineInstance.clearInterruptedSessionUploads(); } // Are we doing a single directory operation (--single-directory) ? if (!appConfig.getValueString("single_directory").empty) { // Set singleDirectoryPath singleDirectoryPath = appConfig.getValueString("single_directory"); // Ensure that this is a normalised relative path to runtimeSyncDirectory string normalisedRelativePath = replace(buildNormalizedPath(absolutePath(singleDirectoryPath)), buildNormalizedPath(absolutePath(runtimeSyncDirectory)), "." ); // The user provided a directory to sync within the configured 'sync_dir' path // This also validates if the path being used exists online and/or does not have a 'case-insensitive match' syncEngineInstance.setSingleDirectoryScope(normalisedRelativePath); // Does the directory we want to sync actually exist locally? if (!exists(singleDirectoryPath)) { // The requested path to use with --single-directory does not exist locally within the configured 'sync_dir' addLogEntry("WARNING: The requested path for --single-directory does not exist locally. Creating requested path within " ~ runtimeSyncDirectory, ["info", "notify"]); // Make the required --single-directory path locally mkdirRecurse(singleDirectoryPath); // Configure the applicable permissions for the folder addLogEntry("Setting directory permissions for: " ~ singleDirectoryPath, ["debug"]); singleDirectoryPath.setAttributes(appConfig.returnRequiredDirectoryPermisions()); } // Update the paths that we use to perform the sync actions localPath = singleDirectoryPath; remotePath = singleDirectoryPath; // Display that we are syncing from a specific path due to --single-directory addLogEntry("Syncing changes from this selected path: " ~ singleDirectoryPath, ["verbose"]); } // Are we doing a --sync operation? This includes doing any --single-directory operations if (appConfig.getValueBool("synchronize")) { // Did the user specify --upload-only? if (appConfig.getValueBool("upload_only")) { // Perform the --upload-only sync process performUploadOnlySyncProcess(localPath); } // Did the user specify --download-only? if (appConfig.getValueBool("download_only")) { // Only download data from OneDrive syncEngineInstance.syncOneDriveAccountToLocalDisk(); // Perform the DB consistency check // This will also delete any out-of-sync flagged items if configured to do so syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); // Do we cleanup local files? // - Deletes of data from online will already have been performed, but what we are now doing is searching the local filesystem // for any new data locally, that usually would be uploaded to OneDrive, but instead, because of the options being // used, will need to be deleted from the local filesystem if (appConfig.getValueBool("cleanup_local_files")) { // Perform the filesystem walk syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); } } // If no use of --upload-only or --download-only if ((!appConfig.getValueBool("upload_only")) && (!appConfig.getValueBool("download_only"))) { // Perform the standard sync process performStandardSyncProcess(localPath); } // Detail the outcome of the sync process displaySyncOutcome(); progressManager.clearAllJobs(); } // Are we doing a --monitor operation? if (appConfig.getValueBool("monitor")) { // What are the current values for the platform we are running on // Max number of open files /proc/sys/fs/file-max string maxOpenFiles = strip(readText("/proc/sys/fs/file-max")); // What is the currently configured maximum inotify watches that can be used // /proc/sys/fs/inotify/max_user_watches string maxInotifyWatches = strip(readText("/proc/sys/fs/inotify/max_user_watches")); // Start the monitor process addLogEntry("OneDrive synchronisation interval (seconds): " ~ to!string(appConfig.getValueLong("monitor_interval"))); // If we are in a --download-only method of operation, the output of these is not required if (!appConfig.getValueBool("download_only")) { addLogEntry("Maximum allowed open files: " ~ maxOpenFiles, ["verbose"]); addLogEntry("Maximum allowed inotify user watches: " ~ maxInotifyWatches, ["verbose"]); } // Configure the monitor class filesystemMonitor = new Monitor(appConfig, selectiveSync); // Delegated function for when inotify detects a new local directory has been created filesystemMonitor.onDirCreated = delegate(string path) { // Handle .folder creation if skip_dotfiles is enabled if ((appConfig.getValueBool("skip_dotfiles")) && (isDotFile(path))) { addLogEntry("[M] Skipping watching local path - .folder found & --skip-dot-files enabled: " ~ path, ["verbose"]); } else { addLogEntry("[M] Local directory created: " ~ path, ["verbose"]); try { syncEngineInstance.scanLocalFilesystemPathForNewData(path); } catch (CurlException e) { addLogEntry("Offline, cannot create remote dir: " ~ path, ["verbose"]); } catch(Exception e) { addLogEntry("Cannot create remote directory: " ~ e.msg, ["info", "notify"]); } } }; // Delegated function for when inotify detects a local file has been changed filesystemMonitor.onFileChanged = delegate(string[] changedLocalFilesToUploadToOneDrive) { // Handle a potentially locally changed file // Logging for this event moved to handleLocalFileTrigger() due to threading and false triggers from scanLocalFilesystemPathForNewData() above addLogEntry("[M] Total number of local file changed: " ~ to!string(changedLocalFilesToUploadToOneDrive.length)); syncEngineInstance.handleLocalFileTrigger(changedLocalFilesToUploadToOneDrive); }; // Delegated function for when inotify detects a delete event filesystemMonitor.onDelete = delegate(string path) { addLogEntry("[M] Local item deleted: " ~ path, ["verbose"]); try { addLogEntry("The operating system sent a deletion notification. Trying to delete the item as requested"); syncEngineInstance.deleteByPath(path); } catch (CurlException e) { addLogEntry("Offline, cannot delete item: " ~ path, ["verbose"]); } catch(SyncException e) { if (e.msg == "The item to delete is not in the local database") { addLogEntry("Item cannot be deleted from Microsoft OneDrive because it was not found in the local database", ["verbose"]); } else { addLogEntry("Cannot delete remote item: " ~ e.msg, ["info", "notify"]); } } catch(Exception e) { addLogEntry("Cannot delete remote item: " ~ e.msg, ["info", "notify"]); } }; // Delegated function for when inotify detects a move event filesystemMonitor.onMove = delegate(string from, string to) { addLogEntry("[M] Local item moved: " ~ from ~ " -> " ~ to, ["verbose"]); try { // Handle .folder -> folder if skip_dotfiles is enabled if ((appConfig.getValueBool("skip_dotfiles")) && (isDotFile(from))) { // .folder -> folder handling - has to be handled as a new folder syncEngineInstance.scanLocalFilesystemPathForNewData(to); } else { syncEngineInstance.uploadMoveItem(from, to); } } catch (CurlException e) { addLogEntry("Offline, cannot move item !", ["verbose"]); } catch(Exception e) { addLogEntry("Cannot move item: " ~ e.msg, ["info", "notify"]); } }; // Handle SIGINT and SIGTERM signal(SIGINT, &exitHandler); signal(SIGTERM, &exitHandler); // Initialise the local filesystem monitor class using inotify to monitor for local filesystem changes // If we are in a --download-only method of operation, we do not enable local filesystem monitoring if (!appConfig.getValueBool("download_only")) { // Not using --download-only try { addLogEntry("Initialising filesystem inotify monitoring ..."); filesystemMonitor.initialise(); addLogEntry("Performing initial syncronisation to ensure consistent local state ..."); } catch (MonitorException e) { // monitor class initialisation failed addLogEntry("ERROR: " ~ e.msg); return EXIT_FAILURE; } } // Filesystem monitor loop variables // Immutables immutable auto checkOnlineInterval = dur!"seconds"(appConfig.getValueLong("monitor_interval")); immutable auto githubCheckInterval = dur!"seconds"(86400); immutable ulong fullScanFrequency = appConfig.getValueLong("monitor_fullscan_frequency"); immutable ulong logOutputSupressionInterval = appConfig.getValueLong("monitor_log_frequency"); immutable bool webhookEnabled = appConfig.getValueBool("webhook_enabled"); immutable string loopStartOutputMessage = "################################################## NEW LOOP ##################################################"; immutable string loopStopOutputMessage = "################################################ LOOP COMPLETE ###############################################"; // Changables bool performMonitor = true; ulong monitorLoopFullCount = 0; ulong fullScanFrequencyLoopCount = 0; ulong monitorLogOutputLoopCount = 0; MonoTime lastCheckTime = MonoTime.currTime(); MonoTime lastGitHubCheckTime = MonoTime.currTime(); // Webhook Notification Handling bool notificationReceived = false; while (performMonitor) { // Do we need to validate the runtimeSyncDirectory to check for the presence of a '.nosync' file - the disk may have been ejected .. checkForNoMountScenario(); // If we are in a --download-only method of operation, there is no filesystem monitoring, so no inotify events to check if (!appConfig.getValueBool("download_only")) { try { // Process any inotify events filesystemMonitor.update(true); } catch (MonitorException e) { // Catch any exceptions thrown by inotify / monitor engine addLogEntry("ERROR: The following inotify error was generated: " ~ e.msg); } } // Check for notifications pushed from Microsoft to the webhook if (webhookEnabled) { // Create a subscription on the first run, or renew the subscription // on subsequent runs when it is about to expire. if (oneDriveWebhook is null) { oneDriveWebhook = new OneDriveWebhook(thisTid, appConfig); oneDriveWebhook.serve(); } else oneDriveWebhook.createOrRenewSubscription(); } // Get the current time this loop is starting auto currentTime = MonoTime.currTime(); // Do we perform a sync with OneDrive? if ((currentTime - lastCheckTime >= checkOnlineInterval) || (monitorLoopFullCount == 0)) { // Increment relevant counters monitorLoopFullCount++; fullScanFrequencyLoopCount++; monitorLogOutputLoopCount++; // If full scan at a specific frequency enabled? if (fullScanFrequency > 0) { // Full Scan set for some 'frequency' - do we flag to perform a full scan of the online data? if (fullScanFrequencyLoopCount > fullScanFrequency) { // set full scan trigger for true up addLogEntry("Enabling Full Scan True Up (fullScanFrequencyLoopCount > fullScanFrequency), resetting fullScanFrequencyLoopCount = 1", ["debug"]); fullScanFrequencyLoopCount = 1; appConfig.fullScanTrueUpRequired = true; } else { // unset full scan trigger for true up addLogEntry("Disabling Full Scan True Up", ["debug"]); appConfig.fullScanTrueUpRequired = false; } } else { // No it is disabled - ensure this is false appConfig.fullScanTrueUpRequired = false; } // Loop Start addLogEntry(loopStartOutputMessage, ["debug"]); addLogEntry("Total Run-Time Loop Number: " ~ to!string(monitorLoopFullCount), ["debug"]); addLogEntry("Full Scan Freqency Loop Number: " ~ to!string(fullScanFrequencyLoopCount), ["debug"]); SysTime startFunctionProcessingTime = Clock.currTime(); addLogEntry("Start Monitor Loop Time: " ~ to!string(startFunctionProcessingTime), ["debug"]); // Do we perform any monitor console logging output surpression? // 'monitor_log_frequency' controls how often, in a non-verbose application output mode, how often // the full output of what is occuring is done. This is done to lessen the 'verbosity' of non-verbose // logging, but only when running in --monitor if (monitorLogOutputLoopCount > logOutputSupressionInterval) { // unsurpress the logging output monitorLogOutputLoopCount = 1; addLogEntry("Unsuppressing initial sync log output", ["debug"]); appConfig.surpressLoggingOutput = false; } else { // do we surpress the logging output to absolute minimal if (monitorLoopFullCount == 1) { // application startup with --monitor addLogEntry("Unsuppressing initial sync log output", ["debug"]); appConfig.surpressLoggingOutput = false; } else { // only surpress if we are not doing --verbose or higher if (appConfig.verbosityCount == 0) { addLogEntry("Suppressing --monitor log output", ["debug"]); appConfig.surpressLoggingOutput = true; } else { addLogEntry("Unsuppressing log output", ["debug"]); appConfig.surpressLoggingOutput = false; } } } // How long has the application been running for? auto elapsedTime = Clock.currTime() - applicationStartTime; addLogEntry("Application run-time thus far: " ~ to!string(elapsedTime), ["debug"]); // Need to re-validate that the client is still online for this loop if (testInternetReachability(appConfig)) { // Starting a sync addLogEntry("Starting a sync with Microsoft OneDrive"); // Attempt to reset syncFailures syncEngineInstance.resetSyncFailures(); // Update cached quota details from online as this may have changed online in the background outside of this application syncEngineInstance.freshenCachedDriveQuotaDetails(); // Did the user specify --upload-only? if (appConfig.getValueBool("upload_only")) { // Perform the --upload-only sync process performUploadOnlySyncProcess(localPath, filesystemMonitor); } else { // Perform the standard sync process performStandardSyncProcess(localPath, filesystemMonitor); } // Handle any new inotify events filesystemMonitor.update(true); // Detail the outcome of the sync process displaySyncOutcome(); if (appConfig.fullScanTrueUpRequired) { // Write WAL and SHM data to file for this loop addLogEntry("Merge contents of WAL and SHM files into main database file", ["debug"]); itemDB.performVacuum(); } } else { // Not online addLogEntry("Microsoft OneDrive service is not reachable at this time. Will re-try on next sync attempt."); } // Output end of loop processing times SysTime endFunctionProcessingTime = Clock.currTime(); addLogEntry("End Monitor Loop Time: " ~ to!string(endFunctionProcessingTime), ["debug"]); addLogEntry("Elapsed Monitor Loop Processing Time: " ~ to!string((endFunctionProcessingTime - startFunctionProcessingTime)), ["debug"]); // Display memory details before cleanup if (displayMemoryUsage) displayMemoryUsagePreGC(); // Perform Garbage Cleanup GC.collect(); // Return free memory to the OS GC.minimize(); // Display memory details after cleanup if (displayMemoryUsage) displayMemoryUsagePostGC(); // Log that this loop is complete addLogEntry(loopStopOutputMessage, ["debug"]); // performSync complete, set lastCheckTime to current time lastCheckTime = MonoTime.currTime(); // Developer break via config option if (appConfig.getValueLong("monitor_max_loop") > 0) { // developer set option to limit --monitor loops if (monitorLoopFullCount == (appConfig.getValueLong("monitor_max_loop"))) { performMonitor = false; addLogEntry("Exiting after " ~ to!string(monitorLoopFullCount) ~ " loops due to developer set option"); } } } progressManager.clearAllJobs(); if (performMonitor) { auto nextCheckTime = lastCheckTime + checkOnlineInterval; currentTime = MonoTime.currTime(); auto sleepTime = nextCheckTime - currentTime; addLogEntry("Sleep for " ~ to!string(sleepTime), ["debug"]); if(filesystemMonitor.initialised || webhookEnabled) { if(filesystemMonitor.initialised) { // If local monitor is on and is waiting (previous event was not from webhook) // start the worker and wait for event if (!notificationReceived) filesystemMonitor.send(true); } if(webhookEnabled) { // if onedrive webhook is enabled // update sleep time based on renew interval Duration nextWebhookCheckDuration = oneDriveWebhook.getNextExpirationCheckDuration(); if (nextWebhookCheckDuration < sleepTime) { sleepTime = nextWebhookCheckDuration; addLogEntry("Update sleeping time to " ~ to!string(sleepTime), ["debug"]); } // Webhook Notification reset to false for this loop notificationReceived = false; } int res = 1; // Process incoming notifications if any. auto signalExists = receiveTimeout(sleepTime, (int msg) { res = msg; }, (ulong _) { notificationReceived = true; } ); // Debug values addLogEntry("signalExists = " ~ to!string(signalExists), ["debug"]); addLogEntry("worker status = " ~ to!string(res), ["debug"]); addLogEntry("notificationReceived = " ~ to!string(notificationReceived), ["debug"]); // Empirical evidence shows that Microsoft often sends multiple // notifications for one single change, so we need a loop to exhaust // all signals that were queued up by the webhook. The notifications // do not contain any actual changes, and we will always rely do the // delta endpoint to sync to latest. Therefore, only one sync run is // good enough to catch up for multiple notifications. if (notificationReceived) { int signalCount = 1; while (true) { signalExists = receiveTimeout(dur!"seconds"(-1), (ulong _) {}); if (signalExists) { signalCount++; } else { addLogEntry("Received " ~ to!string(signalCount) ~ " refresh signals from the webhook"); oneDriveWebhookCallback(); break; } } } if(res == -1) { addLogEntry("ERROR: Monitor worker failed."); monitorFailures = true; performMonitor = false; } } else { // no hooks available, nothing to check Thread.sleep(sleepTime); } } progressManager.clearAllJobs(); } } } else { // Exit application as the sync engine could not be initialised addLogEntry("Application Sync Engine could not be initialised correctly"); // Use exit scope return EXIT_FAILURE; } // Exit application using exit scope if (!syncEngineInstance.syncFailures && !monitorFailures) { return EXIT_SUCCESS; } else { return EXIT_FAILURE; } } void performStandardExitProcess(string scopeCaller = null) { // Who called this function if (!scopeCaller.empty) { addLogEntry("Running performStandardExitProcess due to: " ~ scopeCaller, ["debug"]); } // Shutdown the OneDrive Webhook instance if (oneDriveWebhook !is null) { oneDriveWebhook.stop(); object.destroy(oneDriveWebhook); } // Shutdown the sync engine if (syncEngineInstance !is null) { addLogEntry("Shutdown Sync Engine instance", ["debug"]); object.destroy(syncEngineInstance); } // Shutdown the client side filtering objects if (selectiveSync !is null) { addLogEntry("Shutdown Client Side Filtering instance", ["debug"]); selectiveSync.shutdown(); object.destroy(selectiveSync); } // Shutdown the application configuration objects if (appConfig !is null) { addLogEntry("Shutdown Application Configuration instance", ["debug"]); // Cleanup any existing dry-run elements ... these should never be left hanging around cleanupDryRunDatabaseFiles(appConfig.databaseFilePathDryRun); object.destroy(appConfig); } // Shutdown any local filesystem monitoring if (filesystemMonitor !is null) { addLogEntry("Shutdown Filesystem Monitoring instance", ["debug"]); filesystemMonitor.shutdown(); object.destroy(filesystemMonitor); } // Shutdown the database if (itemDB !is null) { addLogEntry("Shutdown Database instance", ["debug"]); // Make sure the .wal file is incorporated into the main db before we exit if (itemDB.isDatabaseInitialised()) { itemDB.performVacuum(); } object.destroy(itemDB); } // Shutdown cached sockets CurlEngine.releaseAll(); // Set all objects to null if (scopeCaller == "failureScope") { // Set these to be null due to failure scope - prevent 'ERROR: Unable to perform a database vacuum: out of memory' when the exit scope is then called addLogEntry("Setting ALL Class Objects to null due to failure scope", ["debug"]); itemDB = null; appConfig = null; oneDriveWebhook = null; selectiveSync = null; syncEngineInstance = null; } else { addLogEntry("Waiting for all internal threads to complete before exiting application", ["verbose"]); thread_joinAll(); addLogEntry("Application exit", ["debug"]); addLogEntry("#######################################################################################################################################", ["logFileOnly"]); // Destroy the shared logging buffer (cast() logBuffer).shutdown(); object.destroy(logBuffer); } } void oneDriveWebhookCallback() { // If we are in a --download-only method of operation, there is no filesystem monitoring, so no inotify events to check if (!appConfig.getValueBool("download_only")) { try { // Process any inotify events filesystemMonitor.update(true); } catch (MonitorException e) { // Catch any exceptions thrown by inotify / monitor engine addLogEntry("ERROR: The following inotify error was generated: " ~ e.msg); } } // Download data from OneDrive last syncEngineInstance.syncOneDriveAccountToLocalDisk(); if (appConfig.getValueBool("monitor")) { // Handle any new inotify events filesystemMonitor.update(true); } } void performUploadOnlySyncProcess(string localPath, Monitor filesystemMonitor = null) { // Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); if (appConfig.getValueBool("monitor")) { // Handle any inotify events whilst the DB was being scanned filesystemMonitor.update(true); } // Scan the configured 'sync_dir' for new data to upload syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); if (appConfig.getValueBool("monitor")) { // Handle any new inotify events whilst the local filesystem was being scanned filesystemMonitor.update(true); } } void performStandardSyncProcess(string localPath, Monitor filesystemMonitor = null) { // If we are performing log supression, output this message so the user knows what is happening if (appConfig.surpressLoggingOutput) { addLogEntry("Syncing changes from Microsoft OneDrive ..."); } // Zero out these arrays syncEngineInstance.fileDownloadFailures = []; syncEngineInstance.fileUploadFailures = []; // Which way do we sync first? // OneDrive first then local changes (normal operational process that uses OneDrive as the source of truth) // Local First then OneDrive changes (alternate operation process to use local files as source of truth) if (appConfig.getValueBool("local_first")) { // Local data first // Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); if (appConfig.getValueBool("monitor")) { // Handle any inotify events whilst the DB was being scanned filesystemMonitor.update(true); } // Scan the configured 'sync_dir' for new data to upload to OneDrive syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); if (appConfig.getValueBool("monitor")) { // Handle any new inotify events whilst the local filesystem was being scanned filesystemMonitor.update(true); } // Download data from OneDrive last syncEngineInstance.syncOneDriveAccountToLocalDisk(); if (appConfig.getValueBool("monitor")) { // Cancel out any inotify events from downloading data filesystemMonitor.update(false); } } else { // Normal sync // Download data from OneDrive first syncEngineInstance.syncOneDriveAccountToLocalDisk(); if (appConfig.getValueBool("monitor")) { // Cancel out any inotify events from downloading data filesystemMonitor.update(false); } // Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); if (appConfig.getValueBool("monitor")) { // Handle any inotify events whilst the DB was being scanned filesystemMonitor.update(true); } // Is --download-only NOT configured? if (!appConfig.getValueBool("download_only")) { // Scan the configured 'sync_dir' for new data to upload to OneDrive syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); if (appConfig.getValueBool("monitor")) { // Handle any new inotify events whilst the local filesystem was being scanned filesystemMonitor.update(true); } // Make sure we sync any DB data to this point, but only if not in --monitor mode // In --monitor mode, this is handled within the 'loop', based on when the full scan true up is being performed if (!appConfig.getValueBool("monitor")) { itemDB.performVacuum(); } // Perform the final true up scan to ensure we have correctly replicated the current online state locally if (!appConfig.surpressLoggingOutput) { addLogEntry("Performing a last examination of the most recent online data within Microsoft OneDrive to complete the reconciliation process"); } // We pass in the 'appConfig.fullScanTrueUpRequired' value which then flags do we use the configured 'deltaLink' // If 'appConfig.fullScanTrueUpRequired' is true, we do not use the 'deltaLink' if we are in --monitor mode, thus forcing a full scan true up syncEngineInstance.syncOneDriveAccountToLocalDisk(); if (appConfig.getValueBool("monitor")) { // Cancel out any inotify events from downloading data filesystemMonitor.update(false); } } } } void displaySyncOutcome() { // Detail any download or upload transfer failures syncEngineInstance.displaySyncFailures(); // Sync is either complete or partially complete if (!syncEngineInstance.syncFailures) { // No download or upload issues if (!appConfig.getValueBool("monitor")) addLogEntry(); // Add an additional line break so that this is clear when using --sync addLogEntry("Sync with Microsoft OneDrive is complete"); } else { addLogEntry(); addLogEntry("Sync with Microsoft OneDrive has completed, however there are items that failed to sync."); // Due to how the OneDrive API works 'changes' such as add new files online, rename files online, delete files online are only sent once when using the /delta API call. // That we failed to download it, we need to track that, and then issue a --resync to download any of these failed files .. unfortunate, but there is no easy way here if (!syncEngineInstance.fileDownloadFailures.empty) { addLogEntry("To fix any download failures you may need to perform a --resync to ensure this system is correctly synced with your Microsoft OneDrive Account"); } if (!syncEngineInstance.fileUploadFailures.empty) { addLogEntry("To fix any upload failures you may need to perform a --resync to ensure this system is correctly synced with your Microsoft OneDrive Account"); } // So that from a logging perspective these messages are clear, add a line break in addLogEntry(); } } void processResyncDatabaseRemoval(string databaseFilePathToRemove) { addLogEntry("Testing if we have exclusive access to local database file", ["debug"]); // Are we the only running instance? Test that we can open the database file path itemDB = new ItemDatabase(databaseFilePathToRemove); // did we successfully initialise the database class? if (!itemDB.isDatabaseInitialised()) { // no .. destroy class itemDB = null; // exit application - void function, force exit this way exit(-1); } // If we have exclusive access we will not have exited // destroy access test destroy(itemDB); // delete application sync state addLogEntry("Deleting the saved application sync status ..."); if (!appConfig.getValueBool("dry_run")) { safeRemove(databaseFilePathToRemove); } else { // --dry-run scenario ... technically we should not be making any local file changes ....... addLogEntry("DRY RUN: Not removing the saved application sync status"); } } void cleanupDryRunDatabaseFiles(string dryRunDatabaseFile) { // Temp variables string dryRunShmFile = dryRunDatabaseFile ~ "-shm"; string dryRunWalFile = dryRunDatabaseFile ~ "-wal"; // If the dry run database exists, clean this up if (exists(dryRunDatabaseFile)) { // remove the existing file addLogEntry("DRY-RUN: Removing items-dryrun.sqlite3 as it still exists for some reason", ["debug"]); safeRemove(dryRunDatabaseFile); } // silent cleanup of shm files if it exists if (exists(dryRunShmFile)) { // remove items-dryrun.sqlite3-shm addLogEntry("DRY-RUN: Removing items-dryrun.sqlite3-shm as it still exists for some reason", ["debug"]); safeRemove(dryRunShmFile); } // silent cleanup of wal files if it exists if (exists(dryRunWalFile)) { // remove items-dryrun.sqlite3-wal addLogEntry("DRY-RUN: Removing items-dryrun.sqlite3-wal as it still exists for some reason", ["debug"]); safeRemove(dryRunWalFile); } } void checkForNoMountScenario() { // If this is a 'mounted' folder, the 'mount point' should have this file to help the application stop any action to preserve data because the drive to mount is not currently mounted if (appConfig.getValueBool("check_nomount")) { // we were asked to check the mount point for the presence of a '.nosync' file if (exists(".nosync")) { addLogEntry("ERROR: .nosync file found in directory mount point. Aborting application startup process to safeguard data.", ["info", "notify"]); Thread.sleep(dur!("msecs")(500)); exit(EXIT_FAILURE); } } } // Getting around the @nogc problem // https://p0nce.github.io/d-idioms/#Bypassing-@nogc auto assumeNoGC(T) (T t) if (isFunctionPointer!T || isDelegate!T) { enum attrs = functionAttributes!T | FunctionAttribute.nogc; return cast(SetFunctionAttributes!(T, functionLinkage!T, attrs)) t; } // Catch CTRL-C extern(C) nothrow @nogc @system void exitHandler(int value) { try { assumeNoGC ( () { addLogEntry("Got termination signal, performing clean up"); // Wait for all parallel jobs that depend on the database to complete addLogEntry("Waiting for any existing upload|download process to complete"); taskPool.finish(true); // Was itemDb initialised? if (itemDB.isDatabaseInitialised()) { // Make sure the .wal file is incorporated into the main db before we exit addLogEntry("Shutting down DB connection and merging temporary data"); itemDB.performVacuum(); object.destroy(itemDB); } performStandardExitProcess(); })(); } catch(Exception e) {} exit(0); }