diff --git a/src/config.d b/src/config.d index ae9ce58a..d70407f2 100644 --- a/src/config.d +++ b/src/config.d @@ -683,6 +683,7 @@ class ApplicationConfig { auto file = File(filename, "r"); string lineBuffer; + scope(exit) { file.close(); object.destroy(file); diff --git a/src/curlEngine.d b/src/curlEngine.d index e22a0985..22cb172f 100644 --- a/src/curlEngine.d +++ b/src/curlEngine.d @@ -3,7 +3,7 @@ module curlEngine; // What does this module require to function? import std.net.curl; -import etc.c.curl: CurlOption; +import etc.c.curl; import std.datetime; import std.conv; import std.file; @@ -31,18 +31,17 @@ class CurlResponse { void reset() { method = HTTP.Method.undefined; - url = null; + url = ""; requestHeaders = null; - postBody = null; - + postBody = []; hasResponse = false; responseHeaders = null; - object.destroy(statusLine); - content = null; + statusLine.reset(); + content = []; } void addRequestHeader(const(char)[] name, const(char)[] value) { - requestHeaders[name] = value; + requestHeaders[to!string(name)] = to!string(value); } void connect(HTTP.Method method, const(char)[] url) { @@ -74,7 +73,7 @@ class CurlResponse { // Return the current value of retryAfterValue int getRetryAfterValue() { int delayBeforeRetry; - // is retry-after in the response headers + // Is 'retry-after' in the response headers if ("retry-after" in responseHeaders) { // Set the retry-after value addLogEntry("curlEngine.http.perform() => Received a 'Retry-After' Header Response with the following value: " ~ to!string(responseHeaders["retry-after"]), ["debug"]); @@ -85,35 +84,34 @@ class CurlResponse { // This value is based on log files and data when determining correct process for 429 response handling delayBeforeRetry = 120; // Update that we are over-riding the provided value with a default - addLogEntry("HTTP Response Header retry-after value was 0 - Using a preconfigured default of: " ~ to!string(delayBeforeRetry), ["debug"]); + addLogEntry("HTTP Response Header retry-after value was missing - Using a preconfigured default of: " ~ to!string(delayBeforeRetry), ["debug"]); } - return delayBeforeRetry; } - - const string parseHeaders(const(immutable(char)[][immutable(char)[]]) headers) { - string responseHeadersStr = ""; - // Ensure headers is not null and iterate over keys safely. - if (headers !is null) { - foreach (const(char)[] header; headers.byKey()) { - // Check if the key actually exists before accessing it to avoid RangeError. - if (auto val = header in headers) { // 'in' checks for the key and returns a pointer to the value if found. - responseHeadersStr ~= "> " ~ header ~ ": " ~ *val ~ "\n"; // Dereference pointer to get the value. - } - } - } - return responseHeadersStr; - } - - const string parseHeaders(const(const(char)[][const(char)[]]) headers) { - string responseHeadersStr = ""; + + const string parseRequestHeaders(const(const(char)[][const(char)[]]) headers) { + string requestHeadersStr = ""; foreach (string header; headers.byKey()) { if (header == "Authorization") { continue; } // Use the 'in' operator to safely check if the key exists in the associative array. if (auto val = header in headers) { - responseHeadersStr ~= "< " ~ header ~ ": " ~ *val ~ "\n"; + requestHeadersStr ~= "< " ~ header ~ ": " ~ *val ~ "\n"; + } + } + return requestHeadersStr; + } + + const string parseResponseHeaders(const(immutable(char)[][immutable(char)[]]) headers) { + string responseHeadersStr = ""; + // Ensure response headers is not null and iterate over keys safely. + if (headers !is null) { + foreach (const(char)[] header; headers.byKey()) { + // Check if the key actually exists before accessing it to avoid RangeError. + if (auto val = header in headers) { // 'in' checks for the key and returns a pointer to the value if found. + responseHeadersStr ~= "> " ~ header ~ ": " ~ *val ~ "\n"; // Dereference pointer to get the value. + } } } return responseHeadersStr; @@ -126,14 +124,14 @@ class CurlResponse { string str = ""; str ~= format("< %s %s\n", method, url); if (!requestHeaders.empty) { - str ~= parseHeaders(requestHeaders); + str ~= parseRequestHeaders(requestHeaders); } if (!postBody.empty) { - str ~= format("----\n%s\n----\n", postBody); + str ~= format("\n----\n%s\n----\n", postBody); } str ~= format("< %s\n", statusLine); if (!responseHeaders.empty) { - str ~= parseHeaders(responseHeaders); + str ~= parseResponseHeaders(responseHeaders); } return str; } @@ -144,7 +142,7 @@ class CurlResponse { string str = ""; if (!content.empty) { - str ~= format("----\n%s\n----\n", content); + str ~= format("\n----\n%s\n----\n", content); } return str; } @@ -158,75 +156,92 @@ class CurlResponse { } return str; } - - CurlResponse dup() { - CurlResponse copy = new CurlResponse(); - copy.method = method; - copy.url = url; - copy.requestHeaders = requestHeaders; - copy.postBody = postBody; - - copy.responseHeaders = responseHeaders; - copy.statusLine = statusLine; - copy.content = content; - - return copy; - } } class CurlEngine { - __gshared static CurlEngine[] curlEnginePool; // __gshared is used for thread-shared static variables + __gshared static CurlEngine[] curlEnginePool; // __gshared is used for thread-shared static variables - HTTP http; - bool keepAlive; - ulong dnsTimeout; - CurlResponse response; - File uploadFile; - - static CurlEngine getCurlInstance() { - synchronized (CurlEngine.classinfo) { - if (curlEnginePool.empty) { - return new CurlEngine; // Constructs a new CurlEngine with a fresh HTTP instance - } else { - CurlEngine curlEngine = curlEnginePool[$ - 1]; - curlEnginePool = curlEnginePool[0 .. $ - 1]; - return curlEngine; - } - } - } - - static void releaseAll() { - synchronized (CurlEngine.classinfo) { - foreach (CurlEngine curlEngine; curlEnginePool) { - curlEngine.cleanUp(); // Cleanup instance by resetting values - curlEngine.shutdown(); // Assume proper cleanup of any resources used by HTTP - object.destroy(curlEngine); - } - curlEnginePool.length = 0; - } - - // Cleanup curlEnginePool - object.destroy(curlEnginePool); - } - + HTTP http; + bool keepAlive; + ulong dnsTimeout; + CurlResponse response; + File uploadFile; + this() { - http = HTTP(); // Directly initializes HTTP using its default constructor - response = null; // Initialize as null + this.http = HTTP(); // Directly initializes HTTP using its default constructor + this.response = null; // Initialize as null } ~this() { - // The destructor should only clean up resources owned directly by this instance - // Avoid modifying or destroying shared/static resources here - if (uploadFile.isOpen()) - uploadFile.close(); + // The destructor should only clean up resources owned directly by this instance + // Avoid modifying or destroying shared/static resources here + if (uploadFile.isOpen()) { + uploadFile.close(); + } - // Cleanup curlEnginePool - object.destroy(curlEnginePool); + // Cleanup class memory usage + object.destroy(this.uploadFile); // Destroy, however we cant set to null + object.destroy(this.response); // Destroy, then set to null + this.response = null; + + // Is the http instance is stopped? + if (!this.http.isStopped) { + + writeln("TO REMOVE: Calling this.http.shutdown() on this curl instance"); + + this.http.shutdown(); + } + + object.destroy(this.http); // Destroy, however we cant set to null + } - + + static CurlEngine getCurlInstance() { + synchronized (CurlEngine.classinfo) { + if (curlEnginePool.empty) { + return new CurlEngine; // Constructs a new CurlEngine with a fresh HTTP instance + } else { + CurlEngine curlEngine = curlEnginePool[$ - 1]; + curlEnginePool = curlEnginePool[0 .. $ - 1]; + + // Is this engine stopped? + if (curlEngine.http.isStopped) { + // return a new curl engine as a stopped one cannot be used + return new CurlEngine; // Constructs a new CurlEngine with a fresh HTTP instance + } else { + // return an existing curl engine + return curlEngine; + } + } + } + } + + static void releaseAllCurlInstances() { + synchronized (CurlEngine.classinfo) { + // Safely iterate and clean up each CurlEngine instance + foreach (CurlEngine curlEngine; curlEnginePool) { + try { + curlEngine.cleanup(); // Cleanup instance by resetting values + curlEngine.shutdown(); // Assume proper cleanup of any resources used by HTTP + } catch (Exception e) { + // Log the error or handle it appropriately + // e.g., writeln("Error during cleanup/shutdown: ", e.toString()); + } + // It's safe to destroy the object here assuming no other references exist + object.destroy(curlEngine); // Destroy, then set to null + curlEngine = null; + } + // Clear the array after all instances have been handled + curlEnginePool.length = 0; // More explicit than curlEnginePool = []; + } + // Destroy curlEnginePool, set to null + object.destroy(curlEnginePool); + curlEnginePool = null; + } + void release() { - cleanUp(); + cleanup(); synchronized (CurlEngine.classinfo) { curlEnginePool ~= this; } @@ -381,7 +396,7 @@ class CurlEngine { CurlResponse execute() { scope(exit) { - cleanUp(); + cleanup(); } setResponseHolder(null); http.onReceive = (ubyte[] data) { @@ -402,7 +417,7 @@ class CurlEngine { // function scopes scope(exit) { - cleanUp(); + cleanup(); if (file.isOpen()){ // close open file file.close(); @@ -423,17 +438,27 @@ class CurlEngine { return response; } - void cleanUp() { + void cleanup() { // Reset any values to defaults, freeing any set objects - http.clearRequestHeaders(); - http.onSend = null; - http.onReceive = null; - http.onReceiveHeader = null; - http.onReceiveStatusLine = null; - http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) { - return 0; - }; - http.contentLength = 0; + + // Is the instance is stopped? + if (!http.isStopped) { + // A stopped instance is not usable, these cannot be reset + http.clearRequestHeaders(); + http.onSend = null; + http.onReceive = null; + http.onReceiveHeader = null; + http.onReceiveStatusLine = null; + http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) { + return 0; + }; + http.contentLength = 0; + http.flushCookieJar(); + http.clearSessionCookies(); + http.clearAllCookies(); + } + + // set the response to null response = null; // close file if open @@ -445,11 +470,9 @@ class CurlEngine { void shutdown() { // Shut down the curl instance & close any open sockets - http.shutdown(); - } - - void setDisableSSLVerifyPeer() { - addLogEntry("CAUTION: Switching off CurlOption.ssl_verifypeer ... this makes the application insecure.", ["debug"]); - http.handle.set(CurlOption.ssl_verifypeer, 0); + // Is the instance is stopped? + if (!http.isStopped) { + http.shutdown(); + } } } \ No newline at end of file diff --git a/src/log.d b/src/log.d index 09959a01..350c24fe 100644 --- a/src/log.d +++ b/src/log.d @@ -52,7 +52,13 @@ class LogBuffer { flushThread.isDaemon(true); flushThread.start(); } - + + ~this() { + object.destroy(bufferLock); + object.destroy(condReady); + object.destroy(flushThread); + } + void shutdown() { synchronized(bufferLock) { if (!isRunning) return; // Prevent multiple shutdowns @@ -62,6 +68,7 @@ class LogBuffer { flushThread.join(); // Wait for the flush thread to finish flush(); // Perform a final flush to ensure all data is processed } + shared void logThisMessage(string message, string[] levels = ["info"]) { // Generate the timestamp for this log entry auto timeStamp = leftJustify(Clock.currTime().toString(), 28, '0'); diff --git a/src/main.d b/src/main.d index 3f9602d3..22bb4661 100644 --- a/src/main.d +++ b/src/main.d @@ -4,6 +4,7 @@ module main; // What does this module require to function? import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit; import core.stdc.signal; +import core.sys.posix.signal; import core.memory; import core.time; import core.thread; @@ -44,7 +45,19 @@ ItemDatabase itemDB; ClientSideFiltering selectiveSync; Monitor filesystemMonitor; +// Class variables +// Flag for performing a synchronised shutdown +bool shutdownInProgress = false; +// Flag if a --dry-run is being performed, as, on shutdown, once config is destroyed, we have no reference here +bool dryRun = false; +// Configure the runtime database file path so that it is available to us on shutdown so objects can be destroyed and removed if required +// - Typically this will be the default, but in a --dry-run scenario, we use a separate database file +string runtimeDatabaseFile = ""; + int main(string[] cliArgs) { + // Setup CTRL-C handler + setupSignalHandler(); + // Application Start Time - used during monitor loop to detail how long it has been running for auto applicationStartTime = Clock.currTime(); // Disable buffering on stdout - this is needed so that when we are using plain write() it will go to the terminal without flushing @@ -63,8 +76,7 @@ int main(string[] cliArgs) { // What is the runtime syncronisation directory that will be used // Typically this will be '~/OneDrive' .. however tilde expansion is unreliable string runtimeSyncDirectory = ""; - // Configure the runtime database file path. Typically this will be the default, but in a --dry-run scenario, we use a separate database file - string runtimeDatabaseFile = ""; + // Verbosity Logging Count - this defines if verbose or debug logging is being used long verbosityCount = 0; // Application Logging Level @@ -86,15 +98,15 @@ int main(string[] cliArgs) { scope(exit) { // Detail what scope was called addLogEntry("Exit scope was called", ["debug"]); - // Perform exit tasks - performStandardExitProcess("exitScope"); + // Perform synchronised exit + performSynchronisedExitProcess("exitScope"); } scope(failure) { // Detail what scope was called addLogEntry("Failure scope was called", ["debug"]); - // Perform exit tasks - performStandardExitProcess("failureScope"); + // Perform synchronised exit + performSynchronisedExitProcess("failureScope"); } // Read in application options as passed in @@ -140,23 +152,6 @@ int main(string[] cliArgs) { // If we need to enable logging to a file, we can only do this once we know the application configuration which is done slightly later on initialiseLogging(verboseLogging, debugLogging); - /** - // most used - addLogEntry("Basic 'info' message", ["info"]); .... or just use addLogEntry("Basic 'info' message"); - addLogEntry("Basic 'verbose' message", ["verbose"]); - addLogEntry("Basic 'debug' message", ["debug"]); - // GUI notify only - addLogEntry("Basic 'notify' ONLY message and displayed in GUI if notifications are enabled", ["notify"]); - // info and notify - addLogEntry("Basic 'info and notify' message and displayed in GUI if notifications are enabled", ["info", "notify"]); - // log file only - addLogEntry("Information sent to the log file only, and only if logging to a file is enabled", ["logFileOnly"]); - // Console only (session based upload|download) - addLogEntry("Basic 'Console only with new line' message", ["consoleOnly"]); - // Console only with no new line - addLogEntry("Basic 'Console only with no new line' message", ["consoleOnlyNoNewLine"]); - **/ - // Log application start time, log line has start time addLogEntry("Application started", ["debug"]); @@ -188,6 +183,9 @@ int main(string[] cliArgs) { // Update the current runtime application configuration (default or 'config' fileread-in options) from any passed in command line arguments appConfig.updateFromArgs(cliArgs); + // Configure dryRun so that this can be used here & during shutdown + dryRun = appConfig.getValueBool("dry_run"); + // As early as possible, now re-configure the logging class, given that we have read in any applicable 'config' file and updated the application running config from CLI input: // - Enable logging to a file if this is required // - Disable GUI notifications if this has been configured @@ -264,9 +262,9 @@ int main(string[] cliArgs) { // Check for --dry-run operation or a 'no-sync' operation where the 'dry-run' DB copy should be used // If this has been requested, we need to ensure that all actions are performed against the dry-run database copy, and, // no actual action takes place - such as deleting files if deleted online, moving files if moved online or local, downloading new & changed files, uploading new & changed files - if ((appConfig.getValueBool("dry_run")) || (appConfig.hasNoSyncOperationBeenRequested())) { + if (dryRun || (appConfig.hasNoSyncOperationBeenRequested())) { - if (appConfig.getValueBool("dry_run")) { + if (dryRun) { // This is a --dry-run operation addLogEntry("DRY-RUN Configured. Output below shows what 'would' have occurred."); } @@ -279,26 +277,29 @@ int main(string[] cliArgs) { // In a --dry-run --resync scenario, we should not copy the existing database file if (!appConfig.getValueBool("resync")) { // Copy the existing DB file to the dry-run copy - if (appConfig.getValueBool("dry_run")) { + if (dryRun) { addLogEntry("DRY-RUN: Copying items.sqlite3 to items-dryrun.sqlite3 to use for dry run operations"); } copy(appConfig.databaseFilePath,appConfig.databaseFilePathDryRun); } else { // No database copy due to --resync - if (appConfig.getValueBool("dry_run")) { + if (dryRun) { addLogEntry("DRY-RUN: No database copy created for --dry-run due to --resync also being used"); } } } // update runtimeDatabaseFile now that we are using the dry run path runtimeDatabaseFile = appConfig.databaseFilePathDryRun; + } else { + // Cleanup any existing dry-run elements ... these should never be left hanging around + cleanupDryRunDatabaseFiles(appConfig.databaseFilePathDryRun); } // Handle --logout as separate item, do not 'resync' on a --logout if (appConfig.getValueBool("logout")) { addLogEntry("--logout requested", ["debug"]); addLogEntry("Deleting the saved authentication status ..."); - if (!appConfig.getValueBool("dry_run")) { + if (!dryRun) { safeRemove(appConfig.refreshTokenFilePath); } else { // --dry-run scenario ... technically we should not be making any local file changes ....... @@ -312,7 +313,7 @@ int main(string[] cliArgs) { if (appConfig.getValueBool("reauth")) { addLogEntry("--reauth requested", ["debug"]); addLogEntry("Deleting the saved authentication status ... re-authentication requested"); - if (!appConfig.getValueBool("dry_run")) { + if (!dryRun) { safeRemove(appConfig.refreshTokenFilePath); } else { // --dry-run scenario ... technically we should not be making any local file changes ....... @@ -824,10 +825,6 @@ int main(string[] cliArgs) { } }; - // Handle SIGINT and SIGTERM - signal(SIGINT, &exitHandler); - signal(SIGTERM, &exitHandler); - // Initialise the local filesystem monitor class using inotify to monitor for local filesystem changes // If we are in a --download-only method of operation, we do not enable local filesystem monitoring if (!appConfig.getValueBool("download_only")) { @@ -1111,78 +1108,6 @@ int main(string[] cliArgs) { } } -void performStandardExitProcess(string scopeCaller = null) { - // Who called this function - if (!scopeCaller.empty) { - addLogEntry("Running performStandardExitProcess due to: " ~ scopeCaller, ["debug"]); - } - - // Shutdown the OneDrive Webhook instance - if (oneDriveWebhook !is null) { - oneDriveWebhook.stop(); - object.destroy(oneDriveWebhook); - } - - // Shutdown the client side filtering objects - if (selectiveSync !is null) { - addLogEntry("Shutdown Client Side Filtering instance", ["debug"]); - selectiveSync.shutdown(); - object.destroy(selectiveSync); - } - - // Shutdown the application configuration objects - if (appConfig !is null) { - addLogEntry("Shutdown Application Configuration instance", ["debug"]); - // Cleanup any existing dry-run elements ... these should never be left hanging around - cleanupDryRunDatabaseFiles(appConfig.databaseFilePathDryRun); - object.destroy(appConfig); - } - - // Shutdown any local filesystem monitoring - if (filesystemMonitor !is null) { - addLogEntry("Shutdown Filesystem Monitoring instance", ["debug"]); - filesystemMonitor.shutdown(); - object.destroy(filesystemMonitor); - } - - // Shutdown the sync engine - if (syncEngineInstance !is null) { - addLogEntry("Shutdown Sync Engine instance", ["debug"]); - object.destroy(syncEngineInstance); - } - - // Shutdown the database - if (itemDB !is null) { - addLogEntry("Shutdown Database instance", ["debug"]); - // Make sure the .wal file is incorporated into the main db before we exit - if (itemDB.isDatabaseInitialised()) { - itemDB.performVacuum(); - } - object.destroy(itemDB); - } - - // Shutdown cached sockets - CurlEngine.releaseAll(); - - // Set all objects to null - if (scopeCaller == "failureScope") { - // Set these to be null due to failure scope - prevent 'ERROR: Unable to perform a database vacuum: out of memory' when the exit scope is then called - addLogEntry("Setting ALL Class Objects to null due to failure scope", ["debug"]); - itemDB = null; - appConfig = null; - oneDriveWebhook = null; - selectiveSync = null; - syncEngineInstance = null; - } else { - addLogEntry("Waiting for all internal threads to complete before exiting application", ["verbose"]); - addLogEntry("Application exit", ["debug"]); - addLogEntry("#######################################################################################################################################", ["logFileOnly"]); - // Destroy the shared logging buffer - (cast() logBuffer).shutdown(); - object.destroy(logBuffer); - } -} - void oneDriveWebhookCallback() { // If we are in a --download-only method of operation, there is no filesystem monitoring, so no inotify events to check if (!appConfig.getValueBool("download_only")) { @@ -1352,7 +1277,7 @@ void processResyncDatabaseRemoval(string databaseFilePathToRemove) { destroy(itemDB); // delete application sync state addLogEntry("Deleting the saved application sync status ..."); - if (!appConfig.getValueBool("dry_run")) { + if (!dryRun) { safeRemove(databaseFilePathToRemove); } else { // --dry-run scenario ... technically we should not be making any local file changes ....... @@ -1364,7 +1289,7 @@ void cleanupDryRunDatabaseFiles(string dryRunDatabaseFile) { // Temp variables string dryRunShmFile = dryRunDatabaseFile ~ "-shm"; string dryRunWalFile = dryRunDatabaseFile ~ "-wal"; - + // If the dry run database exists, clean this up if (exists(dryRunDatabaseFile)) { // remove the existing file @@ -1406,14 +1331,167 @@ auto assumeNoGC(T) (T t) if (isFunctionPointer!T || isDelegate!T) { return cast(SetFunctionAttributes!(T, functionLinkage!T, attrs)) t; } -// Catch CTRL-C if user pressed this +// Configure the signal handler to catch SIGINT (CTRL-C) and SIGTERM (kill) +void setupSignalHandler() { + sigaction_t sa; + sa.sa_flags = SA_RESETHAND | SA_NODEFER; // Use reset and no defer flags to handle reentrant signals + sa.sa_handler = &exitHandler; // Direct function pointer assignment + sigemptyset(&sa.sa_mask); // Initialize the signal set to empty + + // Register the signal handler for SIGINT + if (sigaction(SIGINT, &sa, null) != 0) { + writeln("FATAL: Failed to install SIGINT handler"); + exit(-1); + } + + // Register the signal handler for SIGTERM + if (sigaction(SIGTERM, &sa, null) != 0) { + writeln("FATAL: Failed to install SIGTERM handler"); + exit(-1); + } +} + +// Catch SIGINT (CTRL-C) and SIGTERM (kill), handle rapid repeat presses extern(C) nothrow @nogc @system void exitHandler(int value) { + + if (shutdownInProgress) { + return; // Ignore subsequent presses + } + shutdownInProgress = true; + try { assumeNoGC ( () { - // Force kill any running threads as ^C was used - taskPool.finish(false); + addLogEntry("\nReceived termination signal, initiating cleanup"); + // Wait for all parallel jobs that depend on the database to complete + + addLogEntry("Waiting for any existing upload|download process to complete"); + taskPool.finish(true); + + // Force kill any running threads + //addLogEntry("Forcing any active thread to exit"); + //taskPool.finish(false); + + // Perform the shutdown process + performSynchronisedExitProcess("exitHandler"); })(); - } catch(Exception e) {} - // Exit with the exitHandler value - exit(value); + } catch(Exception e) { + // Any output here will cause a GC allocation + // - Error: `@nogc` function `main.exitHandler` cannot call non-@nogc function `std.stdio.writeln!string.writeln` + // - Error: cannot use operator `~` in `@nogc` function `main.exitHandler` + // writeln("Exception during shutdown: " ~ e.msg); + } + // Exit the process with the provided exit code + exit(value); + +} + +// Handle application exit +void performSynchronisedExitProcess(string scopeCaller = null) { + synchronized { + // Logging the caller of the shutdown procedure + if (!scopeCaller.empty) { + addLogEntry("performSynchronisedExitProcess called by: " ~ scopeCaller, ["debug"]); + } + + // Perform cleanup and shutdown of various services and resources + try { + // Shutdown the OneDrive Webhook instance + shutdownOneDriveWebhook(); + // Shutdown the client side filtering objects + shutdownSelectiveSync(); + // Shutdown the sync engine + shutdownSyncEngine(); + // Shutdown any local filesystem monitoring + shutdownFilesystemMonitor(); + // Shutdown the database + shutdownDatabase(); + // Shutdown 'curl' instances + shutdownCurlInstances(); + // Shutdown the application configuration objects + shutdownAppConfig(); + + } catch (Exception e) { + addLogEntry("Error during performStandardExitProcess: " ~ e.toString(), ["error"]); + } + + // Finalise all logging and destroy log buffer + shutdownApplicationLogging(); + + // Memory Garbage Collection + GC.collect(); + } +} + +void shutdownOneDriveWebhook() { + if (oneDriveWebhook !is null) { + addLogEntry("Shutdown OneDrive Webhook instance", ["debug"]); + oneDriveWebhook.stop(); + object.destroy(oneDriveWebhook); + oneDriveWebhook = null; + } +} + +void shutdownFilesystemMonitor() { + if (filesystemMonitor !is null) { + addLogEntry("Shutdown Filesystem Monitoring instance", ["debug"]); + filesystemMonitor.shutdown(); + object.destroy(filesystemMonitor); + filesystemMonitor = null; + } +} + +void shutdownSelectiveSync() { + if (selectiveSync !is null) { + addLogEntry("Shutdown Client Side Filtering instance", ["debug"]); + selectiveSync.shutdown(); + object.destroy(selectiveSync); + selectiveSync = null; + } +} + +void shutdownSyncEngine() { + if (syncEngineInstance !is null) { + addLogEntry("Shutdown Sync Engine instance", ["debug"]); + //syncEngineInstance.shutdown(); - potentially need this and also check for a ~this() for class cleanup + object.destroy(syncEngineInstance); + syncEngineInstance = null; + } +} + +void shutdownDatabase() { + if (itemDB !is null && itemDB.isDatabaseInitialised()) { + addLogEntry("Shutdown Database instance", ["debug"]); + itemDB.performVacuum(); + object.destroy(itemDB); + itemDB = null; + } +} + +void shutdownAppConfig() { + if (appConfig !is null) { + addLogEntry("Shutdown Application Configuration instance", ["debug"]); + if (dryRun) { + // We were running with --dry-run , clean up the applicable database + cleanupDryRunDatabaseFiles(runtimeDatabaseFile); + } + object.destroy(appConfig); + appConfig = null; + } +} + +void shutdownCurlInstances() { + CurlEngine.releaseAllCurlInstances(); +} + +void shutdownApplicationLogging() { + // Join all threads to avoid any thread hanging out there + addLogEntry("Waiting for all internal threads to complete before exiting application", ["verbose"]); + // Join all threads + thread_joinAll(); + // Log that we are exitintg + addLogEntry("Application is exiting.", ["debug"]); + addLogEntry("#######################################################################################################################################", ["logFileOnly"]); + // Destroy the shared logging buffer + (cast() logBuffer).shutdown(); + object.destroy(logBuffer); } \ No newline at end of file diff --git a/src/monitor.d b/src/monitor.d index 18861b94..73cb8a79 100644 --- a/src/monitor.d +++ b/src/monitor.d @@ -140,8 +140,7 @@ class MonitorBackgroundWorker { } } -void startMonitorJob(shared(MonitorBackgroundWorker) worker, Tid callerTid) -{ +void startMonitorJob(shared(MonitorBackgroundWorker) worker, Tid callerTid) { try { worker.watch(callerTid); } catch (OwnerTerminated error) { @@ -282,6 +281,11 @@ final class Monitor { this.selectiveSync = selectiveSync; } + // The destructor should only clean up resources owned directly by this instance + ~this() { + object.destroy(worker); + } + // Initialise the monitor class void initialise() { // Configure the variables diff --git a/src/onedrive.d b/src/onedrive.d index 951231ad..11709560 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -63,10 +63,12 @@ class OneDriveError: Error { // Define the 'OneDriveApi' class class OneDriveApi { - // Class variables + // Class variables that use other classes ApplicationConfig appConfig; CurlEngine curlEngine; + CurlResponse response; + // Class variables string clientId = ""; string companyName = ""; string authUrl = ""; @@ -115,7 +117,11 @@ class OneDriveApi { } ~this() { + // We cant destroy 'appConfig' here as this leads to a segfault object.destroy(curlEngine); + object.destroy(response); + curlEngine = null; + response = null; } // Initialise the OneDrive API class @@ -1114,6 +1120,10 @@ class OneDriveApi { // - This should throw a OneDriveException so that this exception can be handled appropriately elsewhere in the application private JSONValue oneDriveErrorHandlerWrapper(CurlResponse delegate(CurlResponse response) executer, bool validateJSONResponse, string callingFunction, int lineno) { + // Create a new 'curl' response + response = new CurlResponse(); + + // Other wrapper variables int retryAttempts = 0; int baseBackoffInterval = 1; // Base backoff interval in seconds int maxRetryCount = 175200; // Approx 365 days based on maxBackoffInterval + appConfig.defaultDataTimeout @@ -1122,12 +1132,10 @@ class OneDriveApi { int thisBackOffInterval = 0; int timestampAlign = 0; JSONValue result; - CurlResponse response = new CurlResponse(); SysTime currentTime; SysTime retryTime; bool retrySuccess = false; bool transientError = false; - auto internalThreadId = generateAlphanumericString(); while (!retrySuccess) { diff --git a/src/sync.d b/src/sync.d index 1603a000..3cb24e0b 100644 --- a/src/sync.d +++ b/src/sync.d @@ -303,9 +303,17 @@ class SyncEngine { } ~this() { - processPool = null; - object.destroy(processPool); - object.destroy(oneDriveApiInstance); + this.processPool.finish(true); + object.destroy(this.processPool); // Destroy, then set to null + this.processPool = null; + object.destroy(this.oneDriveApiInstance); // Destroy, then set to null + this.oneDriveApiInstance = null; + object.destroy(this.appConfig); // Destroy, then set to null + this.appConfig = null; + object.destroy(this.itemDB); // Destroy, then set to null + this.itemDB = null; + object.destroy(this.selectiveSync); // Destroy, then set to null + this.selectiveSync = null; } // Initialise the Sync Engine class diff --git a/src/util.d b/src/util.d index 89f64302..9229b51e 100644 --- a/src/util.d +++ b/src/util.d @@ -201,8 +201,46 @@ Regex!char wild2regex(const(char)[] pattern) { return regex(str, "i"); } -// Test Internet access to Microsoft OneDrive using a simple HTTP HEAD request +// Test Internet access to Microsoft OneDrive bool testInternetReachability(ApplicationConfig appConfig) { + CurlEngine curlEngine; + bool result = false; + try { + // Use preconfigured object with all the correct http values assigned + curlEngine = CurlEngine.getCurlInstance(); + curlEngine.initialise(appConfig.getValueLong("dns_timeout"), appConfig.getValueLong("connect_timeout"), appConfig.getValueLong("data_timeout"), appConfig.getValueLong("operation_timeout"), appConfig.defaultMaxRedirects, appConfig.getValueBool("debug_https"), appConfig.getValueString("user_agent"), appConfig.getValueBool("force_http_11"), appConfig.getValueLong("rate_limit"), appConfig.getValueLong("ip_protocol_version")); + + // Configure the remaining items required + // URL to use + // HTTP connection test method + + curlEngine.connect(HTTP.Method.head, "https://login.microsoftonline.com"); + addLogEntry("Attempting to contact Microsoft OneDrive Login Service", ["debug"]); + curlEngine.http.perform(); + addLogEntry("Shutting down HTTP engine as successfully reached OneDrive Login Service", ["debug"]); + + // Release + curlEngine.release(); // performs curl cleanup() + curlEngine = null; // Clean up this memory variable + + // Set that we are online + result = true; + } catch (SocketException e) { + addLogEntry("HTTP Socket Issue", ["debug"]); + addLogEntry("Cannot connect to Microsoft OneDrive Login Service - Socket Issue"); + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + } catch (CurlException e) { + addLogEntry("No Network Connection", ["debug"]); + addLogEntry("Cannot connect to Microsoft OneDrive Login Service - Network Connection Issue"); + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + } + + // Return test result + return result; +} + +// Test Internet access to Microsoft OneDrive using a simple HTTP HEAD request +bool testInternetReachabilityAlternate(ApplicationConfig appConfig) { auto http = HTTP(); http.url = "https://login.microsoftonline.com"; @@ -684,27 +722,61 @@ string getFunctionName(alias func)() { return __traits(identifier, __traits(parent, func)) ~ "()\n"; } +JSONValue fetchOnlineURLContent(string url) { + // Function variables + char[] content; + JSONValue onlineContent; + + // Setup HTTP request + HTTP http = HTTP(); + + // Create an HTTP object within a scope to ensure cleanup + scope(exit) { + http.shutdown(); + object.destroy(http); + } + + // Configure the URL to access + http.url = url; + // HTTP the connection method + http.method = HTTP.Method.get; + + // Data receive handler + http.onReceive = (ubyte[] data) { + content ~= data; // Append data as it's received + return data.length; + }; + + // Perform HTTP request + http.perform(); + + // Parse Content + onlineContent = parseJSON(to!string(content)); + + // Ensure resources are cleaned up + http.shutdown(); + object.destroy(http); + + // Return onlineResponse + return onlineContent; +} + // Get the latest release version from GitHub JSONValue getLatestReleaseDetails() { - // Import curl just for this function - import std.net.curl; - char[] content; JSONValue githubLatest; JSONValue versionDetails; string latestTag; string publishedDate; // Query GitHub for the 'latest' release details - try { - content = get("https://api.github.com/repos/abraunegg/onedrive/releases/latest"); - githubLatest = content.parseJSON(); + try { + githubLatest = fetchOnlineURLContent("https://api.github.com/repos/abraunegg/onedrive/releases/latest"); } catch (CurlException e) { addLogEntry("CurlException: Unable to query GitHub for latest release - " ~ e.msg, ["debug"]); } catch (JSONException e) { addLogEntry("JSONException: Unable to parse GitHub JSON response - " ~ e.msg, ["debug"]); } - // githubLatest has to be a valid JSON object if (githubLatest.type() == JSONType.object){ // use the returned tag_name @@ -746,9 +818,6 @@ JSONValue getLatestReleaseDetails() { // Get the release details from the 'current' running version JSONValue getCurrentVersionDetails(string thisVersion) { - // Import curl just for this function - import std.net.curl; - char[] content; JSONValue githubDetails; JSONValue versionDetails; string versionTag = "v" ~ thisVersion; @@ -756,9 +825,8 @@ JSONValue getCurrentVersionDetails(string thisVersion) { // Query GitHub for the release details to match the running version try { - content = get("https://api.github.com/repos/abraunegg/onedrive/releases"); - githubDetails = content.parseJSON(); - } catch (CurlException e) { + githubDetails = fetchOnlineURLContent("https://api.github.com/repos/abraunegg/onedrive/releases"); + } catch (CurlException e) { addLogEntry("CurlException: Unable to query GitHub for release details - " ~ e.msg, ["debug"]); return parseJSON(`{"Error": "CurlException", "message": "` ~ e.msg ~ `"}`); } catch (JSONException e) {