From 8801344209ac2079173b4cb4fc165f776a199a90 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Sun, 31 Mar 2024 07:25:33 +1100 Subject: [PATCH 1/4] Update itemdb.d * Add case statement covering ~5 years of user issues when a sqlite error has been generated, so that there is some additional information to point the user in the right direction to resolve --- src/itemdb.d | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/itemdb.d b/src/itemdb.d index b74fff0a..bd14cd06 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -208,13 +208,38 @@ final class ItemDatabase { if (e.msg == "database is locked") { addLogEntry(); addLogEntry("ERROR: The 'onedrive' application is already running - please check system process list for active application instances"); - addLogEntry(" - Use 'sudo ps aufxw | grep onedrive' to potentially determine acive running process", ["verbose"]); + addLogEntry(" - Use 'sudo ps aufxw | grep onedrive' to potentially determine acive running process"); addLogEntry(); } else { // A different error .. detail the message, detail the actual SQLite Error Code to assist with troubleshooting addLogEntry(); addLogEntry("ERROR: An internal database error occurred: " ~ e.msg ~ " (SQLite Error Code: " ~ to!string(e.errorCode) ~ ")"); addLogEntry(); + + // Give the user some additional information and pointers on this error + // The below list is based on user issue / discussion reports since 2018 + switch (e.errorCode) { + case 7: // SQLITE_NOMEM + addLogEntry("The operation could not be completed due to insufficient memory. Please close unnecessary applications to free up memory and try again."); + break; + case 10: // SQLITE_IOERR + addLogEntry("A disk I/O error occurred. This could be due to issues with the storage medium (e.g., disk full, hardware failure, filesystem corruption). Please check your disk's health using a disk utility tool, ensure there is enough free space, and check the filesystem for errors."); + break; + case 11: // SQLITE_CORRUPT + addLogEntry("The database file appears to be corrupt. This could be due to incomplete or failed writes, hardware issues, or unexpected interruptions during database operations. Please perform a --resync operation."); + break; + case 14: // SQLITE_CANTOPEN + addLogEntry("The database file could not be opened. Please check that the database file exists, has the correct permissions, and is not being blocked by another process or security software."); + break; + case 26: // SQLITE_NOTADB + addLogEntry("The file attempted to be opened does not appear to be a valid SQLite database, or it may have been corrupted to a point where it's no longer recognizable. Please check your application configuration directory and/or perform a --resync operation."); + break; + default: + addLogEntry("An unexpected error occurred. Please consult the application documentation or support to resolve this issue."); + break; + } + // Blank line before exit + addLogEntry(); } return; } From a28fe92119a22772266b2cead0e4c6b15d4922d5 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Mon, 1 Apr 2024 06:07:48 +1100 Subject: [PATCH 2/4] Update usage.md * Update how to enable logging --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 91fc3b00..bc23ed11 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -507,7 +507,7 @@ Are you sure you want to proceed with --force-sync [Y/N] To proceed with `--force-sync`, you must type 'y' or 'Y' to allow the application to continue. ### Enabling the Client Activity Log -When running onedrive, all actions can be logged to a separate log file. This can be enabled by using the `--enable-logging` flag. +When running onedrive, all actions can be logged to a separate log file. This can be enabled by using the `--enable-logging` flag or by adding `enable_logging = "true"` to your 'config' file. By default, log files will be written to `/var/log/onedrive/` and will be in the format of `%username%.onedrive.log`, where `%username%` represents the user who ran the client to allow easy sorting of user to client activity log. From c680c75f4f795c91513ed6adfee2738ede2e0aaa Mon Sep 17 00:00:00 2001 From: abraunegg Date: Mon, 8 Apr 2024 13:47:08 +1000 Subject: [PATCH 3/4] Update Makefile.in General Flags: Flags that are always needed, such as warnings (-w) and directory inclusion (-J.), are added outside of the conditionals. Debug Flags: For DMD: When debugging is enabled (DEBUG=yes), the -debug flag for including debug code and -gs for generating standalone debug symbols are added. For LDC or other compilers: -d-debug for debugging and -gc for generating debugging information are added similarly when debugging is enabled. Optimization Flag: The -O flag is only added when debugging is not enabled. This ensures that the program is compiled with optimizations only when it is not in debug mode. --- Makefile.in | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Makefile.in b/Makefile.in index b56ecea5..bf1b0fef 100644 --- a/Makefile.in +++ b/Makefile.in @@ -34,13 +34,18 @@ DEBUG = @DEBUG@ DC = @DC@ DC_TYPE = @DC_TYPE@ DCFLAGS = @DCFLAGS@ -DCFLAGS += -w -g -O -J. +DCFLAGS += -w -J. ifeq ($(DEBUG),yes) ifeq ($(DC_TYPE),dmd) -DCFLAGS += -debug -gs +# Add DMD Debugging Flags +DCFLAGS += -g -debug -gs else -DCFLAGS += -d-debug -gc +# Add LDC Debuggging Flags +DCFLAGS += -g -d-debug -gc endif +else +# Only add optimisation flags if debugging is not enabled +DCFLAGS += -O endif ifeq ($(NOTIFICATIONS),yes) From 3b0674e3c81c50ccad1a1d27faae0c14eaf91f88 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Thu, 11 Apr 2024 17:40:40 +1000 Subject: [PATCH 4/4] Update PR * Update code based on fixing valgrind issues --- src/curlEngine.d | 1 + src/main.d | 1 - src/onedrive.d | 12 +++-- src/sync.d | 119 +++++++++++++++++++++++++++++++++-------------- src/util.d | 15 +++--- 5 files changed, 102 insertions(+), 46 deletions(-) diff --git a/src/curlEngine.d b/src/curlEngine.d index 3875ade6..efc2a6de 100644 --- a/src/curlEngine.d +++ b/src/curlEngine.d @@ -176,6 +176,7 @@ class CurlEngine { synchronized(CurlEngine.classinfo) { foreach(curlEngine; curlEnginePool) { curlEngine.shutdown(); + object.destroy(curlEngine); } curlEnginePool = null; } diff --git a/src/main.d b/src/main.d index 59e370cf..cd5a0a76 100644 --- a/src/main.d +++ b/src/main.d @@ -1175,7 +1175,6 @@ void performStandardExitProcess(string scopeCaller = null) { syncEngineInstance = null; } else { addLogEntry("Waiting for all internal threads to complete before exiting application", ["verbose"]); - thread_joinAll(); addLogEntry("Application exit", ["debug"]); addLogEntry("#######################################################################################################################################", ["logFileOnly"]); // Destroy the shared logging buffer diff --git a/src/onedrive.d b/src/onedrive.d index 385430b3..af22545f 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -676,7 +676,8 @@ class OneDriveApi { curlEngine.http.onSend = data => file.rawRead(data).length; // convert offsetSize to ulong curlEngine.http.contentLength = to!ulong(offsetSize); - auto response = performHTTPOperation(); + JSONValue response; + response = performHTTPOperation(); checkHttpResponseCode(response); return response; } @@ -942,7 +943,8 @@ class OneDriveApi { scope(exit) curlEngine.http.clearRequestHeaders(); curlEngine.connect(HTTP.Method.del, url); addAccessTokenHeader(); - auto response = performHTTPOperation(); + JSONValue response; + response = performHTTPOperation(); checkHttpResponseCode(response); } @@ -1198,7 +1200,8 @@ class OneDriveApi { } else { curlEngine.http.onSend = buf => 0; } - auto response = performHTTPOperation(); + JSONValue response; + response = performHTTPOperation(); return response; } @@ -1407,7 +1410,8 @@ class OneDriveApi { curlEngine.http.addRequestHeader("Content-Type", "application/octet-stream"); curlEngine.http.onSend = data => file.rawRead(data).length; curlEngine.http.contentLength = file.size; - auto response = performHTTPOperation(); + JSONValue response; + response = performHTTPOperation(); checkHttpResponseCode(response); return response; } diff --git a/src/sync.d b/src/sync.d index 9efad8e8..59b4bedb 100644 --- a/src/sync.d +++ b/src/sync.d @@ -178,6 +178,9 @@ class SyncEngine { string latestDeltaLink; // Struct of containing the deltaLink details DeltaLinkDetails deltaLinkCache; + + // Create the specific task pool to process items in parallel + TaskPool processPool; // Configure this class instance this(ApplicationConfig appConfig, ItemDatabase itemDB, ClientSideFiltering selectiveSync) { @@ -298,6 +301,10 @@ class SyncEngine { // Initialise the Sync Engine class bool initialise() { + // Create common parallel thread pool + processPool = taskPool(); + processPool.isDaemon(true); // Control whether the worker threads are daemon threads. A daemon thread is automatically terminated when all non-daemon threads have terminated. + // create a new instance of the OneDrive API oneDriveApiInstance = new OneDriveApi(appConfig); if (oneDriveApiInstance.initialise()) { @@ -307,7 +314,7 @@ class SyncEngine { } catch (accountDetailsException exception) { // details could not be queried addLogEntry(exception.msg); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required oneDriveApiInstance.shutdown(); // Free object and memory object.destroy(oneDriveApiInstance); @@ -321,7 +328,7 @@ class SyncEngine { } catch (accountDetailsException exception) { // details could not be queried addLogEntry(exception.msg); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required oneDriveApiInstance.shutdown(); // Free object and memory object.destroy(oneDriveApiInstance); @@ -335,7 +342,7 @@ class SyncEngine { } catch (accountDetailsException exception) { // details could not be queried addLogEntry(exception.msg); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required oneDriveApiInstance.shutdown(); // Free object and memory object.destroy(oneDriveApiInstance); @@ -345,7 +352,7 @@ class SyncEngine { } else { // API could not be initialised addLogEntry("OneDrive API could not be initialised with previously used details"); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required oneDriveApiInstance.shutdown(); // Free object and memory object.destroy(oneDriveApiInstance); @@ -358,6 +365,7 @@ class SyncEngine { // Shutdown this API instance, as we will create API instances as required, when required oneDriveApiInstance.shutdown(); + // Free object and memory object.destroy(oneDriveApiInstance); return true; @@ -929,7 +937,7 @@ class SyncEngine { // To finish off the JSON processing items, this is needed to reflect this in the log addLogEntry("------------------------------------------------------------------", ["debug"]); - // Shutdown the API + // Shutdown this API instance, as we will create API instances as required, when required getDeltaQueryOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(getDeltaQueryOneDriveApiInstance); @@ -2107,7 +2115,7 @@ class SyncEngine { // Download items in parallel void downloadOneDriveItemsInParallel(JSONValue[] array) { // This function recieved an array of 16 JSON items to download - foreach (i, onedriveJSONItem; taskPool.parallel(array)) { + foreach (i, onedriveJSONItem; processPool.parallel(array)) { // Take each JSON item and downloadFileItem(onedriveJSONItem); } @@ -3690,12 +3698,13 @@ class SyncEngine { // For each batch of files to upload, upload the changed data to OneDrive foreach (chunk; databaseItemsWhereContentHasChanged.chunks(batchSize)) { processChangedLocalItemsToUploadInParallel(chunk); + chunk = null; } } - // Upload the changed file batches in parallel void processChangedLocalItemsToUploadInParallel(string[3][] array) { - foreach (i, localItemDetails; taskPool.parallel(array)) { + + foreach (i, localItemDetails; processPool.parallel(array)) { addLogEntry("Upload Thread " ~ to!string(i) ~ " Starting: " ~ to!string(Clock.currTime()), ["debug"]); uploadChangedLocalFileToOneDrive(localItemDetails); addLogEntry("Upload Thread " ~ to!string(i) ~ " Finished: " ~ to!string(Clock.currTime()), ["debug"]); @@ -4166,7 +4175,7 @@ class SyncEngine { // Debug Log the modified upload response addLogEntry("Modified File Upload Response: " ~ to!string(uploadResponse), ["debug"]); - // Shutdown the API instance + // Shutdown this API instance, as we will create API instances as required, when required instance uploadFileOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(uploadFileOneDriveApiInstance); @@ -4906,7 +4915,7 @@ class SyncEngine { // OneDrive API returned a 404 (above) to say the directory did not exist // but when we attempted to create it, OneDrive responded that it now already exists addLogEntry("OneDrive reported that " ~ thisNewPathToCreate ~ " already exists .. OneDrive API race condition", ["verbose"]); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -4915,7 +4924,7 @@ class SyncEngine { // some other error from OneDrive was returned - display what it is addLogEntry("OneDrive generated an error when creating this path: " ~ thisNewPathToCreate); displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -4931,7 +4940,7 @@ class SyncEngine { saveItem(fakeResponse); } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -5001,7 +5010,7 @@ class SyncEngine { // Add this path to businessSharedFoldersOnlineToSkip businessSharedFoldersOnlineToSkip ~= [thisNewPathToCreate]; // no save to database, no online create - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -5026,7 +5035,7 @@ class SyncEngine { // Is the response a valid JSON object - validation checking done in saveItem saveItem(onlinePathData); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -5040,7 +5049,7 @@ class SyncEngine { addLogEntry("Skipping creating this directory online due to 'case-insensitive match': " ~ thisNewPathToCreate); // Add this path to posixViolationPaths posixViolationPaths ~= [thisNewPathToCreate]; - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -5051,7 +5060,7 @@ class SyncEngine { addLogEntry("ERROR: There was an error performing this operation on Microsoft OneDrive"); addLogEntry("ERROR: Increase logging verbosity to assist determining why."); addLogEntry("Skipping: " ~ buildNormalizedPath(absolutePath(thisNewPathToCreate))); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -5089,7 +5098,7 @@ class SyncEngine { // Upload the file batches in parallel void uploadNewLocalFileItemsInParallel(string[] array) { - foreach (i, fileToUpload; taskPool.parallel(array)) { + foreach (i, fileToUpload; processPool.parallel(array)) { // Add a processing '.' if (appConfig.verbosityCount == 0) addProcessingDotEntry(); @@ -5437,8 +5446,6 @@ class SyncEngine { // Create the OneDriveAPI Upload Instance OneDriveApi uploadFileOneDriveApiInstance; - uploadFileOneDriveApiInstance = new OneDriveApi(appConfig); - uploadFileOneDriveApiInstance.initialise(); // Calculate upload speed auto uploadStartTime = Clock.currTime(); @@ -5458,14 +5465,22 @@ class SyncEngine { if ((thisFileSize == 0) || (useSimpleUpload)) { try { + // Initialise API + uploadFileOneDriveApiInstance = new OneDriveApi(appConfig); + uploadFileOneDriveApiInstance.initialise(); + // Attempt to upload the zero byte file using simpleUpload for all account types uploadResponse = uploadFileOneDriveApiInstance.simpleUpload(fileToUpload, parentItem.driveId, parentItem.id, baseName(fileToUpload)); uploadFailed = false; addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... done."); - // Shutdown the API - uploadFileOneDriveApiInstance.shutdown(); - // Free object and memory - object.destroy(uploadFileOneDriveApiInstance); + + // If the API instance is still valid, shut it down + if (uploadFileOneDriveApiInstance !is null) { + // Shutdown this API instance, as we will create API instances as required, when required + uploadFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadFileOneDriveApiInstance); + } } catch (OneDriveException exception) { // An error was responded with - what was it @@ -5494,21 +5509,49 @@ class SyncEngine { } // re-try original request - retried for 429, 503, 504 - but loop back calling this function performNewFileUpload(parentItem, fileToUpload, thisFileSize); - // Return upload status - return uploadFailed; + + // If the API instance is still valid, shut it down + if (uploadFileOneDriveApiInstance !is null) { + // Shutdown this API instance, as we will create API instances as required, when required + uploadFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadFileOneDriveApiInstance); + } + } else { // Default operation if not 408,429,503,504 errors // display what the error is addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); displayOneDriveErrorMessage(exception.msg, thisFunctionName); + + // If the API instance is still valid, shut it down + if (uploadFileOneDriveApiInstance !is null) { + // Shutdown this API instance, as we will create API instances as required, when required + uploadFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadFileOneDriveApiInstance); + } } } catch (FileException e) { // display the error message addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + + // If the API instance is still valid, shut it down + if (uploadFileOneDriveApiInstance !is null) { + // Shutdown this API instance, as we will create API instances as required, when required + uploadFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadFileOneDriveApiInstance); + } } } else { + + // Initialise API + uploadFileOneDriveApiInstance = new OneDriveApi(appConfig); + uploadFileOneDriveApiInstance.initialise(); + // Session Upload for this criteria: // - Personal Account and file size > 4MB // - All Business | Office365 | SharePoint files > 0 bytes @@ -5642,6 +5685,14 @@ class SyncEngine { // Create session Upload URL failed addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); } + + // If the API instance is still valid, shut it down + if (uploadFileOneDriveApiInstance !is null) { + // Shutdown this API instance, as we will create API instances as required, when required + uploadFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadFileOneDriveApiInstance); + } } } else { // We are in a --dry-run scenario @@ -6055,7 +6106,7 @@ class SyncEngine { // Perform the delete via the default OneDrive API instance performReverseDeletionOneDriveApiInstance.deleteById(itemToDelete.driveId, itemToDelete.id, itemToDelete.eTag); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required performReverseDeletionOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(performReverseDeletionOneDriveApiInstance); @@ -6697,7 +6748,7 @@ class SyncEngine { } } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required queryChildrenOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(queryChildrenOneDriveApiInstance); @@ -7038,7 +7089,7 @@ class SyncEngine { } } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required queryOneDriveForSpecificPath.shutdown(); // Free object and memory object.destroy(queryOneDriveForSpecificPath); @@ -7244,7 +7295,7 @@ class SyncEngine { } } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required movePathOnlineApiInstance.shutdown(); // Free object and memory object.destroy(movePathOnlineApiInstance); @@ -7531,7 +7582,7 @@ class SyncEngine { } } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required querySharePointLibraryNameApiInstance.shutdown(); // Free object and memory object.destroy(querySharePointLibraryNameApiInstance); @@ -7816,7 +7867,7 @@ class SyncEngine { } } - // Shutdown the API access + // Shutdown this API instance, as we will create API instances as required, when required access queryOneDriveForFileDetailsApiInstance.shutdown(); // Free object and memory object.destroy(queryOneDriveForFileDetailsApiInstance); @@ -8050,7 +8101,7 @@ class SyncEngine { return false; } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required validateUploadSessionFileDataApiInstance.shutdown(); // Free object and memory object.destroy(validateUploadSessionFileDataApiInstance); @@ -8091,7 +8142,7 @@ class SyncEngine { void resumeSessionUploadsInParallel(JSONValue[] array) { // This function recieved an array of 16 JSON items to resume upload - foreach (i, jsonItemToResume; taskPool.parallel(array)) { + foreach (i, jsonItemToResume; processPool.parallel(array)) { // Take each JSON item and resume upload using the JSON data JSONValue uploadResponse; @@ -8140,7 +8191,7 @@ class SyncEngine { addLogEntry("CODING TO DO: what to do when session upload resumption JSON data is not valid ... nothing ? error message ?"); } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required uploadFileOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(uploadFileOneDriveApiInstance); diff --git a/src/util.d b/src/util.d index cb6515de..01720fe0 100644 --- a/src/util.d +++ b/src/util.d @@ -227,13 +227,14 @@ bool testInternetReachability(ApplicationConfig appConfig) { addLogEntry("No Network Connection", ["debug"]); addLogEntry("Cannot connect to Microsoft OneDrive Login Service - Network Connection Issue"); displayOneDriveErrorMessage(e.msg, getFunctionName!({})); - } finally { - if (curlEngine) { - curlEngine.release(); - curlEngine = null; - } - } - + } + + // Shutdown engine + curlEngine.http.shutdown(); + curlEngine.releaseAll(); + object.destroy(curlEngine); + curlEngine = null; + // Return test result return result; }