diff --git a/docs/usage.md b/docs/usage.md index 91fc3b00..bc23ed11 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -507,7 +507,7 @@ Are you sure you want to proceed with --force-sync [Y/N] To proceed with `--force-sync`, you must type 'y' or 'Y' to allow the application to continue. ### Enabling the Client Activity Log -When running onedrive, all actions can be logged to a separate log file. This can be enabled by using the `--enable-logging` flag. +When running onedrive, all actions can be logged to a separate log file. This can be enabled by using the `--enable-logging` flag or by adding `enable_logging = "true"` to your 'config' file. By default, log files will be written to `/var/log/onedrive/` and will be in the format of `%username%.onedrive.log`, where `%username%` represents the user who ran the client to allow easy sorting of user to client activity log. diff --git a/src/curlEngine.d b/src/curlEngine.d index b117a4bc..86112ac9 100644 --- a/src/curlEngine.d +++ b/src/curlEngine.d @@ -194,6 +194,7 @@ class CurlEngine { synchronized(CurlEngine.classinfo) { foreach(curlEngine; curlEnginePool) { curlEngine.shutdown(); + object.destroy(curlEngine); } curlEnginePool = null; } diff --git a/src/main.d b/src/main.d index 59e370cf..cd5a0a76 100644 --- a/src/main.d +++ b/src/main.d @@ -1175,7 +1175,6 @@ void performStandardExitProcess(string scopeCaller = null) { syncEngineInstance = null; } else { addLogEntry("Waiting for all internal threads to complete before exiting application", ["verbose"]); - thread_joinAll(); addLogEntry("Application exit", ["debug"]); addLogEntry("#######################################################################################################################################", ["logFileOnly"]); // Destroy the shared logging buffer diff --git a/src/sync.d b/src/sync.d index b59717c6..164a6bc4 100644 --- a/src/sync.d +++ b/src/sync.d @@ -178,6 +178,9 @@ class SyncEngine { string latestDeltaLink; // Struct of containing the deltaLink details DeltaLinkDetails deltaLinkCache; + + // Create the specific task pool to process items in parallel + TaskPool processPool; // Configure this class instance this(ApplicationConfig appConfig, ItemDatabase itemDB, ClientSideFiltering selectiveSync) { @@ -298,6 +301,10 @@ class SyncEngine { // Initialise the Sync Engine class bool initialise() { + // Create common parallel thread pool + processPool = taskPool(); + processPool.isDaemon(true); // Control whether the worker threads are daemon threads. A daemon thread is automatically terminated when all non-daemon threads have terminated. + // create a new instance of the OneDrive API oneDriveApiInstance = new OneDriveApi(appConfig); if (oneDriveApiInstance.initialise()) { @@ -307,7 +314,7 @@ class SyncEngine { } catch (accountDetailsException exception) { // details could not be queried addLogEntry(exception.msg); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required oneDriveApiInstance.shutdown(); // Free object and memory object.destroy(oneDriveApiInstance); @@ -321,7 +328,7 @@ class SyncEngine { } catch (accountDetailsException exception) { // details could not be queried addLogEntry(exception.msg); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required oneDriveApiInstance.shutdown(); // Free object and memory object.destroy(oneDriveApiInstance); @@ -335,7 +342,7 @@ class SyncEngine { } catch (accountDetailsException exception) { // details could not be queried addLogEntry(exception.msg); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required oneDriveApiInstance.shutdown(); // Free object and memory object.destroy(oneDriveApiInstance); @@ -345,7 +352,7 @@ class SyncEngine { } else { // API could not be initialised addLogEntry("OneDrive API could not be initialised with previously used details"); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required oneDriveApiInstance.shutdown(); // Free object and memory object.destroy(oneDriveApiInstance); @@ -358,6 +365,7 @@ class SyncEngine { // Shutdown this API instance, as we will create API instances as required, when required oneDriveApiInstance.shutdown(); + // Free object and memory object.destroy(oneDriveApiInstance); return true; @@ -879,7 +887,7 @@ class SyncEngine { // To finish off the JSON processing items, this is needed to reflect this in the log addLogEntry("------------------------------------------------------------------", ["debug"]); - // Shutdown the API + // Shutdown this API instance, as we will create API instances as required, when required getDeltaQueryOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(getDeltaQueryOneDriveApiInstance); @@ -2057,7 +2065,7 @@ class SyncEngine { // Download items in parallel void downloadOneDriveItemsInParallel(JSONValue[] array) { // This function recieved an array of 16 JSON items to download - foreach (i, onedriveJSONItem; taskPool.parallel(array)) { + foreach (i, onedriveJSONItem; processPool.parallel(array)) { // Take each JSON item and downloadFileItem(onedriveJSONItem); } @@ -3527,12 +3535,13 @@ class SyncEngine { // For each batch of files to upload, upload the changed data to OneDrive foreach (chunk; databaseItemsWhereContentHasChanged.chunks(batchSize)) { processChangedLocalItemsToUploadInParallel(chunk); + chunk = null; } } - // Upload the changed file batches in parallel void processChangedLocalItemsToUploadInParallel(string[3][] array) { - foreach (i, localItemDetails; taskPool.parallel(array)) { + + foreach (i, localItemDetails; processPool.parallel(array)) { addLogEntry("Upload Thread " ~ to!string(i) ~ " Starting: " ~ to!string(Clock.currTime()), ["debug"]); uploadChangedLocalFileToOneDrive(localItemDetails); addLogEntry("Upload Thread " ~ to!string(i) ~ " Finished: " ~ to!string(Clock.currTime()), ["debug"]); @@ -3899,7 +3908,7 @@ class SyncEngine { // Debug Log the modified upload response addLogEntry("Modified File Upload Response: " ~ to!string(uploadResponse), ["debug"]); - // Shutdown the API instance + // Shutdown this API instance, as we will create API instances as required, when required instance uploadFileOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(uploadFileOneDriveApiInstance); @@ -4614,7 +4623,7 @@ class SyncEngine { // OneDrive API returned a 404 (above) to say the directory did not exist // but when we attempted to create it, OneDrive responded that it now already exists addLogEntry("OneDrive reported that " ~ thisNewPathToCreate ~ " already exists .. OneDrive API race condition", ["verbose"]); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -4623,7 +4632,7 @@ class SyncEngine { // some other error from OneDrive was returned - display what it is addLogEntry("OneDrive generated an error when creating this path: " ~ thisNewPathToCreate); displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -4639,7 +4648,7 @@ class SyncEngine { saveItem(fakeResponse); } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -4684,7 +4693,7 @@ class SyncEngine { // Add this path to businessSharedFoldersOnlineToSkip businessSharedFoldersOnlineToSkip ~= [thisNewPathToCreate]; // no save to database, no online create - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -4709,7 +4718,7 @@ class SyncEngine { // Is the response a valid JSON object - validation checking done in saveItem saveItem(onlinePathData); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -4723,7 +4732,7 @@ class SyncEngine { addLogEntry("Skipping creating this directory online due to 'case-insensitive match': " ~ thisNewPathToCreate); // Add this path to posixViolationPaths posixViolationPaths ~= [thisNewPathToCreate]; - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -4734,7 +4743,7 @@ class SyncEngine { addLogEntry("ERROR: There was an error performing this operation on Microsoft OneDrive"); addLogEntry("ERROR: Increase logging verbosity to assist determining why."); addLogEntry("Skipping: " ~ buildNormalizedPath(absolutePath(thisNewPathToCreate))); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required createDirectoryOnlineOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); @@ -4772,7 +4781,7 @@ class SyncEngine { // Upload the file batches in parallel void uploadNewLocalFileItemsInParallel(string[] array) { - foreach (i, fileToUpload; taskPool.parallel(array)) { + foreach (i, fileToUpload; processPool.parallel(array)) { // Add a processing '.' if (appConfig.verbosityCount == 0) addProcessingDotEntry(); @@ -5095,8 +5104,6 @@ class SyncEngine { // Create the OneDriveAPI Upload Instance OneDriveApi uploadFileOneDriveApiInstance; - uploadFileOneDriveApiInstance = new OneDriveApi(appConfig); - uploadFileOneDriveApiInstance.initialise(); // Calculate upload speed auto uploadStartTime = Clock.currTime(); @@ -5116,14 +5123,23 @@ class SyncEngine { if ((thisFileSize == 0) || (useSimpleUpload)) { try { + // Initialise API + uploadFileOneDriveApiInstance = new OneDriveApi(appConfig); + uploadFileOneDriveApiInstance.initialise(); + // Attempt to upload the zero byte file using simpleUpload for all account types uploadResponse = uploadFileOneDriveApiInstance.simpleUpload(fileToUpload, parentItem.driveId, parentItem.id, baseName(fileToUpload)); uploadFailed = false; addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... done."); - // Shutdown the API - uploadFileOneDriveApiInstance.shutdown(); - // Free object and memory - object.destroy(uploadFileOneDriveApiInstance); + + // If the API instance is still valid, shut it down + if (uploadFileOneDriveApiInstance !is null) { + // Shutdown this API instance, as we will create API instances as required, when required + uploadFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadFileOneDriveApiInstance); + } + } catch (OneDriveException exception) { // An error was responded with - what was it @@ -5134,12 +5150,34 @@ class SyncEngine { addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); displayOneDriveErrorMessage(exception.msg, thisFunctionName); + // If the API instance is still valid, shut it down + if (uploadFileOneDriveApiInstance !is null) { + // Shutdown this API instance, as we will create API instances as required, when required + uploadFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadFileOneDriveApiInstance); + } + } catch (FileException e) { // display the error message addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + + // If the API instance is still valid, shut it down + if (uploadFileOneDriveApiInstance !is null) { + addLogEntry("API SHUTDOWN: Shutting down API instance uploadFileOneDriveApiInstance"); + // Shutdown this API instance, as we will create API instances as required, when required + uploadFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadFileOneDriveApiInstance); + } } } else { + + // Initialise API + uploadFileOneDriveApiInstance = new OneDriveApi(appConfig); + uploadFileOneDriveApiInstance.initialise(); + // Session Upload for this criteria: // - Personal Account and file size > 4MB // - All Business | Office365 | SharePoint files > 0 bytes @@ -5222,6 +5260,14 @@ class SyncEngine { // Create session Upload URL failed addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); } + + // If the API instance is still valid, shut it down + if (uploadFileOneDriveApiInstance !is null) { + // Shutdown this API instance, as we will create API instances as required, when required + uploadFileOneDriveApiInstance.shutdown(); + // Free object and memory + object.destroy(uploadFileOneDriveApiInstance); + } } } else { // We are in a --dry-run scenario @@ -5610,7 +5656,7 @@ class SyncEngine { // Perform the delete via the default OneDrive API instance performReverseDeletionOneDriveApiInstance.deleteById(itemToDelete.driveId, itemToDelete.id, itemToDelete.eTag); - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required performReverseDeletionOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(performReverseDeletionOneDriveApiInstance); @@ -6177,7 +6223,7 @@ class SyncEngine { } } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required queryChildrenOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(queryChildrenOneDriveApiInstance); @@ -6443,7 +6489,7 @@ class SyncEngine { } } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required queryOneDriveForSpecificPath.shutdown(); // Free object and memory object.destroy(queryOneDriveForSpecificPath); @@ -6649,7 +6695,7 @@ class SyncEngine { } } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required movePathOnlineApiInstance.shutdown(); // Free object and memory object.destroy(movePathOnlineApiInstance); @@ -6909,7 +6955,7 @@ class SyncEngine { } } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required querySharePointLibraryNameApiInstance.shutdown(); // Free object and memory object.destroy(querySharePointLibraryNameApiInstance); @@ -7194,7 +7240,7 @@ class SyncEngine { } } - // Shutdown the API access + // Shutdown this API instance, as we will create API instances as required, when required access queryOneDriveForFileDetailsApiInstance.shutdown(); // Free object and memory object.destroy(queryOneDriveForFileDetailsApiInstance); @@ -7428,7 +7474,7 @@ class SyncEngine { return false; } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required validateUploadSessionFileDataApiInstance.shutdown(); // Free object and memory object.destroy(validateUploadSessionFileDataApiInstance); @@ -7469,7 +7515,7 @@ class SyncEngine { void resumeSessionUploadsInParallel(JSONValue[] array) { // This function recieved an array of 16 JSON items to resume upload - foreach (i, jsonItemToResume; taskPool.parallel(array)) { + foreach (i, jsonItemToResume; processPool.parallel(array)) { // Take each JSON item and resume upload using the JSON data JSONValue uploadResponse; @@ -7518,7 +7564,7 @@ class SyncEngine { addLogEntry("CODING TO DO: what to do when session upload resumption JSON data is not valid ... nothing ? error message ?"); } - // Shutdown API instance + // Shutdown this API instance, as we will create API instances as required, when required uploadFileOneDriveApiInstance.shutdown(); // Free object and memory object.destroy(uploadFileOneDriveApiInstance); diff --git a/src/util.d b/src/util.d index f2fa0c04..ba89ee66 100644 --- a/src/util.d +++ b/src/util.d @@ -227,13 +227,14 @@ bool testInternetReachability(ApplicationConfig appConfig) { addLogEntry("No Network Connection", ["debug"]); addLogEntry("Cannot connect to Microsoft OneDrive Login Service - Network Connection Issue"); displayOneDriveErrorMessage(e.msg, getFunctionName!({})); - } finally { - if (curlEngine) { - curlEngine.release(); - curlEngine = null; - } - } - + } + + // Shutdown engine + curlEngine.http.shutdown(); + curlEngine.releaseAll(); + object.destroy(curlEngine); + curlEngine = null; + // Return test result return result; }