diff --git a/src/curlEngine.d b/src/curlEngine.d index cd05886b..0bbe3f2a 100644 --- a/src/curlEngine.d +++ b/src/curlEngine.d @@ -60,8 +60,11 @@ class CurlEngine { // Ensure that TCP_NODELAY is set to 0 to ensure that TCP NAGLE is enabled http.handle.set(CurlOption.tcp_nodelay,0); // https://curl.se/libcurl/c/CURLOPT_FORBID_REUSE.html - // Ensure that we ARE reusing connections - setting to 0 ensures that we are reusing connections - http.handle.set(CurlOption.forbid_reuse,0); + // CURLOPT_FORBID_REUSE - make connection get closed at once after use + // Ensure that we ARE NOT reusing TCP sockets connections - setting to 0 ensures that we ARE reusing connections (we did this in v2.4.xx) + // Setting this to 1 ensures that when we close the curl instance, any open sockets are closed - which we need to do when running + // multiple threads and API instances at the same time otherwise we run out of local files | sockets pretty quickly + http.handle.set(CurlOption.forbid_reuse,1); if (httpsDebug) { // Output what options we are using so that in the debug log this can be tracked @@ -73,11 +76,16 @@ class CurlEngine { } } - void setMethodPost(){ + void setMethodPost() { http.method = HTTP.Method.post; } - void setMethodPatch(){ + void setMethodPatch() { http.method = HTTP.Method.patch; } + + void setDisableSSLVerifyPeer() { + log.vdebug("Switching off CurlOption.ssl_verifypeer"); + http.handle.set(CurlOption.ssl_verifypeer, 0); + } } \ No newline at end of file diff --git a/src/main.d b/src/main.d index 77e9f8b2..940e651c 100644 --- a/src/main.d +++ b/src/main.d @@ -415,7 +415,17 @@ int main(string[] cliArgs) { // Are we doing a --monitor operation? if (appConfig.getValueBool("monitor")) { + // What are the current values for the platform we are running on + // Max number of open files /proc/sys/fs/file-max + string maxOpenFiles = strip(readText("/proc/sys/fs/file-max")); + // What is the currently configured maximum inotify watches that can be used + // /proc/sys/user/max_inotify_watches + string maxInotifyWatches = strip(readText("/proc/sys/user/max_inotify_watches")); + + // Start the monitor process log.log("OneDrive syncronisation interval (seconds): ", appConfig.getValueLong("monitor_interval")); + log.vlog("Maximum allowed open files: ", maxOpenFiles); + log.vlog("Maximum allowed inotify watches: ", maxInotifyWatches); // Configure the monitor class Monitor filesystemMonitor = new Monitor(appConfig, selectiveSync); @@ -515,10 +525,9 @@ int main(string[] cliArgs) { string loopStopOutputMessage = "################################################ LOOP COMPLETE ###############################################"; while (performMonitor) { - try { // Process any inotify events - filesystemMonitor.update(online); + filesystemMonitor.update(true); } catch (MonitorException e) { // Catch any exceptions thrown by inotify / monitor engine log.error("ERROR: The following inotify error was generated: ", e.msg); @@ -529,7 +538,6 @@ int main(string[] cliArgs) { // Check here for a webhook notification - // Get the current time this loop is starting auto currentTime = MonoTime.currTime(); @@ -554,12 +562,15 @@ int main(string[] cliArgs) { // Did the user specify --upload-only? if (appConfig.getValueBool("upload_only")) { // Perform the --upload-only sync process - performUploadOnlySyncProcess(localPath); + performUploadOnlySyncProcess(localPath, filesystemMonitor); } else { // Perform the standard sync process - performStandardSyncProcess(localPath); + performStandardSyncProcess(localPath, filesystemMonitor); } + // Discard any inotify events generated as part of any sync operation + filesystemMonitor.update(false); + // Detail the outcome of the sync process displaySyncOutcome(); @@ -568,7 +579,7 @@ int main(string[] cliArgs) { itemDB.performVacuum(); } else { // Not online - log.log("Microsoft OneDrive service is not reachable at this time"); + log.log("Microsoft OneDrive service is not reachable at this time. Will re-try on next loop attempt."); } // Output end of loop processing times @@ -585,7 +596,6 @@ int main(string[] cliArgs) { Thread.sleep(dur!"seconds"(1)); } } - } else { // Exit application as the sync engine could not be initialised log.error("Application Sync Engine could not be initialised correctly"); @@ -603,35 +613,71 @@ int main(string[] cliArgs) { return EXIT_SUCCESS; } -void performUploadOnlySyncProcess(string localPath) { +void performUploadOnlySyncProcess(string localPath, Monitor filesystemMonitor = null) { // Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); + if (appConfig.getValueBool("monitor")) { + // Handle any inotify events whilst the DB was being scanned + filesystemMonitor.update(true); + } + // Scan the configured 'sync_dir' for new data to upload syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); + if (appConfig.getValueBool("monitor")) { + // Handle any new inotify events whilst the local filesystem was being scanned + filesystemMonitor.update(true); + } } -void performStandardSyncProcess(string localPath) { +void performStandardSyncProcess(string localPath, Monitor filesystemMonitor = null) { // Which way do we sync first? // OneDrive first then local changes (normal operational process that uses OneDrive as the source of truth) // Local First then OneDrive changes (alternate operation process to use local files as source of truth) - if (appConfig.getValueBool("local_first")) { // Local data first // Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); + if (appConfig.getValueBool("monitor")) { + // Handle any inotify events whilst the DB was being scanned + filesystemMonitor.update(true); + } + // Scan the configured 'sync_dir' for new data to upload to OneDrive syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); + if (appConfig.getValueBool("monitor")) { + // Handle any new inotify events whilst the local filesystem was being scanned + filesystemMonitor.update(true); + } + // Download data from OneDrive last syncEngineInstance.syncOneDriveAccountToLocalDisk(); + if (appConfig.getValueBool("monitor")) { + // Cancel out any inotify events from downloading data + filesystemMonitor.update(false); + } } else { // Normal sync // Download data from OneDrive first syncEngineInstance.syncOneDriveAccountToLocalDisk(); + if (appConfig.getValueBool("monitor")) { + // Cancel out any inotify events from downloading data + filesystemMonitor.update(false); + } + // Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); + if (appConfig.getValueBool("monitor")) { + // Handle any inotify events whilst the DB was being scanned + filesystemMonitor.update(true); + } + // Scan the configured 'sync_dir' for new data to upload to OneDrive syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); + if (appConfig.getValueBool("monitor")) { + // Handle any new inotify events whilst the local filesystem was being scanned + filesystemMonitor.update(true); + } } } diff --git a/src/monitor.d b/src/monitor.d index e30e0aea..918d4205 100644 --- a/src/monitor.d +++ b/src/monitor.d @@ -14,6 +14,7 @@ import std.path; import std.regex; import std.stdio; import std.string; +import std.conv; // What other modules that we have created do we need to import? import config; @@ -197,11 +198,13 @@ final class Monitor { int wd = inotify_add_watch(fd, toStringz(pathname), mask); if (wd < 0) { if (errno() == ENOSPC) { + // Get the current value + ulong maxInotifyWatches = to!int(strip(readText("/proc/sys/user/max_inotify_watches"))); log.log("The user limit on the total number of inotify watches has been reached."); - log.log("To see the current max number of watches run:"); - log.log("sysctl fs.inotify.max_user_watches"); - log.log("To change the current max number of watches to 524288 run:"); - log.log("sudo sysctl fs.inotify.max_user_watches=524288"); + log.log("Your current limit of inotify watches is: ", maxInotifyWatches); + log.log("It is recommended that you change the max number of inotify watches to at least double your existing value."); + log.log("To change the current max number of watches to " , (maxInotifyWatches * 2) , " run:"); + log.log("EXAMPLE: sudo sysctl fs.inotify.max_user_watches=", (maxInotifyWatches * 2)); } if (errno() == 13) { if ((selectiveSync.getSkipDotfiles()) && (isDotFile(pathname))) { @@ -404,7 +407,7 @@ final class Monitor { cookieToPath.remove(cookie); } - log.vdebug("inotify events flushed"); + log.log("inotify events flushed"); } } } diff --git a/src/onedrive.d b/src/onedrive.d index 613503a1..f79e49e8 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -684,7 +684,12 @@ class OneDriveApi { response = post(tokenUrl, postData); } catch (OneDriveException e) { // an error was generated - displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + if (e.httpStatusCode >= 500) { + // There was a HTTP 5xx Server Side Error - retry + acquireToken(postData); + } else { + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + } } if (response.type() == JSONType.object) { @@ -748,7 +753,7 @@ class OneDriveApi { authorise(); } } else { - log.log("Invalid response from the OneDrive API. Unable to initialise application."); + log.log("Invalid response from the OneDrive API. Unable to initialise OneDrive API instance."); exit(-1); } } @@ -1109,20 +1114,40 @@ class OneDriveApi { throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?"); } } else { - // Log that an error was returned - log.error("ERROR: OneDrive returned an error with the following message:"); - // Some other error was returned - log.error(" Error Message: ", errorMessage); - log.error(" Calling Function: ", getFunctionName!({})); + + // what error was returned? + if (canFind(errorMessage, "Problem with the SSL CA cert (path? access rights?) on handle")) { + // error setting certificate verify locations: + // CAfile: /etc/pki/tls/certs/ca-bundle.crt + // CApath: none + // + // Tell the Curl Engine to bypass SSL check - essentially SSL is passing back a bad value due to 'stdio' compile time option + // Further reading: + // https://github.com/curl/curl/issues/6090 + // https://github.com/openssl/openssl/issues/7536 + // https://stackoverflow.com/questions/45829588/brew-install-fails-curl77-error-setting-certificate-verify + // https://forum.dlang.org/post/vwvkbubufexgeuaxhqfl@forum.dlang.org + + log.vdebug("Problem with reading the SSL CA cert via libcurl - attempting work around"); + curlEngine.setDisableSSLVerifyPeer(); + // retry origional call + performHTTPOperation(); + } else { + // Log that an error was returned + log.error("ERROR: OneDrive returned an error with the following message:"); + // Some other error was returned + log.error(" Error Message: ", errorMessage); + log.error(" Calling Function: ", getFunctionName!({})); - // Was this a curl initialization error? - if (canFind(errorMessage, "Failed initialization on handle")) { - // initialization error ... prevent a run-away process if we have zero disk space - ulong localActualFreeSpace = getAvailableDiskSpace("."); - if (localActualFreeSpace == 0) { - // force exit - shutdown(); - exit(-1); + // Was this a curl initialization error? + if (canFind(errorMessage, "Failed initialization on handle")) { + // initialization error ... prevent a run-away process if we have zero disk space + ulong localActualFreeSpace = getAvailableDiskSpace("."); + if (localActualFreeSpace == 0) { + // force exit + shutdown(); + exit(-1); + } } } } @@ -1179,10 +1204,6 @@ class OneDriveApi { return response; } - - - - private void checkHTTPResponseHeaders() { // Get the HTTP Response headers - needed for correct 429 handling auto responseHeaders = curlEngine.http.responseHeaders(); diff --git a/src/sync.d b/src/sync.d index 31c21511..0e1a6f6b 100644 --- a/src/sync.d +++ b/src/sync.d @@ -4564,11 +4564,16 @@ class SyncEngine { pathToQuery = "."; } + // Create new OneDrive API Instance + OneDriveApi generateDeltaResponseOneDriveApiInstance; + generateDeltaResponseOneDriveApiInstance = new OneDriveApi(appConfig); + generateDeltaResponseOneDriveApiInstance.initialise(); + if (!singleDirectoryScope) { // In a --resync scenario, there is no DB data to query, so we have to query the OneDrive API here to get relevant details try { // Query the OneDrive API - pathData = oneDriveApiInstance.getPathDetails(pathToQuery); + pathData = generateDeltaResponseOneDriveApiInstance.getPathDetails(pathToQuery); // Is the path on OneDrive local or remote to our account drive id? if (isItemRemote(pathData)) { // The path we are seeking is remote to our account drive id @@ -4583,7 +4588,7 @@ class SyncEngine { // Display error message displayOneDriveErrorMessage(e.msg, getFunctionName!({})); // Must exit here - oneDriveApiInstance.shutdown(); + generateDeltaResponseOneDriveApiInstance.shutdown(); exit(-1); } } else { @@ -4610,27 +4615,27 @@ class SyncEngine { // Get drive details for the provided driveId try { - driveData = oneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id); + driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id); } catch (OneDriveException e) { - log.vdebug("driveData = oneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id) generated a OneDriveException"); + log.vdebug("driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id) generated a OneDriveException"); // HTTP request returned status code 504 (Gateway Timeout) or 429 retry if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. if (e.httpStatusCode == 429) { log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - retrying applicable request"); - handleOneDriveThrottleRequest(oneDriveApiInstance); + handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance); } if (e.httpStatusCode == 504) { log.vdebug("Retrying original request that generated the HTTP 504 (Gateway Timeout) - retrying applicable request"); Thread.sleep(dur!"seconds"(30)); } // Retry original request by calling function again to avoid replicating any further error handling - driveData = oneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id); + driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id); } else { // There was a HTTP 5xx Server Side Error displayOneDriveErrorMessage(e.msg, getFunctionName!({})); // Must exit here - oneDriveApiInstance.shutdown(); + generateDeltaResponseOneDriveApiInstance.shutdown(); exit(-1); } } @@ -4639,7 +4644,7 @@ class SyncEngine { if (!isItemRoot(driveData)) { // Get root details for the provided driveId try { - rootData = oneDriveApiInstance.getDriveIdRoot(searchItem.driveId); + rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId); } catch (OneDriveException e) { log.vdebug("rootData = onedrive.getDriveIdRoot(searchItem.driveId) generated a OneDriveException"); // HTTP request returned status code 504 (Gateway Timeout) or 429 retry @@ -4647,20 +4652,20 @@ class SyncEngine { // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. if (e.httpStatusCode == 429) { log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - retrying applicable request"); - handleOneDriveThrottleRequest(oneDriveApiInstance); + handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance); } if (e.httpStatusCode == 504) { log.vdebug("Retrying original request that generated the HTTP 504 (Gateway Timeout) - retrying applicable request"); Thread.sleep(dur!"seconds"(30)); } // Retry original request by calling function again to avoid replicating any further error handling - rootData = oneDriveApiInstance.getDriveIdRoot(searchItem.driveId); + rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId); } else { // There was a HTTP 5xx Server Side Error displayOneDriveErrorMessage(e.msg, getFunctionName!({})); // Must exit here - oneDriveApiInstance.shutdown(); + generateDeltaResponseOneDriveApiInstance.shutdown(); exit(-1); } } @@ -4677,11 +4682,11 @@ class SyncEngine { for (;;) { // query top level children try { - topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink); + topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink); } catch (OneDriveException e) { // OneDrive threw an error log.vdebug("------------------------------------------------------------------"); - log.vdebug("Query Error: topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)"); + log.vdebug("Query Error: topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)"); log.vdebug("driveId: ", searchItem.driveId); log.vdebug("idToQuery: ", searchItem.id); log.vdebug("nextLink: ", nextLink); @@ -4689,7 +4694,7 @@ class SyncEngine { // HTTP request returned status code 429 (Too Many Requests) if (e.httpStatusCode == 429) { // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(oneDriveApiInstance); + handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance); log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children"); } @@ -4711,12 +4716,12 @@ class SyncEngine { } // re-try original request - retried for 429 and 504 try { - log.vdebug("Retrying Query: topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)"); - topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink); - log.vdebug("Query 'topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)' performed successfully on re-try"); + log.vdebug("Retrying Query: topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)"); + topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink); + log.vdebug("Query 'topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)' performed successfully on re-try"); } catch (OneDriveException e) { // display what the error is - log.vdebug("Query Error: topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink) on re-try after delay"); + log.vdebug("Query Error: topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink) on re-try after delay"); // error was not a 504 this time displayOneDriveErrorMessage(e.msg, getFunctionName!({})); } @@ -4843,17 +4848,22 @@ class SyncEngine { // Query the OneDrive API for the child objects for this element JSONValue queryThisLevelChildren(string driveId, string idToQuery, string nextLink) { JSONValue thisLevelChildren; + + // Create new OneDrive API Instance + OneDriveApi queryChildrenOneDriveApiInstance; + queryChildrenOneDriveApiInstance = new OneDriveApi(appConfig); + queryChildrenOneDriveApiInstance.initialise(); // query children try { // attempt API call - log.vdebug("Attempting Query: thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)"); - thisLevelChildren = oneDriveApiInstance.listChildren(driveId, idToQuery, nextLink); - log.vdebug("Query 'thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)' performed successfully"); + log.vdebug("Attempting Query: thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)"); + thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink); + log.vdebug("Query 'thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)' performed successfully"); } catch (OneDriveException e) { // OneDrive threw an error log.vdebug("------------------------------------------------------------------"); - log.vdebug("Query Error: thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)"); + log.vdebug("Query Error: thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)"); log.vdebug("driveId: ", driveId); log.vdebug("idToQuery: ", idToQuery); log.vdebug("nextLink: ", nextLink); @@ -4861,7 +4871,7 @@ class SyncEngine { // HTTP request returned status code 429 (Too Many Requests) if (e.httpStatusCode == 429) { // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(oneDriveApiInstance); + handleOneDriveThrottleRequest(queryChildrenOneDriveApiInstance); log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children"); } @@ -4871,7 +4881,7 @@ class SyncEngine { if (e.httpStatusCode == 504) { // transient error - try again in 30 seconds log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query OneDrive drive children - retrying applicable request"); - log.vdebug("thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink) previously threw an error - retrying"); + log.vdebug("thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink) previously threw an error - retrying"); // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); Thread.sleep(dur!"seconds"(30));