diff --git a/src/onedrive.d b/src/onedrive.d index e2fe18e4..7f76a30a 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -12,11 +12,18 @@ static import log; shared bool debugResponse = false; private bool dryRun = false; private bool simulateNoRefreshTokenFile = false; +private ulong retryAfterValue = 0; private immutable { // Client Identifier + // Client ID (skilion) string clientId = "22c49a0d-d21c-4792-aed1-8f163c982546"; + // Default User Agent configuration + string isvTag = "ISV"; + string companyName = "abraunegg"; + string appTitle = "OneDrive_Client_for_Linux"; + // Personal & Business Queries string authUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize"; string redirectUrl = "https://login.microsoftonline.com/common/oauth2/nativeclient"; @@ -101,11 +108,18 @@ final class OneDriveApi .debugResponse = true; } - // Custom User Agent - if (cfg.getValueString("user_agent") != "") { - http.setUserAgent = cfg.getValueString("user_agent"); + // Configure the User Agent string + if (cfg.getValueString("user_agent") == "") { + // Application defaults + // Comply with traffic decoration requirements + // https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online + // - Identify as ISV and include Company Name, App Name separated by a pipe character and then adding Version number separated with a slash character + // Note: If you've created an application, the recommendation is to register and use AppID and AppTitle + // The issue here is that currently the application is still using the 'skilion' application ID, thus no idea what the AppTitle used was. + http.setUserAgent = isvTag ~ "|" ~ companyName ~ "|" ~ appTitle ~ "/" ~ strip(import("version")); } else { - http.setUserAgent = "OneDrive Client for Linux " ~ strip(import("version")); + // Use the value entered by the user + http.setUserAgent = cfg.getValueString("user_agent"); } // What version of HTTP protocol do we use? @@ -221,6 +235,18 @@ final class OneDriveApi return true; } + ulong getRetryAfterValue() + { + // Return the current value of retryAfterValue if it has been set to something other than 0 + return .retryAfterValue; + } + + void resetRetryAfterValue() + { + // Reset the current value of retryAfterValue to 0 after it has been used + .retryAfterValue = 0; + } + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get JSONValue getDefaultDrive() { @@ -488,9 +514,9 @@ final class OneDriveApi if (!skipToken) addAccessTokenHeader(); // HACK: requestUploadStatus auto response = perform(); checkHttpCode(response); - // OneDrive API Response Debugging + // OneDrive API Response Debugging if --https-debug is being used if (.debugResponse){ - writeln("OneDrive API Response: ", response); + log.vdebug("OneDrive API Response: ", response); } return response; } @@ -639,22 +665,38 @@ final class OneDriveApi char[] content; http.onReceive = (ubyte[] data) { content ~= data; - // HTTP Server Response Code Debugging + // HTTP Server Response Code Debugging if --https-debug is being used if (.debugResponse){ - writeln("OneDrive HTTP Server Response: ", http.statusLine.code); + log.vdebug("onedrive.perform() => OneDrive HTTP Server Response: ", http.statusLine.code); } return data.length; }; JSONValue json; + try { http.perform(); + // Get the HTTP Response headers - needed for correct 429 handling + auto responseHeaders = http.responseHeaders(); + // HTTP Server Response Headers Debugging if --https-debug is being used + if (.debugResponse){ + log.vdebug("onedrive.perform() => HTTP Response Headers: ", responseHeaders); + } + + if ("retry-after" in http.responseHeaders) { + // retry-after as in the response headers + // Set the value + log.vdebug("onedrive.perform() => Received a 'Retry-After' Header Response with the following value: ", http.responseHeaders["retry-after"]); + log.vdebug("onedrive.perform() => Setting retryAfterValue to: ", http.responseHeaders["retry-after"]); + .retryAfterValue = to!ulong(http.responseHeaders["retry-after"]); + } + } catch (CurlException e) { // Parse and display error message received from OneDrive log.error("ERROR: OneDrive returned an error with the following message:"); auto errorArray = splitLines(e.msg); string errorMessage = errorArray[0]; - + if (canFind(errorMessage, "Couldn't connect to server on handle") || canFind(errorMessage, "Couldn't resolve host name on handle")) { // This is a curl timeout @@ -830,7 +872,7 @@ final class OneDriveApi // Too many requests in a certain time window // https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online log.vlog("OneDrive returned a 'HTTP 429 - Too Many Requests' - gracefully handling error"); - break; + throw new OneDriveException(http.statusLine.code, http.statusLine.reason); // Server side (OneDrive) Errors // 500 - Internal Server Error diff --git a/src/sync.d b/src/sync.d index e3445472..80912fce 100644 --- a/src/sync.d +++ b/src/sync.d @@ -276,6 +276,7 @@ final class SyncEngine try { oneDriveDetails = onedrive.getDefaultDrive(); } catch (OneDriveException e) { + log.vdebug("oneDriveDetails = onedrive.getDefaultDrive() generated a OneDriveException"); if (e.httpStatusCode == 400) { // OneDrive responded with 400 error: Bad Request log.error("\nERROR: OneDrive returned a 'HTTP 400 Bad Request' - Cannot Initialize Sync Engine"); @@ -294,11 +295,13 @@ final class SyncEngine exit(-1); } if (e.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests) - log.error("\nERROR: OneDrive returned a 'HTTP 429 - Too Many Requests' - Cannot Initialize Sync Engine"); - log.error("ERROR: Please try to access OneDrive again later\n"); - // Must exit here - exit(-1); + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling init();"); + init(); + // return back to original call + return; } if (e.httpStatusCode >= 500) { // There was a HTTP 5xx Server Side Error @@ -312,6 +315,7 @@ final class SyncEngine try { oneDriveRootDetails = onedrive.getDefaultRoot(); } catch (OneDriveException e) { + log.vdebug("oneDriveRootDetails = onedrive.getDefaultRoot() generated a OneDriveException"); if (e.httpStatusCode == 400) { // OneDrive responded with 400 error: Bad Request log.error("\nERROR: OneDrive returned a 'HTTP 400 Bad Request' - Cannot Initialize Sync Engine"); @@ -330,11 +334,13 @@ final class SyncEngine exit(-1); } if (e.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests) - log.error("\nERROR: OneDrive returned a 'HTTP 429 - Too Many Requests' - Cannot Initialize Sync Engine"); - log.error("ERROR: Please try to access OneDrive again later\n"); - // Must exit here - exit(-1); + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling init();"); + init(); + // return back to original call + return; } if (e.httpStatusCode >= 500) { // There was a HTTP 5xx Server Side Error @@ -494,12 +500,23 @@ final class SyncEngine try { onedrivePathDetails = onedrive.getPathDetails(path); // Returns a JSON String for the OneDrive Path } catch (OneDriveException e) { + log.vdebug("onedrivePathDetails = onedrive.getPathDetails(path) generated a OneDriveException"); if (e.httpStatusCode == 404) { // The directory was not found log.error("ERROR: The requested single directory to sync was not found on OneDrive"); return; } + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling applyDifferencesSingleDirectory(path);"); + applyDifferencesSingleDirectory(path); + // return back to original call + return; + } + if (e.httpStatusCode >= 500) { // OneDrive returned a 'HTTP 5xx Server Side Error' - gracefully handling error - error message already logged return; @@ -592,12 +609,23 @@ final class SyncEngine try { onedrive.getPathDetails(path); } catch (OneDriveException e) { + log.vdebug("onedrive.getPathDetails(path) generated a OneDriveException"); if (e.httpStatusCode == 404) { // The directory was not found on OneDrive - no need to delete it log.vlog("The requested directory to delete was not found on OneDrive - skipping removing the remote directory as it doesn't exist"); return; } + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling deleteDirectoryNoSync(path);"); + deleteDirectoryNoSync(path); + // return back to original call + return; + } + if (e.httpStatusCode >= 500) { // OneDrive returned a 'HTTP 5xx Server Side Error' - gracefully handling error - error message already logged return; @@ -624,12 +652,23 @@ final class SyncEngine // test if the local path exists on OneDrive onedrive.getPathDetails(source); } catch (OneDriveException e) { + log.vdebug("onedrive.getPathDetails(source); generated a OneDriveException"); if (e.httpStatusCode == 404) { // The directory was not found log.vlog("The requested directory to rename was not found on OneDrive"); return; } + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling renameDirectoryNoSync(source, destination);"); + renameDirectoryNoSync(source, destination); + // return back to original call + return; + } + if (e.httpStatusCode >= 500) { // OneDrive returned a 'HTTP 5xx Server Side Error' - gracefully handling error - error message already logged return; @@ -656,12 +695,23 @@ final class SyncEngine try { idDetails = onedrive.getPathDetailsById(driveId, id); } catch (OneDriveException e) { + log.vdebug("idDetails = onedrive.getPathDetailsById(driveId, id) generated a OneDriveException"); if (e.httpStatusCode == 404) { // id was not found - possibly a remote (shared) folder log.vlog("No details returned for given Path ID"); return; } + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling applyDifferences(driveId, id, performFullItemScan);"); + applyDifferences(driveId, id, performFullItemScan); + // return back to original call + return; + } + if (e.httpStatusCode >= 500) { // OneDrive returned a 'HTTP 5xx Server Side Error' - gracefully handling error - error message already logged return; @@ -861,6 +911,17 @@ final class SyncEngine continue; } + // HTTP request returned status code 429 (Too Many Requests) + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling applyDifferences(driveId, idToQuery, performFullItemScan);"); + applyDifferences(driveId, idToQuery, performFullItemScan); + // return back to original call + return; + } + // HTTP request returned status code 500 (Internal Server Error) if (e.httpStatusCode == 500) { // display what the error is @@ -868,13 +929,13 @@ final class SyncEngine return; } + // HTTP request returned status code 504 (Gateway Timeout) if (e.httpStatusCode == 504) { - // HTTP request returned status code 504 (Gateway Timeout) // Retry by calling applyDifferences() again - log.vlog("OneDrive returned a 'HTTP 504 - Gateway Timeout' - gracefully handling error"); + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' - retrying request"); applyDifferences(driveId, idToQuery, performFullItemScan); } else { - // Default operation if not 404, 410, 500, 504 errors + // Default operation if not 404, 410, 429, 500 or 504 errors // display what the error is displayOneDriveErrorMessage(e.msg); log.log("\nRemove your '", cfg.databaseFilePath, "' file and try to sync again\n"); @@ -1045,14 +1106,36 @@ final class SyncEngine try { oneDriveMovedNotDeleted = onedrive.getPathDetailsById(driveId, item["id"].str); } catch (OneDriveException e) { + log.vdebug("oneDriveMovedNotDeleted = onedrive.getPathDetailsById(driveId, item['id'].str); generated a OneDriveException"); if (e.httpStatusCode == 404) { // No .. that ID is GONE log.vlog("Remote change discarded - item cannot be found"); return; } - if (e.httpStatusCode >= 500) { - // OneDrive returned a 'HTTP 5xx Server Side Error' - gracefully handling error - error message already logged + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry request after delay + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling oneDriveMovedNotDeleted = onedrive.getPathDetailsById(driveId, item['id'].str);"); + try { + oneDriveMovedNotDeleted = onedrive.getPathDetailsById(driveId, item["id"].str); + } catch (OneDriveException e) { + // A further error was generated + // Rather than retry original function, retry the actual call and replicate error handling + if (e.httpStatusCode == 404) { + // No .. that ID is GONE + log.vlog("Remote change discarded - item cannot be found"); + return; + } else { + // not a 404 + displayOneDriveErrorMessage(e.msg); + return; + } + } + } else { + // not a 404 or a 429 + displayOneDriveErrorMessage(e.msg); return; } } @@ -1618,25 +1701,77 @@ final class SyncEngine try { onedrive.downloadById(item.driveId, item.id, path, fileSize); } catch (OneDriveException e) { - if ((e.httpStatusCode == 429) || (e.httpStatusCode == 408)) { - // HTTP request returned status code 429 (Too Many Requests) - // https://github.com/abraunegg/onedrive/issues/133 - // or 408 request timeout + log.vdebug("onedrive.downloadById(item.driveId, item.id, path, fileSize); generated a OneDriveException"); + // 408 = Request Time Out + // 429 = Too Many Requests - need to delay + + if (e.httpStatusCode == 408) { + // 408 error handling - request time out // https://github.com/abraunegg/onedrive/issues/694 // Back off & retry with incremental delay int retryCount = 10; int retryAttempts = 1; int backoffInterval = 2; while (retryAttempts < retryCount){ + // retry in 2,4,8,16,32,64,128,256,512,1024 seconds Thread.sleep(dur!"seconds"(retryAttempts*backoffInterval)); try { onedrive.downloadById(item.driveId, item.id, path, fileSize); // successful download retryAttempts = retryCount; } catch (OneDriveException e) { + log.vdebug("onedrive.downloadById(item.driveId, item.id, path, fileSize); generated a OneDriveException"); if ((e.httpStatusCode == 429) || (e.httpStatusCode == 408)) { - // Increment & loop around - retryAttempts++; + // If another 408 .. + if (e.httpStatusCode == 408) { + // Increment & loop around + log.vdebug("HTTP 408 generated - incrementing retryAttempts"); + retryAttempts++; + } + // If a 429 .. + if (e.httpStatusCode == 429) { + // Increment & loop around + handleOneDriveThrottleRequest(); + log.vdebug("HTTP 429 generated - incrementing retryAttempts"); + retryAttempts++; + } + } else { + displayOneDriveErrorMessage(e.msg); + } + } + } + } + + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests) + // https://github.com/abraunegg/onedrive/issues/133 + int retryCount = 10; + int retryAttempts = 1; + while (retryAttempts < retryCount){ + // retry after waiting the timeout value from the 429 HTTP response header Retry-After + handleOneDriveThrottleRequest(); + try { + onedrive.downloadById(item.driveId, item.id, path, fileSize); + // successful download + retryAttempts = retryCount; + } catch (OneDriveException e) { + log.vdebug("onedrive.downloadById(item.driveId, item.id, path, fileSize); generated a OneDriveException"); + if ((e.httpStatusCode == 429) || (e.httpStatusCode == 408)) { + // If another 408 .. + if (e.httpStatusCode == 408) { + // Increment & loop around + log.vdebug("HTTP 408 generated - incrementing retryAttempts"); + retryAttempts++; + } + // If a 429 .. + if (e.httpStatusCode == 429) { + // Increment & loop around + handleOneDriveThrottleRequest(); + log.vdebug("HTTP 429 generated - incrementing retryAttempts"); + retryAttempts++; + } + } else { + displayOneDriveErrorMessage(e.msg); } } } @@ -2029,6 +2164,7 @@ final class SyncEngine } if (e.httpStatusCode == 504) { // HTTP request returned status code 504 (Gateway Timeout) + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' - retrying upload request as a session"); // Try upload as a session response = session.upload(path, item.driveId, item.parentId, baseName(path), item.eTag); } else { @@ -2491,6 +2627,7 @@ final class SyncEngine log.vdebug("Attempting to query OneDrive for this parent path: ", parentPath); onedrivePathDetails = onedrive.getPathDetails(parentPath); } catch (OneDriveException e) { + log.vdebug("onedrivePathDetails = onedrive.getPathDetails(parentPath); generated a OneDriveException"); // exception - set onedriveParentRootDetails to a blank valid JSON onedrivePathDetails = parseJSON("{}"); if (e.httpStatusCode == 404) { @@ -2499,6 +2636,16 @@ final class SyncEngine uploadCreateDir(parentPath); } + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling uploadCreateDir(path);"); + uploadCreateDir(path); + // return back to original call + return; + } + if (e.httpStatusCode >= 500) { // OneDrive returned a 'HTTP 5xx Server Side Error' - gracefully handling error - error message already logged return; @@ -2525,6 +2672,7 @@ final class SyncEngine log.vdebug("Attempting to query OneDrive for this path: ", path); response = onedrive.getPathDetails(path); } catch (OneDriveException e) { + log.vdebug("response = onedrive.getPathDetails(path); generated a OneDriveException"); if (e.httpStatusCode == 404) { // The directory was not found log.vlog("The requested directory to create was not found on OneDrive - creating remote directory: ", path); @@ -2567,6 +2715,16 @@ final class SyncEngine return; } + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling uploadCreateDir(path);"); + uploadCreateDir(path); + // return back to original call + return; + } + if (e.httpStatusCode >= 500) { // OneDrive returned a 'HTTP 5xx Server Side Error' - gracefully handling error - error message already logged return; @@ -2648,6 +2806,7 @@ final class SyncEngine // test if the local path exists on OneDrive fileDetailsFromOneDrive = onedrive.getPathDetails(path); } catch (OneDriveException e) { + log.vdebug("fileDetailsFromOneDrive = onedrive.getPathDetails(path); generated a OneDriveException"); if (e.httpStatusCode == 401) { // OneDrive returned a 'HTTP/1.1 401 Unauthorized Error' log.vlog("Skipping item - OneDrive returned a 'HTTP 401 - Unauthorized' when attempting to query if file exists"); @@ -2673,11 +2832,35 @@ final class SyncEngine response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path)); } catch (OneDriveException e) { // error uploading file - // display what the error is - writeln("skipped."); - displayOneDriveErrorMessage(e.msg); - uploadFailed = true; - return; + if (e.httpStatusCode == 401) { + // OneDrive returned a 'HTTP/1.1 401 Unauthorized Error' - file failed to be uploaded + writeln("skipped."); + log.vlog("OneDrive returned a 'HTTP 401 - Unauthorized' - gracefully handling error"); + uploadFailed = true; + return; + } + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + uploadNewFile(path); + // return back to original call + return; + } + if (e.httpStatusCode == 504) { + // HTTP request returned status code 504 (Gateway Timeout) + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' - retrying upload request"); + // Retry original request by calling function again to avoid replicating any further error handling + uploadNewFile(path); + // return back to original call + return; + } else { + // display what the error is + writeln("skipped."); + displayOneDriveErrorMessage(e.msg); + uploadFailed = true; + return; + } } catch (FileException e) { // display the error message writeln("skipped."); @@ -2703,18 +2886,38 @@ final class SyncEngine uploadFailed = true; return; } + + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + uploadNewFile(path); + // return back to original call + return; + } + if (e.httpStatusCode == 504) { // HTTP request returned status code 504 (Gateway Timeout) + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' - retrying upload request as a session"); // Try upload as a session try { response = session.upload(path, parent.driveId, parent.id, baseName(path)); } catch (OneDriveException e) { // error uploading file - // display what the error is - writeln("skipped."); - displayOneDriveErrorMessage(e.msg); - uploadFailed = true; - return; + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + uploadNewFile(path); + // return back to original call + return; + } else { + // display what the error is + writeln("skipped."); + displayOneDriveErrorMessage(e.msg); + uploadFailed = true; + return; + } } } else { // display what the error is @@ -2742,6 +2945,22 @@ final class SyncEngine log.vlog("OneDrive returned a 'HTTP 401 - Unauthorized' - gracefully handling error"); uploadFailed = true; return; + } + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + uploadNewFile(path); + // return back to original call + return; + } + if (e.httpStatusCode == 504) { + // HTTP request returned status code 504 (Gateway Timeout) + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' - retrying upload request"); + // Retry original request by calling function again to avoid replicating any further error handling + uploadNewFile(path); + // return back to original call + return; } else { // display what the error is writeln("skipped."); @@ -2769,6 +2988,22 @@ final class SyncEngine log.vlog("OneDrive returned a 'HTTP 401 - Unauthorized' - gracefully handling error"); uploadFailed = true; return; + } + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + uploadNewFile(path); + // return back to original call + return; + } + if (e.httpStatusCode == 504) { + // HTTP request returned status code 504 (Gateway Timeout) + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' - retrying upload request"); + // Retry original request by calling function again to avoid replicating any further error handling + uploadNewFile(path); + // return back to original call + return; } else { // display what the error is writeln("skipped."); @@ -2870,6 +3105,16 @@ final class SyncEngine return; } } + + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling uploadNewFile(path);"); + uploadNewFile(path); + // return back to original call + return; + } if (e.httpStatusCode >= 500) { // OneDrive returned a 'HTTP 5xx Server Side Error' - gracefully handling error - error message already logged @@ -2910,6 +3155,7 @@ final class SyncEngine response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path)); writeln("done."); } catch (OneDriveException e) { + log.vdebug("response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path)); generated a OneDriveException"); if (e.httpStatusCode == 401) { // OneDrive returned a 'HTTP/1.1 401 Unauthorized Error' - file failed to be uploaded writeln("skipped."); @@ -2917,19 +3163,40 @@ final class SyncEngine uploadFailed = true; return; } + + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling uploadNewFile(path);"); + uploadNewFile(path); + // return back to original call + return; + } + if (e.httpStatusCode == 504) { // HTTP request returned status code 504 (Gateway Timeout) + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' - retrying upload request as a session"); // Try upload as a session try { response = session.upload(path, parent.driveId, parent.id, baseName(path)); writeln("done."); } catch (OneDriveException e) { - // error uploading file - // display what the error is - writeln("skipped."); - displayOneDriveErrorMessage(e.msg); - uploadFailed = true; - return; + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + uploadNewFile(path); + // return back to original call + return; + } else { + // error uploading file + // display what the error is + writeln("skipped."); + displayOneDriveErrorMessage(e.msg); + uploadFailed = true; + return; + } } } else { // display what the error is @@ -2952,13 +3219,32 @@ final class SyncEngine response = session.upload(path, parent.driveId, parent.id, baseName(path)); writeln("done."); } catch (OneDriveException e) { + log.vdebug("response = session.upload(path, parent.driveId, parent.id, baseName(path)); generated a OneDriveException"); if (e.httpStatusCode == 401) { // OneDrive returned a 'HTTP/1.1 401 Unauthorized Error' - file failed to be uploaded writeln("skipped."); log.vlog("OneDrive returned a 'HTTP 401 - Unauthorized' - gracefully handling error"); uploadFailed = true; return; + } + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling uploadNewFile(path);"); + uploadNewFile(path); + // return back to original call + return; + } + if (e.httpStatusCode == 504) { + // HTTP request returned status code 504 (Gateway Timeout) + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' - retrying upload request"); + // Retry original request by calling function again to avoid replicating any further error handling + uploadNewFile(path); + // return back to original call + return; } else { + // error uploading file // display what the error is writeln("skipped."); displayOneDriveErrorMessage(e.msg); @@ -3015,13 +3301,32 @@ final class SyncEngine try { response = session.upload(path, parent.driveId, parent.id, baseName(path), fileDetailsFromOneDrive["eTag"].str); } catch (OneDriveException e) { + log.vdebug("response = session.upload(path, parent.driveId, parent.id, baseName(path), fileDetailsFromOneDrive['eTag'].str); generated a OneDriveException"); if (e.httpStatusCode == 401) { // OneDrive returned a 'HTTP/1.1 401 Unauthorized Error' - file failed to be uploaded writeln("skipped."); log.vlog("OneDrive returned a 'HTTP 401 - Unauthorized' - gracefully handling error"); uploadFailed = true; return; + } + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling uploadNewFile(path);"); + uploadNewFile(path); + // return back to original call + return; + } + if (e.httpStatusCode == 504) { + // HTTP request returned status code 504 (Gateway Timeout) + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' - retrying upload request"); + // Retry original request by calling function again to avoid replicating any further error handling + uploadNewFile(path); + // return back to original call + return; } else { + // error uploading file // display what the error is writeln("skipped."); displayOneDriveErrorMessage(e.msg); @@ -3601,11 +3906,35 @@ final class SyncEngine try { onedrivePathDetails = onedrive.getPathDetails(path); // Returns a JSON String for the OneDrive Path } catch (OneDriveException e) { + log.vdebug("onedrivePathDetails = onedrive.getPathDetails(path); generated a OneDriveException"); if (e.httpStatusCode == 404) { // Requested path could not be found log.error("ERROR: The requested path to query was not found on OneDrive"); return; } + + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling queryDriveForChanges(path);"); + queryDriveForChanges(path); + // return back to original call + return; + } + + if (e.httpStatusCode == 504) { + // HTTP request returned status code 504 (Gateway Timeout) + log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' - retrying request"); + // Retry original request by calling function again to avoid replicating any further error handling + queryDriveForChanges(path); + // return back to original call + return; + } else { + // display what the error is + displayOneDriveErrorMessage(e.msg); + return; + } } if(isItemRemote(onedrivePathDetails)){ @@ -3645,7 +3974,28 @@ final class SyncEngine } // Query OneDrive changes - changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink); + try { + changes = onedrive.viewChangesById(driveId, idToQuery, deltaLink); + } catch (OneDriveException e) { + // OneDrive threw an error + log.vdebug("OneDrive threw an error when querying for these changes:"); + log.vdebug("driveId: ", driveId); + log.vdebug("idToQuery: ", idToQuery); + log.vdebug("deltaLink: ", deltaLink); + + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. + handleOneDriveThrottleRequest(); + // Retry original request by calling function again to avoid replicating any further error handling + log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling queryDriveForChanges(path);"); + queryDriveForChanges(path); + // return back to original call + return; + } else { + displayOneDriveErrorMessage(e.msg); + return; + } + } // Are there any changes on OneDrive? if (count(changes["value"].array) != 0) { @@ -3778,5 +4128,36 @@ final class SyncEngine log.vdebug("Generated Fake OneDrive Response: ", fakeResponse); return fakeResponse; } - + + void handleOneDriveThrottleRequest() { + // If OneDrive sends a status code 429 then this function will be used to process the Retry-After response header which contains the value by which we need to wait + log.vdebug("Handling a OneDrive HTTP 429 Response Code (Too Many Requests)"); + // Read in the Retry-After HTTP header as set and delay as per this value before retrying the request + auto retryAfterValue = onedrive.getRetryAfterValue(); + log.vdebug("Using Retry-After Value = ", retryAfterValue); + + // HTTP request returned status code 429 (Too Many Requests) + // https://github.com/abraunegg/onedrive/issues/133 + // https://github.com/abraunegg/onedrive/issues/815 + + ulong delayBeforeRetry = 0; + if (retryAfterValue != 0) { + // Use the HTTP Response Header Value + delayBeforeRetry = retryAfterValue; + } else { + // Use a 120 second delay as a default given header value was zero + // This value is based on log files and data when determining correct process for 429 response handling + delayBeforeRetry = 120; + // Update that we are over-riding the provided value with a default + log.vdebug("HTTP Response Header retry-after value was 0 - Using a preconfigured default of: ", delayBeforeRetry); + } + + // Sleep thread as per request + log.log("Thread sleeping due to 'HTTP request returned status code 429' - The request has been throttled"); + log.log("Sleeping for ", delayBeforeRetry, " seconds"); + Thread.sleep(dur!"seconds"(delayBeforeRetry)); + + // Reset retry-after value to zero as we have used this value now and it may be changed in the future to a different value + onedrive.resetRetryAfterValue(); + } } diff --git a/src/upload.d b/src/upload.d index 97644dea..275ac46a 100644 --- a/src/upload.d +++ b/src/upload.d @@ -190,12 +190,25 @@ struct UploadSession ); } catch (OneDriveException e) { // there was an error response from OneDrive when uploading the file fragment - // insert a new line as well, so that the below error is inserted on the console in the right location - log.vlog("\nFragment upload failed - received an exception response from OneDrive"); - // display what the error is - displayOneDriveErrorMessage(e.msg); - // retry fragment upload in case error is transient - log.vlog("Retrying fragment upload"); + // handle 'HTTP request returned status code 429 (Too Many Requests)' first + if (e.httpStatusCode == 429) { + auto retryAfterValue = onedrive.getRetryAfterValue(); + log.vdebug("Fragment upload failed - received throttle request response from OneDrive"); + log.vdebug("Using Retry-After Value = ", retryAfterValue); + // Sleep thread as per request + log.log("\nThread sleeping due to 'HTTP request returned status code 429' - The request has been throttled"); + log.log("Sleeping for ", retryAfterValue, " seconds"); + Thread.sleep(dur!"seconds"(retryAfterValue)); + log.log("Retrying fragment upload"); + } else { + // insert a new line as well, so that the below error is inserted on the console in the right location + log.vlog("\nFragment upload failed - received an exception response from OneDrive"); + // display what the error is + displayOneDriveErrorMessage(e.msg); + // retry fragment upload in case error is transient + log.vlog("Retrying fragment upload"); + } + try { response = onedrive.uploadFragment( session["uploadUrl"].str,