deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-06-15 20:25:18 +02:00
Code Issues Projects Releases Wiki Activity

Update PR

Browse source 
* Update PR
This commit is contained in:
abraunegg 2023-08-31 06:29:24 +10:00
parent 44be502c62
commit eb837dbe1f
5 changed files with 150 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
src/curlEngine.d
View file

@ -60,8 +60,11 @@ class CurlEngine {
// Ensure that TCP_NODELAY is set to 0 to ensure that TCP NAGLE is enabled // Ensure that TCP_NODELAY is set to 0 to ensure that TCP NAGLE is enabled
http.handle.set(CurlOption.tcp_nodelay,0); http.handle.set(CurlOption.tcp_nodelay,0);
// https://curl.se/libcurl/c/CURLOPT_FORBID_REUSE.html // https://curl.se/libcurl/c/CURLOPT_FORBID_REUSE.html
// Ensure that we ARE reusing connections - setting to 0 ensures that we are reusing connections // CURLOPT_FORBID_REUSE - make connection get closed at once after use
http.handle.set(CurlOption.forbid_reuse,0); // Ensure that we ARE NOT reusing TCP sockets connections - setting to 0 ensures that we ARE reusing connections (we did this in v2.4.xx)
// Setting this to 1 ensures that when we close the curl instance, any open sockets are closed - which we need to do when running
// multiple threads and API instances at the same time otherwise we run out of local files | sockets pretty quickly
http.handle.set(CurlOption.forbid_reuse,1);
if (httpsDebug) { if (httpsDebug) {
// Output what options we are using so that in the debug log this can be tracked // Output what options we are using so that in the debug log this can be tracked
@ -73,11 +76,16 @@ class CurlEngine {
} }
} }
void setMethodPost(){ void setMethodPost() {
http.method = HTTP.Method.post; http.method = HTTP.Method.post;
} }
void setMethodPatch(){ void setMethodPatch() {
http.method = HTTP.Method.patch; http.method = HTTP.Method.patch;
} }
void setDisableSSLVerifyPeer() {
log.vdebug("Switching off CurlOption.ssl_verifypeer");
http.handle.set(CurlOption.ssl_verifypeer, 0);
}
} }

66
src/main.d
View file

@ -415,7 +415,17 @@ int main(string[] cliArgs) {
// Are we doing a --monitor operation? // Are we doing a --monitor operation?
if (appConfig.getValueBool("monitor")) { if (appConfig.getValueBool("monitor")) {
// What are the current values for the platform we are running on
// Max number of open files /proc/sys/fs/file-max
string maxOpenFiles = strip(readText("/proc/sys/fs/file-max"));
// What is the currently configured maximum inotify watches that can be used
// /proc/sys/user/max_inotify_watches
string maxInotifyWatches = strip(readText("/proc/sys/user/max_inotify_watches"));
// Start the monitor process
log.log("OneDrive syncronisation interval (seconds): ", appConfig.getValueLong("monitor_interval")); log.log("OneDrive syncronisation interval (seconds): ", appConfig.getValueLong("monitor_interval"));
log.vlog("Maximum allowed open files: ", maxOpenFiles);
log.vlog("Maximum allowed inotify watches: ", maxInotifyWatches);
// Configure the monitor class // Configure the monitor class
Monitor filesystemMonitor = new Monitor(appConfig, selectiveSync); Monitor filesystemMonitor = new Monitor(appConfig, selectiveSync);
@ -515,10 +525,9 @@ int main(string[] cliArgs) {
string loopStopOutputMessage = "################################################ LOOP COMPLETE ###############################################"; string loopStopOutputMessage = "################################################ LOOP COMPLETE ###############################################";
while (performMonitor) { while (performMonitor) {
try { try {
// Process any inotify events // Process any inotify events
filesystemMonitor.update(online); filesystemMonitor.update(true);
} catch (MonitorException e) { } catch (MonitorException e) {
// Catch any exceptions thrown by inotify / monitor engine // Catch any exceptions thrown by inotify / monitor engine
log.error("ERROR: The following inotify error was generated: ", e.msg); log.error("ERROR: The following inotify error was generated: ", e.msg);
@ -529,7 +538,6 @@ int main(string[] cliArgs) {
// Check here for a webhook notification // Check here for a webhook notification
// Get the current time this loop is starting // Get the current time this loop is starting
auto currentTime = MonoTime.currTime(); auto currentTime = MonoTime.currTime();
@ -554,12 +562,15 @@ int main(string[] cliArgs) {
// Did the user specify --upload-only? // Did the user specify --upload-only?
if (appConfig.getValueBool("upload_only")) { if (appConfig.getValueBool("upload_only")) {
// Perform the --upload-only sync process // Perform the --upload-only sync process
performUploadOnlySyncProcess(localPath); performUploadOnlySyncProcess(localPath, filesystemMonitor);
} else { } else {
// Perform the standard sync process // Perform the standard sync process
performStandardSyncProcess(localPath); performStandardSyncProcess(localPath, filesystemMonitor);
} }
// Discard any inotify events generated as part of any sync operation
filesystemMonitor.update(false);
// Detail the outcome of the sync process // Detail the outcome of the sync process
displaySyncOutcome(); displaySyncOutcome();
@ -568,7 +579,7 @@ int main(string[] cliArgs) {
itemDB.performVacuum(); itemDB.performVacuum();
} else { } else {
// Not online // Not online
log.log("Microsoft OneDrive service is not reachable at this time"); log.log("Microsoft OneDrive service is not reachable at this time. Will re-try on next loop attempt.");
} }
// Output end of loop processing times // Output end of loop processing times
@ -585,7 +596,6 @@ int main(string[] cliArgs) {
Thread.sleep(dur!"seconds"(1)); Thread.sleep(dur!"seconds"(1));
} }
} }
} else { } else {
// Exit application as the sync engine could not be initialised // Exit application as the sync engine could not be initialised
log.error("Application Sync Engine could not be initialised correctly"); log.error("Application Sync Engine could not be initialised correctly");
@ -603,35 +613,71 @@ int main(string[] cliArgs) {
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }
void performUploadOnlySyncProcess(string localPath) { void performUploadOnlySyncProcess(string localPath, Monitor filesystemMonitor = null) {
// Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive // Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive
syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck();
if (appConfig.getValueBool("monitor")) {
// Handle any inotify events whilst the DB was being scanned
filesystemMonitor.update(true);
}
// Scan the configured 'sync_dir' for new data to upload // Scan the configured 'sync_dir' for new data to upload
syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); syncEngineInstance.scanLocalFilesystemPathForNewData(localPath);
if (appConfig.getValueBool("monitor")) {
// Handle any new inotify events whilst the local filesystem was being scanned
filesystemMonitor.update(true);
}
} }
void performStandardSyncProcess(string localPath) { void performStandardSyncProcess(string localPath, Monitor filesystemMonitor = null) {
// Which way do we sync first? // Which way do we sync first?
// OneDrive first then local changes (normal operational process that uses OneDrive as the source of truth) // OneDrive first then local changes (normal operational process that uses OneDrive as the source of truth)
// Local First then OneDrive changes (alternate operation process to use local files as source of truth) // Local First then OneDrive changes (alternate operation process to use local files as source of truth)
if (appConfig.getValueBool("local_first")) { if (appConfig.getValueBool("local_first")) {
// Local data first // Local data first
// Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive // Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive
syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck();
if (appConfig.getValueBool("monitor")) {
// Handle any inotify events whilst the DB was being scanned
filesystemMonitor.update(true);
}
// Scan the configured 'sync_dir' for new data to upload to OneDrive // Scan the configured 'sync_dir' for new data to upload to OneDrive
syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); syncEngineInstance.scanLocalFilesystemPathForNewData(localPath);
if (appConfig.getValueBool("monitor")) {
// Handle any new inotify events whilst the local filesystem was being scanned
filesystemMonitor.update(true);
}
// Download data from OneDrive last // Download data from OneDrive last
syncEngineInstance.syncOneDriveAccountToLocalDisk(); syncEngineInstance.syncOneDriveAccountToLocalDisk();
if (appConfig.getValueBool("monitor")) {
// Cancel out any inotify events from downloading data
filesystemMonitor.update(false);
}
} else { } else {
// Normal sync // Normal sync
// Download data from OneDrive first // Download data from OneDrive first
syncEngineInstance.syncOneDriveAccountToLocalDisk(); syncEngineInstance.syncOneDriveAccountToLocalDisk();
if (appConfig.getValueBool("monitor")) {
// Cancel out any inotify events from downloading data
filesystemMonitor.update(false);
}
// Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive // Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive
syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck(); syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck();
if (appConfig.getValueBool("monitor")) {
// Handle any inotify events whilst the DB was being scanned
filesystemMonitor.update(true);
}
// Scan the configured 'sync_dir' for new data to upload to OneDrive // Scan the configured 'sync_dir' for new data to upload to OneDrive
syncEngineInstance.scanLocalFilesystemPathForNewData(localPath); syncEngineInstance.scanLocalFilesystemPathForNewData(localPath);
if (appConfig.getValueBool("monitor")) {
// Handle any new inotify events whilst the local filesystem was being scanned
filesystemMonitor.update(true);
}
} }
} }

13
src/monitor.d
View file

@ -14,6 +14,7 @@ import std.path;
import std.regex; import std.regex;
import std.stdio; import std.stdio;
import std.string; import std.string;
import std.conv;
// What other modules that we have created do we need to import? // What other modules that we have created do we need to import?
import config; import config;
@ -197,11 +198,13 @@ final class Monitor {
int wd = inotify_add_watch(fd, toStringz(pathname), mask); int wd = inotify_add_watch(fd, toStringz(pathname), mask);
if (wd < 0) { if (wd < 0) {
if (errno() == ENOSPC) { if (errno() == ENOSPC) {
// Get the current value
ulong maxInotifyWatches = to!int(strip(readText("/proc/sys/user/max_inotify_watches")));
log.log("The user limit on the total number of inotify watches has been reached."); log.log("The user limit on the total number of inotify watches has been reached.");
log.log("To see the current max number of watches run:"); log.log("Your current limit of inotify watches is: ", maxInotifyWatches);
log.log("sysctl fs.inotify.max_user_watches"); log.log("It is recommended that you change the max number of inotify watches to at least double your existing value.");
log.log("To change the current max number of watches to 524288 run:"); log.log("To change the current max number of watches to " , (maxInotifyWatches * 2) , " run:");
log.log("sudo sysctl fs.inotify.max_user_watches=524288"); log.log("EXAMPLE: sudo sysctl fs.inotify.max_user_watches=", (maxInotifyWatches * 2));
} }
if (errno() == 13) { if (errno() == 13) {
if ((selectiveSync.getSkipDotfiles()) && (isDotFile(pathname))) { if ((selectiveSync.getSkipDotfiles()) && (isDotFile(pathname))) {
@ -404,7 +407,7 @@ final class Monitor {
cookieToPath.remove(cookie); cookieToPath.remove(cookie);
} }
log.vdebug("inotify events flushed"); log.log("inotify events flushed");
} }
} }
} }

59
src/onedrive.d
View file

@ -684,7 +684,12 @@ class OneDriveApi {
response = post(tokenUrl, postData); response = post(tokenUrl, postData);
} catch (OneDriveException e) { } catch (OneDriveException e) {
// an error was generated // an error was generated
displayOneDriveErrorMessage(e.msg, getFunctionName!({})); if (e.httpStatusCode >= 500) {
// There was a HTTP 5xx Server Side Error - retry
acquireToken(postData);
} else {
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
}
} }
if (response.type() == JSONType.object) { if (response.type() == JSONType.object) {
@ -748,7 +753,7 @@ class OneDriveApi {
authorise(); authorise();
} }
} else { } else {
log.log("Invalid response from the OneDrive API. Unable to initialise application."); log.log("Invalid response from the OneDrive API. Unable to initialise OneDrive API instance.");
exit(-1); exit(-1);
} }
} }
@ -1109,20 +1114,40 @@ class OneDriveApi {
throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?"); throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?");
} }
} else { } else {
// Log that an error was returned
log.error("ERROR: OneDrive returned an error with the following message:"); // what error was returned?
// Some other error was returned if (canFind(errorMessage, "Problem with the SSL CA cert (path? access rights?) on handle")) {
log.error(" Error Message: ", errorMessage); // error setting certificate verify locations:
log.error(" Calling Function: ", getFunctionName!({})); // CAfile: /etc/pki/tls/certs/ca-bundle.crt
// CApath: none
//
// Tell the Curl Engine to bypass SSL check - essentially SSL is passing back a bad value due to 'stdio' compile time option
// Further reading:
// https://github.com/curl/curl/issues/6090
// https://github.com/openssl/openssl/issues/7536
// https://stackoverflow.com/questions/45829588/brew-install-fails-curl77-error-setting-certificate-verify
// https://forum.dlang.org/post/vwvkbubufexgeuaxhqfl@forum.dlang.org
log.vdebug("Problem with reading the SSL CA cert via libcurl - attempting work around");
curlEngine.setDisableSSLVerifyPeer();
// retry origional call
performHTTPOperation();
} else {
// Log that an error was returned
log.error("ERROR: OneDrive returned an error with the following message:");
// Some other error was returned
log.error(" Error Message: ", errorMessage);
log.error(" Calling Function: ", getFunctionName!({}));
// Was this a curl initialization error? // Was this a curl initialization error?
if (canFind(errorMessage, "Failed initialization on handle")) { if (canFind(errorMessage, "Failed initialization on handle")) {
// initialization error ... prevent a run-away process if we have zero disk space // initialization error ... prevent a run-away process if we have zero disk space
ulong localActualFreeSpace = getAvailableDiskSpace("."); ulong localActualFreeSpace = getAvailableDiskSpace(".");
if (localActualFreeSpace == 0) { if (localActualFreeSpace == 0) {
// force exit // force exit
shutdown(); shutdown();
exit(-1); exit(-1);
}
} }
} }
} }
@ -1179,10 +1204,6 @@ class OneDriveApi {
return response; return response;
} }
private void checkHTTPResponseHeaders() { private void checkHTTPResponseHeaders() {
// Get the HTTP Response headers - needed for correct 429 handling // Get the HTTP Response headers - needed for correct 429 handling
auto responseHeaders = curlEngine.http.responseHeaders(); auto responseHeaders = curlEngine.http.responseHeaders();

58
src/sync.d
View file

@ -4564,11 +4564,16 @@ class SyncEngine {
pathToQuery = "."; pathToQuery = ".";
} }
// Create new OneDrive API Instance
OneDriveApi generateDeltaResponseOneDriveApiInstance;
generateDeltaResponseOneDriveApiInstance = new OneDriveApi(appConfig);
generateDeltaResponseOneDriveApiInstance.initialise();
if (!singleDirectoryScope) { if (!singleDirectoryScope) {
// In a --resync scenario, there is no DB data to query, so we have to query the OneDrive API here to get relevant details // In a --resync scenario, there is no DB data to query, so we have to query the OneDrive API here to get relevant details
try { try {
// Query the OneDrive API // Query the OneDrive API
pathData = oneDriveApiInstance.getPathDetails(pathToQuery); pathData = generateDeltaResponseOneDriveApiInstance.getPathDetails(pathToQuery);
// Is the path on OneDrive local or remote to our account drive id? // Is the path on OneDrive local or remote to our account drive id?
if (isItemRemote(pathData)) { if (isItemRemote(pathData)) {
// The path we are seeking is remote to our account drive id // The path we are seeking is remote to our account drive id
@ -4583,7 +4588,7 @@ class SyncEngine {
// Display error message // Display error message
displayOneDriveErrorMessage(e.msg, getFunctionName!({})); displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
// Must exit here // Must exit here
oneDriveApiInstance.shutdown(); generateDeltaResponseOneDriveApiInstance.shutdown();
exit(-1); exit(-1);
} }
} else { } else {
@ -4610,27 +4615,27 @@ class SyncEngine {
// Get drive details for the provided driveId // Get drive details for the provided driveId
try { try {
driveData = oneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id); driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id);
} catch (OneDriveException e) { } catch (OneDriveException e) {
log.vdebug("driveData = oneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id) generated a OneDriveException"); log.vdebug("driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id) generated a OneDriveException");
// HTTP request returned status code 504 (Gateway Timeout) or 429 retry // HTTP request returned status code 504 (Gateway Timeout) or 429 retry
if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) {
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
if (e.httpStatusCode == 429) { if (e.httpStatusCode == 429) {
log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - retrying applicable request"); log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - retrying applicable request");
handleOneDriveThrottleRequest(oneDriveApiInstance); handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance);
} }
if (e.httpStatusCode == 504) { if (e.httpStatusCode == 504) {
log.vdebug("Retrying original request that generated the HTTP 504 (Gateway Timeout) - retrying applicable request"); log.vdebug("Retrying original request that generated the HTTP 504 (Gateway Timeout) - retrying applicable request");
Thread.sleep(dur!"seconds"(30)); Thread.sleep(dur!"seconds"(30));
} }
// Retry original request by calling function again to avoid replicating any further error handling // Retry original request by calling function again to avoid replicating any further error handling
driveData = oneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id); driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id);
} else { } else {
// There was a HTTP 5xx Server Side Error // There was a HTTP 5xx Server Side Error
displayOneDriveErrorMessage(e.msg, getFunctionName!({})); displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
// Must exit here // Must exit here
oneDriveApiInstance.shutdown(); generateDeltaResponseOneDriveApiInstance.shutdown();
exit(-1); exit(-1);
} }
} }
@ -4639,7 +4644,7 @@ class SyncEngine {
if (!isItemRoot(driveData)) { if (!isItemRoot(driveData)) {
// Get root details for the provided driveId // Get root details for the provided driveId
try { try {
rootData = oneDriveApiInstance.getDriveIdRoot(searchItem.driveId); rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId);
} catch (OneDriveException e) { } catch (OneDriveException e) {
log.vdebug("rootData = onedrive.getDriveIdRoot(searchItem.driveId) generated a OneDriveException"); log.vdebug("rootData = onedrive.getDriveIdRoot(searchItem.driveId) generated a OneDriveException");
// HTTP request returned status code 504 (Gateway Timeout) or 429 retry // HTTP request returned status code 504 (Gateway Timeout) or 429 retry
@ -4647,20 +4652,20 @@ class SyncEngine {
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
if (e.httpStatusCode == 429) { if (e.httpStatusCode == 429) {
log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - retrying applicable request"); log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - retrying applicable request");
handleOneDriveThrottleRequest(oneDriveApiInstance); handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance);
} }
if (e.httpStatusCode == 504) { if (e.httpStatusCode == 504) {
log.vdebug("Retrying original request that generated the HTTP 504 (Gateway Timeout) - retrying applicable request"); log.vdebug("Retrying original request that generated the HTTP 504 (Gateway Timeout) - retrying applicable request");
Thread.sleep(dur!"seconds"(30)); Thread.sleep(dur!"seconds"(30));
} }
// Retry original request by calling function again to avoid replicating any further error handling // Retry original request by calling function again to avoid replicating any further error handling
rootData = oneDriveApiInstance.getDriveIdRoot(searchItem.driveId); rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId);
} else { } else {
// There was a HTTP 5xx Server Side Error // There was a HTTP 5xx Server Side Error
displayOneDriveErrorMessage(e.msg, getFunctionName!({})); displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
// Must exit here // Must exit here
oneDriveApiInstance.shutdown(); generateDeltaResponseOneDriveApiInstance.shutdown();
exit(-1); exit(-1);
} }
} }
@ -4677,11 +4682,11 @@ class SyncEngine {
for (;;) { for (;;) {
// query top level children // query top level children
try { try {
topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink); topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink);
} catch (OneDriveException e) { } catch (OneDriveException e) {
// OneDrive threw an error // OneDrive threw an error
log.vdebug("------------------------------------------------------------------"); log.vdebug("------------------------------------------------------------------");
log.vdebug("Query Error: topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)"); log.vdebug("Query Error: topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)");
log.vdebug("driveId: ", searchItem.driveId); log.vdebug("driveId: ", searchItem.driveId);
log.vdebug("idToQuery: ", searchItem.id); log.vdebug("idToQuery: ", searchItem.id);
log.vdebug("nextLink: ", nextLink); log.vdebug("nextLink: ", nextLink);
@ -4689,7 +4694,7 @@ class SyncEngine {
// HTTP request returned status code 429 (Too Many Requests) // HTTP request returned status code 429 (Too Many Requests)
if (e.httpStatusCode == 429) { if (e.httpStatusCode == 429) {
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
handleOneDriveThrottleRequest(oneDriveApiInstance); handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance);
log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children"); log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children");
} }
@ -4711,12 +4716,12 @@ class SyncEngine {
} }
// re-try original request - retried for 429 and 504 // re-try original request - retried for 429 and 504
try { try {
log.vdebug("Retrying Query: topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)"); log.vdebug("Retrying Query: topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)");
topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink); topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink);
log.vdebug("Query 'topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)' performed successfully on re-try"); log.vdebug("Query 'topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)' performed successfully on re-try");
} catch (OneDriveException e) { } catch (OneDriveException e) {
// display what the error is // display what the error is
log.vdebug("Query Error: topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink) on re-try after delay"); log.vdebug("Query Error: topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink) on re-try after delay");
// error was not a 504 this time // error was not a 504 this time
displayOneDriveErrorMessage(e.msg, getFunctionName!({})); displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
} }
@ -4843,17 +4848,22 @@ class SyncEngine {
// Query the OneDrive API for the child objects for this element // Query the OneDrive API for the child objects for this element
JSONValue queryThisLevelChildren(string driveId, string idToQuery, string nextLink) { JSONValue queryThisLevelChildren(string driveId, string idToQuery, string nextLink) {
JSONValue thisLevelChildren; JSONValue thisLevelChildren;
// Create new OneDrive API Instance
OneDriveApi queryChildrenOneDriveApiInstance;
queryChildrenOneDriveApiInstance = new OneDriveApi(appConfig);
queryChildrenOneDriveApiInstance.initialise();
// query children // query children
try { try {
// attempt API call // attempt API call
log.vdebug("Attempting Query: thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)"); log.vdebug("Attempting Query: thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)");
thisLevelChildren = oneDriveApiInstance.listChildren(driveId, idToQuery, nextLink); thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink);
log.vdebug("Query 'thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)' performed successfully"); log.vdebug("Query 'thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)' performed successfully");
} catch (OneDriveException e) { } catch (OneDriveException e) {
// OneDrive threw an error // OneDrive threw an error
log.vdebug("------------------------------------------------------------------"); log.vdebug("------------------------------------------------------------------");
log.vdebug("Query Error: thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)"); log.vdebug("Query Error: thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)");
log.vdebug("driveId: ", driveId); log.vdebug("driveId: ", driveId);
log.vdebug("idToQuery: ", idToQuery); log.vdebug("idToQuery: ", idToQuery);
log.vdebug("nextLink: ", nextLink); log.vdebug("nextLink: ", nextLink);
@ -4861,7 +4871,7 @@ class SyncEngine {
// HTTP request returned status code 429 (Too Many Requests) // HTTP request returned status code 429 (Too Many Requests)
if (e.httpStatusCode == 429) { if (e.httpStatusCode == 429) {
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
handleOneDriveThrottleRequest(oneDriveApiInstance); handleOneDriveThrottleRequest(queryChildrenOneDriveApiInstance);
log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children"); log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children");
} }
@ -4871,7 +4881,7 @@ class SyncEngine {
if (e.httpStatusCode == 504) { if (e.httpStatusCode == 504) {
// transient error - try again in 30 seconds // transient error - try again in 30 seconds
log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query OneDrive drive children - retrying applicable request"); log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query OneDrive drive children - retrying applicable request");
log.vdebug("thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink) previously threw an error - retrying"); log.vdebug("thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink) previously threw an error - retrying");
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request"); log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request");
Thread.sleep(dur!"seconds"(30)); Thread.sleep(dur!"seconds"(30));