deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-22 15:46:36 +02:00
Code Issues Projects Releases Wiki Activity

Update PR

Browse source 
* Update PR
This commit is contained in:
abraunegg 2023-08-31 06:29:24 +10:00
parent 44be502c62
commit eb837dbe1f
5 changed files with 150 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
src/curlEngine.d
View file

@ -60,8 +60,11 @@ class CurlEngine {
// Ensure that TCP_NODELAY is set to 0 to ensure that TCP NAGLE is enabled
http.handle.set(CurlOption.tcp_nodelay,0);
// https://curl.se/libcurl/c/CURLOPT_FORBID_REUSE.html
// Ensure that we ARE reusing connections - setting to 0 ensures that we are reusing connections
http.handle.set(CurlOption.forbid_reuse,0);
// CURLOPT_FORBID_REUSE - make connection get closed at once after use
// Ensure that we ARE NOT reusing TCP sockets connections - setting to 0 ensures that we ARE reusing connections (we did this in v2.4.xx)
// Setting this to 1 ensures that when we close the curl instance, any open sockets are closed - which we need to do when running
// multiple threads and API instances at the same time otherwise we run out of local files | sockets pretty quickly
http.handle.set(CurlOption.forbid_reuse,1);
if (httpsDebug) {
// Output what options we are using so that in the debug log this can be tracked
@ -73,11 +76,16 @@ class CurlEngine {
}
}
void setMethodPost(){
void setMethodPost() {
http.method = HTTP.Method.post;
}
void setMethodPatch(){
void setMethodPatch() {
http.method = HTTP.Method.patch;
}
void setDisableSSLVerifyPeer() {
log.vdebug("Switching off CurlOption.ssl_verifypeer");
http.handle.set(CurlOption.ssl_verifypeer, 0);
}
}

66
src/main.d
View file

@ -415,7 +415,17 @@ int main(string[] cliArgs) {
// Are we doing a --monitor operation?
if (appConfig.getValueBool("monitor")) {
// What are the current values for the platform we are running on
// Max number of open files /proc/sys/fs/file-max
string maxOpenFiles = strip(readText("/proc/sys/fs/file-max"));
// What is the currently configured maximum inotify watches that can be used
// /proc/sys/user/max_inotify_watches
string maxInotifyWatches = strip(readText("/proc/sys/user/max_inotify_watches"));
// Start the monitor process
log.log("OneDrive syncronisation interval (seconds): ", appConfig.getValueLong("monitor_interval"));
log.vlog("Maximum allowed open files: ", maxOpenFiles);
log.vlog("Maximum allowed inotify watches: ", maxInotifyWatches);
// Configure the monitor class
Monitor filesystemMonitor = new Monitor(appConfig, selectiveSync);
@ -515,10 +525,9 @@ int main(string[] cliArgs) {
string loopStopOutputMessage = "################################################ LOOP COMPLETE ###############################################";
while (performMonitor) {
try {
// Process any inotify events
filesystemMonitor.update(online);
filesystemMonitor.update(true);
} catch (MonitorException e) {
// Catch any exceptions thrown by inotify / monitor engine
log.error("ERROR: The following inotify error was generated: ", e.msg);
@ -529,7 +538,6 @@ int main(string[] cliArgs) {
// Check here for a webhook notification
// Get the current time this loop is starting
auto currentTime = MonoTime.currTime();
@ -554,12 +562,15 @@ int main(string[] cliArgs) {
// Did the user specify --upload-only?
if (appConfig.getValueBool("upload_only")) {
// Perform the --upload-only sync process
performUploadOnlySyncProcess(localPath);
performUploadOnlySyncProcess(localPath, filesystemMonitor);
} else {
// Perform the standard sync process
performStandardSyncProcess(localPath);
performStandardSyncProcess(localPath, filesystemMonitor);
}
// Discard any inotify events generated as part of any sync operation
filesystemMonitor.update(false);
// Detail the outcome of the sync process
displaySyncOutcome();
@ -568,7 +579,7 @@ int main(string[] cliArgs) {
itemDB.performVacuum();
} else {
// Not online
log.log("Microsoft OneDrive service is not reachable at this time");
log.log("Microsoft OneDrive service is not reachable at this time. Will re-try on next loop attempt.");
}
// Output end of loop processing times
@ -585,7 +596,6 @@ int main(string[] cliArgs) {
Thread.sleep(dur!"seconds"(1));
}
}
} else {
// Exit application as the sync engine could not be initialised
log.error("Application Sync Engine could not be initialised correctly");
@ -603,35 +613,71 @@ int main(string[] cliArgs) {
return EXIT_SUCCESS;
}
void performUploadOnlySyncProcess(string localPath) {
void performUploadOnlySyncProcess(string localPath, Monitor filesystemMonitor = null) {
// Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive
syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck();
if (appConfig.getValueBool("monitor")) {
// Handle any inotify events whilst the DB was being scanned
filesystemMonitor.update(true);
}
// Scan the configured 'sync_dir' for new data to upload
syncEngineInstance.scanLocalFilesystemPathForNewData(localPath);
if (appConfig.getValueBool("monitor")) {
// Handle any new inotify events whilst the local filesystem was being scanned
filesystemMonitor.update(true);
}
}
void performStandardSyncProcess(string localPath) {
void performStandardSyncProcess(string localPath, Monitor filesystemMonitor = null) {
// Which way do we sync first?
// OneDrive first then local changes (normal operational process that uses OneDrive as the source of truth)
// Local First then OneDrive changes (alternate operation process to use local files as source of truth)
if (appConfig.getValueBool("local_first")) {
// Local data first
// Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive
syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck();
if (appConfig.getValueBool("monitor")) {
// Handle any inotify events whilst the DB was being scanned
filesystemMonitor.update(true);
}
// Scan the configured 'sync_dir' for new data to upload to OneDrive
syncEngineInstance.scanLocalFilesystemPathForNewData(localPath);
if (appConfig.getValueBool("monitor")) {
// Handle any new inotify events whilst the local filesystem was being scanned
filesystemMonitor.update(true);
}
// Download data from OneDrive last
syncEngineInstance.syncOneDriveAccountToLocalDisk();
if (appConfig.getValueBool("monitor")) {
// Cancel out any inotify events from downloading data
filesystemMonitor.update(false);
}
} else {
// Normal sync
// Download data from OneDrive first
syncEngineInstance.syncOneDriveAccountToLocalDisk();
if (appConfig.getValueBool("monitor")) {
// Cancel out any inotify events from downloading data
filesystemMonitor.update(false);
}
// Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive
syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck();
if (appConfig.getValueBool("monitor")) {
// Handle any inotify events whilst the DB was being scanned
filesystemMonitor.update(true);
}
// Scan the configured 'sync_dir' for new data to upload to OneDrive
syncEngineInstance.scanLocalFilesystemPathForNewData(localPath);
if (appConfig.getValueBool("monitor")) {
// Handle any new inotify events whilst the local filesystem was being scanned
filesystemMonitor.update(true);
}
}
}

13
src/monitor.d
View file

@ -14,6 +14,7 @@ import std.path;
import std.regex;
import std.stdio;
import std.string;
import std.conv;
// What other modules that we have created do we need to import?
import config;
@ -197,11 +198,13 @@ final class Monitor {
int wd = inotify_add_watch(fd, toStringz(pathname), mask);
if (wd < 0) {
if (errno() == ENOSPC) {
// Get the current value
ulong maxInotifyWatches = to!int(strip(readText("/proc/sys/user/max_inotify_watches")));
log.log("The user limit on the total number of inotify watches has been reached.");
log.log("To see the current max number of watches run:");
log.log("sysctl fs.inotify.max_user_watches");
log.log("To change the current max number of watches to 524288 run:");
log.log("sudo sysctl fs.inotify.max_user_watches=524288");
log.log("Your current limit of inotify watches is: ", maxInotifyWatches);
log.log("It is recommended that you change the max number of inotify watches to at least double your existing value.");
log.log("To change the current max number of watches to " , (maxInotifyWatches * 2) , " run:");
log.log("EXAMPLE: sudo sysctl fs.inotify.max_user_watches=", (maxInotifyWatches * 2));
}
if (errno() == 13) {
if ((selectiveSync.getSkipDotfiles()) && (isDotFile(pathname))) {
@ -404,7 +407,7 @@ final class Monitor {
cookieToPath.remove(cookie);
}
log.vdebug("inotify events flushed");
log.log("inotify events flushed");
}
}
}

59
src/onedrive.d
View file

@ -684,7 +684,12 @@ class OneDriveApi {
response = post(tokenUrl, postData);
} catch (OneDriveException e) {
// an error was generated
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
if (e.httpStatusCode >= 500) {
// There was a HTTP 5xx Server Side Error - retry
acquireToken(postData);
} else {
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
}
}
if (response.type() == JSONType.object) {
@ -748,7 +753,7 @@ class OneDriveApi {
authorise();
}
} else {
log.log("Invalid response from the OneDrive API. Unable to initialise application.");
log.log("Invalid response from the OneDrive API. Unable to initialise OneDrive API instance.");
exit(-1);
}
}
@ -1109,20 +1114,40 @@ class OneDriveApi {
throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?");
}
} else {
// Log that an error was returned
log.error("ERROR: OneDrive returned an error with the following message:");
// Some other error was returned
log.error(" Error Message: ", errorMessage);
log.error(" Calling Function: ", getFunctionName!({}));
// what error was returned?
if (canFind(errorMessage, "Problem with the SSL CA cert (path? access rights?) on handle")) {
// error setting certificate verify locations:
// CAfile: /etc/pki/tls/certs/ca-bundle.crt
// CApath: none
//
// Tell the Curl Engine to bypass SSL check - essentially SSL is passing back a bad value due to 'stdio' compile time option
// Further reading:
// https://github.com/curl/curl/issues/6090
// https://github.com/openssl/openssl/issues/7536
// https://stackoverflow.com/questions/45829588/brew-install-fails-curl77-error-setting-certificate-verify
// https://forum.dlang.org/post/vwvkbubufexgeuaxhqfl@forum.dlang.org
log.vdebug("Problem with reading the SSL CA cert via libcurl - attempting work around");
curlEngine.setDisableSSLVerifyPeer();
// retry origional call
performHTTPOperation();
} else {
// Log that an error was returned
log.error("ERROR: OneDrive returned an error with the following message:");
// Some other error was returned
log.error(" Error Message: ", errorMessage);
log.error(" Calling Function: ", getFunctionName!({}));
// Was this a curl initialization error?
if (canFind(errorMessage, "Failed initialization on handle")) {
// initialization error ... prevent a run-away process if we have zero disk space
ulong localActualFreeSpace = getAvailableDiskSpace(".");
if (localActualFreeSpace == 0) {
// force exit
shutdown();
exit(-1);
// Was this a curl initialization error?
if (canFind(errorMessage, "Failed initialization on handle")) {
// initialization error ... prevent a run-away process if we have zero disk space
ulong localActualFreeSpace = getAvailableDiskSpace(".");
if (localActualFreeSpace == 0) {
// force exit
shutdown();
exit(-1);
}
}
}
}
@ -1179,10 +1204,6 @@ class OneDriveApi {
return response;
}
private void checkHTTPResponseHeaders() {
// Get the HTTP Response headers - needed for correct 429 handling
auto responseHeaders = curlEngine.http.responseHeaders();

58
src/sync.d
View file

@ -4564,11 +4564,16 @@ class SyncEngine {
pathToQuery = ".";
}
// Create new OneDrive API Instance
OneDriveApi generateDeltaResponseOneDriveApiInstance;
generateDeltaResponseOneDriveApiInstance = new OneDriveApi(appConfig);
generateDeltaResponseOneDriveApiInstance.initialise();
if (!singleDirectoryScope) {
// In a --resync scenario, there is no DB data to query, so we have to query the OneDrive API here to get relevant details
try {
// Query the OneDrive API
pathData = oneDriveApiInstance.getPathDetails(pathToQuery);
pathData = generateDeltaResponseOneDriveApiInstance.getPathDetails(pathToQuery);
// Is the path on OneDrive local or remote to our account drive id?
if (isItemRemote(pathData)) {
// The path we are seeking is remote to our account drive id
@ -4583,7 +4588,7 @@ class SyncEngine {
// Display error message
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
// Must exit here
oneDriveApiInstance.shutdown();
generateDeltaResponseOneDriveApiInstance.shutdown();
exit(-1);
}
} else {
@ -4610,27 +4615,27 @@ class SyncEngine {
// Get drive details for the provided driveId
try {
driveData = oneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id);
driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id);
} catch (OneDriveException e) {
log.vdebug("driveData = oneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id) generated a OneDriveException");
log.vdebug("driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id) generated a OneDriveException");
// HTTP request returned status code 504 (Gateway Timeout) or 429 retry
if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) {
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
if (e.httpStatusCode == 429) {
log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - retrying applicable request");
handleOneDriveThrottleRequest(oneDriveApiInstance);
handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance);
}
if (e.httpStatusCode == 504) {
log.vdebug("Retrying original request that generated the HTTP 504 (Gateway Timeout) - retrying applicable request");
Thread.sleep(dur!"seconds"(30));
}
// Retry original request by calling function again to avoid replicating any further error handling
driveData = oneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id);
driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id);
} else {
// There was a HTTP 5xx Server Side Error
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
// Must exit here
oneDriveApiInstance.shutdown();
generateDeltaResponseOneDriveApiInstance.shutdown();
exit(-1);
}
}
@ -4639,7 +4644,7 @@ class SyncEngine {
if (!isItemRoot(driveData)) {
// Get root details for the provided driveId
try {
rootData = oneDriveApiInstance.getDriveIdRoot(searchItem.driveId);
rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId);
} catch (OneDriveException e) {
log.vdebug("rootData = onedrive.getDriveIdRoot(searchItem.driveId) generated a OneDriveException");
// HTTP request returned status code 504 (Gateway Timeout) or 429 retry
@ -4647,20 +4652,20 @@ class SyncEngine {
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
if (e.httpStatusCode == 429) {
log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - retrying applicable request");
handleOneDriveThrottleRequest(oneDriveApiInstance);
handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance);
}
if (e.httpStatusCode == 504) {
log.vdebug("Retrying original request that generated the HTTP 504 (Gateway Timeout) - retrying applicable request");
Thread.sleep(dur!"seconds"(30));
}
// Retry original request by calling function again to avoid replicating any further error handling
rootData = oneDriveApiInstance.getDriveIdRoot(searchItem.driveId);
rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId);
} else {
// There was a HTTP 5xx Server Side Error
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
// Must exit here
oneDriveApiInstance.shutdown();
generateDeltaResponseOneDriveApiInstance.shutdown();
exit(-1);
}
}
@ -4677,11 +4682,11 @@ class SyncEngine {
for (;;) {
// query top level children
try {
topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink);
topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink);
} catch (OneDriveException e) {
// OneDrive threw an error
log.vdebug("------------------------------------------------------------------");
log.vdebug("Query Error: topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)");
log.vdebug("Query Error: topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)");
log.vdebug("driveId: ", searchItem.driveId);
log.vdebug("idToQuery: ", searchItem.id);
log.vdebug("nextLink: ", nextLink);
@ -4689,7 +4694,7 @@ class SyncEngine {
// HTTP request returned status code 429 (Too Many Requests)
if (e.httpStatusCode == 429) {
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
handleOneDriveThrottleRequest(oneDriveApiInstance);
handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance);
log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children");
}
@ -4711,12 +4716,12 @@ class SyncEngine {
}
// re-try original request - retried for 429 and 504
try {
log.vdebug("Retrying Query: topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)");
topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink);
log.vdebug("Query 'topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)' performed successfully on re-try");
log.vdebug("Retrying Query: topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)");
topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink);
log.vdebug("Query 'topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)' performed successfully on re-try");
} catch (OneDriveException e) {
// display what the error is
log.vdebug("Query Error: topLevelChildren = oneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink) on re-try after delay");
log.vdebug("Query Error: topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink) on re-try after delay");
// error was not a 504 this time
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
}
@ -4843,17 +4848,22 @@ class SyncEngine {
// Query the OneDrive API for the child objects for this element
JSONValue queryThisLevelChildren(string driveId, string idToQuery, string nextLink) {
JSONValue thisLevelChildren;
// Create new OneDrive API Instance
OneDriveApi queryChildrenOneDriveApiInstance;
queryChildrenOneDriveApiInstance = new OneDriveApi(appConfig);
queryChildrenOneDriveApiInstance.initialise();
// query children
try {
// attempt API call
log.vdebug("Attempting Query: thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)");
thisLevelChildren = oneDriveApiInstance.listChildren(driveId, idToQuery, nextLink);
log.vdebug("Query 'thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)' performed successfully");
log.vdebug("Attempting Query: thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)");
thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink);
log.vdebug("Query 'thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)' performed successfully");
} catch (OneDriveException e) {
// OneDrive threw an error
log.vdebug("------------------------------------------------------------------");
log.vdebug("Query Error: thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink)");
log.vdebug("Query Error: thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)");
log.vdebug("driveId: ", driveId);
log.vdebug("idToQuery: ", idToQuery);
log.vdebug("nextLink: ", nextLink);
@ -4861,7 +4871,7 @@ class SyncEngine {
// HTTP request returned status code 429 (Too Many Requests)
if (e.httpStatusCode == 429) {
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
handleOneDriveThrottleRequest(oneDriveApiInstance);
handleOneDriveThrottleRequest(queryChildrenOneDriveApiInstance);
log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children");
}
@ -4871,7 +4881,7 @@ class SyncEngine {
if (e.httpStatusCode == 504) {
// transient error - try again in 30 seconds
log.log("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query OneDrive drive children - retrying applicable request");
log.vdebug("thisLevelChildren = onedrive.listChildren(driveId, idToQuery, nextLink) previously threw an error - retrying");
log.vdebug("thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink) previously threw an error - retrying");
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
log.vdebug("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request");
Thread.sleep(dur!"seconds"(30));