deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-23 08:02:16 +02:00
Code Issues Projects Releases Wiki Activity

Update PR

Browse source 
* Update PR
This commit is contained in:
abraunegg 2023-08-29 09:56:55 +10:00
parent 3f5a4fa29c
commit 44be502c62
6 changed files with 97 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/config.d
View file

@ -732,7 +732,7 @@ class ApplicationConfig {
}
// Azure AD Configuration
if (key == "azure_ad_endpoint") {
string azureConfigValue = c.front.dup;
string azureConfigValue = strip(c.front.dup);
switch(azureConfigValue) {
case "":
log.log("Using config option for Global Azure AD Endpoints");
@ -754,6 +754,18 @@ class ApplicationConfig {
log.log("Unknown Azure AD Endpoint - using Global Azure AD Endpoints");
}
}
// Application ID
if (key == "application_id") {
// This key cannot be empty
string tempApplicationId = strip(c.front.dup);
if (tempApplicationId.empty) {
log.vdebug("application_id in config file cannot be empty - using default application_id");
setValueString("application_id", defaultApplicationId);
} else {
setValueString("application_id", tempApplicationId);
}
}
} else {
auto ppp = key in longValues;
if (ppp) {

15
src/curlEngine.d
View file

@ -39,29 +39,14 @@ class CurlEngine {
// What version of HTTP protocol do we use?
// Curl >= 7.62.0 defaults to http2 for a significant number of operations
if (httpProtocol) {
// Downgrade to curl to use HTTP 1.1 for all operations
log.vlog("Downgrading all HTTP operations to HTTP/1.1 due to user configuration");
// Downgrade to HTTP 1.1 - yes version = 2 is HTTP 1.1
http.handle.set(CurlOption.http_version,2);
} else {
// Use curl defaults
log.vdebug("Using Curl defaults for HTTP operational protocol version (potentially HTTP/2)");
}
// Configure upload / download rate limits if configured
// 131072 = 128 KB/s - minimum for basic application operations to prevent timeouts
// A 0 value means rate is unlimited, and is the curl default
if (userRateLimit > 0) {
// User configured rate limit
log.log("User Configured Rate Limit: ", userRateLimit);
// If user provided rate limit is < 131072, flag that this is too low, setting to the minimum of 131072
if (userRateLimit < 131072) {
// user provided limit too low
log.log("WARNING: User configured rate limit too low for normal application processing and preventing application timeouts. Overriding to default minimum of 131072 (128KB/s)");
userRateLimit = 131072;
}
// set rate limit
http.handle.set(CurlOption.max_send_speed_large,userRateLimit);
http.handle.set(CurlOption.max_recv_speed_large,userRateLimit);

47
src/main.d
View file

@ -57,7 +57,10 @@ int main(string[] cliArgs) {
string runtimeSyncDirectory;
// Configure the runtime database file path. Typically this will be the default, but in a --dry-run scenario, we use a separate database file
string runtimeDatabaseFile;
// Application Start Time - used during monitor loop to detail how long it has been running for
auto applicationStartTime = Clock.currTime();
// DEVELOPER OPTIONS OUTPUT VARIABLES
bool displayMemoryUsage = false;
bool displaySyncOptions = false;
@ -535,27 +538,39 @@ int main(string[] cliArgs) {
// Increment monitorLoopFullCount
monitorLoopFullCount++;
log.vdebug(loopStartOutputMessage);
log.vdebug("Loop Number: ", monitorLoopFullCount);
log.log("Starting a sync with Microsoft OneDrive");
log.log("Loop Number: ", monitorLoopFullCount);
SysTime startFunctionProcessingTime = Clock.currTime();
log.vdebug("Start Monitor Loop Time: ", startFunctionProcessingTime);
// Did the user specify --upload-only?
if (appConfig.getValueBool("upload_only")) {
// Perform the --upload-only sync process
performUploadOnlySyncProcess(localPath);
// How long has the application been running for?
auto elapsedTime = Clock.currTime() - applicationStartTime;
log.log("Application run-time thus far: ", elapsedTime);
// Need to re-validate that the client is still online for this loop
if (testInternetReachability(appConfig)) {
// Starting a sync
log.log("Starting a sync with Microsoft OneDrive");
// Did the user specify --upload-only?
if (appConfig.getValueBool("upload_only")) {
// Perform the --upload-only sync process
performUploadOnlySyncProcess(localPath);
} else {
// Perform the standard sync process
performStandardSyncProcess(localPath);
}
// Detail the outcome of the sync process
displaySyncOutcome();
// Write WAL and SHM data to file for this loop
log.vdebug("Merge contents of WAL and SHM files into main database file");
itemDB.performVacuum();
} else {
// Perform the standard sync process
performStandardSyncProcess(localPath);
// Not online
log.log("Microsoft OneDrive service is not reachable at this time");
}
// Detail the outcome of the sync process
displaySyncOutcome();
// Write WAL and SHM data to file for this loop
log.vdebug("Merge contents of WAL and SHM files into main database file");
itemDB.performVacuum();
// Output end of loop processing times
SysTime endFunctionProcessingTime = Clock.currTime();
log.vdebug("End Monitor Loop Time: ", endFunctionProcessingTime);

2
src/monitor.d
View file

@ -404,7 +404,7 @@ final class Monitor {
cookieToPath.remove(cookie);
}
log.log("inotify events flushed");
log.vdebug("inotify events flushed");
}
}
}

14
src/onedrive.d
View file

@ -285,12 +285,12 @@ class OneDriveApi {
// Try and read the value from the appConfig if it is set, rather than trying to read the value from disk
if (!appConfig.refreshToken.empty) {
log.vdebug("read token from appConfig");
refreshToken = appConfig.refreshToken;
refreshToken = strip(appConfig.refreshToken);
authorised = true;
} else {
// Try and read the file from disk
try {
refreshToken = readText(appConfig.refreshTokenFilePath);
refreshToken = strip(readText(appConfig.refreshTokenFilePath));
// is the refresh_token empty?
if (refreshToken.empty) {
log.error("refreshToken exists but is empty: ", appConfig.refreshTokenFilePath);
@ -712,24 +712,27 @@ class OneDriveApi {
}
if ("access_token" in response){
accessToken = "bearer " ~ response["access_token"].str();
refreshToken = response["refresh_token"].str();
accessToken = "bearer " ~ strip(response["access_token"].str);
refreshToken = strip(response["refresh_token"].str);
accessTokenExpiration = Clock.currTime() + dur!"seconds"(response["expires_in"].integer());
if (!dryRun) {
// Update the refreshToken in appConfig so that we can reuse it
if (appConfig.refreshToken.empty) {
// The access token is empty
log.vdebug("Updating appConfig.refreshToken with new refreshToken as appConfig.refreshToken is empty");
appConfig.refreshToken = refreshToken;
} else {
// Is the access token different?
if (appConfig.refreshToken != refreshToken) {
// Update the memory version
log.vdebug("Updating appConfig.refreshToken with updated refreshToken");
appConfig.refreshToken = refreshToken;
}
}
// try and update the refresh_token file on disk
try {
log.vdebug("Updating refreshToken on disk");
std.file.write(appConfig.refreshTokenFilePath, refreshToken);
log.vdebug("Setting file permissions for: ", appConfig.refreshTokenFilePath);
appConfig.refreshTokenFilePath.setAttributes(appConfig.returnRequiredFilePermisions());
@ -737,9 +740,8 @@ class OneDriveApi {
// display the error message
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
}
}
if (printAccessToken) writeln("New access token: ", accessToken);
if (printAccessToken) writeln("Current access token: ", accessToken);
} else {
log.error("\nInvalid authentication response from OneDrive. Please check the response uri\n");
// re-authorize

54
src/sync.d
View file

@ -134,6 +134,11 @@ class SyncEngine {
this.itemDB = itemDB;
// Configure the class variable to consume the selective sync (skip_dir, skip_file and sync_list) configuration
this.selectiveSync = selectiveSync;
// Configure the dryRun flag to capture if --dry-run was used
// Application startup already flagged we are also in a --dry-run state, so no need to output anything else here
this.dryRun = appConfig.getValueBool("dry_run");
// Configure file size limit
if (appConfig.getValueLong("skip_size") != 0) {
fileSizeLimit = appConfig.getValueLong("skip_size") * 2^^20;
@ -143,10 +148,6 @@ class SyncEngine {
// Is there a sync_list file present?
if (exists(appConfig.syncListFilePath)) this.syncListConfigured = true;
// Configure the dryRun flag to capture if --dry-run was used
// Application startup already flagged we are also in a --dry-run state, so no need to output anything else here
this.dryRun = appConfig.getValueBool("dry_run");
// Configure the uploadOnly flag to capture if --upload-only was used
if (appConfig.getValueBool("upload_only")) {
log.vdebug("Configuring uploadOnly flag to TRUE as --upload-only passed in or configured");
@ -195,6 +196,31 @@ class SyncEngine {
log.log("WARNING: Local data loss MAY occur in this scenario.");
this.bypassDataPreservation = true;
}
// Did the user configure a specific rate limit for the application?
if (appConfig.getValueLong("rate_limit") > 0) {
// User configured rate limit
log.log("User Configured Rate Limit: ", appConfig.getValueLong("rate_limit"));
// If user provided rate limit is < 131072, flag that this is too low, setting to the minimum of 131072
if (appConfig.getValueLong("rate_limit") < 131072) {
// user provided limit too low
log.log("WARNING: User configured rate limit too low for normal application processing and preventing application timeouts. Overriding to default minimum of 131072 (128KB/s)");
appConfig.setValueLong("rate_limit", 131072);
}
}
// Did the user downgrade all HTTP operations to force HTTP 1.1
if (appConfig.getValueBool("force_http_11")) {
// User is forcing downgrade to curl to use HTTP 1.1 for all operations
log.vlog("Downgrading all HTTP operations to HTTP/1.1 due to user configuration");
} else {
// Use curl defaults
log.vdebug("Using Curl defaults for HTTP operational protocol version (potentially HTTP/2)");
}
}
// Initialise the Sync Engine class
@ -1779,8 +1805,9 @@ class SyncEngine {
getDeltaQueryOneDriveApiInstance.initialise();
try {
deltaChangesBundle = getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink);
getDeltaQueryOneDriveApiInstance.shutdown();
} catch (OneDriveException exception) {
log.vdebug("deltaChangesBundle = oneDriveApiInstance.viewChangesByItemId() generated a OneDriveException");
log.vdebug("deltaChangesBundle = getDeltaQueryOneDriveApiInstance.viewChangesByItemId() generated a OneDriveException");
// display error message
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
}
@ -4173,7 +4200,6 @@ class SyncEngine {
log.error("ERROR: An attempt to remove a large volume of data from OneDrive has been detected. Exiting client to preserve data on OneDrive");
log.error("ERROR: To delete a large volume of data use --force or increase the config value 'classify_as_big_delete' to a larger value");
// Must exit here to preserve data on OneDrive
oneDriveApiInstance.shutdown();
exit(-1);
}
}
@ -4183,12 +4209,18 @@ class SyncEngine {
if (!dryRun) {
// We are not in a dry run scenario
log.vdebug("itemToDelete: ", itemToDelete);
// Create new OneDrive API Instance
OneDriveApi uploadDeletedItemOneDriveApiInstance;
uploadDeletedItemOneDriveApiInstance = new OneDriveApi(appConfig);
uploadDeletedItemOneDriveApiInstance.initialise();
// Attempt to do the delete
try {
// what item are we trying to delete?
log.vdebug("Attempting to delete this item id: ", itemToDelete.id, " from drive: ", itemToDelete.driveId);
// perform the delete via the default OneDrive API instance
oneDriveApiInstance.deleteById(itemToDelete.driveId, itemToDelete.id);
uploadDeletedItemOneDriveApiInstance.deleteById(itemToDelete.driveId, itemToDelete.id);
// Delete the reference in the local database
itemDB.deleteById(itemToDelete.driveId, itemToDelete.id);
@ -4196,6 +4228,8 @@ class SyncEngine {
// If the item is a remote item, delete the reference in the local database
itemDB.deleteById(itemToDelete.remoteDriveId, itemToDelete.remoteId);
}
// Shutdown API
uploadDeletedItemOneDriveApiInstance.shutdown();
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
// item.id, item.eTag could not be found on the specified driveId
@ -4206,7 +4240,7 @@ class SyncEngine {
// Issue #1041 - Unable to delete OneDrive content when permissions prevent deletion
if (appConfig.accountType != "personal") {
if (e.httpStatusCode == 401) {
log.vdebug("oneDriveApiInstance.deleteById generated a 401 error response when attempting to delete object by item id");
log.vdebug("uploadDeletedItemOneDriveApiInstance.deleteById generated a 401 error response when attempting to delete object by item id");
try {
performReverseDeletionOfOneDriveItems(children, itemToDelete);
} catch (OneDriveException e) {
@ -4217,7 +4251,7 @@ class SyncEngine {
}
if (e.httpStatusCode == 403) {
log.vdebug("oneDriveApiInstance.deleteById generated a 403 error response when attempting to delete object by item id");
log.vdebug("uploadDeletedItemOneDriveApiInstance.deleteById generated a 403 error response when attempting to delete object by item id");
try {
performReverseDeletionOfOneDriveItems(children, itemToDelete);
} catch (OneDriveException e) {
@ -4227,7 +4261,7 @@ class SyncEngine {
}
} else {
// Not a 401 or 403 error response & OneDrive Business Account / O365 Shared Folder / Library
log.vdebug("oneDriveApiInstance.deleteById generated an error response when attempting to delete object by item id");
log.vdebug("uploadDeletedItemOneDriveApiInstance.deleteById generated an error response when attempting to delete object by item id");
// display what the error is
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
}