deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-04 15:03:24 +02:00
Code Issues Projects Releases Wiki Activity

Merge latest RC1 into this PR

Browse source 
* Merge latest RC1 into this PR
This commit is contained in:
abraunegg 2024-04-12 08:04:10 +10:00
parent 7be19abde8 3b0674e3c8
commit 2bb82f718f
5 changed files with 88 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/usage.md
View file

@ -507,7 +507,7 @@ Are you sure you want to proceed with --force-sync [Y/N]
To proceed with `--force-sync`, you must type 'y' or 'Y' to allow the application to continue.
### Enabling the Client Activity Log
When running onedrive, all actions can be logged to a separate log file. This can be enabled by using the `--enable-logging` flag.
When running onedrive, all actions can be logged to a separate log file. This can be enabled by using the `--enable-logging` flag or by adding `enable_logging = "true"` to your 'config' file.
By default, log files will be written to `/var/log/onedrive/` and will be in the format of `%username%.onedrive.log`, where `%username%` represents the user who ran the client to allow easy sorting of user to client activity log.

1
src/curlEngine.d
View file

@ -194,6 +194,7 @@ class CurlEngine {
synchronized(CurlEngine.classinfo) {
foreach(curlEngine; curlEnginePool) {
curlEngine.shutdown();
object.destroy(curlEngine);
}
curlEnginePool = null;
}

1
src/main.d
View file

@ -1175,7 +1175,6 @@ void performStandardExitProcess(string scopeCaller = null) {
syncEngineInstance = null;
} else {
addLogEntry("Waiting for all internal threads to complete before exiting application", ["verbose"]);
thread_joinAll();
addLogEntry("Application exit", ["debug"]);
addLogEntry("#######################################################################################################################################", ["logFileOnly"]);
// Destroy the shared logging buffer

110
src/sync.d
View file

@ -178,6 +178,9 @@ class SyncEngine {
string latestDeltaLink;
// Struct of containing the deltaLink details
DeltaLinkDetails deltaLinkCache;
// Create the specific task pool to process items in parallel
TaskPool processPool;
// Configure this class instance
this(ApplicationConfig appConfig, ItemDatabase itemDB, ClientSideFiltering selectiveSync) {
@ -298,6 +301,10 @@ class SyncEngine {
// Initialise the Sync Engine class
bool initialise() {
// Create common parallel thread pool
processPool = taskPool();
processPool.isDaemon(true); // Control whether the worker threads are daemon threads. A daemon thread is automatically terminated when all non-daemon threads have terminated.
// create a new instance of the OneDrive API
oneDriveApiInstance = new OneDriveApi(appConfig);
if (oneDriveApiInstance.initialise()) {
@ -307,7 +314,7 @@ class SyncEngine {
} catch (accountDetailsException exception) {
// details could not be queried
addLogEntry(exception.msg);
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
oneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(oneDriveApiInstance);
@ -321,7 +328,7 @@ class SyncEngine {
} catch (accountDetailsException exception) {
// details could not be queried
addLogEntry(exception.msg);
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
oneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(oneDriveApiInstance);
@ -335,7 +342,7 @@ class SyncEngine {
} catch (accountDetailsException exception) {
// details could not be queried
addLogEntry(exception.msg);
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
oneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(oneDriveApiInstance);
@ -345,7 +352,7 @@ class SyncEngine {
} else {
// API could not be initialised
addLogEntry("OneDrive API could not be initialised with previously used details");
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
oneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(oneDriveApiInstance);
@ -358,6 +365,7 @@ class SyncEngine {
// Shutdown this API instance, as we will create API instances as required, when required
oneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(oneDriveApiInstance);
return true;
@ -879,7 +887,7 @@ class SyncEngine {
// To finish off the JSON processing items, this is needed to reflect this in the log
addLogEntry("------------------------------------------------------------------", ["debug"]);
// Shutdown the API
// Shutdown this API instance, as we will create API instances as required, when required
getDeltaQueryOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(getDeltaQueryOneDriveApiInstance);
@ -2057,7 +2065,7 @@ class SyncEngine {
// Download items in parallel
void downloadOneDriveItemsInParallel(JSONValue[] array) {
// This function recieved an array of 16 JSON items to download
foreach (i, onedriveJSONItem; taskPool.parallel(array)) {
foreach (i, onedriveJSONItem; processPool.parallel(array)) {
// Take each JSON item and
downloadFileItem(onedriveJSONItem);
}
@ -3527,12 +3535,13 @@ class SyncEngine {
// For each batch of files to upload, upload the changed data to OneDrive
foreach (chunk; databaseItemsWhereContentHasChanged.chunks(batchSize)) {
processChangedLocalItemsToUploadInParallel(chunk);
chunk = null;
}
}
// Upload the changed file batches in parallel
void processChangedLocalItemsToUploadInParallel(string[3][] array) {
foreach (i, localItemDetails; taskPool.parallel(array)) {
foreach (i, localItemDetails; processPool.parallel(array)) {
addLogEntry("Upload Thread " ~ to!string(i) ~ " Starting: " ~ to!string(Clock.currTime()), ["debug"]);
uploadChangedLocalFileToOneDrive(localItemDetails);
addLogEntry("Upload Thread " ~ to!string(i) ~ " Finished: " ~ to!string(Clock.currTime()), ["debug"]);
@ -3899,7 +3908,7 @@ class SyncEngine {
// Debug Log the modified upload response
addLogEntry("Modified File Upload Response: " ~ to!string(uploadResponse), ["debug"]);
// Shutdown the API instance
// Shutdown this API instance, as we will create API instances as required, when required instance
uploadFileOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(uploadFileOneDriveApiInstance);
@ -4614,7 +4623,7 @@ class SyncEngine {
// OneDrive API returned a 404 (above) to say the directory did not exist
// but when we attempted to create it, OneDrive responded that it now already exists
addLogEntry("OneDrive reported that " ~ thisNewPathToCreate ~ " already exists .. OneDrive API race condition", ["verbose"]);
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
createDirectoryOnlineOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(createDirectoryOnlineOneDriveApiInstance);
@ -4623,7 +4632,7 @@ class SyncEngine {
// some other error from OneDrive was returned - display what it is
addLogEntry("OneDrive generated an error when creating this path: " ~ thisNewPathToCreate);
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
createDirectoryOnlineOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(createDirectoryOnlineOneDriveApiInstance);
@ -4639,7 +4648,7 @@ class SyncEngine {
saveItem(fakeResponse);
}
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
createDirectoryOnlineOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(createDirectoryOnlineOneDriveApiInstance);
@ -4684,7 +4693,7 @@ class SyncEngine {
// Add this path to businessSharedFoldersOnlineToSkip
businessSharedFoldersOnlineToSkip ~= [thisNewPathToCreate];
// no save to database, no online create
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
createDirectoryOnlineOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(createDirectoryOnlineOneDriveApiInstance);
@ -4709,7 +4718,7 @@ class SyncEngine {
// Is the response a valid JSON object - validation checking done in saveItem
saveItem(onlinePathData);
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
createDirectoryOnlineOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(createDirectoryOnlineOneDriveApiInstance);
@ -4723,7 +4732,7 @@ class SyncEngine {
addLogEntry("Skipping creating this directory online due to 'case-insensitive match': " ~ thisNewPathToCreate);
// Add this path to posixViolationPaths
posixViolationPaths ~= [thisNewPathToCreate];
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
createDirectoryOnlineOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(createDirectoryOnlineOneDriveApiInstance);
@ -4734,7 +4743,7 @@ class SyncEngine {
addLogEntry("ERROR: There was an error performing this operation on Microsoft OneDrive");
addLogEntry("ERROR: Increase logging verbosity to assist determining why.");
addLogEntry("Skipping: " ~ buildNormalizedPath(absolutePath(thisNewPathToCreate)));
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
createDirectoryOnlineOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(createDirectoryOnlineOneDriveApiInstance);
@ -4772,7 +4781,7 @@ class SyncEngine {
// Upload the file batches in parallel
void uploadNewLocalFileItemsInParallel(string[] array) {
foreach (i, fileToUpload; taskPool.parallel(array)) {
foreach (i, fileToUpload; processPool.parallel(array)) {
// Add a processing '.'
if (appConfig.verbosityCount == 0)
addProcessingDotEntry();
@ -5095,8 +5104,6 @@ class SyncEngine {
// Create the OneDriveAPI Upload Instance
OneDriveApi uploadFileOneDriveApiInstance;
uploadFileOneDriveApiInstance = new OneDriveApi(appConfig);
uploadFileOneDriveApiInstance.initialise();
// Calculate upload speed
auto uploadStartTime = Clock.currTime();
@ -5116,14 +5123,23 @@ class SyncEngine {
if ((thisFileSize == 0) || (useSimpleUpload)) {
try {
// Initialise API
uploadFileOneDriveApiInstance = new OneDriveApi(appConfig);
uploadFileOneDriveApiInstance.initialise();
// Attempt to upload the zero byte file using simpleUpload for all account types
uploadResponse = uploadFileOneDriveApiInstance.simpleUpload(fileToUpload, parentItem.driveId, parentItem.id, baseName(fileToUpload));
uploadFailed = false;
addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... done.");
// Shutdown the API
uploadFileOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(uploadFileOneDriveApiInstance);
// If the API instance is still valid, shut it down
if (uploadFileOneDriveApiInstance !is null) {
// Shutdown this API instance, as we will create API instances as required, when required
uploadFileOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(uploadFileOneDriveApiInstance);
}
} catch (OneDriveException exception) {
// An error was responded with - what was it
@ -5134,12 +5150,34 @@ class SyncEngine {
addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed.");
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
// If the API instance is still valid, shut it down
if (uploadFileOneDriveApiInstance !is null) {
// Shutdown this API instance, as we will create API instances as required, when required
uploadFileOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(uploadFileOneDriveApiInstance);
}
} catch (FileException e) {
// display the error message
addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed.");
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
// If the API instance is still valid, shut it down
if (uploadFileOneDriveApiInstance !is null) {
addLogEntry("API SHUTDOWN: Shutting down API instance uploadFileOneDriveApiInstance");
// Shutdown this API instance, as we will create API instances as required, when required
uploadFileOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(uploadFileOneDriveApiInstance);
}
}
} else {
// Initialise API
uploadFileOneDriveApiInstance = new OneDriveApi(appConfig);
uploadFileOneDriveApiInstance.initialise();
// Session Upload for this criteria:
// - Personal Account and file size > 4MB
// - All Business | Office365 | SharePoint files > 0 bytes
@ -5222,6 +5260,14 @@ class SyncEngine {
// Create session Upload URL failed
addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed.");
}
// If the API instance is still valid, shut it down
if (uploadFileOneDriveApiInstance !is null) {
// Shutdown this API instance, as we will create API instances as required, when required
uploadFileOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(uploadFileOneDriveApiInstance);
}
}
} else {
// We are in a --dry-run scenario
@ -5610,7 +5656,7 @@ class SyncEngine {
// Perform the delete via the default OneDrive API instance
performReverseDeletionOneDriveApiInstance.deleteById(itemToDelete.driveId, itemToDelete.id, itemToDelete.eTag);
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
performReverseDeletionOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(performReverseDeletionOneDriveApiInstance);
@ -6177,7 +6223,7 @@ class SyncEngine {
}
}
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
queryChildrenOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(queryChildrenOneDriveApiInstance);
@ -6443,7 +6489,7 @@ class SyncEngine {
}
}
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
queryOneDriveForSpecificPath.shutdown();
// Free object and memory
object.destroy(queryOneDriveForSpecificPath);
@ -6649,7 +6695,7 @@ class SyncEngine {
}
}
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
movePathOnlineApiInstance.shutdown();
// Free object and memory
object.destroy(movePathOnlineApiInstance);
@ -6909,7 +6955,7 @@ class SyncEngine {
}
}
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
querySharePointLibraryNameApiInstance.shutdown();
// Free object and memory
object.destroy(querySharePointLibraryNameApiInstance);
@ -7194,7 +7240,7 @@ class SyncEngine {
}
}
// Shutdown the API access
// Shutdown this API instance, as we will create API instances as required, when required access
queryOneDriveForFileDetailsApiInstance.shutdown();
// Free object and memory
object.destroy(queryOneDriveForFileDetailsApiInstance);
@ -7428,7 +7474,7 @@ class SyncEngine {
return false;
}
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
validateUploadSessionFileDataApiInstance.shutdown();
// Free object and memory
object.destroy(validateUploadSessionFileDataApiInstance);
@ -7469,7 +7515,7 @@ class SyncEngine {
void resumeSessionUploadsInParallel(JSONValue[] array) {
// This function recieved an array of 16 JSON items to resume upload
foreach (i, jsonItemToResume; taskPool.parallel(array)) {
foreach (i, jsonItemToResume; processPool.parallel(array)) {
// Take each JSON item and resume upload using the JSON data
JSONValue uploadResponse;
@ -7518,7 +7564,7 @@ class SyncEngine {
addLogEntry("CODING TO DO: what to do when session upload resumption JSON data is not valid ... nothing ? error message ?");
}
// Shutdown API instance
// Shutdown this API instance, as we will create API instances as required, when required
uploadFileOneDriveApiInstance.shutdown();
// Free object and memory
object.destroy(uploadFileOneDriveApiInstance);

15
src/util.d
View file

@ -227,13 +227,14 @@ bool testInternetReachability(ApplicationConfig appConfig) {
addLogEntry("No Network Connection", ["debug"]);
addLogEntry("Cannot connect to Microsoft OneDrive Login Service - Network Connection Issue");
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
} finally {
if (curlEngine) {
curlEngine.release();
curlEngine = null;
}
}
}
// Shutdown engine
curlEngine.http.shutdown();
curlEngine.releaseAll();
object.destroy(curlEngine);
curlEngine = null;
// Return test result
return result;
}