mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-08 08:52:15 +02:00
06026c2c5b
* Add missing log output for new file adds
7544 lines
352 KiB
D
7544 lines
352 KiB
D
// What is this module called?
|
|
module syncEngine;
|
|
|
|
// What does this module require to function?
|
|
import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit;
|
|
import core.thread;
|
|
import core.time;
|
|
import std.algorithm;
|
|
import std.array;
|
|
import std.concurrency;
|
|
import std.container.rbtree;
|
|
import std.conv;
|
|
import std.datetime;
|
|
import std.encoding;
|
|
import std.exception;
|
|
import std.file;
|
|
import std.json;
|
|
import std.parallelism;
|
|
import std.path;
|
|
import std.range;
|
|
import std.regex;
|
|
import std.stdio;
|
|
import std.string;
|
|
import std.uni;
|
|
import std.uri;
|
|
import std.utf;
|
|
import std.math;
|
|
|
|
// What other modules that we have created do we need to import?
|
|
import config;
|
|
import log;
|
|
import util;
|
|
import onedrive;
|
|
import itemdb;
|
|
import clientSideFiltering;
|
|
|
|
class jsonResponseException: Exception {
|
|
@safe pure this(string inputMessage) {
|
|
string msg = format(inputMessage);
|
|
super(msg);
|
|
}
|
|
}
|
|
|
|
class posixException: Exception {
|
|
@safe pure this(string localTargetName, string remoteTargetName) {
|
|
string msg = format("POSIX 'case-insensitive match' between '%s' (local) and '%s' (online) which violates the Microsoft OneDrive API namespace convention", localTargetName, remoteTargetName);
|
|
super(msg);
|
|
}
|
|
}
|
|
|
|
class accountDetailsException: Exception {
|
|
@safe pure this() {
|
|
string msg = format("Unable to query OneDrive API to obtain required account details");
|
|
super(msg);
|
|
}
|
|
}
|
|
|
|
class SyncException: Exception {
|
|
@nogc @safe pure nothrow this(string msg, string file = __FILE__, size_t line = __LINE__) {
|
|
super(msg, file, line);
|
|
}
|
|
}
|
|
|
|
void forceExit() {
|
|
// Allow logging to flush and complete
|
|
Thread.sleep(dur!("msecs")(500));
|
|
// Force Exit
|
|
exit(EXIT_FAILURE);
|
|
}
|
|
|
|
class SyncEngine {
|
|
// Class Variables
|
|
ApplicationConfig appConfig;
|
|
OneDriveApi oneDriveApiInstance;
|
|
ItemDatabase itemDB;
|
|
ClientSideFiltering selectiveSync;
|
|
|
|
// Array of directory databaseItem.id to skip while applying the changes.
|
|
// These are the 'parent path' id's that are being excluded, so if the parent id is in here, the child needs to be skipped as well
|
|
RedBlackTree!string skippedItems = redBlackTree!string();
|
|
// Array of databaseItem.id to delete after the changes have been downloaded
|
|
string[2][] idsToDelete;
|
|
// Array of JSON items which are files or directories that are not 'root', skipped or to be deleted, that need to be processed
|
|
JSONValue[] jsonItemsToProcess;
|
|
// Array of JSON items which are files that are not 'root', skipped or to be deleted, that need to be downloaded
|
|
JSONValue[] fileJSONItemsToDownload;
|
|
// Array of paths that failed to download
|
|
string[] fileDownloadFailures;
|
|
// Array of all OneDrive driveId's that have been seen
|
|
string[] driveIDsArray;
|
|
// List of items we fake created when using --dry-run
|
|
string[2][] idsFaked;
|
|
// List of paths we fake deleted when using --dry-run
|
|
string[] pathFakeDeletedArray;
|
|
// Array of database Parent Item ID, Item ID & Local Path where the content has changed and needs to be uploaded
|
|
string[3][] databaseItemsWhereContentHasChanged;
|
|
// Array of local file paths that need to be uploaded as new itemts to OneDrive
|
|
string[] newLocalFilesToUploadToOneDrive;
|
|
// Array of local file paths that failed to be uploaded to OneDrive
|
|
string[] fileUploadFailures;
|
|
// List of path names changed online, but not changed locally when using --dry-run
|
|
string[] pathsRenamed;
|
|
// List of paths that were a POSIX case-insensitive match, thus could not be created online
|
|
string[] posixViolationPaths;
|
|
// List of local paths, that, when using the OneDrive Business Shared Folders feature, then diabling it, folder still exists locally and online
|
|
// This list of local paths need to be skipped
|
|
string[] businessSharedFoldersOnlineToSkip;
|
|
// List of interrupted uploads session files that need to be resumed
|
|
string[] interruptedUploadsSessionFiles;
|
|
// List of validated interrupted uploads session JSON items to resume
|
|
JSONValue[] jsonItemsToResumeUpload;
|
|
|
|
// Flag that there were upload or download failures listed
|
|
bool syncFailures = false;
|
|
// Is sync_list configured
|
|
bool syncListConfigured = false;
|
|
// Was --dry-run used?
|
|
bool dryRun = false;
|
|
// Was --upload-only used?
|
|
bool uploadOnly = false;
|
|
// Was --remove-source-files used?
|
|
// Flag to set whether the local file should be deleted once it is successfully uploaded to OneDrive
|
|
bool localDeleteAfterUpload = false;
|
|
|
|
// Do we configure to disable the download validation routine due to --disable-download-validation
|
|
// We will always validate our downloads
|
|
// However, when downloading files from SharePoint, the OneDrive API will not advise the correct file size
|
|
// which means that the application thinks the file download has failed as the size is different / hash is different
|
|
// See: https://github.com/abraunegg/onedrive/discussions/1667
|
|
bool disableDownloadValidation = false;
|
|
|
|
// Do we configure to disable the upload validation routine due to --disable-upload-validation
|
|
// We will always validate our uploads
|
|
// However, when uploading a file that can contain metadata SharePoint will associate some
|
|
// metadata from the library the file is uploaded to directly in the file which breaks this validation.
|
|
// See: https://github.com/abraunegg/onedrive/issues/205
|
|
// See: https://github.com/OneDrive/onedrive-api-docs/issues/935
|
|
bool disableUploadValidation = false;
|
|
|
|
// Do we perform a local cleanup of files that are 'extra' on the local file system, when using --download-only
|
|
bool cleanupLocalFiles = false;
|
|
// Are we performing a --single-directory sync ?
|
|
bool singleDirectoryScope = false;
|
|
string singleDirectoryScopeDriveId;
|
|
string singleDirectoryScopeItemId;
|
|
// Is National Cloud Deployments configured ?
|
|
bool nationalCloudDeployment = false;
|
|
// Do we configure not to perform a remote file delete if --upload-only & --no-remote-delete configured
|
|
bool noRemoteDelete = false;
|
|
// Is bypass_data_preservation set via config file
|
|
// Local data loss MAY occur in this scenario
|
|
bool bypassDataPreservation = false;
|
|
// Maximum file size upload
|
|
// https://support.microsoft.com/en-us/office/invalid-file-names-and-file-types-in-onedrive-and-sharepoint-64883a5d-228e-48f5-b3d2-eb39e07630fa?ui=en-us&rs=en-us&ad=us
|
|
// July 2020, maximum file size for all accounts is 100GB
|
|
// January 2021, maximum file size for all accounts is 250GB
|
|
ulong maxUploadFileSize = 268435456000; // 250GB
|
|
// Threshold after which files will be uploaded using an upload session
|
|
ulong sessionThresholdFileSize = 4 * 2^^20; // 4 MiB
|
|
// File size limit for file operations that the user has configured
|
|
ulong fileSizeLimit;
|
|
// Total data to upload
|
|
ulong totalDataToUpload;
|
|
// How many items have been processed for the active operation
|
|
ulong processedCount;
|
|
// Are we creating a simulated /delta response? This is critically important in terms of how we 'update' the database
|
|
bool generateSimulatedDeltaResponse = false;
|
|
// Store the latest DeltaLink
|
|
string latestDeltaLink;
|
|
|
|
// Configure this class instance
|
|
this(ApplicationConfig appConfig, ItemDatabase itemDB, ClientSideFiltering selectiveSync) {
|
|
// Configure the class varaible to consume the application configuration
|
|
this.appConfig = appConfig;
|
|
// Configure the class varaible to consume the database configuration
|
|
this.itemDB = itemDB;
|
|
// Configure the class variable to consume the selective sync (skip_dir, skip_file and sync_list) configuration
|
|
this.selectiveSync = selectiveSync;
|
|
|
|
// Configure the dryRun flag to capture if --dry-run was used
|
|
// Application startup already flagged we are also in a --dry-run state, so no need to output anything else here
|
|
this.dryRun = appConfig.getValueBool("dry_run");
|
|
|
|
// Configure file size limit
|
|
if (appConfig.getValueLong("skip_size") != 0) {
|
|
fileSizeLimit = appConfig.getValueLong("skip_size") * 2^^20;
|
|
fileSizeLimit = (fileSizeLimit == 0) ? ulong.max : fileSizeLimit;
|
|
}
|
|
|
|
// Is there a sync_list file present?
|
|
if (exists(appConfig.syncListFilePath)) this.syncListConfigured = true;
|
|
|
|
// Configure the uploadOnly flag to capture if --upload-only was used
|
|
if (appConfig.getValueBool("upload_only")) {
|
|
addLogEntry("Configuring uploadOnly flag to TRUE as --upload-only passed in or configured", ["debug"]);
|
|
this.uploadOnly = true;
|
|
}
|
|
|
|
// Configure the localDeleteAfterUpload flag
|
|
if (appConfig.getValueBool("remove_source_files")) {
|
|
addLogEntry("Configuring localDeleteAfterUpload flag to TRUE as --remove-source-files passed in or configured", ["debug"]);
|
|
this.localDeleteAfterUpload = true;
|
|
}
|
|
|
|
// Configure the disableDownloadValidation flag
|
|
if (appConfig.getValueBool("disable_download_validation")) {
|
|
addLogEntry("Configuring disableDownloadValidation flag to TRUE as --disable-download-validation passed in or configured", ["debug"]);
|
|
this.disableDownloadValidation = true;
|
|
}
|
|
|
|
// Configure the disableUploadValidation flag
|
|
if (appConfig.getValueBool("disable_upload_validation")) {
|
|
addLogEntry("Configuring disableUploadValidation flag to TRUE as --disable-upload-validation passed in or configured", ["debug"]);
|
|
this.disableUploadValidation = true;
|
|
}
|
|
|
|
// Do we configure to clean up local files if using --download-only ?
|
|
if ((appConfig.getValueBool("download_only")) && (appConfig.getValueBool("cleanup_local_files"))) {
|
|
// --download-only and --cleanup-local-files were passed in
|
|
addLogEntry("WARNING: Application has been configured to cleanup local files that are not present online.");
|
|
addLogEntry("WARNING: Local data loss MAY occur in this scenario if you are expecting data to remain archived locally.");
|
|
// Set the flag
|
|
this.cleanupLocalFiles = true;
|
|
}
|
|
|
|
// Do we configure to NOT perform a remote delete if --upload-only & --no-remote-delete configured ?
|
|
if ((appConfig.getValueBool("upload_only")) && (appConfig.getValueBool("no_remote_delete"))) {
|
|
// --upload-only and --no-remote-delete were passed in
|
|
addLogEntry("WARNING: Application has been configured NOT to cleanup remote files that are deleted locally.");
|
|
// Set the flag
|
|
this.noRemoteDelete = true;
|
|
}
|
|
|
|
// Are we forcing to use /children scan instead of /delta to simulate National Cloud Deployment use of /children?
|
|
if (appConfig.getValueBool("force_children_scan")) {
|
|
addLogEntry("Forcing client to use /children API call rather than /delta API to retrieve objects from the OneDrive API");
|
|
this.nationalCloudDeployment = true;
|
|
}
|
|
|
|
// Are we forcing the client to bypass any data preservation techniques to NOT rename any local files if there is a conflict?
|
|
// The enabling of this function could lead to data loss
|
|
if (appConfig.getValueBool("bypass_data_preservation")) {
|
|
addLogEntry("WARNING: Application has been configured to bypass local data preservation in the event of file conflict.");
|
|
addLogEntry("WARNING: Local data loss MAY occur in this scenario.");
|
|
this.bypassDataPreservation = true;
|
|
}
|
|
|
|
// Did the user configure a specific rate limit for the application?
|
|
if (appConfig.getValueLong("rate_limit") > 0) {
|
|
// User configured rate limit
|
|
addLogEntry("User Configured Rate Limit: " ~ to!string(appConfig.getValueLong("rate_limit")));
|
|
|
|
// If user provided rate limit is < 131072, flag that this is too low, setting to the recommended minimum of 131072
|
|
if (appConfig.getValueLong("rate_limit") < 131072) {
|
|
// user provided limit too low
|
|
addLogEntry("WARNING: User configured rate limit too low for normal application processing and preventing application timeouts. Overriding to recommended minimum of 131072 (128KB/s)");
|
|
appConfig.setValueLong("rate_limit", 131072);
|
|
}
|
|
}
|
|
|
|
// Did the user downgrade all HTTP operations to force HTTP 1.1
|
|
if (appConfig.getValueBool("force_http_11")) {
|
|
// User is forcing downgrade to curl to use HTTP 1.1 for all operations
|
|
addLogEntry("Downgrading all HTTP operations to HTTP/1.1 due to user configuration", ["verbose"]);
|
|
} else {
|
|
// Use curl defaults
|
|
addLogEntry("Using Curl defaults for HTTP operational protocol version (potentially HTTP/2)", ["debug"]);
|
|
}
|
|
}
|
|
|
|
// Initialise the Sync Engine class
|
|
bool initialise() {
|
|
|
|
// create a new instance of the OneDrive API
|
|
oneDriveApiInstance = new OneDriveApi(appConfig);
|
|
if (oneDriveApiInstance.initialise()) {
|
|
try {
|
|
// Get the relevant default account & drive details
|
|
getDefaultDriveDetails();
|
|
} catch (accountDetailsException exception) {
|
|
// details could not be queried
|
|
addLogEntry(exception.msg);
|
|
// Shutdown API instance
|
|
oneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(oneDriveApiInstance);
|
|
// Must force exit here, allow logging to be done
|
|
forceExit();
|
|
}
|
|
|
|
try {
|
|
// Get the relevant default account & drive details
|
|
getDefaultRootDetails();
|
|
} catch (accountDetailsException exception) {
|
|
// details could not be queried
|
|
addLogEntry(exception.msg);
|
|
// Shutdown API instance
|
|
oneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(oneDriveApiInstance);
|
|
// Must force exit here, allow logging to be done
|
|
forceExit();
|
|
}
|
|
|
|
try {
|
|
// Display details
|
|
displaySyncEngineDetails();
|
|
} catch (accountDetailsException exception) {
|
|
// details could not be queried
|
|
addLogEntry(exception.msg);
|
|
// Shutdown API instance
|
|
oneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(oneDriveApiInstance);
|
|
// Must force exit here, allow logging to be done
|
|
forceExit();
|
|
}
|
|
} else {
|
|
// API could not be initialised
|
|
addLogEntry("OneDrive API could not be initialised with previously used details");
|
|
// Shutdown API instance
|
|
oneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(oneDriveApiInstance);
|
|
// Must force exit here, allow logging to be done
|
|
forceExit();
|
|
}
|
|
|
|
// API was initialised
|
|
addLogEntry("Sync Engine Initialised with new Onedrive API instance", ["verbose"]);
|
|
|
|
// Shutdown this API instance, as we will create API instances as required, when required
|
|
oneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(oneDriveApiInstance);
|
|
return true;
|
|
}
|
|
|
|
// Get Default Drive Details for this Account
|
|
void getDefaultDriveDetails() {
|
|
|
|
// Function variables
|
|
JSONValue defaultOneDriveDriveDetails;
|
|
|
|
// Get Default Drive Details for this Account
|
|
try {
|
|
addLogEntry("Getting Account Default Drive Details", ["debug"]);
|
|
defaultOneDriveDriveDetails = oneDriveApiInstance.getDefaultDriveDetails();
|
|
} catch (OneDriveException exception) {
|
|
addLogEntry("defaultOneDriveDriveDetails = oneDriveApiInstance.getDefaultDriveDetails() generated a OneDriveException", ["debug"]);
|
|
string thisFunctionName = getFunctionName!({});
|
|
|
|
if ((exception.httpStatusCode == 400) || (exception.httpStatusCode == 401)) {
|
|
// Handle the 400 | 401 error
|
|
handleClientUnauthorised(exception.httpStatusCode, exception.msg);
|
|
}
|
|
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(oneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) ||(exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to query Account Default Drive Details - retrying applicable request in 30 seconds");
|
|
addLogEntry("defaultOneDriveDriveDetails = oneDriveApiInstance.getDefaultDriveDetails() previously threw an error - retrying", ["debug"]);
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429 and 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: getDefaultDriveDetails()", ["debug"]);
|
|
getDefaultDriveDetails();
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
}
|
|
}
|
|
|
|
// If the JSON response is a correct JSON object, and has an 'id' we can set these details
|
|
if ((defaultOneDriveDriveDetails.type() == JSONType.object) && (hasId(defaultOneDriveDriveDetails))) {
|
|
addLogEntry("OneDrive Account Default Drive Details: " ~ to!string(defaultOneDriveDriveDetails), ["debug"]);
|
|
appConfig.accountType = defaultOneDriveDriveDetails["driveType"].str;
|
|
appConfig.defaultDriveId = defaultOneDriveDriveDetails["id"].str;
|
|
|
|
// Get the initial remaining size from OneDrive API response JSON
|
|
// This will be updated as we upload data to OneDrive
|
|
if (hasQuota(defaultOneDriveDriveDetails)) {
|
|
if ("remaining" in defaultOneDriveDriveDetails["quota"]){
|
|
// use the value provided
|
|
appConfig.remainingFreeSpace = defaultOneDriveDriveDetails["quota"]["remaining"].integer;
|
|
}
|
|
}
|
|
|
|
// In some cases OneDrive Business configurations 'restrict' quota details thus is empty / blank / negative value / zero
|
|
if (appConfig.remainingFreeSpace <= 0) {
|
|
// free space is <= 0 .. why ?
|
|
if ("remaining" in defaultOneDriveDriveDetails["quota"]) {
|
|
if (appConfig.accountType == "personal") {
|
|
// zero space available
|
|
addLogEntry("ERROR: OneDrive account currently has zero space available. Please free up some space online.");
|
|
appConfig.quotaAvailable = false;
|
|
} else {
|
|
// zero space available is being reported, maybe being restricted?
|
|
addLogEntry("WARNING: OneDrive quota information is being restricted or providing a zero value. Please fix by speaking to your OneDrive / Office 365 Administrator.");
|
|
appConfig.quotaRestricted = true;
|
|
}
|
|
} else {
|
|
// json response was missing a 'remaining' value
|
|
if (appConfig.accountType == "personal") {
|
|
addLogEntry("ERROR: OneDrive quota information is missing. Potentially your OneDrive account currently has zero space available. Please free up some space online.");
|
|
appConfig.quotaAvailable = false;
|
|
} else {
|
|
// quota details not available
|
|
addLogEntry("ERROR: OneDrive quota information is being restricted. Please fix by speaking to your OneDrive / Office 365 Administrator.");
|
|
appConfig.quotaRestricted = true;
|
|
}
|
|
}
|
|
}
|
|
// What did we set based on the data from the JSON
|
|
addLogEntry("appConfig.accountType = " ~ appConfig.accountType, ["debug"]);
|
|
addLogEntry("appConfig.defaultDriveId = " ~ appConfig.defaultDriveId, ["debug"]);
|
|
addLogEntry("appConfig.remainingFreeSpace = " ~ to!string(appConfig.remainingFreeSpace), ["debug"]);
|
|
addLogEntry("appConfig.quotaAvailable = " ~ appConfig.quotaAvailable, ["debug"]);
|
|
addLogEntry("appConfig.quotaRestricted = " ~ appConfig.quotaRestricted, ["debug"]);
|
|
|
|
// Make sure that appConfig.defaultDriveId is in our driveIDs array to use when checking if item is in database
|
|
// Keep the driveIDsArray with unique entries only
|
|
if (!canFind(driveIDsArray, appConfig.defaultDriveId)) {
|
|
// Add this drive id to the array to search with
|
|
driveIDsArray ~= appConfig.defaultDriveId;
|
|
}
|
|
} else {
|
|
// Handle the invalid JSON response
|
|
throw new accountDetailsException();
|
|
}
|
|
}
|
|
|
|
// Get Default Root Details for this Account
|
|
void getDefaultRootDetails() {
|
|
|
|
// Function variables
|
|
JSONValue defaultOneDriveRootDetails;
|
|
|
|
// Get Default Root Details for this Account
|
|
try {
|
|
addLogEntry("Getting Account Default Root Details", ["debug"]);
|
|
defaultOneDriveRootDetails = oneDriveApiInstance.getDefaultRootDetails();
|
|
} catch (OneDriveException exception) {
|
|
addLogEntry("defaultOneDriveRootDetails = oneDriveApiInstance.getDefaultRootDetails() generated a OneDriveException", ["debug"]);
|
|
string thisFunctionName = getFunctionName!({});
|
|
|
|
if ((exception.httpStatusCode == 400) || (exception.httpStatusCode == 401)) {
|
|
// Handle the 400 | 401 error
|
|
handleClientUnauthorised(exception.httpStatusCode, exception.msg);
|
|
}
|
|
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(oneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to query Account Default Root Details - retrying applicable request in 30 seconds");
|
|
addLogEntry("defaultOneDriveRootDetails = oneDriveApiInstance.getDefaultRootDetails() previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
getDefaultRootDetails();
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
}
|
|
}
|
|
|
|
// If the JSON response is a correct JSON object, and has an 'id' we can set these details
|
|
if ((defaultOneDriveRootDetails.type() == JSONType.object) && (hasId(defaultOneDriveRootDetails))) {
|
|
addLogEntry("OneDrive Account Default Root Details: " ~ to!string(defaultOneDriveRootDetails), ["debug"]);
|
|
appConfig.defaultRootId = defaultOneDriveRootDetails["id"].str;
|
|
addLogEntry("appConfig.defaultRootId = " ~ appConfig.defaultRootId, ["debug"]);
|
|
|
|
// Save the item to the database, so the account root drive is is always going to be present in the DB
|
|
saveItem(defaultOneDriveRootDetails);
|
|
} else {
|
|
// Handle the invalid JSON response
|
|
throw new accountDetailsException();
|
|
}
|
|
}
|
|
|
|
// Reset syncFailures to false
|
|
void resetSyncFailures() {
|
|
// Reset syncFailures to false if these are both empty
|
|
if (syncFailures) {
|
|
if ((fileDownloadFailures.empty) && (fileUploadFailures.empty)) {
|
|
addLogEntry("Resetting syncFailures = false");
|
|
syncFailures = false;
|
|
} else {
|
|
addLogEntry("File activity array's not empty - not resetting syncFailures");
|
|
}
|
|
}
|
|
}
|
|
|
|
// Perform a sync of the OneDrive Account
|
|
// - Query /delta
|
|
// - If singleDirectoryScope or nationalCloudDeployment is used we need to generate a /delta like response
|
|
// - Process changes (add, changes, moves, deletes)
|
|
// - Process any items to add (download data to local)
|
|
// - Detail any files that we failed to download
|
|
// - Process any deletes (remove local data)
|
|
void syncOneDriveAccountToLocalDisk() {
|
|
|
|
// performFullScanTrueUp value
|
|
addLogEntry("Perform a Full Scan True-Up: " ~ appConfig.fullScanTrueUpRequired, ["debug"]);
|
|
|
|
// Fetch the API response of /delta to track changes on OneDrive
|
|
fetchOneDriveDeltaAPIResponse(null, null, null);
|
|
// Process any download activities or cleanup actions
|
|
processDownloadActivities();
|
|
|
|
// If singleDirectoryScope is false, we are not targeting a single directory
|
|
// but if true, the target 'could' be a shared folder - so dont try and scan it again
|
|
if (!singleDirectoryScope) {
|
|
// OneDrive Shared Folder Handling
|
|
if (appConfig.accountType == "personal") {
|
|
// Personal Account Type
|
|
// https://github.com/OneDrive/onedrive-api-docs/issues/764
|
|
|
|
// Get the Remote Items from the Database
|
|
Item[] remoteItems = itemDB.selectRemoteItems();
|
|
foreach (remoteItem; remoteItems) {
|
|
// Check if this path is specifically excluded by 'skip_dir', but only if 'skip_dir' is not empty
|
|
if (appConfig.getValueString("skip_dir") != "") {
|
|
// The path that needs to be checked needs to include the '/'
|
|
// This due to if the user has specified in skip_dir an exclusive path: '/path' - that is what must be matched
|
|
if (selectiveSync.isDirNameExcluded(remoteItem.name)) {
|
|
// This directory name is excluded
|
|
addLogEntry("Skipping item - excluded by skip_dir config: " ~ remoteItem.name, ["verbose"]);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Directory name is not excluded or skip_dir is not populated
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
addLogEntry("Syncing this OneDrive Personal Shared Folder: " ~ remoteItem.name);
|
|
}
|
|
// Check this OneDrive Personal Shared Folder for changes
|
|
fetchOneDriveDeltaAPIResponse(remoteItem.remoteDriveId, remoteItem.remoteId, remoteItem.name);
|
|
// Process any download activities or cleanup actions for this OneDrive Personal Shared Folder
|
|
processDownloadActivities();
|
|
}
|
|
} else {
|
|
// Is this a Business Account with Sync Business Shared Items enabled?
|
|
if ((appConfig.accountType == "business") && ( appConfig.getValueBool("sync_business_shared_items"))) {
|
|
|
|
// Business Account Shared Items Handling
|
|
// - OneDrive Business Shared Folder
|
|
// - OneDrive Business Shared Files ??
|
|
// - SharePoint Links
|
|
|
|
// Get the Remote Items from the Database
|
|
Item[] remoteItems = itemDB.selectRemoteItems();
|
|
|
|
foreach (remoteItem; remoteItems) {
|
|
// Check if this path is specifically excluded by 'skip_dir', but only if 'skip_dir' is not empty
|
|
if (appConfig.getValueString("skip_dir") != "") {
|
|
// The path that needs to be checked needs to include the '/'
|
|
// This due to if the user has specified in skip_dir an exclusive path: '/path' - that is what must be matched
|
|
if (selectiveSync.isDirNameExcluded(remoteItem.name)) {
|
|
// This directory name is excluded
|
|
addLogEntry("Skipping item - excluded by skip_dir config: " ~ remoteItem.name, ["verbose"]);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Directory name is not excluded or skip_dir is not populated
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
addLogEntry("Syncing this OneDrive Business Shared Folder: " ~ remoteItem.name);
|
|
}
|
|
|
|
// Debug log output
|
|
addLogEntry("Fetching /delta API response for:", ["debug"]);
|
|
addLogEntry(" remoteItem.remoteDriveId: " ~ remoteItem.remoteDriveId, ["debug"]);
|
|
addLogEntry(" remoteItem.remoteId: " ~ remoteItem.remoteId, ["debug"]);
|
|
|
|
// Check this OneDrive Personal Shared Folder for changes
|
|
fetchOneDriveDeltaAPIResponse(remoteItem.remoteDriveId, remoteItem.remoteId, remoteItem.name);
|
|
|
|
// Process any download activities or cleanup actions for this OneDrive Personal Shared Folder
|
|
processDownloadActivities();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Configure singleDirectoryScope = true if this function is called
|
|
// By default, singleDirectoryScope = false
|
|
void setSingleDirectoryScope(string normalisedSingleDirectoryPath) {
|
|
|
|
// Function variables
|
|
Item searchItem;
|
|
JSONValue onlinePathData;
|
|
|
|
// Set the main flag
|
|
singleDirectoryScope = true;
|
|
|
|
// What are we doing?
|
|
addLogEntry("The OneDrive Client was asked to search for this directory online and create it if it's not located: " ~ normalisedSingleDirectoryPath);
|
|
|
|
// Query the OneDrive API for the specified path online
|
|
// In a --single-directory scenario, we need to travervse the entire path that we are wanting to sync
|
|
// and then check the path element does it exist online, if it does, is it a POSIX match, or if it does not, create the path
|
|
// Once we have searched online, we have the right drive id and item id so that we can downgrade the sync status, then build up
|
|
// any object items from that location
|
|
// This is because, in a --single-directory scenario, any folder in the entire path tree could be a 'case-insensitive match'
|
|
|
|
try {
|
|
onlinePathData = queryOneDriveForSpecificPathAndCreateIfMissing(normalisedSingleDirectoryPath, true);
|
|
} catch (posixException e) {
|
|
displayPosixErrorMessage(e.msg);
|
|
addLogEntry("ERROR: Requested directory to search for and potentially create has a 'case-insensitive match' to an existing directory on OneDrive online.");
|
|
}
|
|
|
|
// Was a valid JSON response provided?
|
|
if (onlinePathData.type() == JSONType.object) {
|
|
// Valid JSON item was returned
|
|
searchItem = makeItem(onlinePathData);
|
|
addLogEntry("searchItem: " ~ to!string(searchItem), ["debug"]);
|
|
|
|
// Is this item a potential Shared Folder?
|
|
// Is this JSON a remote object
|
|
if (isItemRemote(onlinePathData)) {
|
|
// The path we are seeking is remote to our account drive id
|
|
searchItem.driveId = onlinePathData["remoteItem"]["parentReference"]["driveId"].str;
|
|
searchItem.id = onlinePathData["remoteItem"]["id"].str;
|
|
}
|
|
|
|
// Set these items so that these can be used as required
|
|
singleDirectoryScopeDriveId = searchItem.driveId;
|
|
singleDirectoryScopeItemId = searchItem.id;
|
|
} else {
|
|
addLogEntry();
|
|
addLogEntry("The requested --single-directory path to sync has generated an error. Please correct this error and try again.");
|
|
addLogEntry();
|
|
Thread.sleep(dur!("msecs")(500));
|
|
exit(EXIT_FAILURE);
|
|
}
|
|
}
|
|
|
|
// Query OneDrive API for /delta changes and iterate through items online
|
|
void fetchOneDriveDeltaAPIResponse(string driveIdToQuery = null, string itemIdToQuery = null, string sharedFolderName = null) {
|
|
|
|
string deltaLink = null;
|
|
string currentDeltaLink = null;
|
|
string deltaLinkAvailable;
|
|
JSONValue deltaChanges;
|
|
ulong responseBundleCount;
|
|
ulong jsonItemsReceived = 0;
|
|
|
|
// Reset jsonItemsToProcess & processedCount
|
|
jsonItemsToProcess = [];
|
|
processedCount = 0;
|
|
|
|
// Was a driveId provided as an input
|
|
//if (driveIdToQuery == "") {
|
|
if (strip(driveIdToQuery).empty) {
|
|
// No provided driveId to query, use the account default
|
|
addLogEntry("driveIdToQuery was empty, setting to appConfig.defaultDriveId", ["debug"]);
|
|
driveIdToQuery = appConfig.defaultDriveId;
|
|
addLogEntry("driveIdToQuery: " ~ driveIdToQuery, ["debug"]);
|
|
}
|
|
|
|
// Was an itemId provided as an input
|
|
//if (itemIdToQuery == "") {
|
|
if (strip(itemIdToQuery).empty) {
|
|
// No provided itemId to query, use the account default
|
|
addLogEntry("itemIdToQuery was empty, setting to appConfig.defaultRootId", ["debug"]);
|
|
itemIdToQuery = appConfig.defaultRootId;
|
|
addLogEntry("itemIdToQuery: " ~ itemIdToQuery, ["debug"]);
|
|
}
|
|
|
|
// What OneDrive API query do we use?
|
|
// - Are we running against a National Cloud Deployments that does not support /delta ?
|
|
// National Cloud Deployments do not support /delta as a query
|
|
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
|
|
//
|
|
// - Are we performing a --single-directory sync, which will exclude many items online, focusing in on a specific online directory
|
|
//
|
|
// - Are we performing a --download-only --cleanup-local-files action?
|
|
// - If we are, and we use a normal /delta query, we get all the local 'deleted' objects as well.
|
|
// - If the user deletes a folder online, then replaces it online, we download the deletion events and process the new 'upload' via the web iterface ..
|
|
// the net effect of this, is that the valid local files we want to keep, are actually deleted ...... not desirable
|
|
if ((singleDirectoryScope) || (nationalCloudDeployment) || (cleanupLocalFiles)) {
|
|
// Generate a simulated /delta response so that we correctly capture the current online state, less any 'online' delete and replace activity
|
|
generateSimulatedDeltaResponse = true;
|
|
}
|
|
|
|
// What /delta query do we use?
|
|
if (!generateSimulatedDeltaResponse) {
|
|
// This should be the majority default pathway application use
|
|
// Get the current delta link from the database for this DriveID and RootID
|
|
deltaLinkAvailable = itemDB.getDeltaLink(driveIdToQuery, itemIdToQuery);
|
|
if (!deltaLinkAvailable.empty) {
|
|
addLogEntry("Using database stored deltaLink", ["debug"]);
|
|
currentDeltaLink = deltaLinkAvailable;
|
|
}
|
|
|
|
// Do we need to perform a Full Scan True Up? Is 'appConfig.fullScanTrueUpRequired' set to 'true'?
|
|
if (appConfig.fullScanTrueUpRequired) {
|
|
addLogEntry("Performing a full scan of online data to ensure consistent local state");
|
|
addLogEntry("Setting currentDeltaLink = null", ["debug"]);
|
|
currentDeltaLink = null;
|
|
}
|
|
|
|
// Dynamic output for non-verbose and verbose run so that the user knows something is being retreived from the OneDrive API
|
|
if (appConfig.verbosityCount == 0) {
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
addLogEntry("Fetching items from the OneDrive API for Drive ID: " ~ driveIdToQuery, ["logFileOnly"]);
|
|
|
|
// Use the dots to show the application is 'doing something'
|
|
addLogEntry("Fetching items from the OneDrive API for Drive ID: " ~ driveIdToQuery ~ " .", ["consoleOnlyNoNewLine"]);
|
|
}
|
|
} else {
|
|
addLogEntry("Fetching /delta response from the OneDrive API for Drive ID: " ~ driveIdToQuery, ["verbose"]);
|
|
}
|
|
|
|
// Create a new API Instance for querying /delta and initialise it
|
|
// Reuse the socket to speed up
|
|
bool keepAlive = true;
|
|
OneDriveApi getDeltaQueryOneDriveApiInstance;
|
|
getDeltaQueryOneDriveApiInstance = new OneDriveApi(appConfig);
|
|
getDeltaQueryOneDriveApiInstance.initialise(keepAlive);
|
|
|
|
for (;;) {
|
|
responseBundleCount++;
|
|
// Get the /delta changes via the OneDrive API
|
|
// getDeltaChangesByItemId has the re-try logic for transient errors
|
|
deltaChanges = getDeltaChangesByItemId(driveIdToQuery, itemIdToQuery, currentDeltaLink, getDeltaQueryOneDriveApiInstance);
|
|
|
|
// If the initial deltaChanges response is an invalid JSON object, keep trying ..
|
|
if (deltaChanges.type() != JSONType.object) {
|
|
while (deltaChanges.type() != JSONType.object) {
|
|
// Handle the invalid JSON response adn retry
|
|
addLogEntry("ERROR: Query of the OneDrive API via deltaChanges = getDeltaChangesByItemId() returned an invalid JSON response", ["debug"]);
|
|
deltaChanges = getDeltaChangesByItemId(driveIdToQuery, itemIdToQuery, currentDeltaLink, getDeltaQueryOneDriveApiInstance);
|
|
}
|
|
}
|
|
|
|
ulong nrChanges = count(deltaChanges["value"].array);
|
|
int changeCount = 0;
|
|
|
|
if (appConfig.verbosityCount == 0) {
|
|
// Dynamic output for a non-verbose run so that the user knows something is happening
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
addLogEntry(".", ["consoleOnlyNoNewLine"]);
|
|
}
|
|
} else {
|
|
addLogEntry("Processing API Response Bundle: " ~ to!string(responseBundleCount) ~ " - Quantity of 'changes|items' in this bundle to process: " ~ to!string(nrChanges), ["verbose"]);
|
|
}
|
|
|
|
jsonItemsReceived = jsonItemsReceived + nrChanges;
|
|
|
|
// We have a valid deltaChanges JSON array. This means we have at least 200+ JSON items to process.
|
|
// The API response however cannot be run in parallel as the OneDrive API sends the JSON items in the order in which they must be processed
|
|
foreach (onedriveJSONItem; deltaChanges["value"].array) {
|
|
// increment change count for this item
|
|
changeCount++;
|
|
// Process the OneDrive object item JSON
|
|
processDeltaJSONItem(onedriveJSONItem, nrChanges, changeCount, responseBundleCount, singleDirectoryScope);
|
|
}
|
|
|
|
// The response may contain either @odata.deltaLink or @odata.nextLink
|
|
if ("@odata.deltaLink" in deltaChanges) {
|
|
// Log action
|
|
addLogEntry("Setting next currentDeltaLink to (@odata.deltaLink): " ~ deltaChanges["@odata.deltaLink"].str, ["debug"]);
|
|
// Update currentDeltaLink
|
|
currentDeltaLink = deltaChanges["@odata.deltaLink"].str;
|
|
// Store this for later use post processing jsonItemsToProcess items
|
|
latestDeltaLink = deltaChanges["@odata.deltaLink"].str;
|
|
}
|
|
|
|
// Update deltaLink to next changeSet bundle
|
|
if ("@odata.nextLink" in deltaChanges) {
|
|
// Log action
|
|
addLogEntry("Setting next currentDeltaLink & deltaLinkAvailable to (@odata.nextLink): " ~ deltaChanges["@odata.nextLink"].str, ["debug"]);
|
|
// Update currentDeltaLink
|
|
currentDeltaLink = deltaChanges["@odata.nextLink"].str;
|
|
// Update deltaLinkAvailable to next changeSet bundle to quantify how many changes we have to process
|
|
deltaLinkAvailable = deltaChanges["@odata.nextLink"].str;
|
|
// Store this for later use post processing jsonItemsToProcess items
|
|
latestDeltaLink = deltaChanges["@odata.nextLink"].str;
|
|
}
|
|
else break;
|
|
}
|
|
|
|
// To finish off the JSON processing items, this is needed to reflect this in the log
|
|
addLogEntry("------------------------------------------------------------------", ["debug"]);
|
|
|
|
// Shutdown the API
|
|
getDeltaQueryOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(getDeltaQueryOneDriveApiInstance);
|
|
|
|
// Log that we have finished querying the /delta API
|
|
if (appConfig.verbosityCount == 0) {
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
// Close out the '....' being printed to the console
|
|
addLogEntry("\n", ["consoleOnlyNoNewLine"]);
|
|
}
|
|
} else {
|
|
addLogEntry("Finished processing /delta JSON response from the OneDrive API", ["verbose"]);
|
|
}
|
|
|
|
// If this was set, now unset it, as this will have been completed, so that for a true up, we dont do a double full scan
|
|
if (appConfig.fullScanTrueUpRequired) {
|
|
addLogEntry("Unsetting fullScanTrueUpRequired as this has been performed", ["debug"]);
|
|
appConfig.fullScanTrueUpRequired = false;
|
|
}
|
|
} else {
|
|
// Why are are generating a /delta response
|
|
addLogEntry("Why are we generating a /delta response:", ["debug"]);
|
|
addLogEntry(" singleDirectoryScope: " ~ to!string(singleDirectoryScope), ["debug"]);
|
|
addLogEntry(" nationalCloudDeployment: " ~ to!string(nationalCloudDeployment), ["debug"]);
|
|
addLogEntry(" cleanupLocalFiles: " ~ to!string(cleanupLocalFiles), ["debug"]);
|
|
|
|
// What 'path' are we going to start generating the response for
|
|
string pathToQuery;
|
|
|
|
// If --single-directory has been called, use the value that has been set
|
|
if (singleDirectoryScope) {
|
|
pathToQuery = appConfig.getValueString("single_directory");
|
|
}
|
|
|
|
// We could also be syncing a Shared Folder of some description
|
|
if (!sharedFolderName.empty) {
|
|
pathToQuery = sharedFolderName;
|
|
}
|
|
|
|
// Generate the simulated /delta response
|
|
//
|
|
// The generated /delta response however contains zero deleted JSON items, so the only way that we can track this, is if the object was in sync
|
|
// we have the object in the database, thus, what we need to do is for every DB object in the tree of items, flag 'syncStatus' as 'N', then when we process
|
|
// the returned JSON items from the API, we flag the item as back in sync, then we can cleanup any out-of-sync items
|
|
//
|
|
// The flagging of the local database items to 'N' is handled within the generateDeltaResponse() function
|
|
//
|
|
// When these JSON items are then processed, if the item exists online, and is in the DB, and that the values match, the DB item is flipped back to 'Y'
|
|
// This then allows the application to look for any remaining 'N' values, and delete these as no longer needed locally
|
|
deltaChanges = generateDeltaResponse(pathToQuery);
|
|
|
|
ulong nrChanges = count(deltaChanges["value"].array);
|
|
int changeCount = 0;
|
|
addLogEntry("API Response Bundle: " ~ to!string(responseBundleCount) ~ " - Quantity of 'changes|items' in this bundle to process: " ~ to!string(nrChanges), ["debug"]);
|
|
jsonItemsReceived = jsonItemsReceived + nrChanges;
|
|
|
|
// The API response however cannot be run in parallel as the OneDrive API sends the JSON items in the order in which they must be processed
|
|
foreach (onedriveJSONItem; deltaChanges["value"].array) {
|
|
// increment change count for this item
|
|
changeCount++;
|
|
// Process the OneDrive object item JSON
|
|
processDeltaJSONItem(onedriveJSONItem, nrChanges, changeCount, responseBundleCount, singleDirectoryScope);
|
|
}
|
|
|
|
// To finish off the JSON processing items, this is needed to reflect this in the log
|
|
addLogEntry("------------------------------------------------------------------", ["debug"]);
|
|
|
|
// Log that we have finished generating our self generated /delta response
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
addLogEntry("Finished processing self generated /delta JSON response from the OneDrive API");
|
|
}
|
|
}
|
|
|
|
// Cleanup deltaChanges as this is no longer needed
|
|
object.destroy(deltaChanges);
|
|
|
|
// We have JSON items received from the OneDrive API
|
|
addLogEntry("Number of JSON Objects received from OneDrive API: " ~ to!string(jsonItemsReceived), ["debug"]);
|
|
addLogEntry("Number of JSON Objects already processed (root and deleted items): " ~ to!string((jsonItemsReceived - jsonItemsToProcess.length)), ["debug"]);
|
|
|
|
// We should have now at least processed all the JSON items as returned by the /delta call
|
|
// Additionally, we should have a new array, that now contains all the JSON items we need to process that are non 'root' or deleted items
|
|
addLogEntry("Number of JSON items to process is: " ~ to!string(jsonItemsToProcess.length), ["debug"]);
|
|
|
|
// Are there items to process?
|
|
if (jsonItemsToProcess.length > 0) {
|
|
// Lets deal with the JSON items in a batch process
|
|
ulong batchSize = 500;
|
|
ulong batchCount = (jsonItemsToProcess.length + batchSize - 1) / batchSize;
|
|
ulong batchesProcessed = 0;
|
|
|
|
// Dynamic output for a non-verbose run so that the user knows something is happening
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
// Logfile entry
|
|
addLogEntry("Processing " ~ to!string(jsonItemsToProcess.length) ~ " applicable changes and items received from Microsoft OneDrive", ["logFileOnly"]);
|
|
// Console only output
|
|
addLogEntry("Processing " ~ to!string(jsonItemsToProcess.length) ~ " applicable changes and items received from Microsoft OneDrive ", ["consoleOnlyNoNewLine"]);
|
|
|
|
if (appConfig.verbosityCount != 0) {
|
|
// Close out the console only processing line above, if we are doing verbose or above logging
|
|
addLogEntry("\n", ["consoleOnlyNoNewLine"]);
|
|
}
|
|
}
|
|
|
|
// For each batch, process the JSON items that need to be now processed.
|
|
// 'root' and deleted objects have already been handled
|
|
foreach (batchOfJSONItems; jsonItemsToProcess.chunks(batchSize)) {
|
|
// Chunk the total items to process into 500 lot items
|
|
batchesProcessed++;
|
|
|
|
if (appConfig.verbosityCount == 0) {
|
|
// Dynamic output for a non-verbose run so that the user knows something is happening
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
addLogEntry(".", ["consoleOnlyNoNewLine"]);
|
|
}
|
|
} else {
|
|
addLogEntry("Processing OneDrive JSON item batch [" ~ to!string(batchesProcessed) ~ "/" ~ to!string(batchCount) ~ "] to ensure consistent local state", ["verbose"]);
|
|
}
|
|
|
|
// Process the batch
|
|
processJSONItemsInBatch(batchOfJSONItems, batchesProcessed, batchCount);
|
|
|
|
// To finish off the JSON processing items, this is needed to reflect this in the log
|
|
addLogEntry("------------------------------------------------------------------", ["debug"]);
|
|
}
|
|
|
|
if (appConfig.verbosityCount == 0) {
|
|
// close off '.' output
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
addLogEntry("\n", ["consoleOnlyNoNewLine"]);
|
|
}
|
|
}
|
|
|
|
// Free up memory and items processed as it is pointless now having this data around
|
|
jsonItemsToProcess = [];
|
|
|
|
// Debug output - what was processed
|
|
addLogEntry("Number of JSON items to process is: " ~ to!string(jsonItemsToProcess.length), ["debug"]);
|
|
addLogEntry("Number of JSON items processed was: " ~ to!string(processedCount), ["debug"]);
|
|
} else {
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
addLogEntry("No additional changes or items that can be applied were discovered while processing the data received from Microsoft OneDrive");
|
|
}
|
|
}
|
|
|
|
// Update the deltaLink in the database so that we can reuse this now that jsonItemsToProcess has been processed
|
|
if (!latestDeltaLink.empty) {
|
|
addLogEntry("Updating completed deltaLink in DB to: " ~ latestDeltaLink, ["debug"]);
|
|
itemDB.setDeltaLink(driveIdToQuery, itemIdToQuery, latestDeltaLink);
|
|
}
|
|
|
|
// Keep the driveIDsArray with unique entries only
|
|
if (!canFind(driveIDsArray, driveIdToQuery)) {
|
|
// Add this driveId to the array of driveId's we know about
|
|
driveIDsArray ~= driveIdToQuery;
|
|
}
|
|
}
|
|
|
|
// Process the /delta API JSON response items
|
|
void processDeltaJSONItem(JSONValue onedriveJSONItem, ulong nrChanges, int changeCount, ulong responseBundleCount, bool singleDirectoryScope) {
|
|
|
|
// Variables for this foreach loop
|
|
string thisItemId;
|
|
bool itemIsRoot = false;
|
|
bool handleItemAsRootObject = false;
|
|
bool itemIsDeletedOnline = false;
|
|
bool itemHasParentReferenceId = false;
|
|
bool itemHasParentReferencePath = false;
|
|
bool itemIdMatchesDefaultRootId = false;
|
|
bool itemNameExplicitMatchRoot = false;
|
|
string objectParentDriveId;
|
|
|
|
addLogEntry("------------------------------------------------------------------", ["debug"]);
|
|
addLogEntry("Processing OneDrive Item " ~ to!string(changeCount) ~ " of " ~ to!string(nrChanges) ~ " from API Response Bundle " ~ to!string(responseBundleCount), ["debug"]);
|
|
addLogEntry("Raw JSON OneDrive Item: " ~ to!string(onedriveJSONItem), ["debug"]);
|
|
|
|
// What is this item's id
|
|
thisItemId = onedriveJSONItem["id"].str;
|
|
// Is this a deleted item - only calculate this once
|
|
itemIsDeletedOnline = isItemDeleted(onedriveJSONItem);
|
|
|
|
if(!itemIsDeletedOnline){
|
|
// This is not a deleted item
|
|
addLogEntry("This item is not a OneDrive deletion change", ["debug"]);
|
|
|
|
// Only calculate this once
|
|
itemIsRoot = isItemRoot(onedriveJSONItem);
|
|
itemHasParentReferenceId = hasParentReferenceId(onedriveJSONItem);
|
|
itemIdMatchesDefaultRootId = (thisItemId == appConfig.defaultRootId);
|
|
itemNameExplicitMatchRoot = (onedriveJSONItem["name"].str == "root");
|
|
objectParentDriveId = onedriveJSONItem["parentReference"]["driveId"].str;
|
|
|
|
// Test is this is the OneDrive Users Root?
|
|
// Debug output of change evaluation items
|
|
addLogEntry("defaultRootId = " ~ appConfig.defaultRootId, ["debug"]);
|
|
addLogEntry("'search id' = " ~ thisItemId, ["debug"]);
|
|
addLogEntry("id == defaultRootId = " ~ to!string(itemIdMatchesDefaultRootId), ["debug"]);
|
|
addLogEntry("isItemRoot(onedriveJSONItem) = " ~ to!string(itemIsRoot), ["debug"]);
|
|
addLogEntry("onedriveJSONItem['name'].str == 'root' = " ~ to!string(itemNameExplicitMatchRoot), ["debug"]);
|
|
addLogEntry("itemHasParentReferenceId = " ~ to!string(itemHasParentReferenceId), ["debug"]);
|
|
|
|
if ( (itemIdMatchesDefaultRootId || singleDirectoryScope) && itemIsRoot && itemNameExplicitMatchRoot) {
|
|
// This IS a OneDrive Root item or should be classified as such in the case of 'singleDirectoryScope'
|
|
addLogEntry("JSON item will flagged as a 'root' item", ["debug"]);
|
|
handleItemAsRootObject = true;
|
|
}
|
|
}
|
|
|
|
// How do we handle this JSON item from the OneDrive API?
|
|
// Is this a confirmed 'root' item, has no Parent ID, or is a Deleted Item
|
|
if (handleItemAsRootObject || !itemHasParentReferenceId || itemIsDeletedOnline){
|
|
// Is a root item, has no id in parentReference or is a OneDrive deleted item
|
|
addLogEntry("objectParentDriveId = " ~ objectParentDriveId, ["debug"]);
|
|
addLogEntry("handleItemAsRootObject = " ~ to!string(handleItemAsRootObject), ["debug"]);
|
|
addLogEntry("itemHasParentReferenceId = " ~ to!string(itemHasParentReferenceId), ["debug"]);
|
|
addLogEntry("itemIsDeletedOnline = " ~ to!string(itemIsDeletedOnline), ["debug"]);
|
|
addLogEntry("Handling change immediately as 'root item', or has no parent reference id or is a deleted item", ["debug"]);
|
|
|
|
// OK ... do something with this JSON post here ....
|
|
processRootAndDeletedJSONItems(onedriveJSONItem, objectParentDriveId, handleItemAsRootObject, itemIsDeletedOnline, itemHasParentReferenceId);
|
|
} else {
|
|
// Do we need to update this RAW JSON from OneDrive?
|
|
if ( (objectParentDriveId != appConfig.defaultDriveId) && (appConfig.accountType == "business") && (appConfig.getValueBool("sync_business_shared_items")) ) {
|
|
// Potentially need to update this JSON data
|
|
addLogEntry("Potentially need to update this source JSON .... need to check the database", ["debug"]);
|
|
|
|
// Check the DB for 'remote' objects, searching 'remoteDriveId' and 'remoteId' items for this remoteItem.driveId and remoteItem.id
|
|
Item remoteDBItem;
|
|
itemDB.selectByRemoteId(objectParentDriveId, thisItemId, remoteDBItem);
|
|
|
|
// Is the data that was returned from the database what we are looking for?
|
|
if ((remoteDBItem.remoteDriveId == objectParentDriveId) && (remoteDBItem.remoteId == thisItemId)) {
|
|
// Yes, this is the record we are looking for
|
|
addLogEntry("DB Item response for remoteDBItem: " ~ to!string(remoteDBItem), ["debug"]);
|
|
|
|
// Must compare remoteDBItem.name with remoteItem.name
|
|
if (remoteDBItem.name != onedriveJSONItem["name"].str) {
|
|
// Update JSON Item
|
|
string actualOnlineName = onedriveJSONItem["name"].str;
|
|
addLogEntry("Updating source JSON 'name' to that which is the actual local directory", ["debug"]);
|
|
addLogEntry("onedriveJSONItem['name'] was: " ~ onedriveJSONItem["name"].str, ["debug"]);
|
|
addLogEntry("Updating onedriveJSONItem['name'] to: " ~ remoteDBItem.name, ["debug"]);
|
|
onedriveJSONItem["name"] = remoteDBItem.name;
|
|
addLogEntry("onedriveJSONItem['name'] now: " ~ onedriveJSONItem["name"].str, ["debug"]);
|
|
// Add the original name to the JSON
|
|
onedriveJSONItem["actualOnlineName"] = actualOnlineName;
|
|
}
|
|
}
|
|
}
|
|
|
|
// If we are not self-generating a /delta response, check this initial /delta JSON bundle item against the basic checks
|
|
// of applicability against 'skip_file', 'skip_dir' and 'sync_list'
|
|
// We only do this if we did not generate a /delta response, as generateDeltaResponse() performs the checkJSONAgainstClientSideFiltering()
|
|
// against elements as it is building the /delta compatible response
|
|
// If we blindly just 'check again' all JSON responses then there is potentially double JSON processing going on if we used generateDeltaResponse()
|
|
bool discardDeltaJSONItem = false;
|
|
if (!generateSimulatedDeltaResponse) {
|
|
// Check applicability against 'skip_file', 'skip_dir' and 'sync_list'
|
|
discardDeltaJSONItem = checkJSONAgainstClientSideFiltering(onedriveJSONItem);
|
|
}
|
|
|
|
// Add this JSON item for further processing if this is not being discarded
|
|
if (!discardDeltaJSONItem) {
|
|
addLogEntry("Adding this Raw JSON OneDrive Item to jsonItemsToProcess array for further processing", ["debug"]);
|
|
jsonItemsToProcess ~= onedriveJSONItem;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Process 'root' and 'deleted' OneDrive JSON items
|
|
void processRootAndDeletedJSONItems(JSONValue onedriveJSONItem, string driveId, bool handleItemAsRootObject, bool itemIsDeletedOnline, bool itemHasParentReferenceId) {
|
|
|
|
// Use the JSON elements rather can computing a DB struct via makeItem()
|
|
string thisItemId = onedriveJSONItem["id"].str;
|
|
string thisItemDriveId = onedriveJSONItem["parentReference"]["driveId"].str;
|
|
|
|
// Check if the item has been seen before
|
|
Item existingDatabaseItem;
|
|
bool existingDBEntry = itemDB.selectById(thisItemDriveId, thisItemId, existingDatabaseItem);
|
|
|
|
// Is the item deleted online?
|
|
if(!itemIsDeletedOnline) {
|
|
|
|
// Is the item a confirmed root object?
|
|
|
|
// The JSON item should be considered a 'root' item if:
|
|
// 1. Contains a ["root"] element
|
|
// 2. Has no ["parentReference"]["id"] ... #323 & #324 highlighted that this is false as some 'root' shared objects now can have an 'id' element .. OneDrive API change
|
|
// 2. Has no ["parentReference"]["path"]
|
|
// 3. Was detected by an input flag as to be handled as a root item regardless of actual status
|
|
|
|
if ((handleItemAsRootObject) || (!itemHasParentReferenceId)) {
|
|
addLogEntry("Handing JSON object as OneDrive 'root' object", ["debug"]);
|
|
if (!existingDBEntry) {
|
|
// we have not seen this item before
|
|
saveItem(onedriveJSONItem);
|
|
}
|
|
}
|
|
} else {
|
|
// Change is to delete an item
|
|
addLogEntry("Handing a OneDrive Deleted Item", ["debug"]);
|
|
if (existingDBEntry) {
|
|
// Flag to delete
|
|
addLogEntry("Flagging to delete item locally: " ~ to!string(onedriveJSONItem), ["debug"]);
|
|
idsToDelete ~= [thisItemDriveId, thisItemId];
|
|
} else {
|
|
// Flag to ignore
|
|
addLogEntry("Flagging item to skip: " ~ to!string(onedriveJSONItem), ["debug"]);
|
|
skippedItems.insert(thisItemId);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Process each of the elements contained in jsonItemsToProcess[]
|
|
void processJSONItemsInBatch(JSONValue[] array, ulong batchGroup, ulong batchCount) {
|
|
|
|
ulong batchElementCount = array.length;
|
|
|
|
foreach (i, onedriveJSONItem; array.enumerate) {
|
|
// Use the JSON elements rather can computing a DB struct via makeItem()
|
|
ulong elementCount = i +1;
|
|
|
|
// To show this is the processing for this particular item, start off with this breaker line
|
|
addLogEntry("------------------------------------------------------------------", ["debug"]);
|
|
addLogEntry("Processing OneDrive JSON item " ~ to!string(elementCount) ~ " of " ~ to!string(batchElementCount) ~ " as part of JSON Item Batch " ~ to!string(batchGroup) ~ " of " ~ to!string(batchCount), ["debug"]);
|
|
addLogEntry("Raw JSON OneDrive Item: " ~ to!string(onedriveJSONItem), ["debug"]);
|
|
|
|
string thisItemId = onedriveJSONItem["id"].str;
|
|
string thisItemDriveId = onedriveJSONItem["parentReference"]["driveId"].str;
|
|
string thisItemParentId = onedriveJSONItem["parentReference"]["id"].str;
|
|
string thisItemName = onedriveJSONItem["name"].str;
|
|
|
|
// Create an empty item struct for an existing DB item
|
|
Item existingDatabaseItem;
|
|
|
|
// Do we NOT want this item?
|
|
bool unwanted = false; // meaning by default we will WANT this item
|
|
// Is this parent is in the database
|
|
bool parentInDatabase = false;
|
|
// What is the path of the new item
|
|
string newItemPath;
|
|
|
|
// Configure the remoteItem - so if it is used, it can be utilised later
|
|
Item remoteItem;
|
|
|
|
// Check the database for an existing entry for this JSON item
|
|
bool existingDBEntry = itemDB.selectById(thisItemDriveId, thisItemId, existingDatabaseItem);
|
|
|
|
// Calculate if the Parent Item is in the database so that it can be re-used
|
|
parentInDatabase = itemDB.idInLocalDatabase(thisItemDriveId, thisItemParentId);
|
|
|
|
// Calculate the path of this JSON item, but we can only do this if the parent is in the database
|
|
if (parentInDatabase) {
|
|
// Calculate this items path
|
|
newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName;
|
|
addLogEntry("New Item calculated full path is: " ~ newItemPath, ["debug"]);
|
|
} else {
|
|
// Parent not in the database
|
|
// Is the parent a 'folder' from another user? ie - is this a 'shared folder' that has been shared with us?
|
|
addLogEntry("Parent ID is not in DB .. ", ["debug"]);
|
|
|
|
// Why?
|
|
if (thisItemDriveId == appConfig.defaultDriveId) {
|
|
// Flagging as unwanted
|
|
addLogEntry("Flagging as unwanted: thisItemDriveId (" ~ thisItemDriveId ~ "), thisItemParentId (" ~ thisItemParentId ~ ") not in local database", ["debug"]);
|
|
|
|
if (thisItemParentId in skippedItems) {
|
|
addLogEntry("Reason: thisItemParentId listed within skippedItems", ["debug"]);
|
|
}
|
|
unwanted = true;
|
|
} else {
|
|
// Edge case as the parent (from another users OneDrive account) will never be in the database - potentially a shared object?
|
|
addLogEntry("The reported parentId is not in the database. This potentially is a shared folder as 'remoteItem.driveId' != 'appConfig.defaultDriveId'. Relevant Details: remoteItem.driveId (" ~ remoteItem.driveId ~ "), remoteItem.parentId (" ~ remoteItem.parentId ~ ")", ["debug"]);
|
|
addLogEntry("Potential Shared Object JSON: " ~ to!string(onedriveJSONItem), ["debug"]);
|
|
|
|
// Format the OneDrive change into a consumable object for the database
|
|
remoteItem = makeItem(onedriveJSONItem);
|
|
|
|
if (appConfig.accountType == "personal") {
|
|
// Personal Account Handling
|
|
addLogEntry("Handling a Personal Shared Item JSON object", ["debug"]);
|
|
|
|
if (hasSharedElement(onedriveJSONItem)) {
|
|
// Has the Shared JSON structure
|
|
addLogEntry("Personal Shared Item JSON object has the 'shared' JSON structure", ["debug"]);
|
|
|
|
// Create a DB Tie Record for this parent object
|
|
addLogEntry("Creating a DB Tie for this Personal Shared Folder", ["debug"]);
|
|
|
|
// DB Tie
|
|
Item parentItem;
|
|
parentItem.driveId = onedriveJSONItem["parentReference"]["driveId"].str;
|
|
parentItem.id = onedriveJSONItem["parentReference"]["id"].str;
|
|
parentItem.name = "root";
|
|
parentItem.type = ItemType.dir;
|
|
parentItem.mtime = remoteItem.mtime;
|
|
parentItem.parentId = null;
|
|
|
|
// Add this DB Tie parent record to the local database
|
|
addLogEntry("Insert local database with remoteItem parent details: " ~ to!string(parentItem), ["debug"]);
|
|
itemDB.upsert(parentItem);
|
|
}
|
|
|
|
// Ensure that this item has no parent
|
|
addLogEntry("Setting remoteItem.parentId to be null", ["debug"]);
|
|
remoteItem.parentId = null;
|
|
// Add this record to the local database
|
|
addLogEntry("Update/Insert local database with remoteItem details with remoteItem.parentId as null: " ~ to!string(remoteItem), ["debug"]);
|
|
itemDB.upsert(remoteItem);
|
|
} else {
|
|
// Business or SharePoint Account Handling
|
|
addLogEntry("Handling a Business or SharePoint Shared Item JSON object", ["debug"]);
|
|
|
|
if (appConfig.accountType == "business") {
|
|
// Create a DB Tie Record for this parent object
|
|
addLogEntry("Creating a DB Tie for this Business Shared Folder", ["debug"]);
|
|
|
|
// DB Tie
|
|
Item parentItem;
|
|
parentItem.driveId = onedriveJSONItem["parentReference"]["driveId"].str;
|
|
parentItem.id = onedriveJSONItem["parentReference"]["id"].str;
|
|
parentItem.name = "root";
|
|
parentItem.type = ItemType.dir;
|
|
parentItem.mtime = remoteItem.mtime;
|
|
parentItem.parentId = null;
|
|
|
|
// Add this DB Tie parent record to the local database
|
|
addLogEntry("Insert local database with remoteItem parent details: " ~ to!string(parentItem), ["debug"]);
|
|
itemDB.upsert(parentItem);
|
|
|
|
// Ensure that this item has no parent
|
|
addLogEntry("Setting remoteItem.parentId to be null", ["debug"]);
|
|
remoteItem.parentId = null;
|
|
|
|
// Check the DB for 'remote' objects, searching 'remoteDriveId' and 'remoteId' items for this remoteItem.driveId and remoteItem.id
|
|
Item remoteDBItem;
|
|
itemDB.selectByRemoteId(remoteItem.driveId, remoteItem.id, remoteDBItem);
|
|
|
|
// Must compare remoteDBItem.name with remoteItem.name
|
|
if ((!remoteDBItem.name.empty) && (remoteDBItem.name != remoteItem.name)) {
|
|
// Update DB Item
|
|
addLogEntry("The shared item stored in OneDrive, has a different name to the actual name on the remote drive", ["debug"]);
|
|
addLogEntry("Updating remoteItem.name JSON data with the actual name being used on account drive and local folder", ["debug"]);
|
|
addLogEntry("remoteItem.name was: " ~ remoteItem.name, ["debug"]);
|
|
addLogEntry("Updating remoteItem.name to: " ~ remoteDBItem.name, ["debug"]);
|
|
remoteItem.name = remoteDBItem.name;
|
|
addLogEntry("Setting remoteItem.remoteName to: " ~ onedriveJSONItem["name"].str, ["debug"]);
|
|
|
|
// Update JSON Item
|
|
remoteItem.remoteName = onedriveJSONItem["name"].str;
|
|
addLogEntry("Updating source JSON 'name' to that which is the actual local directory", ["debug"]);
|
|
addLogEntry("onedriveJSONItem['name'] was: " ~ onedriveJSONItem["name"].str, ["debug"]);
|
|
addLogEntry("Updating onedriveJSONItem['name'] to: " ~ remoteDBItem.name, ["debug"]);
|
|
onedriveJSONItem["name"] = remoteDBItem.name;
|
|
addLogEntry("onedriveJSONItem['name'] now: " ~ onedriveJSONItem["name"].str, ["debug"]);
|
|
|
|
// Update newItemPath value
|
|
newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ remoteDBItem.name;
|
|
addLogEntry("New Item updated calculated full path is: " ~ newItemPath, ["debug"]);
|
|
}
|
|
|
|
// Add this record to the local database
|
|
addLogEntry("Update/Insert local database with remoteItem details: " ~ to!string(remoteItem), ["debug"]);
|
|
itemDB.upsert(remoteItem);
|
|
} else {
|
|
// Sharepoint account type
|
|
addLogEntry("Handling a SharePoint Shared Item JSON object - NOT IMPLEMENTED ........ ", ["debug"]);
|
|
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check the skippedItems array for the parent id of this JSONItem if this is something we need to skip
|
|
if (!unwanted) {
|
|
if (thisItemParentId in skippedItems) {
|
|
// Flag this JSON item as unwanted
|
|
addLogEntry("Flagging as unwanted: find(thisItemParentId).length != 0", ["debug"]);
|
|
unwanted = true;
|
|
|
|
// Is this item id in the database?
|
|
if (existingDBEntry) {
|
|
// item exists in database, most likely moved out of scope for current client configuration
|
|
addLogEntry("This item was previously synced / seen by the client", ["debug"]);
|
|
|
|
if (("name" in onedriveJSONItem["parentReference"]) != null) {
|
|
|
|
// How is this out of scope?
|
|
// is sync_list configured
|
|
if (syncListConfigured) {
|
|
// sync_list configured and in use
|
|
if (selectiveSync.isPathExcludedViaSyncList(onedriveJSONItem["parentReference"]["name"].str)) {
|
|
// Previously synced item is now out of scope as it has been moved out of what is included in sync_list
|
|
addLogEntry("This previously synced item is now excluded from being synced due to sync_list exclusion", ["debug"]);
|
|
}
|
|
}
|
|
// flag to delete local file as it now is no longer in sync with OneDrive
|
|
addLogEntry("Flagging to delete item locally: ", ["debug"]);
|
|
idsToDelete ~= [thisItemDriveId, thisItemId];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check the item type - if it not an item type that we support, we cant process the JSON item
|
|
if (!unwanted) {
|
|
if (isItemFile(onedriveJSONItem)) {
|
|
addLogEntry("The item we are syncing is a file", ["debug"]);
|
|
} else if (isItemFolder(onedriveJSONItem)) {
|
|
addLogEntry("The item we are syncing is a folder", ["debug"]);
|
|
} else if (isItemRemote(onedriveJSONItem)) {
|
|
addLogEntry("The item we are syncing is a remote item", ["debug"]);
|
|
} else {
|
|
// Why was this unwanted?
|
|
if (newItemPath.empty) {
|
|
// Compute this item path & need the full path for this file
|
|
newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName;
|
|
addLogEntry("New Item calculated full path is: " ~ newItemPath, ["debug"]);
|
|
}
|
|
// Microsoft OneNote container objects present as neither folder or file but has file size
|
|
if ((!isItemFile(onedriveJSONItem)) && (!isItemFolder(onedriveJSONItem)) && (hasFileSize(onedriveJSONItem))) {
|
|
// Log that this was skipped as this was a Microsoft OneNote item and unsupported
|
|
addLogEntry("The Microsoft OneNote Notebook '" ~ newItemPath ~ "' is not supported by this client", ["verbose"]);
|
|
} else {
|
|
// Log that this item was skipped as unsupported
|
|
addLogEntry("The OneDrive item '" ~ newItemPath ~ "' is not supported by this client", ["verbose"]);
|
|
}
|
|
unwanted = true;
|
|
addLogEntry("Flagging as unwanted: item type is not supported", ["debug"]);
|
|
}
|
|
}
|
|
|
|
// Check if this is excluded by config option: skip_dir
|
|
if (!unwanted) {
|
|
// Only check path if config is != ""
|
|
if (!appConfig.getValueString("skip_dir").empty) {
|
|
// Is the item a folder?
|
|
if (isItemFolder(onedriveJSONItem)) {
|
|
// work out the 'snippet' path where this folder would be created
|
|
string simplePathToCheck = "";
|
|
string complexPathToCheck = "";
|
|
string matchDisplay = "";
|
|
|
|
if (hasParentReference(onedriveJSONItem)) {
|
|
// we need to workout the FULL path for this item
|
|
// simple path
|
|
if (("name" in onedriveJSONItem["parentReference"]) != null) {
|
|
simplePathToCheck = onedriveJSONItem["parentReference"]["name"].str ~ "/" ~ onedriveJSONItem["name"].str;
|
|
} else {
|
|
simplePathToCheck = onedriveJSONItem["name"].str;
|
|
}
|
|
addLogEntry("skip_dir path to check (simple): " ~ simplePathToCheck, ["debug"]);
|
|
|
|
// complex path
|
|
if (parentInDatabase) {
|
|
// build up complexPathToCheck
|
|
complexPathToCheck = buildNormalizedPath(newItemPath);
|
|
} else {
|
|
addLogEntry("Parent details not in database - unable to compute complex path to check", ["debug"]);
|
|
}
|
|
if (!complexPathToCheck.empty) {
|
|
addLogEntry("skip_dir path to check (complex): " ~ complexPathToCheck, ["debug"]);
|
|
}
|
|
} else {
|
|
simplePathToCheck = onedriveJSONItem["name"].str;
|
|
}
|
|
|
|
// If 'simplePathToCheck' or 'complexPathToCheck' is of the following format: root:/folder
|
|
// then isDirNameExcluded matching will not work
|
|
// Clean up 'root:' if present
|
|
if (startsWith(simplePathToCheck, "root:")){
|
|
addLogEntry("Updating simplePathToCheck to remove 'root:'", ["debug"]);
|
|
simplePathToCheck = strip(simplePathToCheck, "root:");
|
|
}
|
|
if (startsWith(complexPathToCheck, "root:")){
|
|
addLogEntry("Updating complexPathToCheck to remove 'root:'", ["debug"]);
|
|
complexPathToCheck = strip(complexPathToCheck, "root:");
|
|
}
|
|
|
|
// OK .. what checks are we doing?
|
|
if ((!simplePathToCheck.empty) && (complexPathToCheck.empty)) {
|
|
// just a simple check
|
|
addLogEntry("Performing a simple check only", ["debug"]);
|
|
unwanted = selectiveSync.isDirNameExcluded(simplePathToCheck);
|
|
} else {
|
|
// simple and complex
|
|
addLogEntry("Performing a simple then complex path match if required", ["debug"]);
|
|
|
|
// simple first
|
|
addLogEntry("Performing a simple check first", ["debug"]);
|
|
unwanted = selectiveSync.isDirNameExcluded(simplePathToCheck);
|
|
matchDisplay = simplePathToCheck;
|
|
if (!unwanted) {
|
|
// simple didnt match, perform a complex check
|
|
addLogEntry("Simple match was false, attempting complex match", ["debug"]);
|
|
unwanted = selectiveSync.isDirNameExcluded(complexPathToCheck);
|
|
matchDisplay = complexPathToCheck;
|
|
}
|
|
}
|
|
// result
|
|
addLogEntry("skip_dir exclude result (directory based): " ~ to!string(unwanted), ["debug"]);
|
|
if (unwanted) {
|
|
// This path should be skipped
|
|
addLogEntry("Skipping item - excluded by skip_dir config: " ~ matchDisplay, ["verbose"]);
|
|
}
|
|
}
|
|
// Is the item a file?
|
|
// We need to check to see if this files path is excluded as well
|
|
if (isItemFile(onedriveJSONItem)) {
|
|
|
|
string pathToCheck;
|
|
// does the newItemPath start with '/'?
|
|
if (!startsWith(newItemPath, "/")){
|
|
// path does not start with '/', but we need to check skip_dir entries with and without '/'
|
|
// so always make sure we are checking a path with '/'
|
|
pathToCheck = '/' ~ dirName(newItemPath);
|
|
} else {
|
|
pathToCheck = dirName(newItemPath);
|
|
}
|
|
|
|
// perform the check
|
|
unwanted = selectiveSync.isDirNameExcluded(pathToCheck);
|
|
// result
|
|
addLogEntry("skip_dir exclude result (file based): " ~ to!string(unwanted), ["debug"]);
|
|
if (unwanted) {
|
|
// this files path should be skipped
|
|
addLogEntry("Skipping item - file path is excluded by skip_dir config: " ~ newItemPath, ["verbose"]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if this is excluded by config option: skip_file
|
|
if (!unwanted) {
|
|
// Is the JSON item a file?
|
|
if (isItemFile(onedriveJSONItem)) {
|
|
// skip_file can contain 4 types of entries:
|
|
// - wildcard - *.txt
|
|
// - text + wildcard - name*.txt
|
|
// - full path + combination of any above two - /path/name*.txt
|
|
// - full path to file - /path/to/file.txt
|
|
|
|
// is the parent id in the database?
|
|
if (parentInDatabase) {
|
|
// Compute this item path & need the full path for this file
|
|
if (newItemPath.empty) {
|
|
newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName;
|
|
addLogEntry("New Item calculated full path is: " ~ newItemPath, ["debug"]);
|
|
}
|
|
|
|
// The path that needs to be checked needs to include the '/'
|
|
// This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched
|
|
// However, as 'path' used throughout, use a temp variable with this modification so that we use the temp variable for exclusion checks
|
|
string exclusionTestPath = "";
|
|
if (!startsWith(newItemPath, "/")){
|
|
// Add '/' to the path
|
|
exclusionTestPath = '/' ~ newItemPath;
|
|
}
|
|
|
|
addLogEntry("skip_file item to check: " ~ exclusionTestPath, ["debug"]);
|
|
unwanted = selectiveSync.isFileNameExcluded(exclusionTestPath);
|
|
addLogEntry("Result: " ~ to!string(unwanted), ["debug"]);
|
|
if (unwanted) addLogEntry("Skipping item - excluded by skip_file config: " ~ thisItemName, ["verbose"]);
|
|
} else {
|
|
// parent id is not in the database
|
|
unwanted = true;
|
|
addLogEntry("Skipping file - parent path not present in local database", ["verbose"]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if this is included or excluded by use of sync_list
|
|
if (!unwanted) {
|
|
// No need to try and process something against a sync_list if it has been configured
|
|
if (syncListConfigured) {
|
|
// Compute the item path if empty - as to check sync_list we need an actual path to check
|
|
if (newItemPath.empty) {
|
|
// Calculate this items path
|
|
newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName;
|
|
addLogEntry("New Item calculated full path is: " ~ newItemPath, ["debug"]);
|
|
}
|
|
|
|
// What path are we checking?
|
|
addLogEntry("sync_list item to check: " ~ newItemPath, ["debug"]);
|
|
|
|
// Unfortunatly there is no avoiding this call to check if the path is excluded|included via sync_list
|
|
if (selectiveSync.isPathExcludedViaSyncList(newItemPath)) {
|
|
// selective sync advised to skip, however is this a file and are we configured to upload / download files in the root?
|
|
if ((isItemFile(onedriveJSONItem)) && (appConfig.getValueBool("sync_root_files")) && (rootName(newItemPath) == "") ) {
|
|
// This is a file
|
|
// We are configured to sync all files in the root
|
|
// This is a file in the logical root
|
|
unwanted = false;
|
|
} else {
|
|
// path is unwanted
|
|
unwanted = true;
|
|
addLogEntry("Skipping item - excluded by sync_list config: " ~ newItemPath, ["verbose"]);
|
|
// flagging to skip this item now, but does this exist in the DB thus needs to be removed / deleted?
|
|
if (existingDBEntry) {
|
|
// flag to delete
|
|
addLogEntry("Flagging item for local delete as item exists in database: " ~ newItemPath, ["verbose"]);
|
|
idsToDelete ~= [thisItemDriveId, thisItemId];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if the user has configured to skip downloading .files or .folders: skip_dotfiles
|
|
if (!unwanted) {
|
|
if (appConfig.getValueBool("skip_dotfiles")) {
|
|
if (isDotFile(newItemPath)) {
|
|
addLogEntry("Skipping item - .file or .folder: " ~ newItemPath, ["verbose"]);
|
|
unwanted = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if this should be skipped due to a --check-for-nosync directive (.nosync)?
|
|
if (!unwanted) {
|
|
if (appConfig.getValueBool("check_nosync")) {
|
|
// need the parent path for this object
|
|
string parentPath = dirName(newItemPath);
|
|
// Check for the presence of a .nosync in the parent path
|
|
if (exists(parentPath ~ "/.nosync")) {
|
|
addLogEntry("Skipping downloading item - .nosync found in parent folder & --check-for-nosync is enabled: " ~ newItemPath, ["verbose"]);
|
|
unwanted = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if this is excluded by a user set maximum filesize to download
|
|
if (!unwanted) {
|
|
if (isItemFile(onedriveJSONItem)) {
|
|
if (fileSizeLimit != 0) {
|
|
if (onedriveJSONItem["size"].integer >= fileSizeLimit) {
|
|
addLogEntry("Skipping item - excluded by skip_size config: " ~ thisItemName ~ " (" ~ to!string(onedriveJSONItem["size"].integer/2^^20) ~ " MB)", ["verbose"]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// At this point all the applicable checks on this JSON object from OneDrive are complete:
|
|
// - skip_file
|
|
// - skip_dir
|
|
// - sync_list
|
|
// - skip_dotfiles
|
|
// - check_nosync
|
|
// - skip_size
|
|
// - We know if this item exists in the DB or not in the DB
|
|
|
|
// We know if this JSON item is unwanted or not
|
|
if (unwanted) {
|
|
// This JSON item is NOT wanted - it is excluded
|
|
addLogEntry("Skipping OneDrive change as this is determined to be unwanted", ["debug"]);
|
|
|
|
// Add to the skippedItems array, but only if it is a directory ... pointless adding 'files' here, as it is the 'id' we check as the parent path which can only be a directory
|
|
if (!isItemFile(onedriveJSONItem)) {
|
|
skippedItems.insert(thisItemId);
|
|
}
|
|
} else {
|
|
// This JSON item is wanted - we need to process this JSON item further
|
|
// Take the JSON item and create a consumable object for eventual database insertion
|
|
Item newDatabaseItem = makeItem(onedriveJSONItem);
|
|
|
|
if (existingDBEntry) {
|
|
// The details of this JSON item are already in the DB
|
|
// Is the item in the DB the same as the JSON data provided - or is the JSON data advising this is an updated file?
|
|
addLogEntry("OneDrive change is an update to an existing local item", ["debug"]);
|
|
|
|
// Compute the existing item path
|
|
// NOTE:
|
|
// string existingItemPath = computeItemPath(existingDatabaseItem.driveId, existingDatabaseItem.id);
|
|
//
|
|
// This will calculate the path as follows:
|
|
//
|
|
// existingItemPath: Document.txt
|
|
//
|
|
// Whereas above we use the following
|
|
//
|
|
// newItemPath = computeItemPath(newDatabaseItem.driveId, newDatabaseItem.parentId) ~ "/" ~ newDatabaseItem.name;
|
|
//
|
|
// Which generates the following path:
|
|
//
|
|
// changedItemPath: ./Document.txt
|
|
//
|
|
// Need to be consistent here with how 'newItemPath' was calculated
|
|
string existingItemPath = computeItemPath(existingDatabaseItem.driveId, existingDatabaseItem.parentId) ~ "/" ~ existingDatabaseItem.name;
|
|
// Attempt to apply this changed item
|
|
applyPotentiallyChangedItem(existingDatabaseItem, existingItemPath, newDatabaseItem, newItemPath, onedriveJSONItem);
|
|
} else {
|
|
// Action this JSON item as a new item as we have no DB record of it
|
|
// The actual item may actually exist locally already, meaning that just the database is out-of-date or missing the data due to --resync
|
|
// But we also cannot compute the newItemPath as the parental objects may not exist as well
|
|
addLogEntry("OneDrive change is potentially a new local item", ["debug"]);
|
|
|
|
// Attempt to apply this potentially new item
|
|
applyPotentiallyNewLocalItem(newDatabaseItem, onedriveJSONItem, newItemPath);
|
|
}
|
|
}
|
|
|
|
// Tracking as to if this item was processed
|
|
processedCount++;
|
|
}
|
|
}
|
|
|
|
// Perform the download of any required objects in parallel
|
|
void processDownloadActivities() {
|
|
|
|
// Are there any items to delete locally? Cleanup space locally first
|
|
if (!idsToDelete.empty) {
|
|
// There are elements that potentially need to be deleted locally
|
|
addLogEntry("Items to potentially delete locally: " ~ to!string(idsToDelete.length), ["verbose"]);
|
|
|
|
if (appConfig.getValueBool("download_only")) {
|
|
// Download only has been configured
|
|
if (cleanupLocalFiles) {
|
|
// Process online deleted items
|
|
addLogEntry("Processing local deletion activity as --download-only & --cleanup-local-files configured", ["verbose"]);
|
|
processDeleteItems();
|
|
} else {
|
|
// Not cleaning up local files
|
|
addLogEntry("Skipping local deletion activity as --download-only has been used", ["verbose"]);
|
|
}
|
|
} else {
|
|
// Not using --download-only process normally
|
|
processDeleteItems();
|
|
}
|
|
// Cleanup array memory
|
|
idsToDelete = [];
|
|
}
|
|
|
|
// Are there any items to download post fetching and processing the /delta data?
|
|
if (!fileJSONItemsToDownload.empty) {
|
|
// There are elements to download
|
|
addLogEntry("Number of items to download from OneDrive: " ~ to!string(fileJSONItemsToDownload.length), ["verbose"]);
|
|
downloadOneDriveItems();
|
|
// Cleanup array memory
|
|
fileJSONItemsToDownload = [];
|
|
}
|
|
|
|
// Are there any skipped items still?
|
|
if (!skippedItems.empty) {
|
|
// Cleanup array memory
|
|
skippedItems.clear();
|
|
}
|
|
}
|
|
|
|
// If the JSON item is not in the database, it is potentially a new item that we need to action
|
|
void applyPotentiallyNewLocalItem(Item newDatabaseItem, JSONValue onedriveJSONItem, string newItemPath) {
|
|
|
|
// The JSON and Database items being passed in here have passed the following checks:
|
|
// - skip_file
|
|
// - skip_dir
|
|
// - sync_list
|
|
// - skip_dotfiles
|
|
// - check_nosync
|
|
// - skip_size
|
|
// - Is not currently cached in the local database
|
|
// As such, we should not be doing any other checks here to determine if the JSON item is wanted .. it is
|
|
|
|
if (exists(newItemPath)) {
|
|
// Issue #2209 fix - test if path is a bad symbolic link
|
|
if (isSymlink(newItemPath)) {
|
|
addLogEntry("Path on local disk is a symbolic link ........", ["debug"]);
|
|
if (!exists(readLink(newItemPath))) {
|
|
// reading the symbolic link failed
|
|
addLogEntry("Reading the symbolic link target failed ........ ", ["debug"]);
|
|
addLogEntry("Skipping item - invalid symbolic link: " ~ newItemPath, ["info", "notify"]);
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Path exists locally, is not a bad symbolic link
|
|
// Test if this item is actually in-sync
|
|
// What is the source of this item data?
|
|
string itemSource = "remote";
|
|
if (isItemSynced(newDatabaseItem, newItemPath, itemSource)) {
|
|
// Item details from OneDrive and local item details in database are in-sync
|
|
addLogEntry("The item to sync is already present on the local filesystem and is in-sync with what is reported online", ["debug"]);
|
|
addLogEntry("Update/Insert local database with item details: " ~ to!string(newDatabaseItem), ["debug"]);
|
|
itemDB.upsert(newDatabaseItem);
|
|
return;
|
|
} else {
|
|
// Item details from OneDrive and local item details in database are NOT in-sync
|
|
addLogEntry("The item to sync exists locally but is NOT in the local database - otherwise this would be handled as changed item", ["debug"]);
|
|
|
|
// Which object is newer? The local file or the remote file?
|
|
SysTime localModifiedTime = timeLastModified(newItemPath).toUTC();
|
|
SysTime itemModifiedTime = newDatabaseItem.mtime;
|
|
// Reduce time resolution to seconds before comparing
|
|
localModifiedTime.fracSecs = Duration.zero;
|
|
itemModifiedTime.fracSecs = Duration.zero;
|
|
|
|
// Is the local modified time greater than that from OneDrive?
|
|
if (localModifiedTime > itemModifiedTime) {
|
|
// Local file is newer than item on OneDrive based on file modified time
|
|
// Is this item id in the database?
|
|
if (itemDB.idInLocalDatabase(newDatabaseItem.driveId, newDatabaseItem.id)) {
|
|
// item id is in the database
|
|
// no local rename
|
|
// no download needed
|
|
addLogEntry("Local item modified time is newer based on UTC time conversion - keeping local item as this exists in the local database", ["verbose"]);
|
|
addLogEntry("Skipping OneDrive change as this is determined to be unwanted due to local item modified time being newer than OneDrive item and present in the sqlite database", ["debug"]);
|
|
} else {
|
|
// item id is not in the database .. maybe a --resync ?
|
|
// file exists locally but is not in the sqlite database - maybe a failed download?
|
|
addLogEntry("Local item does not exist in local database - replacing with file from OneDrive - failed download?", ["verbose"]);
|
|
|
|
// In a --resync scenario or if items.sqlite3 was deleted before startup we have zero way of knowing IF the local file is meant to be the right file
|
|
// To this pint we have passed the following checks:
|
|
// 1. Any client side filtering checks - this determined this is a file that is wanted
|
|
// 2. A file with the exact name exists locally
|
|
// 3. The local modified time > remote modified time
|
|
// 4. The id of the item from OneDrive is not in the database
|
|
|
|
// Has the user configured to IGNORE local data protection rules?
|
|
if (bypassDataPreservation) {
|
|
// The user has configured to ignore data safety checks and overwrite local data rather than preserve & rename
|
|
addLogEntry("WARNING: Local Data Protection has been disabled. You may experience data loss on this file: " ~ newItemPath, ["info", "notify"]);
|
|
} else {
|
|
// local data protection is configured, rename the local file, passing in if we are performing a --dry-run or not
|
|
safeBackup(newItemPath, dryRun);
|
|
}
|
|
}
|
|
} else {
|
|
// Remote file is newer than the existing local item
|
|
addLogEntry("Remote item modified time is newer based on UTC time conversion", ["verbose"]); // correct message, remote item is newer
|
|
addLogEntry("localModifiedTime (local file): " ~ to!string(localModifiedTime), ["debug"]);
|
|
addLogEntry("itemModifiedTime (OneDrive item): " ~ to!string(itemModifiedTime), ["debug"]);
|
|
|
|
// Has the user configured to IGNORE local data protection rules?
|
|
if (bypassDataPreservation) {
|
|
// The user has configured to ignore data safety checks and overwrite local data rather than preserve & rename
|
|
addLogEntry("WARNING: Local Data Protection has been disabled. You may experience data loss on this file: " ~ newItemPath, ["info", "notify"]);
|
|
} else {
|
|
// local data protection is configured, rename the local file, passing in if we are performing a --dry-run or not
|
|
safeBackup(newItemPath, dryRun);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Path does not exist locally (should not exist locally if renamed file) - this will be a new file download or new folder creation
|
|
// How to handle this Potentially New Local Item JSON ?
|
|
final switch (newDatabaseItem.type) {
|
|
case ItemType.file:
|
|
// Add to the items to download array for processing
|
|
fileJSONItemsToDownload ~= onedriveJSONItem;
|
|
break;
|
|
case ItemType.dir:
|
|
case ItemType.remote:
|
|
addLogEntry("Creating local directory: " ~ newItemPath);
|
|
if (!dryRun) {
|
|
try {
|
|
// Create the new directory
|
|
addLogEntry("Requested path does not exist, creating directory structure: " ~ newItemPath, ["debug"]);
|
|
mkdirRecurse(newItemPath);
|
|
// Configure the applicable permissions for the folder
|
|
addLogEntry("Setting directory permissions for: " ~ newItemPath, ["debug"]);
|
|
newItemPath.setAttributes(appConfig.returnRequiredDirectoryPermisions());
|
|
// Update the time of the folder to match the last modified time as is provided by OneDrive
|
|
// If there are any files then downloaded into this folder, the last modified time will get
|
|
// updated by the local Operating System with the latest timestamp - as this is normal operation
|
|
// as the directory has been modified
|
|
addLogEntry("Setting directory lastModifiedDateTime for: " ~ newItemPath ~ " to " ~ to!string(newDatabaseItem.mtime), ["debug"]);
|
|
addLogEntry("Calling setTimes() for this file: " ~ newItemPath, ["debug"]);
|
|
setTimes(newItemPath, newDatabaseItem.mtime, newDatabaseItem.mtime);
|
|
// Save the item to the database
|
|
saveItem(onedriveJSONItem);
|
|
} catch (FileException e) {
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
} else {
|
|
// we dont create the directory, but we need to track that we 'faked it'
|
|
idsFaked ~= [newDatabaseItem.driveId, newDatabaseItem.id];
|
|
// Save the item to the dry-run database
|
|
saveItem(onedriveJSONItem);
|
|
}
|
|
break;
|
|
case ItemType.unknown:
|
|
// Unknown type - we dont action or sync these items
|
|
break;
|
|
}
|
|
}
|
|
|
|
// If the JSON item IS in the database, this will be an update to an existing in-sync item
|
|
void applyPotentiallyChangedItem(Item existingDatabaseItem, string existingItemPath, Item changedOneDriveItem, string changedItemPath, JSONValue onedriveJSONItem) {
|
|
|
|
// If we are moving the item, we do not need to download it again
|
|
bool itemWasMoved = false;
|
|
|
|
// Do we need to actually update the database with the details that were provided by the OneDrive API?
|
|
// Calculate these time items from the provided items
|
|
SysTime existingItemModifiedTime = existingDatabaseItem.mtime;
|
|
existingItemModifiedTime.fracSecs = Duration.zero;
|
|
SysTime changedOneDriveItemModifiedTime = changedOneDriveItem.mtime;
|
|
changedOneDriveItemModifiedTime.fracSecs = Duration.zero;
|
|
|
|
if (existingDatabaseItem.eTag != changedOneDriveItem.eTag) {
|
|
// The eTag has changed to what we previously cached
|
|
if (existingItemPath != changedItemPath) {
|
|
// Log that we are changing / moving an item to a new name
|
|
addLogEntry("Moving " ~ existingItemPath ~ " to " ~ changedItemPath);
|
|
// Is the destination path empty .. or does something exist at that location?
|
|
if (exists(changedItemPath)) {
|
|
// Destination we are moving to exists ...
|
|
Item changedLocalItem;
|
|
// Query DB for this changed item in specified path that exists and see if it is in-sync
|
|
if (itemDB.selectByPath(changedItemPath, changedOneDriveItem.driveId, changedLocalItem)) {
|
|
// The 'changedItemPath' is in the database
|
|
string itemSource = "database";
|
|
if (isItemSynced(changedLocalItem, changedItemPath, itemSource)) {
|
|
// The destination item is in-sync
|
|
addLogEntry("Destination is in sync and will be overwritten", ["verbose"]);
|
|
} else {
|
|
// The destination item is different
|
|
addLogEntry("The destination is occupied with a different item, renaming the conflicting file...", ["verbose"]);
|
|
// Backup this item, passing in if we are performing a --dry-run or not
|
|
safeBackup(changedItemPath, dryRun);
|
|
}
|
|
} else {
|
|
// The to be overwritten item is not already in the itemdb, so it should saved to avoid data loss
|
|
addLogEntry("The destination is occupied by an existing un-synced file, renaming the conflicting file...", ["verbose"]);
|
|
// Backup this item, passing in if we are performing a --dry-run or not
|
|
safeBackup(changedItemPath, dryRun);
|
|
}
|
|
}
|
|
|
|
// Try and rename path, catch any exception generated
|
|
try {
|
|
// Rename this item, passing in if we are performing a --dry-run or not
|
|
safeBackup(changedItemPath, dryRun);
|
|
|
|
// If the item is a file, make sure that the local timestamp now is the same as the timestamp online
|
|
// Otherwise when we do the DB check, the move on the file system, the file technically has a newer timestamp
|
|
// which is 'correct' .. but we need to report locally the online timestamp here as the move was made online
|
|
if (changedOneDriveItem.type == ItemType.file) {
|
|
setTimes(changedItemPath, changedOneDriveItem.mtime, changedOneDriveItem.mtime);
|
|
}
|
|
|
|
// Flag that the item was moved | renamed
|
|
itemWasMoved = true;
|
|
|
|
// If we are in a --dry-run situation, the actual rename did not occur - but we need to track like it did
|
|
if (dryRun) {
|
|
// Track this as a faked id item
|
|
idsFaked ~= [changedOneDriveItem.driveId, changedOneDriveItem.id];
|
|
// We also need to track that we did not rename this path
|
|
pathsRenamed ~= [existingItemPath];
|
|
}
|
|
} catch (FileException e) {
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
}
|
|
|
|
// What sort of changed item is this?
|
|
// Is it a file, and we did not move it ..
|
|
if ((changedOneDriveItem.type == ItemType.file) && (!itemWasMoved)) {
|
|
// The eTag is notorious for being 'changed' online by some backend Microsoft process
|
|
if (existingDatabaseItem.quickXorHash != changedOneDriveItem.quickXorHash) {
|
|
// Add to the items to download array for processing - the file hash we previously recorded is not the same as online
|
|
fileJSONItemsToDownload ~= onedriveJSONItem;
|
|
} else {
|
|
// If the timestamp is different, or we are running a client operational mode that does not support /delta queries - we have to update the DB with the details from OneDrive
|
|
// Unfortunatly because of the consequence of Nataional Cloud Deployments not supporting /delta queries, the application uses the local database to flag what is out-of-date / track changes
|
|
// This means that the constant disk writing to the database fix implemented with https://github.com/abraunegg/onedrive/pull/2004 cannot be utilised when using these operational modes
|
|
// as all records are touched / updated when performing the OneDrive sync operations. The impacted operational modes are:
|
|
// - National Cloud Deployments do not support /delta as a query
|
|
// - When using --single-directory
|
|
// - When using --download-only --cleanup-local-files
|
|
|
|
// Is the last modified timestamp in the DB the same as the API data or are we running an operational mode where we simulated the /delta response?
|
|
if ((existingItemModifiedTime != changedOneDriveItemModifiedTime) || (generateSimulatedDeltaResponse)) {
|
|
// Save this item in the database
|
|
// Add to the local database
|
|
addLogEntry("Adding changed OneDrive Item to database: " ~ to!string(changedOneDriveItem), ["debug"]);
|
|
itemDB.upsert(changedOneDriveItem);
|
|
}
|
|
}
|
|
} else {
|
|
// Save this item in the database
|
|
saveItem(onedriveJSONItem);
|
|
|
|
// If the 'Add shortcut to My files' link was the item that was actually renamed .. we have to update our DB records
|
|
if (changedOneDriveItem.type == ItemType.remote) {
|
|
// Select remote item data from the database
|
|
Item existingRemoteDbItem;
|
|
itemDB.selectById(changedOneDriveItem.remoteDriveId, changedOneDriveItem.remoteId, existingRemoteDbItem);
|
|
// Update the 'name' in existingRemoteDbItem and save it back to the database
|
|
// This is the local name stored on disk that was just 'moved'
|
|
existingRemoteDbItem.name = changedOneDriveItem.name;
|
|
itemDB.upsert(existingRemoteDbItem);
|
|
}
|
|
}
|
|
} else {
|
|
// The existingDatabaseItem.eTag == changedOneDriveItem.eTag .. nothing has changed eTag wise
|
|
|
|
// If the timestamp is different, or we are running a client operational mode that does not support /delta queries - we have to update the DB with the details from OneDrive
|
|
// Unfortunatly because of the consequence of Nataional Cloud Deployments not supporting /delta queries, the application uses the local database to flag what is out-of-date / track changes
|
|
// This means that the constant disk writing to the database fix implemented with https://github.com/abraunegg/onedrive/pull/2004 cannot be utilised when using these operational modes
|
|
// as all records are touched / updated when performing the OneDrive sync operations. The impacted operational modes are:
|
|
// - National Cloud Deployments do not support /delta as a query
|
|
// - When using --single-directory
|
|
// - When using --download-only --cleanup-local-files
|
|
|
|
// Is the last modified timestamp in the DB the same as the API data or are we running an operational mode where we simulated the /delta response?
|
|
if ((existingItemModifiedTime != changedOneDriveItemModifiedTime) || (generateSimulatedDeltaResponse)) {
|
|
// Database update needed for this item because our local record is out-of-date
|
|
// Add to the local database
|
|
addLogEntry("Adding changed OneDrive Item to database: " ~ to!string(changedOneDriveItem), ["debug"]);
|
|
itemDB.upsert(changedOneDriveItem);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Download new file items as identified
|
|
void downloadOneDriveItems() {
|
|
// Lets deal with all the JSON items that need to be downloaded in a batch process
|
|
ulong batchSize = appConfig.concurrentThreads;
|
|
ulong batchCount = (fileJSONItemsToDownload.length + batchSize - 1) / batchSize;
|
|
ulong batchesProcessed = 0;
|
|
|
|
foreach (chunk; fileJSONItemsToDownload.chunks(batchSize)) {
|
|
// send an array containing 'appConfig.concurrentThreads' (16) JSON items to download
|
|
downloadOneDriveItemsInParallel(chunk);
|
|
}
|
|
}
|
|
|
|
// Download items in parallel
|
|
void downloadOneDriveItemsInParallel(JSONValue[] array) {
|
|
// This function recieved an array of 16 JSON items to download
|
|
foreach (i, onedriveJSONItem; taskPool.parallel(array)) {
|
|
// Take each JSON item and
|
|
downloadFileItem(onedriveJSONItem);
|
|
}
|
|
}
|
|
|
|
// Perform the actual download of an object from OneDrive
|
|
void downloadFileItem(JSONValue onedriveJSONItem) {
|
|
|
|
bool downloadFailed = false;
|
|
string OneDriveFileXORHash;
|
|
string OneDriveFileSHA256Hash;
|
|
ulong jsonFileSize = 0;
|
|
|
|
// Download item specifics
|
|
string downloadDriveId = onedriveJSONItem["parentReference"]["driveId"].str;
|
|
string downloadParentId = onedriveJSONItem["parentReference"]["id"].str;
|
|
string downloadItemName = onedriveJSONItem["name"].str;
|
|
string downloadItemId = onedriveJSONItem["id"].str;
|
|
|
|
// Calculate this items path
|
|
string newItemPath = computeItemPath(downloadDriveId, downloadParentId) ~ "/" ~ downloadItemName;
|
|
addLogEntry("New Item calculated full path is: " ~ newItemPath, ["debug"]);
|
|
|
|
// Is the item reported as Malware ?
|
|
if (isMalware(onedriveJSONItem)){
|
|
// OneDrive reports that this file is malware
|
|
addLogEntry("ERROR: MALWARE DETECTED IN FILE - DOWNLOAD SKIPPED: " ~ newItemPath, ["info", "notify"]);
|
|
downloadFailed = true;
|
|
} else {
|
|
// Grab this file's filesize
|
|
if (hasFileSize(onedriveJSONItem)) {
|
|
// Use the configured filesize as reported by OneDrive
|
|
jsonFileSize = onedriveJSONItem["size"].integer;
|
|
} else {
|
|
// filesize missing
|
|
addLogEntry("ERROR: onedriveJSONItem['size'] is missing", ["debug"]);
|
|
}
|
|
|
|
// Configure the hashes for comparison post download
|
|
if (hasHashes(onedriveJSONItem)) {
|
|
// File details returned hash details
|
|
// QuickXorHash
|
|
if (hasQuickXorHash(onedriveJSONItem)) {
|
|
// Use the provided quickXorHash as reported by OneDrive
|
|
if (onedriveJSONItem["file"]["hashes"]["quickXorHash"].str != "") {
|
|
OneDriveFileXORHash = onedriveJSONItem["file"]["hashes"]["quickXorHash"].str;
|
|
}
|
|
} else {
|
|
// Fallback: Check for SHA256Hash
|
|
if (hasSHA256Hash(onedriveJSONItem)) {
|
|
// Use the provided sha256Hash as reported by OneDrive
|
|
if (onedriveJSONItem["file"]["hashes"]["sha256Hash"].str != "") {
|
|
OneDriveFileSHA256Hash = onedriveJSONItem["file"]["hashes"]["sha256Hash"].str;
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// file hash data missing
|
|
addLogEntry("ERROR: onedriveJSONItem['file']['hashes'] is missing - unable to compare file hash after download", ["debug"]);
|
|
}
|
|
|
|
// Is this a --download-only scenario?
|
|
if (appConfig.getValueBool("download_only")) {
|
|
if (exists(newItemPath)) {
|
|
// file exists locally already
|
|
Item databaseItem;
|
|
bool fileFoundInDB = false;
|
|
|
|
foreach (driveId; driveIDsArray) {
|
|
if (itemDB.selectByPath(newItemPath, driveId, databaseItem)) {
|
|
fileFoundInDB = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Log the DB details
|
|
addLogEntry("File to download exists locally and this is the DB record: " ~ to!string(databaseItem), ["debug"]);
|
|
|
|
// Does the DB (what we think is in sync) hash match the existing local file hash?
|
|
if (!testFileHash(newItemPath, databaseItem)) {
|
|
// local file is different to what we know to be true
|
|
addLogEntry("The local file to replace (" ~ newItemPath ~ ") has been modified locally since the last download. Renaming it to avoid potential local data loss.");
|
|
|
|
// Perform the local rename of the existing local file, passing in if we are performing a --dry-run or not
|
|
safeBackup(newItemPath, dryRun);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Is there enough free space locally to download the file
|
|
// - We can use '.' here as we change the current working directory to the configured 'sync_dir'
|
|
ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace("."));
|
|
// So that we are not responsible in making the disk 100% full if we can download the file, compare the current available space against the reservation set and file size
|
|
// The reservation value is user configurable in the config file, 50MB by default
|
|
ulong freeSpaceReservation = appConfig.getValueLong("space_reservation");
|
|
// debug output
|
|
addLogEntry("Local Disk Space Actual: " ~ to!string(localActualFreeSpace), ["debug"]);
|
|
addLogEntry("Free Space Reservation: " ~ to!string(freeSpaceReservation), ["debug"]);
|
|
addLogEntry("File Size to Download: " ~ to!string(jsonFileSize), ["debug"]);
|
|
|
|
// Calculate if we can actually download file - is there enough free space?
|
|
if ((localActualFreeSpace < freeSpaceReservation) || (jsonFileSize > localActualFreeSpace)) {
|
|
// localActualFreeSpace is less than freeSpaceReservation .. insufficient free space
|
|
// jsonFileSize is greater than localActualFreeSpace .. insufficient free space
|
|
addLogEntry("Downloading file " ~ newItemPath ~ " ... failed!");
|
|
addLogEntry("Insufficient local disk space to download file");
|
|
downloadFailed = true;
|
|
} else {
|
|
// If we are in a --dry-run situation - if not, actually perform the download
|
|
if (!dryRun) {
|
|
// Attempt to download the file as there is enough free space locally
|
|
OneDriveApi downloadFileOneDriveApiInstance;
|
|
downloadFileOneDriveApiInstance = new OneDriveApi(appConfig);
|
|
try {
|
|
downloadFileOneDriveApiInstance.initialise();
|
|
downloadFileOneDriveApiInstance.downloadById(downloadDriveId, downloadItemId, newItemPath, jsonFileSize);
|
|
downloadFileOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(downloadFileOneDriveApiInstance);
|
|
} catch (OneDriveException exception) {
|
|
addLogEntry("downloadFileOneDriveApiInstance.downloadById(downloadDriveId, downloadItemId, newItemPath, jsonFileSize); generated a OneDriveException", ["debug"]);
|
|
string thisFunctionName = getFunctionName!({});
|
|
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(downloadFileOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to download an item from OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
downloadFileItem(onedriveJSONItem);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
}
|
|
|
|
} catch (FileException e) {
|
|
// There was a file system error
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
downloadFailed = true;
|
|
} catch (ErrnoException e) {
|
|
// There was a file system error
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
downloadFailed = true;
|
|
}
|
|
|
|
// If we get to this point, something was downloaded .. does it match what we expected?
|
|
if (exists(newItemPath)) {
|
|
// When downloading some files from SharePoint, the OneDrive API reports one file size,
|
|
// but the SharePoint HTTP Server sends a totally different byte count for the same file
|
|
// we have implemented --disable-download-validation to disable these checks
|
|
|
|
if (!disableDownloadValidation) {
|
|
// A 'file' was downloaded - does what we downloaded = reported jsonFileSize or if there is some sort of funky local disk compression going on
|
|
// Does the file hash OneDrive reports match what we have locally?
|
|
string onlineFileHash;
|
|
string downloadedFileHash;
|
|
ulong downloadFileSize = getSize(newItemPath);
|
|
|
|
if (!OneDriveFileXORHash.empty) {
|
|
onlineFileHash = OneDriveFileXORHash;
|
|
// Calculate the QuickXOHash for this file
|
|
downloadedFileHash = computeQuickXorHash(newItemPath);
|
|
} else {
|
|
onlineFileHash = OneDriveFileSHA256Hash;
|
|
// Fallback: Calculate the SHA256 Hash for this file
|
|
downloadedFileHash = computeSHA256Hash(newItemPath);
|
|
}
|
|
|
|
if ((downloadFileSize == jsonFileSize) && (downloadedFileHash == onlineFileHash)) {
|
|
// Downloaded file matches size and hash
|
|
addLogEntry("Downloaded file matches reported size and reported file hash", ["debug"]);
|
|
|
|
try {
|
|
// get the mtime from the JSON data
|
|
SysTime itemModifiedTime;
|
|
if (isItemRemote(onedriveJSONItem)) {
|
|
// remote file item
|
|
itemModifiedTime = SysTime.fromISOExtString(onedriveJSONItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str);
|
|
} else {
|
|
// not a remote item
|
|
itemModifiedTime = SysTime.fromISOExtString(onedriveJSONItem["fileSystemInfo"]["lastModifiedDateTime"].str);
|
|
}
|
|
|
|
// set the correct time on the downloaded file
|
|
addLogEntry("Calling setTimes() for this file: " ~ newItemPath, ["debug"]);
|
|
setTimes(newItemPath, itemModifiedTime, itemModifiedTime);
|
|
} catch (FileException e) {
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
} else {
|
|
// Downloaded file does not match size or hash .. which is it?
|
|
bool downloadValueMismatch = false;
|
|
// Size error?
|
|
if (downloadFileSize != jsonFileSize) {
|
|
// downloaded file size does not match
|
|
downloadValueMismatch = true;
|
|
addLogEntry("Actual file size on disk: " ~ to!string(downloadFileSize), ["debug"]);
|
|
addLogEntry("OneDrive API reported size: " ~ to!string(jsonFileSize), ["debug"]);
|
|
addLogEntry("ERROR: File download size mis-match. Increase logging verbosity to determine why.");
|
|
}
|
|
// Hash Error
|
|
if (downloadedFileHash != onlineFileHash) {
|
|
// downloaded file hash does not match
|
|
downloadValueMismatch = true;
|
|
addLogEntry("Actual local file hash: " ~ downloadedFileHash, ["debug"]);
|
|
addLogEntry("OneDrive API reported hash: " ~ onlineFileHash, ["debug"]);
|
|
addLogEntry("ERROR: File download hash mis-match. Increase logging verbosity to determine why.");
|
|
}
|
|
// .heic data loss check
|
|
// - https://github.com/abraunegg/onedrive/issues/2471
|
|
// - https://github.com/OneDrive/onedrive-api-docs/issues/1532
|
|
// - https://github.com/OneDrive/onedrive-api-docs/issues/1723
|
|
if (downloadValueMismatch && (toLower(extension(newItemPath)) == ".heic")) {
|
|
// Need to display a message to the user that they have experienced data loss
|
|
addLogEntry("DATA-LOSS: File downloaded has experienced data loss due to a Microsoft OneDrive API bug. DO NOT DELETE THIS FILE ONLINE: " ~ newItemPath, ["info", "notify"]);
|
|
addLogEntry(" Please read https://github.com/OneDrive/onedrive-api-docs/issues/1723 for more details.", ["verbose"]);
|
|
}
|
|
|
|
// Add some workaround messaging for SharePoint
|
|
if (appConfig.accountType == "documentLibrary"){
|
|
// It has been seen where SharePoint / OneDrive API reports one size via the JSON
|
|
// but the content length and file size written to disk is totally different - example:
|
|
// From JSON: "size": 17133
|
|
// From HTTPS Server: < Content-Length: 19340
|
|
// with no logical reason for the difference, except for a 302 redirect before file download
|
|
addLogEntry("INFO: It is most likely that a SharePoint OneDrive API issue is the root cause. Add --disable-download-validation to work around this issue but downloaded data integrity cannot be guaranteed.");
|
|
} else {
|
|
// other account types
|
|
addLogEntry("INFO: Potentially add --disable-download-validation to work around this issue but downloaded data integrity cannot be guaranteed.");
|
|
}
|
|
// We do not want this local file to remain on the local file system as it failed the integrity checks
|
|
addLogEntry("Removing file " ~ newItemPath ~ " due to failed integrity checks");
|
|
if (!dryRun) {
|
|
safeRemove(newItemPath);
|
|
}
|
|
downloadFailed = true;
|
|
}
|
|
} else {
|
|
// Download validation checks were disabled
|
|
addLogEntry("Downloaded file validation disabled due to --disable-download-validation", ["debug"]);
|
|
addLogEntry("WARNING: Skipping download integrity check for: " ~ newItemPath, ["verbose"]);
|
|
} // end of (!disableDownloadValidation)
|
|
} else {
|
|
addLogEntry("ERROR: File failed to download. Increase logging verbosity to determine why.");
|
|
downloadFailed = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// File should have been downloaded
|
|
if (!downloadFailed) {
|
|
// Download did not fail
|
|
addLogEntry("Downloading file " ~ newItemPath ~ " ... done");
|
|
// Save this item into the database
|
|
saveItem(onedriveJSONItem);
|
|
|
|
// If we are in a --dry-run situation - if we are, we need to track that we faked the download
|
|
if (dryRun) {
|
|
// track that we 'faked it'
|
|
idsFaked ~= [downloadDriveId, downloadItemId];
|
|
}
|
|
} else {
|
|
// Output download failed
|
|
addLogEntry("Downloading file " ~ newItemPath ~ " ... failed!");
|
|
// Add the path to a list of items that failed to download
|
|
fileDownloadFailures ~= newItemPath;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Test if the given item is in-sync. Returns true if the given item corresponds to the local one
|
|
bool isItemSynced(Item item, string path, string itemSource) {
|
|
|
|
if (!exists(path)) return false;
|
|
final switch (item.type) {
|
|
case ItemType.file:
|
|
if (isFile(path)) {
|
|
// can we actually read the local file?
|
|
if (readLocalFile(path)){
|
|
// local file is readable
|
|
SysTime localModifiedTime = timeLastModified(path).toUTC();
|
|
SysTime itemModifiedTime = item.mtime;
|
|
// Reduce time resolution to seconds before comparing
|
|
localModifiedTime.fracSecs = Duration.zero;
|
|
itemModifiedTime.fracSecs = Duration.zero;
|
|
if (localModifiedTime == itemModifiedTime) {
|
|
return true;
|
|
} else {
|
|
addLogEntry("Local item time discrepancy detected: " ~ path, ["verbose"]);
|
|
addLogEntry("This local item has a different modified time " ~ to!string(localModifiedTime) ~ " when compared to " ~ itemSource ~ " modified time " ~ to!string(itemModifiedTime), ["verbose"]);
|
|
|
|
// The file has been modified ... is the hash the same?
|
|
// Test the file hash as the date / time stamp is different
|
|
// Generating a hash is computationally expensive - we only generate the hash if timestamp was different
|
|
if (testFileHash(path, item)) {
|
|
// The hash is the same .. so we need to fix-up the timestamp depending on where it is wrong
|
|
addLogEntry("Local item has the same hash value as the item online - correcting timestamp", ["verbose"]);
|
|
// Test if the local timestamp is newer
|
|
if (localModifiedTime > itemModifiedTime) {
|
|
// The source of the out-of-date timestamp was OneDrive and this needs to be corrected to avoid always generating a hash test if timestamp is different
|
|
addLogEntry("The source of the incorrect timestamp was OneDrive online - correcting timestamp online", ["verbose"]);
|
|
if (!dryRun) {
|
|
// Attempt to update the online date time stamp
|
|
uploadLastModifiedTime(item.driveId, item.id, localModifiedTime.toUTC(), item.eTag);
|
|
}
|
|
} else {
|
|
// The source of the out-of-date timestamp was the local file and this needs to be corrected to avoid always generating a hash test if timestamp is different
|
|
addLogEntry("The source of the incorrect timestamp was the local file - correcting timestamp locally", ["verbose"]);
|
|
if (!dryRun) {
|
|
addLogEntry("Calling setTimes() for this file: " ~ path, ["debug"]);
|
|
setTimes(path, item.mtime, item.mtime);
|
|
}
|
|
}
|
|
return true;
|
|
} else {
|
|
// The hash is different so the content of the file has to be different as to what is stored online
|
|
addLogEntry("The local item has a different hash when compared to " ~ itemSource ~ " item hash", ["verbose"]);
|
|
return false;
|
|
}
|
|
}
|
|
} else {
|
|
// Unable to read local file
|
|
addLogEntry("Unable to determine the sync state of this file as it cannot be read (file permissions or file corruption): " ~ path);
|
|
return false;
|
|
}
|
|
} else {
|
|
addLogEntry("The local item is a directory but should be a file", ["verbose"]);
|
|
}
|
|
break;
|
|
case ItemType.dir:
|
|
case ItemType.remote:
|
|
if (isDir(path)) {
|
|
return true;
|
|
} else {
|
|
addLogEntry("The local item is a file but should be a directory", ["verbose"]);
|
|
}
|
|
break;
|
|
case ItemType.unknown:
|
|
// Unknown type - return true but we dont action or sync these items
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Get the /delta data using the provided details
|
|
JSONValue getDeltaChangesByItemId(string selectedDriveId, string selectedItemId, string providedDeltaLink, OneDriveApi getDeltaQueryOneDriveApiInstance) {
|
|
|
|
// Function variables
|
|
JSONValue deltaChangesBundle;
|
|
|
|
// Get the /delta data for this account | driveId | deltaLink combination
|
|
addLogEntry("------------------------------------------------------------------", ["debug"]);
|
|
addLogEntry("selectedDriveId: " ~ selectedDriveId, ["debug"]);
|
|
addLogEntry("selectedItemId: " ~ selectedItemId, ["debug"]);
|
|
addLogEntry("providedDeltaLink: " ~ providedDeltaLink, ["debug"]);
|
|
addLogEntry("------------------------------------------------------------------", ["debug"]);
|
|
|
|
try {
|
|
deltaChangesBundle = getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink);
|
|
} catch (OneDriveException exception) {
|
|
// caught an exception
|
|
addLogEntry("getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink) generated a OneDriveException", ["debug"]);
|
|
|
|
auto errorArray = splitLines(exception.msg);
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(getDeltaQueryOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to query OneDrive API for Delta Changes - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// dont retry request, loop back to calling function
|
|
addLogEntry("Looping back after failure", ["debug"]);
|
|
deltaChangesBundle = null;
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
if (exception.httpStatusCode == 410) {
|
|
addLogEntry();
|
|
addLogEntry("WARNING: The OneDrive API responded with an error that indicates the locally stored deltaLink value is invalid");
|
|
// Essentially the 'providedDeltaLink' that we have stored is no longer available ... re-try without the stored deltaLink
|
|
addLogEntry("WARNING: Retrying OneDrive API call without using the locally stored deltaLink value");
|
|
// Configure an empty deltaLink
|
|
addLogEntry("Delta link expired for 'getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink)', setting 'deltaLink = null'", ["debug"]);
|
|
string emptyDeltaLink = "";
|
|
// retry with empty deltaLink
|
|
deltaChangesBundle = getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, emptyDeltaLink);
|
|
} else {
|
|
// display what the error is
|
|
addLogEntry("CODING TO DO: Hitting this failure error output");
|
|
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
|
|
deltaChangesBundle = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
return deltaChangesBundle;
|
|
}
|
|
|
|
// Common code to handle a 408 or 429 response from the OneDrive API
|
|
void handleOneDriveThrottleRequest(OneDriveApi activeOneDriveApiInstance) {
|
|
|
|
// If OneDrive sends a status code 429 then this function will be used to process the Retry-After response header which contains the value by which we need to wait
|
|
addLogEntry("Handling a OneDrive HTTP 429 Response Code (Too Many Requests)", ["debug"]);
|
|
|
|
// Read in the Retry-After HTTP header as set and delay as per this value before retrying the request
|
|
auto retryAfterValue = activeOneDriveApiInstance.getRetryAfterValue();
|
|
addLogEntry("Using Retry-After Value = " ~ to!string(retryAfterValue), ["debug"]);
|
|
|
|
// HTTP request returned status code 429 (Too Many Requests)
|
|
// https://github.com/abraunegg/onedrive/issues/133
|
|
// https://github.com/abraunegg/onedrive/issues/815
|
|
|
|
ulong delayBeforeRetry = 0;
|
|
if (retryAfterValue != 0) {
|
|
// Use the HTTP Response Header Value
|
|
delayBeforeRetry = retryAfterValue;
|
|
} else {
|
|
// Use a 120 second delay as a default given header value was zero
|
|
// This value is based on log files and data when determining correct process for 429 response handling
|
|
delayBeforeRetry = 120;
|
|
// Update that we are over-riding the provided value with a default
|
|
addLogEntry("HTTP Response Header retry-after value was 0 - Using a preconfigured default of: " ~ to!string(delayBeforeRetry), ["debug"]);
|
|
}
|
|
|
|
// Sleep thread as per request
|
|
addLogEntry("Thread sleeping due to 'HTTP request returned status code 429' - The request has been throttled");
|
|
addLogEntry("Sleeping for " ~ to!string(delayBeforeRetry) ~ " seconds");
|
|
Thread.sleep(dur!"seconds"(delayBeforeRetry));
|
|
|
|
// Reset retry-after value to zero as we have used this value now and it may be changed in the future to a different value
|
|
activeOneDriveApiInstance.resetRetryAfterValue();
|
|
}
|
|
|
|
// If the JSON response is not correct JSON object, exit
|
|
void invalidJSONResponseFromOneDriveAPI() {
|
|
addLogEntry("ERROR: Query of the OneDrive API returned an invalid JSON response");
|
|
// Must force exit here, allow logging to be done
|
|
forceExit();
|
|
}
|
|
|
|
// Handle an unhandled API error
|
|
void defaultUnhandledHTTPErrorCode(OneDriveException exception) {
|
|
// display error
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
// Must force exit here, allow logging to be done
|
|
forceExit();
|
|
}
|
|
|
|
// Display the pertinant details of the sync engine
|
|
void displaySyncEngineDetails() {
|
|
// Display accountType, defaultDriveId, defaultRootId & remainingFreeSpace for verbose logging purposes
|
|
addLogEntry("Application Version: " ~ appConfig.applicationVersion, ["verbose"]);
|
|
addLogEntry("Account Type: " ~ appConfig.accountType, ["verbose"]);
|
|
addLogEntry("Default Drive ID: " ~ appConfig.defaultDriveId, ["verbose"]);
|
|
addLogEntry("Default Root ID: " ~ appConfig.defaultRootId, ["verbose"]);
|
|
|
|
// What do we display here for space remaining
|
|
if (appConfig.remainingFreeSpace > 0) {
|
|
// Display the actual value
|
|
addLogEntry("Remaining Free Space: " ~ to!string(byteToGibiByte(appConfig.remainingFreeSpace)) ~ " GB (" ~ to!string(appConfig.remainingFreeSpace) ~ " bytes)", ["verbose"]);
|
|
} else {
|
|
// zero or non-zero value or restricted
|
|
if (!appConfig.quotaRestricted){
|
|
addLogEntry("Remaining Free Space: 0 KB", ["verbose"]);
|
|
} else {
|
|
addLogEntry("Remaining Free Space: Not Available", ["verbose"]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Query itemdb.computePath() and catch potential assert when DB consistency issue occurs
|
|
string computeItemPath(string thisDriveId, string thisItemId) {
|
|
|
|
// static declare this for this function
|
|
static import core.exception;
|
|
string calculatedPath;
|
|
addLogEntry("Attempting to calculate local filesystem path for " ~ thisDriveId ~ " and " ~ thisItemId, ["debug"]);
|
|
|
|
try {
|
|
calculatedPath = itemDB.computePath(thisDriveId, thisItemId);
|
|
} catch (core.exception.AssertError) {
|
|
// broken tree in the database, we cant compute the path for this item id, exit
|
|
addLogEntry("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database.");
|
|
// Must force exit here, allow logging to be done
|
|
forceExit();
|
|
}
|
|
|
|
// return calculated path as string
|
|
return calculatedPath;
|
|
}
|
|
|
|
// Try and compute the file hash for the given item
|
|
bool testFileHash(string path, Item item) {
|
|
|
|
// Generate QuickXORHash first before attempting to generate any other type of hash
|
|
if (item.quickXorHash) {
|
|
if (item.quickXorHash == computeQuickXorHash(path)) return true;
|
|
} else if (item.sha256Hash) {
|
|
if (item.sha256Hash == computeSHA256Hash(path)) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Process items that need to be removed
|
|
void processDeleteItems() {
|
|
|
|
foreach_reverse (i; idsToDelete) {
|
|
Item item;
|
|
string path;
|
|
if (!itemDB.selectById(i[0], i[1], item)) continue; // check if the item is in the db
|
|
// Compute this item path
|
|
path = computeItemPath(i[0], i[1]);
|
|
|
|
// Log the action if the path exists .. it may of already been removed and this is a legacy array item
|
|
if (exists(path)) {
|
|
if (item.type == ItemType.file) {
|
|
addLogEntry("Trying to delete file " ~ path);
|
|
} else {
|
|
addLogEntry("Trying to delete directory " ~ path);
|
|
}
|
|
}
|
|
|
|
// Process the database entry removal. In a --dry-run scenario, this is being done against a DB copy
|
|
itemDB.deleteById(item.driveId, item.id);
|
|
if (item.remoteDriveId != null) {
|
|
// delete the linked remote folder
|
|
itemDB.deleteById(item.remoteDriveId, item.remoteId);
|
|
}
|
|
|
|
// Add to pathFakeDeletedArray
|
|
// We dont want to try and upload this item again, so we need to track this object
|
|
if (dryRun) {
|
|
// We need to add './' here so that it can be correctly searched to ensure it is not uploaded
|
|
string pathToAdd = "./" ~ path;
|
|
pathFakeDeletedArray ~= pathToAdd;
|
|
}
|
|
|
|
bool needsRemoval = false;
|
|
if (exists(path)) {
|
|
// path exists on the local system
|
|
// make sure that the path refers to the correct item
|
|
Item pathItem;
|
|
if (itemDB.selectByPath(path, item.driveId, pathItem)) {
|
|
if (pathItem.id == item.id) {
|
|
needsRemoval = true;
|
|
} else {
|
|
addLogEntry("Skipped due to id difference!");
|
|
}
|
|
} else {
|
|
// item has disappeared completely
|
|
needsRemoval = true;
|
|
}
|
|
}
|
|
if (needsRemoval) {
|
|
// Log the action
|
|
if (item.type == ItemType.file) {
|
|
addLogEntry("Deleting file " ~ path);
|
|
} else {
|
|
addLogEntry("Deleting directory " ~ path);
|
|
}
|
|
|
|
// Perform the action
|
|
if (!dryRun) {
|
|
if (isFile(path)) {
|
|
remove(path);
|
|
} else {
|
|
try {
|
|
// Remove any children of this path if they still exist
|
|
// Resolve 'Directory not empty' error when deleting local files
|
|
foreach (DirEntry child; dirEntries(path, SpanMode.depth, false)) {
|
|
attrIsDir(child.linkAttributes) ? rmdir(child.name) : remove(child.name);
|
|
}
|
|
// Remove the path now that it is empty of children
|
|
rmdirRecurse(path);
|
|
} catch (FileException e) {
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!dryRun) {
|
|
// Cleanup array memory
|
|
idsToDelete = [];
|
|
}
|
|
}
|
|
|
|
// Update the timestamp of an object online
|
|
void uploadLastModifiedTime(string driveId, string id, SysTime mtime, string eTag) {
|
|
|
|
string itemModifiedTime;
|
|
itemModifiedTime = mtime.toISOExtString();
|
|
JSONValue data = [
|
|
"fileSystemInfo": JSONValue([
|
|
"lastModifiedDateTime": itemModifiedTime
|
|
])
|
|
];
|
|
|
|
// What eTag value do we use?
|
|
string eTagValue;
|
|
if (appConfig.accountType == "personal") {
|
|
eTagValue = null;
|
|
} else {
|
|
eTagValue = eTag;
|
|
}
|
|
|
|
JSONValue response;
|
|
// Create a new OneDrive API instance
|
|
OneDriveApi uploadLastModifiedTimeApiInstance;
|
|
uploadLastModifiedTimeApiInstance = new OneDriveApi(appConfig);
|
|
uploadLastModifiedTimeApiInstance.initialise();
|
|
|
|
// Try and update the online last modified time
|
|
try {
|
|
// Use this instance
|
|
response = uploadLastModifiedTimeApiInstance.updateById(driveId, id, data, eTagValue);
|
|
// Shut the instance down
|
|
uploadLastModifiedTimeApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(uploadLastModifiedTimeApiInstance);
|
|
// Is the response a valid JSON object - validation checking done in saveItem
|
|
saveItem(response);
|
|
} catch (OneDriveException exception) {
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(uploadLastModifiedTimeApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to update the timestamp on an item on OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
uploadLastModifiedTime(driveId, id, mtime, eTag);
|
|
return;
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
if (exception.httpStatusCode == 409) {
|
|
// ETag does not match current item's value - use a null eTag
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
uploadLastModifiedTime(driveId, id, mtime, null);
|
|
} else {
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Perform a database integrity check - checking all the items that are in-sync at the moment, validating what we know should be on disk, to what is actually on disk
|
|
void performDatabaseConsistencyAndIntegrityCheck() {
|
|
|
|
// Log what we are doing
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
addLogEntry("Performing a database consistency and integrity check on locally stored data ... ");
|
|
}
|
|
|
|
// What driveIDsArray do we use? If we are doing a --single-directory we need to use just the drive id associated with that operation
|
|
string[] consistencyCheckDriveIdsArray;
|
|
if (singleDirectoryScope) {
|
|
consistencyCheckDriveIdsArray ~= singleDirectoryScopeDriveId;
|
|
} else {
|
|
consistencyCheckDriveIdsArray = driveIDsArray;
|
|
}
|
|
|
|
// Create a new DB blank item
|
|
Item item;
|
|
// Use the array we populate, rather than selecting all distinct driveId's from the database
|
|
foreach (driveId; consistencyCheckDriveIdsArray) {
|
|
// Make the logging more accurate - we cant update driveId as this then breaks the below queries
|
|
addLogEntry("Processing DB entries for this Drive ID: " ~ driveId, ["verbose"]);
|
|
|
|
// What OneDrive API query do we use?
|
|
// - Are we running against a National Cloud Deployments that does not support /delta ?
|
|
// National Cloud Deployments do not support /delta as a query
|
|
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
|
|
//
|
|
// - Are we performing a --single-directory sync, which will exclude many items online, focusing in on a specific online directory
|
|
//
|
|
// - Are we performing a --download-only --cleanup-local-files action?
|
|
//
|
|
// If we did, we self generated a /delta response, thus need to now process elements that are still flagged as out-of-sync
|
|
if ((singleDirectoryScope) || (nationalCloudDeployment) || (cleanupLocalFiles)) {
|
|
// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
|
|
// Normally, this is done at the end of processing all /delta queries, however when using --single-directory or a National Cloud Deployments is configured
|
|
// We cant use /delta to query the OneDrive API as National Cloud Deployments dont support /delta
|
|
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
|
|
// We dont use /delta for --single-directory as, in order to sync a single path with /delta, we need to query the entire OneDrive API JSON data to then filter out
|
|
// objects that we dont want, thus, it is easier to use the same method as National Cloud Deployments, but query just the objects we are after
|
|
|
|
// For each unique OneDrive driveID we know about
|
|
Item[] outOfSyncItems = itemDB.selectOutOfSyncItems(driveId);
|
|
foreach (outOfSyncItem; outOfSyncItems) {
|
|
if (!dryRun) {
|
|
// clean up idsToDelete
|
|
idsToDelete.length = 0;
|
|
assumeSafeAppend(idsToDelete);
|
|
// flag to delete local file as it now is no longer in sync with OneDrive
|
|
addLogEntry("Flagging to delete local item as it now is no longer in sync with OneDrive", ["debug"]);
|
|
addLogEntry("outOfSyncItem: " ~ to!string(outOfSyncItem), ["debug"]);
|
|
idsToDelete ~= [outOfSyncItem.driveId, outOfSyncItem.id];
|
|
// delete items in idsToDelete
|
|
if (idsToDelete.length > 0) processDeleteItems();
|
|
}
|
|
}
|
|
|
|
// Fetch database items associated with this path
|
|
Item[] driveItems;
|
|
if (singleDirectoryScope) {
|
|
// Use the --single-directory items we previously configured
|
|
// - query database for children objects using those items
|
|
driveItems = getChildren(singleDirectoryScopeDriveId, singleDirectoryScopeItemId);
|
|
} else {
|
|
// Check everything associated with each driveId we know about
|
|
addLogEntry("Selecting DB items via itemDB.selectByDriveId(driveId)", ["debug"]);
|
|
// Query database
|
|
driveItems = itemDB.selectByDriveId(driveId);
|
|
}
|
|
|
|
// Log DB items to process
|
|
addLogEntry("Database items to process for this driveId: " ~ to!string(driveItems.count), ["debug"]);
|
|
|
|
// Process each database database item associated with the driveId
|
|
foreach(dbItem; driveItems) {
|
|
// Does it still exist on disk in the location the DB thinks it is
|
|
checkDatabaseItemForConsistency(dbItem);
|
|
}
|
|
} else {
|
|
// Check everything associated with each driveId we know about
|
|
addLogEntry("Selecting DB items via itemDB.selectByDriveId(driveId)", ["debug"]);
|
|
|
|
// Query database
|
|
auto driveItems = itemDB.selectByDriveId(driveId);
|
|
addLogEntry("Database items to process for this driveId: " ~ to!string(driveItems.count), ["debug"]);
|
|
|
|
// Process each database database item associated with the driveId
|
|
foreach(dbItem; driveItems) {
|
|
// Does it still exist on disk in the location the DB thinks it is
|
|
checkDatabaseItemForConsistency(dbItem);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Are we doing a --download-only sync?
|
|
if (!appConfig.getValueBool("download_only")) {
|
|
// Do we have any known items, where the content has changed locally, that needs to be uploaded?
|
|
if (!databaseItemsWhereContentHasChanged.empty) {
|
|
// There are changed local files that were in the DB to upload
|
|
addLogEntry("Changed local items to upload to OneDrive: " ~ to!string(databaseItemsWhereContentHasChanged.length));
|
|
processChangedLocalItemsToUpload();
|
|
// Cleanup array memory
|
|
databaseItemsWhereContentHasChanged = [];
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check this Database Item for its consistency on disk
|
|
void checkDatabaseItemForConsistency(Item dbItem) {
|
|
|
|
// What is the local path item
|
|
string localFilePath;
|
|
// Do we want to onward process this item?
|
|
bool unwanted = false;
|
|
|
|
// Compute this dbItem path early as we we use this path often
|
|
localFilePath = buildNormalizedPath(computeItemPath(dbItem.driveId, dbItem.id));
|
|
|
|
// To improve logging output for this function, what is the 'logical path'?
|
|
string logOutputPath;
|
|
if (localFilePath == ".") {
|
|
// get the configured sync_dir
|
|
logOutputPath = buildNormalizedPath(appConfig.getValueString("sync_dir"));
|
|
} else {
|
|
// Use the path that was computed
|
|
logOutputPath = localFilePath;
|
|
}
|
|
|
|
// Log what we are doing
|
|
addLogEntry("Processing " ~ logOutputPath, ["verbose"]);
|
|
|
|
// Determine which action to take
|
|
final switch (dbItem.type) {
|
|
case ItemType.file:
|
|
// Logging output
|
|
checkFileDatabaseItemForConsistency(dbItem, localFilePath);
|
|
break;
|
|
case ItemType.dir:
|
|
// Logging output
|
|
checkDirectoryDatabaseItemForConsistency(dbItem, localFilePath);
|
|
break;
|
|
case ItemType.remote:
|
|
// checkRemoteDirectoryDatabaseItemForConsistency(dbItem, localFilePath);
|
|
break;
|
|
case ItemType.unknown:
|
|
// Unknown type - we dont action these items
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Perform the database consistency check on this file item
|
|
void checkFileDatabaseItemForConsistency(Item dbItem, string localFilePath) {
|
|
|
|
// What is the source of this item data?
|
|
string itemSource = "database";
|
|
|
|
// Does this item|file still exist on disk?
|
|
if (exists(localFilePath)) {
|
|
// Path exists locally, is this path a file?
|
|
if (isFile(localFilePath)) {
|
|
// Can we actually read the local file?
|
|
if (readLocalFile(localFilePath)){
|
|
// File is readable
|
|
SysTime localModifiedTime = timeLastModified(localFilePath).toUTC();
|
|
SysTime itemModifiedTime = dbItem.mtime;
|
|
// Reduce time resolution to seconds before comparing
|
|
itemModifiedTime.fracSecs = Duration.zero;
|
|
localModifiedTime.fracSecs = Duration.zero;
|
|
|
|
if (localModifiedTime != itemModifiedTime) {
|
|
// The modified dates are different
|
|
addLogEntry("The local item has a different modified time " ~ to!string(localModifiedTime) ~ " when compared to " ~ itemSource ~ " modified time " ~ to!string(itemModifiedTime), ["debug"]);
|
|
|
|
// Test the file hash
|
|
if (!testFileHash(localFilePath, dbItem)) {
|
|
// Is the local file 'newer' or 'older' (ie was an old file 'restored locally' by a different backup / replacement process?)
|
|
if (localModifiedTime >= itemModifiedTime) {
|
|
// Local file is newer
|
|
if (!appConfig.getValueBool("download_only")) {
|
|
addLogEntry("The file content has changed locally and has a newer timestamp, thus needs to be uploaded to OneDrive", ["verbose"]);
|
|
// Add to an array of files we need to upload as this file has changed locally in-between doing the /delta check and performing this check
|
|
databaseItemsWhereContentHasChanged ~= [dbItem.driveId, dbItem.id, localFilePath];
|
|
} else {
|
|
addLogEntry("The file content has changed locally and has a newer timestamp. The file will remain different to online file due to --download-only being used", ["verbose"]);
|
|
}
|
|
} else {
|
|
// Local file is older - data recovery process? something else?
|
|
if (!appConfig.getValueBool("download_only")) {
|
|
addLogEntry("The file content has changed locally and file now has a older timestamp. Uploading this file to OneDrive may potentially cause data-loss online", ["verbose"]);
|
|
// Add to an array of files we need to upload as this file has changed locally in-between doing the /delta check and performing this check
|
|
databaseItemsWhereContentHasChanged ~= [dbItem.driveId, dbItem.id, localFilePath];
|
|
} else {
|
|
addLogEntry("The file content has changed locally and file now has a older timestamp. The file will remain different to online file due to --download-only being used", ["verbose"]);
|
|
}
|
|
}
|
|
} else {
|
|
// The file contents have not changed, but the modified timestamp has
|
|
addLogEntry("The last modified timestamp has changed however the file content has not changed", ["verbose"]);
|
|
addLogEntry("The local item has the same hash value as the item online - correcting timestamp online", ["verbose"]);
|
|
if (!dryRun) {
|
|
// Attempt to update the online date time stamp
|
|
uploadLastModifiedTime(dbItem.driveId, dbItem.id, localModifiedTime.toUTC(), dbItem.eTag);
|
|
}
|
|
}
|
|
} else {
|
|
// The file has not changed
|
|
addLogEntry("The file has not changed", ["verbose"]);
|
|
}
|
|
} else {
|
|
//The file is not readable - skipped
|
|
addLogEntry("Skipping processing this file as it cannot be read (file permissions or file corruption): " ~ localFilePath);
|
|
}
|
|
} else {
|
|
// The item was a file but now is a directory
|
|
addLogEntry("The item was a file but now is a directory", ["verbose"]);
|
|
}
|
|
} else {
|
|
// File does not exist locally, but is in our database as a dbItem containing all the data was passed into this function
|
|
// If we are in a --dry-run situation - this file may never have existed as we never downloaded it
|
|
if (!dryRun) {
|
|
// Not --dry-run situation
|
|
addLogEntry("The file has been deleted locally", ["verbose"]);
|
|
// Upload to OneDrive the instruction to delete this item. This will handle the 'noRemoteDelete' flag if set
|
|
uploadDeletedItem(dbItem, localFilePath);
|
|
} else {
|
|
// We are in a --dry-run situation, file appears to have been deleted locally - this file may never have existed locally as we never downloaded it due to --dry-run
|
|
// Did we 'fake create it' as part of --dry-run ?
|
|
bool idsFakedMatch = false;
|
|
foreach (i; idsFaked) {
|
|
if (i[1] == dbItem.id) {
|
|
addLogEntry("Matched faked file which is 'supposed' to exist but not created due to --dry-run use", ["debug"]);
|
|
addLogEntry("The file has not changed", ["verbose"]);
|
|
idsFakedMatch = true;
|
|
}
|
|
}
|
|
if (!idsFakedMatch) {
|
|
// dbItem.id did not match a 'faked' download new file creation - so this in-sync object was actually deleted locally, but we are in a --dry-run situation
|
|
addLogEntry("The file has been deleted locally", ["verbose"]);
|
|
// Upload to OneDrive the instruction to delete this item. This will handle the 'noRemoteDelete' flag if set
|
|
uploadDeletedItem(dbItem, localFilePath);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Perform the database consistency check on this directory item
|
|
void checkDirectoryDatabaseItemForConsistency(Item dbItem, string localFilePath) {
|
|
|
|
// What is the source of this item data?
|
|
string itemSource = "database";
|
|
|
|
// Does this item|directory still exist on disk?
|
|
if (exists(localFilePath)) {
|
|
// Fix https://github.com/abraunegg/onedrive/issues/1915
|
|
try {
|
|
if (!isDir(localFilePath)) {
|
|
addLogEntry("The item was a directory but now it is a file", ["verbose"]);
|
|
uploadDeletedItem(dbItem, localFilePath);
|
|
uploadNewFile(localFilePath);
|
|
} else {
|
|
// Directory still exists locally
|
|
addLogEntry("The directory has not changed", ["verbose"]);
|
|
// When we are using --single-directory, we use a the getChildren() call to get all children of a path, meaning all children are already traversed
|
|
// Thus, if we traverse the path of this directory .. we end up with double processing & log output .. which is not ideal
|
|
if (!singleDirectoryScope) {
|
|
// loop through the children
|
|
foreach (Item child; itemDB.selectChildren(dbItem.driveId, dbItem.id)) {
|
|
checkDatabaseItemForConsistency(child);
|
|
}
|
|
}
|
|
}
|
|
} catch (FileException e) {
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
} else {
|
|
// Directory does not exist locally, but it is in our database as a dbItem containing all the data was passed into this function
|
|
// If we are in a --dry-run situation - this directory may never have existed as we never created it
|
|
if (!dryRun) {
|
|
// Not --dry-run situation
|
|
if (!appConfig.getValueBool("monitor")) {
|
|
// Not in --monitor mode
|
|
addLogEntry("The directory has been deleted locally", ["verbose"]);
|
|
} else {
|
|
// Appropriate message as we are in --monitor mode
|
|
addLogEntry("The directory appears to have been deleted locally .. but we are running in --monitor mode. This may have been 'moved' on the local filesystem rather than being 'deleted'", ["verbose"]);
|
|
addLogEntry("Most likely cause - 'inotify' event was missing for whatever action was taken locally or action taken when application was stopped", ["debug"]);
|
|
}
|
|
// A moved directory will be uploaded as 'new', delete the old directory and database reference
|
|
// Upload to OneDrive the instruction to delete this item. This will handle the 'noRemoteDelete' flag if set
|
|
uploadDeletedItem(dbItem, localFilePath);
|
|
} else {
|
|
// We are in a --dry-run situation, directory appears to have been deleted locally - this directory may never have existed locally as we never created it due to --dry-run
|
|
// Did we 'fake create it' as part of --dry-run ?
|
|
bool idsFakedMatch = false;
|
|
foreach (i; idsFaked) {
|
|
if (i[1] == dbItem.id) {
|
|
addLogEntry("Matched faked dir which is 'supposed' to exist but not created due to --dry-run use", ["debug"]);
|
|
addLogEntry("The directory has not changed", ["verbose"]);
|
|
idsFakedMatch = true;
|
|
}
|
|
}
|
|
if (!idsFakedMatch) {
|
|
// dbItem.id did not match a 'faked' download new directory creation - so this in-sync object was actually deleted locally, but we are in a --dry-run situation
|
|
addLogEntry("The directory has been deleted locally", ["verbose"]);
|
|
// Upload to OneDrive the instruction to delete this item. This will handle the 'noRemoteDelete' flag if set
|
|
uploadDeletedItem(dbItem, localFilePath);
|
|
} else {
|
|
// When we are using --single-directory, we use a the getChildren() call to get all children of a path, meaning all children are already traversed
|
|
// Thus, if we traverse the path of this directory .. we end up with double processing & log output .. which is not ideal
|
|
if (!singleDirectoryScope) {
|
|
// loop through the children
|
|
foreach (Item child; itemDB.selectChildren(dbItem.driveId, dbItem.id)) {
|
|
checkDatabaseItemForConsistency(child);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Does this local path (directory or file) conform with the Microsoft Naming Restrictions? It needs to conform otherwise we cannot create the directory or upload the file.
|
|
bool checkPathAgainstMicrosoftNamingRestrictions(string localFilePath) {
|
|
|
|
// Check if the given path violates certain Microsoft restrictions and limitations
|
|
// Return a true|false response
|
|
bool invalidPath = false;
|
|
|
|
// Check path against Microsoft OneDrive restriction and limitations about Windows naming for files and folders
|
|
if (!invalidPath) {
|
|
if (!isValidName(localFilePath)) { // This will return false if this is not a valid name according to the OneDrive API specifications
|
|
addLogEntry("Skipping item - invalid name (Microsoft Naming Convention): " ~ localFilePath, ["info", "notify"]);
|
|
invalidPath = true;
|
|
}
|
|
}
|
|
|
|
// Check path for bad whitespace items
|
|
if (!invalidPath) {
|
|
if (containsBadWhiteSpace(localFilePath)) { // This will return true if this contains a bad whitespace item
|
|
addLogEntry("Skipping item - invalid name (Contains an invalid whitespace item): " ~ localFilePath, ["info", "notify"]);
|
|
invalidPath = true;
|
|
}
|
|
}
|
|
|
|
// Check path for HTML ASCII Codes
|
|
if (!invalidPath) {
|
|
if (containsASCIIHTMLCodes(localFilePath)) { // This will return true if this contains HTML ASCII Codes
|
|
addLogEntry("Skipping item - invalid name (Contains HTML ASCII Code): " ~ localFilePath, ["info", "notify"]);
|
|
invalidPath = true;
|
|
}
|
|
}
|
|
|
|
// Check path for ASCII Control Codes
|
|
if (!invalidPath) {
|
|
if (containsASCIIControlCodes(localFilePath)) { // This will return true if this contains ASCII Control Codes
|
|
addLogEntry("Skipping item - invalid name (Contains ASCII Control Codes): " ~ localFilePath, ["info", "notify"]);
|
|
invalidPath = true;
|
|
}
|
|
}
|
|
|
|
// Return if this is a valid path
|
|
return invalidPath;
|
|
}
|
|
|
|
// Does this local path (directory or file) get excluded from any operation based on any client side filtering rules?
|
|
bool checkPathAgainstClientSideFiltering(string localFilePath) {
|
|
|
|
// Check the path against client side filtering rules
|
|
// - check_nosync
|
|
// - skip_dotfiles
|
|
// - skip_symlinks
|
|
// - skip_file
|
|
// - skip_dir
|
|
// - sync_list
|
|
// - skip_size
|
|
// Return a true|false response
|
|
|
|
bool clientSideRuleExcludesPath = false;
|
|
|
|
// does the path exist?
|
|
if (!exists(localFilePath)) {
|
|
// path does not exist - we cant review any client side rules on something that does not exist locally
|
|
return clientSideRuleExcludesPath;
|
|
}
|
|
|
|
// - check_nosync
|
|
if (!clientSideRuleExcludesPath) {
|
|
// Do we need to check for .nosync? Only if --check-for-nosync was passed in
|
|
if (appConfig.getValueBool("check_nosync")) {
|
|
if (exists(localFilePath ~ "/.nosync")) {
|
|
addLogEntry("Skipping item - .nosync found & --check-for-nosync enabled: " ~ localFilePath, ["verbose"]);
|
|
clientSideRuleExcludesPath = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// - skip_dotfiles
|
|
if (!clientSideRuleExcludesPath) {
|
|
// Do we need to check skip dot files if configured
|
|
if (appConfig.getValueBool("skip_dotfiles")) {
|
|
if (isDotFile(localFilePath)) {
|
|
addLogEntry("Skipping item - .file or .folder: " ~ localFilePath, ["verbose"]);
|
|
clientSideRuleExcludesPath = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// - skip_symlinks
|
|
if (!clientSideRuleExcludesPath) {
|
|
// Is the path a symbolic link
|
|
if (isSymlink(localFilePath)) {
|
|
// if config says so we skip all symlinked items
|
|
if (appConfig.getValueBool("skip_symlinks")) {
|
|
addLogEntry("Skipping item - skip symbolic links configured: " ~ localFilePath, ["verbose"]);
|
|
clientSideRuleExcludesPath = true;
|
|
}
|
|
// skip unexisting symbolic links
|
|
else if (!exists(readLink(localFilePath))) {
|
|
// reading the symbolic link failed - is the link a relative symbolic link
|
|
// drwxrwxr-x. 2 alex alex 46 May 30 09:16 .
|
|
// drwxrwxr-x. 3 alex alex 35 May 30 09:14 ..
|
|
// lrwxrwxrwx. 1 alex alex 61 May 30 09:16 absolute.txt -> /home/alex/OneDrivePersonal/link_tests/intercambio/prueba.txt
|
|
// lrwxrwxrwx. 1 alex alex 13 May 30 09:16 relative.txt -> ../prueba.txt
|
|
//
|
|
// absolute links will be able to be read, but 'relative' links will fail, because they cannot be read based on the current working directory 'sync_dir'
|
|
string currentSyncDir = getcwd();
|
|
string fullLinkPath = buildNormalizedPath(absolutePath(localFilePath));
|
|
string fileName = baseName(fullLinkPath);
|
|
string parentLinkPath = dirName(fullLinkPath);
|
|
// test if this is a 'relative' symbolic link
|
|
chdir(parentLinkPath);
|
|
auto relativeLink = readLink(fileName);
|
|
auto relativeLinkTest = exists(readLink(fileName));
|
|
// reset back to our 'sync_dir'
|
|
chdir(currentSyncDir);
|
|
// results
|
|
if (relativeLinkTest) {
|
|
addLogEntry("Not skipping item - symbolic link is a 'relative link' to target ('" ~ relativeLink ~ "') which can be supported: " ~ localFilePath, ["debug"]);
|
|
} else {
|
|
addLogEntry("Skipping item - invalid symbolic link: "~ localFilePath, ["info", "notify"]);
|
|
clientSideRuleExcludesPath = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Is this item excluded by user configuration of skip_dir or skip_file?
|
|
if (!clientSideRuleExcludesPath) {
|
|
if (localFilePath != ".") {
|
|
// skip_dir handling
|
|
if (isDir(localFilePath)) {
|
|
addLogEntry("Checking local path: " ~ localFilePath, ["debug"]);
|
|
|
|
// Only check path if config is != ""
|
|
if (appConfig.getValueString("skip_dir") != "") {
|
|
// The path that needs to be checked needs to include the '/'
|
|
// This due to if the user has specified in skip_dir an exclusive path: '/path' - that is what must be matched
|
|
if (selectiveSync.isDirNameExcluded(localFilePath.strip('.'))) {
|
|
addLogEntry("Skipping item - excluded by skip_dir config: " ~ localFilePath, ["verbose"]);
|
|
clientSideRuleExcludesPath = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// skip_file handling
|
|
if (isFile(localFilePath)) {
|
|
addLogEntry("Checking file: " ~ localFilePath, ["debug"]);
|
|
|
|
// The path that needs to be checked needs to include the '/'
|
|
// This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched
|
|
if (selectiveSync.isFileNameExcluded(localFilePath.strip('.'))) {
|
|
addLogEntry("Skipping item - excluded by skip_file config: " ~ localFilePath, ["verbose"]);
|
|
clientSideRuleExcludesPath = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Is this item excluded by user configuration of sync_list?
|
|
if (!clientSideRuleExcludesPath) {
|
|
if (localFilePath != ".") {
|
|
if (syncListConfigured) {
|
|
// sync_list configured and in use
|
|
if (selectiveSync.isPathExcludedViaSyncList(localFilePath)) {
|
|
if ((isFile(localFilePath)) && (appConfig.getValueBool("sync_root_files")) && (rootName(localFilePath.strip('.').strip('/')) == "")) {
|
|
addLogEntry("Not skipping path due to sync_root_files inclusion: " ~ localFilePath, ["debug"]);
|
|
} else {
|
|
if (exists(appConfig.syncListFilePath)){
|
|
// skipped most likely due to inclusion in sync_list
|
|
addLogEntry("Skipping item - excluded by sync_list config: " ~ localFilePath, ["verbose"]);
|
|
clientSideRuleExcludesPath = true;
|
|
} else {
|
|
// skipped for some other reason
|
|
addLogEntry("Skipping item - path excluded by user config: " ~ localFilePath, ["verbose"]);
|
|
clientSideRuleExcludesPath = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if this is excluded by a user set maximum filesize to upload
|
|
if (!clientSideRuleExcludesPath) {
|
|
if (isFile(localFilePath)) {
|
|
if (fileSizeLimit != 0) {
|
|
// Get the file size
|
|
ulong thisFileSize = getSize(localFilePath);
|
|
if (thisFileSize >= fileSizeLimit) {
|
|
addLogEntry("Skipping item - excluded by skip_size config: " ~ localFilePath ~ " (" ~ to!string(thisFileSize/2^^20) ~ " MB)", ["verbose"]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return clientSideRuleExcludesPath;
|
|
}
|
|
|
|
// Does this JSON item (as received from OneDrive API) get excluded from any operation based on any client side filtering rules?
|
|
// This function is only used when we are fetching objects from the OneDrive API using a /children query to help speed up what object we query
|
|
bool checkJSONAgainstClientSideFiltering(JSONValue onedriveJSONItem) {
|
|
|
|
bool clientSideRuleExcludesPath = false;
|
|
|
|
// Check the path against client side filtering rules
|
|
// - check_nosync (MISSING)
|
|
// - skip_dotfiles (MISSING)
|
|
// - skip_symlinks (MISSING)
|
|
// - skip_file
|
|
// - skip_dir
|
|
// - sync_list
|
|
// - skip_size (MISSING)
|
|
// Return a true|false response
|
|
|
|
// Use the JSON elements rather can computing a DB struct via makeItem()
|
|
string thisItemId = onedriveJSONItem["id"].str;
|
|
string thisItemDriveId = onedriveJSONItem["parentReference"]["driveId"].str;
|
|
string thisItemParentId = onedriveJSONItem["parentReference"]["id"].str;
|
|
string thisItemName = onedriveJSONItem["name"].str;
|
|
|
|
// Is this parent is in the database
|
|
bool parentInDatabase = false;
|
|
|
|
// Calculate if the Parent Item is in the database so that it can be re-used
|
|
parentInDatabase = itemDB.idInLocalDatabase(thisItemDriveId, thisItemParentId);
|
|
|
|
// Check if this is excluded by config option: skip_dir
|
|
if (!clientSideRuleExcludesPath) {
|
|
// Is the item a folder?
|
|
if (isItemFolder(onedriveJSONItem)) {
|
|
// Only check path if config is != ""
|
|
if (!appConfig.getValueString("skip_dir").empty) {
|
|
// work out the 'snippet' path where this folder would be created
|
|
string simplePathToCheck = "";
|
|
string complexPathToCheck = "";
|
|
string matchDisplay = "";
|
|
|
|
if (hasParentReference(onedriveJSONItem)) {
|
|
// we need to workout the FULL path for this item
|
|
// simple path
|
|
if (("name" in onedriveJSONItem["parentReference"]) != null) {
|
|
simplePathToCheck = onedriveJSONItem["parentReference"]["name"].str ~ "/" ~ onedriveJSONItem["name"].str;
|
|
} else {
|
|
simplePathToCheck = onedriveJSONItem["name"].str;
|
|
}
|
|
addLogEntry("skip_dir path to check (simple): " ~ simplePathToCheck, ["debug"]);
|
|
|
|
// complex path
|
|
if (parentInDatabase) {
|
|
// build up complexPathToCheck
|
|
//complexPathToCheck = buildNormalizedPath(newItemPath);
|
|
complexPathToCheck = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName;
|
|
} else {
|
|
addLogEntry("Parent details not in database - unable to compute complex path to check", ["debug"]);
|
|
}
|
|
if (!complexPathToCheck.empty) {
|
|
addLogEntry("skip_dir path to check (complex): " ~ complexPathToCheck, ["debug"]);
|
|
}
|
|
} else {
|
|
simplePathToCheck = onedriveJSONItem["name"].str;
|
|
}
|
|
|
|
// If 'simplePathToCheck' or 'complexPathToCheck' is of the following format: root:/folder
|
|
// then isDirNameExcluded matching will not work
|
|
// Clean up 'root:' if present
|
|
if (startsWith(simplePathToCheck, "root:")){
|
|
addLogEntry("Updating simplePathToCheck to remove 'root:'", ["debug"]);
|
|
simplePathToCheck = strip(simplePathToCheck, "root:");
|
|
}
|
|
if (startsWith(complexPathToCheck, "root:")){
|
|
addLogEntry("Updating complexPathToCheck to remove 'root:'", ["debug"]);
|
|
complexPathToCheck = strip(complexPathToCheck, "root:");
|
|
}
|
|
|
|
// OK .. what checks are we doing?
|
|
if ((!simplePathToCheck.empty) && (complexPathToCheck.empty)) {
|
|
// just a simple check
|
|
addLogEntry("Performing a simple check only", ["debug"]);
|
|
clientSideRuleExcludesPath = selectiveSync.isDirNameExcluded(simplePathToCheck);
|
|
} else {
|
|
// simple and complex
|
|
addLogEntry("Performing a simple then complex path match if required", ["debug"]);
|
|
|
|
// simple first
|
|
addLogEntry("Performing a simple check first", ["debug"]);
|
|
clientSideRuleExcludesPath = selectiveSync.isDirNameExcluded(simplePathToCheck);
|
|
matchDisplay = simplePathToCheck;
|
|
if (!clientSideRuleExcludesPath) {
|
|
addLogEntry("Simple match was false, attempting complex match", ["debug"]);
|
|
// simple didnt match, perform a complex check
|
|
clientSideRuleExcludesPath = selectiveSync.isDirNameExcluded(complexPathToCheck);
|
|
matchDisplay = complexPathToCheck;
|
|
}
|
|
}
|
|
// End Result
|
|
addLogEntry("skip_dir exclude result (directory based): " ~ clientSideRuleExcludesPath, ["debug"]);
|
|
if (clientSideRuleExcludesPath) {
|
|
// This path should be skipped
|
|
addLogEntry("Skipping item - excluded by skip_dir config: " ~ matchDisplay, ["verbose"]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if this is excluded by config option: skip_file
|
|
if (!clientSideRuleExcludesPath) {
|
|
// is the item a file ?
|
|
if (isFileItem(onedriveJSONItem)) {
|
|
// JSON item is a file
|
|
|
|
// skip_file can contain 4 types of entries:
|
|
// - wildcard - *.txt
|
|
// - text + wildcard - name*.txt
|
|
// - full path + combination of any above two - /path/name*.txt
|
|
// - full path to file - /path/to/file.txt
|
|
|
|
string exclusionTestPath = "";
|
|
|
|
// is the parent id in the database?
|
|
if (parentInDatabase) {
|
|
// parent id is in the database, so we can try and calculate the full file path
|
|
string jsonItemPath = "";
|
|
|
|
// Compute this item path & need the full path for this file
|
|
jsonItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName;
|
|
// Log the calculation
|
|
addLogEntry("New Item calculated full path is: " ~ jsonItemPath, ["debug"]);
|
|
|
|
// The path that needs to be checked needs to include the '/'
|
|
// This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched
|
|
// However, as 'path' used throughout, use a temp variable with this modification so that we use the temp variable for exclusion checks
|
|
if (!startsWith(jsonItemPath, "/")){
|
|
// Add '/' to the path
|
|
exclusionTestPath = '/' ~ jsonItemPath;
|
|
}
|
|
|
|
// what are we checking
|
|
addLogEntry("skip_file item to check (full calculated path): " ~ exclusionTestPath, ["debug"]);
|
|
} else {
|
|
// parent not in database, we can only check using this JSON item's name
|
|
if (!startsWith(thisItemName, "/")){
|
|
// Add '/' to the path
|
|
exclusionTestPath = '/' ~ thisItemName;
|
|
}
|
|
|
|
// what are we checking
|
|
addLogEntry("skip_file item to check (file name only - parent path not in database): " ~ exclusionTestPath, ["debug"]);
|
|
clientSideRuleExcludesPath = selectiveSync.isFileNameExcluded(exclusionTestPath);
|
|
}
|
|
|
|
// Perform the 'skip_file' evaluation
|
|
clientSideRuleExcludesPath = selectiveSync.isFileNameExcluded(exclusionTestPath);
|
|
addLogEntry("Result: " ~ to!string(clientSideRuleExcludesPath), ["debug"]);
|
|
|
|
if (clientSideRuleExcludesPath) {
|
|
// This path should be skipped
|
|
addLogEntry("Skipping item - excluded by skip_file config: " ~ exclusionTestPath, ["verbose"]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if this is included or excluded by use of sync_list
|
|
if (!clientSideRuleExcludesPath) {
|
|
// No need to try and process something against a sync_list if it has been configured
|
|
if (syncListConfigured) {
|
|
// Compute the item path if empty - as to check sync_list we need an actual path to check
|
|
|
|
// What is the path of the new item
|
|
string newItemPath;
|
|
|
|
// Is the parent in the database? If not, we cannot compute the the full path based on the database entries
|
|
// In a --resync scenario - the database is empty
|
|
if (parentInDatabase) {
|
|
// Calculate this items path based on database entries
|
|
newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ thisItemName;
|
|
} else {
|
|
// parent not in the database
|
|
if (("path" in onedriveJSONItem["parentReference"]) != null) {
|
|
// If there is a parent reference path, try and use it
|
|
string selfBuiltPath = onedriveJSONItem["parentReference"]["path"].str ~ "/" ~ onedriveJSONItem["name"].str;
|
|
|
|
// Check for ':' and split if present
|
|
auto splitIndex = selfBuiltPath.indexOf(":");
|
|
if (splitIndex != -1) {
|
|
// Keep only the part after ':'
|
|
selfBuiltPath = selfBuiltPath[splitIndex + 1 .. $];
|
|
}
|
|
|
|
// Set newItemPath to the self built path
|
|
newItemPath = selfBuiltPath;
|
|
} else {
|
|
// no parent reference path available in provided JSON
|
|
newItemPath = thisItemName;
|
|
}
|
|
}
|
|
|
|
// Check for HTML entities (e.g., '%20' for space) in newItemPath
|
|
if (containsURLEncodedItems(newItemPath)) {
|
|
addLogEntry("CAUTION: The JSON element transmitted by the Microsoft OneDrive API includes HTML URL encoded items, which may complicate pattern matching and potentially lead to synchronisation problems for this item.");
|
|
addLogEntry("WORKAROUND: An alternative solution could be to change the name of this item through the online platform: " ~ newItemPath, ["verbose"]);
|
|
addLogEntry("See: https://github.com/OneDrive/onedrive-api-docs/issues/1765 for further details", ["verbose"]);
|
|
}
|
|
|
|
// Update newItemPath
|
|
if(newItemPath[0] == '/') {
|
|
newItemPath = newItemPath[1..$];
|
|
}
|
|
|
|
// What path are we checking?
|
|
addLogEntry("sync_list item to check: " ~ newItemPath, ["debug"]);
|
|
|
|
// Unfortunatly there is no avoiding this call to check if the path is excluded|included via sync_list
|
|
if (selectiveSync.isPathExcludedViaSyncList(newItemPath)) {
|
|
// selective sync advised to skip, however is this a file and are we configured to upload / download files in the root?
|
|
if ((isItemFile(onedriveJSONItem)) && (appConfig.getValueBool("sync_root_files")) && (rootName(newItemPath) == "") ) {
|
|
// This is a file
|
|
// We are configured to sync all files in the root
|
|
// This is a file in the logical root
|
|
clientSideRuleExcludesPath = false;
|
|
} else {
|
|
// path is unwanted
|
|
clientSideRuleExcludesPath = true;
|
|
addLogEntry("Skipping item - excluded by sync_list config: " ~ newItemPath, ["verbose"]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// return if path is excluded
|
|
return clientSideRuleExcludesPath;
|
|
}
|
|
|
|
// Process the list of local changes to upload to OneDrive
|
|
void processChangedLocalItemsToUpload() {
|
|
|
|
// Each element in this array 'databaseItemsWhereContentHasChanged' is an Database Item ID that has been modified locally
|
|
ulong batchSize = appConfig.concurrentThreads;
|
|
ulong batchCount = (databaseItemsWhereContentHasChanged.length + batchSize - 1) / batchSize;
|
|
ulong batchesProcessed = 0;
|
|
|
|
// For each batch of files to upload, upload the changed data to OneDrive
|
|
foreach (chunk; databaseItemsWhereContentHasChanged.chunks(batchSize)) {
|
|
uploadChangedLocalFileToOneDrive(chunk);
|
|
}
|
|
}
|
|
|
|
// Upload changed local files to OneDrive in parallel
|
|
void uploadChangedLocalFileToOneDrive(string[3][] array) {
|
|
|
|
foreach (i, localItemDetails; taskPool.parallel(array)) {
|
|
|
|
addLogEntry("Thread " ~ to!string(i) ~ " Starting: " ~ to!string(Clock.currTime()), ["debug"]);
|
|
|
|
// These are the details of the item we need to upload
|
|
string changedItemParentId = localItemDetails[0];
|
|
string changedItemId = localItemDetails[1];
|
|
string localFilePath = localItemDetails[2];
|
|
|
|
// How much space is remaining on OneDrive
|
|
ulong remainingFreeSpace;
|
|
// Did the upload fail?
|
|
bool uploadFailed = false;
|
|
// Did we skip due to exceeding maximum allowed size?
|
|
bool skippedMaxSize = false;
|
|
// Did we skip to an exception error?
|
|
bool skippedExceptionError = false;
|
|
|
|
// Unfortunatly, we cant store an array of Item's ... so we have to re-query the DB again - unavoidable extra processing here
|
|
// This is because the Item[] has no other functions to allow is to parallel process those elements, so we have to use a string array as input to this function
|
|
Item dbItem;
|
|
itemDB.selectById(changedItemParentId, changedItemId, dbItem);
|
|
|
|
// Query the available space online
|
|
// This will update appConfig.quotaAvailable & appConfig.quotaRestricted values
|
|
remainingFreeSpace = getRemainingFreeSpace(dbItem.driveId);
|
|
|
|
// Get the file size from the actual file
|
|
ulong thisFileSizeLocal = getSize(localFilePath);
|
|
// Get the file size from the DB data
|
|
ulong thisFileSizeFromDB = to!ulong(dbItem.size);
|
|
|
|
// remainingFreeSpace online includes the current file online
|
|
// we need to remove the online file (add back the existing file size) then take away the new local file size to get a new approximate value
|
|
ulong calculatedSpaceOnlinePostUpload = (remainingFreeSpace + thisFileSizeFromDB) - thisFileSizeLocal;
|
|
|
|
// Based on what we know, for this thread - can we safely upload this modified local file?
|
|
addLogEntry("This Thread Current Free Space Online: " ~ to!string(remainingFreeSpace), ["debug"]);
|
|
addLogEntry("This Thread Calculated Free Space Online Post Upload: " ~ to!string(calculatedSpaceOnlinePostUpload), ["debug"]);
|
|
|
|
JSONValue uploadResponse;
|
|
|
|
bool spaceAvailableOnline = false;
|
|
// If 'personal' accounts, if driveId == defaultDriveId, then we will have data - appConfig.quotaAvailable will be updated
|
|
// If 'personal' accounts, if driveId != defaultDriveId, then we will not have quota data - appConfig.quotaRestricted will be set as true
|
|
// If 'business' accounts, if driveId == defaultDriveId, then we will have data
|
|
// If 'business' accounts, if driveId != defaultDriveId, then we will have data, but it will be a 0 value - appConfig.quotaRestricted will be set as true
|
|
|
|
// What was the latest getRemainingFreeSpace() value?
|
|
if (appConfig.quotaAvailable) {
|
|
// Our query told us we have free space online .. if we upload this file, will we exceed space online - thus upload will fail during upload?
|
|
if (calculatedSpaceOnlinePostUpload > 0) {
|
|
// Based on this thread action, we beleive that there is space available online to upload - proceed
|
|
spaceAvailableOnline = true;
|
|
}
|
|
}
|
|
// Is quota being restricted?
|
|
if (appConfig.quotaRestricted) {
|
|
// Space available online is being restricted - so we have no way to really know if there is space available online
|
|
spaceAvailableOnline = true;
|
|
}
|
|
|
|
// Do we have space available or is space available being restricted (so we make the blind assumption that there is space available)
|
|
if (spaceAvailableOnline) {
|
|
// Does this file exceed the maximum file size to upload to OneDrive?
|
|
if (thisFileSizeLocal <= maxUploadFileSize) {
|
|
// Attempt to upload the modified file
|
|
// Error handling is in performModifiedFileUpload(), and the JSON that is responded with - will either be null or a valid JSON object containing the upload result
|
|
uploadResponse = performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal);
|
|
|
|
// Evaluate the returned JSON uploadResponse
|
|
// If there was an error uploading the file, uploadResponse should be empty and invalid
|
|
if (uploadResponse.type() != JSONType.object) {
|
|
uploadFailed = true;
|
|
skippedExceptionError = true;
|
|
}
|
|
|
|
} else {
|
|
// Skip file - too large
|
|
uploadFailed = true;
|
|
skippedMaxSize = true;
|
|
}
|
|
} else {
|
|
// Cant upload this file - no space available
|
|
uploadFailed = true;
|
|
}
|
|
|
|
// Did the upload fail?
|
|
if (uploadFailed) {
|
|
// Upload failed .. why?
|
|
// No space available online
|
|
if (!spaceAvailableOnline) {
|
|
addLogEntry("Skipping uploading modified file " ~ localFilePath ~ " due to insufficient free space available on Microsoft OneDrive", ["info", "notify"]);
|
|
}
|
|
// File exceeds max allowed size
|
|
if (skippedMaxSize) {
|
|
addLogEntry("Skipping uploading this modified file as it exceeds the maximum size allowed by OneDrive: " ~ localFilePath, ["info", "notify"]);
|
|
}
|
|
// Generic message
|
|
if (skippedExceptionError) {
|
|
// normal failure message if API or exception error generated
|
|
addLogEntry("Uploading modified file " ~ localFilePath ~ " ... failed!", ["info", "notify"]);
|
|
}
|
|
} else {
|
|
// Upload was successful
|
|
addLogEntry("Uploading modified file " ~ localFilePath ~ " ... done.", ["info", "notify"]);
|
|
|
|
// Save JSON item in database
|
|
saveItem(uploadResponse);
|
|
|
|
if (!dryRun) {
|
|
// Check the integrity of the uploaded modified file
|
|
performUploadIntegrityValidationChecks(uploadResponse, localFilePath, thisFileSizeLocal);
|
|
|
|
// Update the date / time of the file online to match the local item
|
|
// Get the local file last modified time
|
|
SysTime localModifiedTime = timeLastModified(localFilePath).toUTC();
|
|
localModifiedTime.fracSecs = Duration.zero;
|
|
// Get the latest eTag, and use that
|
|
string etagFromUploadResponse = uploadResponse["eTag"].str;
|
|
// Attempt to update the online date time stamp based on our local data
|
|
uploadLastModifiedTime(dbItem.driveId, dbItem.id, localModifiedTime, etagFromUploadResponse);
|
|
}
|
|
}
|
|
|
|
addLogEntry("Thread " ~ to!string(i) ~ " Finished: " ~ to!string(Clock.currTime()), ["debug"]);
|
|
|
|
} // end of 'foreach (i, localItemDetails; array.enumerate)'
|
|
}
|
|
|
|
// Perform the upload of a locally modified file to OneDrive
|
|
JSONValue performModifiedFileUpload(Item dbItem, string localFilePath, ulong thisFileSizeLocal) {
|
|
|
|
JSONValue uploadResponse;
|
|
OneDriveApi uploadFileOneDriveApiInstance;
|
|
uploadFileOneDriveApiInstance = new OneDriveApi(appConfig);
|
|
uploadFileOneDriveApiInstance.initialise();
|
|
|
|
// Is this a dry-run scenario?
|
|
if (!dryRun) {
|
|
// Do we use simpleUpload or create an upload session?
|
|
bool useSimpleUpload = false;
|
|
|
|
//if ((appConfig.accountType == "personal") && (thisFileSizeLocal <= sessionThresholdFileSize)) {
|
|
|
|
if (thisFileSizeLocal <= sessionThresholdFileSize) {
|
|
useSimpleUpload = true;
|
|
}
|
|
|
|
// We can only upload zero size files via simpleFileUpload regardless of account type
|
|
// Reference: https://github.com/OneDrive/onedrive-api-docs/issues/53
|
|
// Additionally, all files where file size is < 4MB should be uploaded by simpleUploadReplace - everything else should use a session to upload the modified file
|
|
|
|
if ((thisFileSizeLocal == 0) || (useSimpleUpload)) {
|
|
// Must use Simple Upload to replace the file online
|
|
try {
|
|
uploadResponse = uploadFileOneDriveApiInstance.simpleUploadReplace(localFilePath, dbItem.driveId, dbItem.id);
|
|
} catch (OneDriveException exception) {
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to upload a modified file to OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
}
|
|
|
|
} catch (FileException e) {
|
|
// filesystem error
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
} else {
|
|
// Configure JSONValue variables we use for a session upload
|
|
JSONValue currentOnlineData;
|
|
JSONValue uploadSessionData;
|
|
string currentETag;
|
|
|
|
// As this is a unique thread, the sessionFilePath for where we save the data needs to be unique
|
|
// The best way to do this is generate a 10 digit alphanumeric string, and use this as the file extention
|
|
string threadUploadSessionFilePath = appConfig.uploadSessionFilePath ~ "." ~ generateAlphanumericString();
|
|
|
|
// Get the absolute latest object details from online
|
|
try {
|
|
currentOnlineData = uploadFileOneDriveApiInstance.getPathDetailsByDriveId(dbItem.driveId, localFilePath);
|
|
} catch (OneDriveException exception) {
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to obtain latest file details from OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
}
|
|
|
|
}
|
|
|
|
// Was a valid JSON response provided?
|
|
if (currentOnlineData.type() == JSONType.object) {
|
|
// Does the response contain an eTag?
|
|
if (hasETag(currentOnlineData)) {
|
|
// Use the value returned from online
|
|
currentETag = currentOnlineData["eTag"].str;
|
|
} else {
|
|
// Use the database value
|
|
currentETag = dbItem.eTag;
|
|
}
|
|
} else {
|
|
// no valid JSON response
|
|
currentETag = dbItem.eTag;
|
|
}
|
|
|
|
// Create the Upload Session
|
|
try {
|
|
uploadSessionData = createSessionFileUpload(uploadFileOneDriveApiInstance, localFilePath, dbItem.driveId, dbItem.parentId, baseName(localFilePath), currentETag, threadUploadSessionFilePath);
|
|
} catch (OneDriveException exception) {
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to create an upload session on OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
}
|
|
|
|
} catch (FileException e) {
|
|
writeln("DEBUG TO REMOVE: Modified file upload FileException Handling (Create the Upload Session)");
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
|
|
// Perform the Upload using the session
|
|
try {
|
|
uploadResponse = performSessionFileUpload(uploadFileOneDriveApiInstance, thisFileSizeLocal, uploadSessionData, threadUploadSessionFilePath);
|
|
} catch (OneDriveException exception) {
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to upload a file via a session to OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
}
|
|
|
|
} catch (FileException e) {
|
|
writeln("DEBUG TO REMOVE: Modified file upload FileException Handling (Perform the Upload using the session)");
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
}
|
|
|
|
} else {
|
|
// We are in a --dry-run scenario
|
|
uploadResponse = createFakeResponse(localFilePath);
|
|
}
|
|
|
|
// Debug Log the modified upload response
|
|
addLogEntry("Modified File Upload Response: " ~ to!string(uploadResponse), ["debug"]);
|
|
|
|
// Shutdown the API instance
|
|
uploadFileOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(uploadFileOneDriveApiInstance);
|
|
// Return JSON
|
|
return uploadResponse;
|
|
}
|
|
|
|
// Query the OneDrive API using the provided driveId to get the latest quota details
|
|
ulong getRemainingFreeSpace(string driveId) {
|
|
|
|
// Get the quota details for this driveId, as this could have changed since we started the application - the user could have added / deleted data online, or purchased additional storage
|
|
// Quota details are ONLY available for the main default driveId, as the OneDrive API does not provide quota details for shared folders
|
|
|
|
JSONValue currentDriveQuota;
|
|
ulong remainingQuota;
|
|
|
|
try {
|
|
// Create a new OneDrive API instance
|
|
OneDriveApi getCurrentDriveQuotaApiInstance;
|
|
getCurrentDriveQuotaApiInstance = new OneDriveApi(appConfig);
|
|
getCurrentDriveQuotaApiInstance.initialise();
|
|
addLogEntry("Seeking available quota for this drive id: " ~ driveId, ["debug"]);
|
|
currentDriveQuota = getCurrentDriveQuotaApiInstance.getDriveQuota(driveId);
|
|
// Shut this API instance down
|
|
getCurrentDriveQuotaApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(getCurrentDriveQuotaApiInstance);
|
|
} catch (OneDriveException e) {
|
|
addLogEntry("currentDriveQuota = onedrive.getDriveQuota(driveId) generated a OneDriveException", ["debug"]);
|
|
}
|
|
|
|
// validate that currentDriveQuota is a JSON value
|
|
if (currentDriveQuota.type() == JSONType.object) {
|
|
// Response from API contains valid data
|
|
// If 'personal' accounts, if driveId == defaultDriveId, then we will have data
|
|
// If 'personal' accounts, if driveId != defaultDriveId, then we will not have quota data
|
|
// If 'business' accounts, if driveId == defaultDriveId, then we will have data
|
|
// If 'business' accounts, if driveId != defaultDriveId, then we will have data, but it will be a 0 value
|
|
|
|
if ("quota" in currentDriveQuota){
|
|
if (driveId == appConfig.defaultDriveId) {
|
|
// We potentially have updated quota remaining details available
|
|
// However in some cases OneDrive Business configurations 'restrict' quota details thus is empty / blank / negative value / zero
|
|
if ("remaining" in currentDriveQuota["quota"]){
|
|
// We have valid quota remaining details returned for the provided drive id
|
|
remainingQuota = currentDriveQuota["quota"]["remaining"].integer;
|
|
|
|
if (remainingQuota <= 0) {
|
|
if (appConfig.accountType == "personal"){
|
|
// zero space available
|
|
addLogEntry("ERROR: OneDrive account currently has zero space available. Please free up some space online or purchase additional space.");
|
|
remainingQuota = 0;
|
|
appConfig.quotaAvailable = false;
|
|
} else {
|
|
// zero space available is being reported, maybe being restricted?
|
|
addLogEntry("WARNING: OneDrive quota information is being restricted or providing a zero value. Please fix by speaking to your OneDrive / Office 365 Administrator.");
|
|
remainingQuota = 0;
|
|
appConfig.quotaRestricted = true;
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// quota details returned, but for a drive id that is not ours
|
|
if ("remaining" in currentDriveQuota["quota"]){
|
|
// remaining is in the quota JSON response
|
|
if (currentDriveQuota["quota"]["remaining"].integer <= 0) {
|
|
// value returned is 0 or less than 0
|
|
addLogEntry("OneDrive quota information is set at zero, as this is not our drive id, ignoring", ["verbose"]);
|
|
remainingQuota = 0;
|
|
appConfig.quotaRestricted = true;
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// No quota details returned
|
|
if (driveId == appConfig.defaultDriveId) {
|
|
// no quota details returned for current drive id
|
|
addLogEntry("ERROR: OneDrive quota information is missing. Potentially your OneDrive account currently has zero space available. Please free up some space online or purchase additional space.");
|
|
remainingQuota = 0;
|
|
appConfig.quotaRestricted = true;
|
|
} else {
|
|
// quota details not available
|
|
addLogEntry("WARNING: OneDrive quota information is being restricted as this is not our drive id.", ["debug"]);
|
|
remainingQuota = 0;
|
|
appConfig.quotaRestricted = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// what was the determined available quota?
|
|
addLogEntry("Available quota: " ~ to!string(remainingQuota), ["debug"]);
|
|
return remainingQuota;
|
|
}
|
|
|
|
// Perform a filesystem walk to uncover new data to upload to OneDrive
|
|
void scanLocalFilesystemPathForNewData(string path) {
|
|
|
|
// To improve logging output for this function, what is the 'logical path' we are scanning for file & folder differences?
|
|
string logPath;
|
|
if (path == ".") {
|
|
// get the configured sync_dir
|
|
logPath = buildNormalizedPath(appConfig.getValueString("sync_dir"));
|
|
} else {
|
|
// use what was passed in
|
|
if (!appConfig.getValueBool("monitor")) {
|
|
logPath = buildNormalizedPath(appConfig.getValueString("sync_dir")) ~ "/" ~ path;
|
|
} else {
|
|
logPath = path;
|
|
}
|
|
}
|
|
|
|
// Log the action that we are performing, however only if this is a directory
|
|
if (isDir(path)) {
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
if (!cleanupLocalFiles) {
|
|
addLogEntry("Scanning the local file system '" ~ logPath ~ "' for new data to upload ...");
|
|
} else {
|
|
addLogEntry("Scanning the local file system '" ~ logPath ~ "' for data to cleanup ...");
|
|
}
|
|
}
|
|
}
|
|
|
|
auto startTime = Clock.currTime();
|
|
addLogEntry("Starting Filesystem Walk: " ~ to!string(startTime), ["debug"]);
|
|
|
|
// Perform the filesystem walk of this path, building an array of new items to upload
|
|
scanPathForNewData(path);
|
|
|
|
// To finish off the processing items, this is needed to reflect this in the log
|
|
addLogEntry("------------------------------------------------------------------", ["debug"]);
|
|
|
|
auto finishTime = Clock.currTime();
|
|
addLogEntry("Finished Filesystem Walk: " ~ to!string(finishTime), ["debug"]);
|
|
|
|
auto elapsedTime = finishTime - startTime;
|
|
addLogEntry("Elapsed Time Filesystem Walk: " ~ to!string(elapsedTime), ["debug"]);
|
|
|
|
// Upload new data that has been identified
|
|
// Are there any items to download post fetching the /delta data?
|
|
if (!newLocalFilesToUploadToOneDrive.empty) {
|
|
// There are elements to upload
|
|
addLogEntry("New items to upload to OneDrive: " ~ to!string(newLocalFilesToUploadToOneDrive.length), ["verbose"]);
|
|
|
|
// Reset totalDataToUpload
|
|
totalDataToUpload = 0;
|
|
|
|
// How much data do we need to upload? This is important, as, we need to know how much data to determine if all the files can be uploaded
|
|
foreach (uploadFilePath; newLocalFilesToUploadToOneDrive) {
|
|
// validate that the path actually exists so that it can be counted
|
|
if (exists(uploadFilePath)) {
|
|
totalDataToUpload = totalDataToUpload + getSize(uploadFilePath);
|
|
}
|
|
}
|
|
|
|
// How much data is there to upload
|
|
if (totalDataToUpload < 1024) {
|
|
// Display as Bytes to upload
|
|
addLogEntry("Total New Data to Upload: " ~ to!string(totalDataToUpload) ~ " Bytes", ["verbose"]);
|
|
} else {
|
|
if ((totalDataToUpload > 1024) && (totalDataToUpload < 1048576)) {
|
|
// Display as KB to upload
|
|
addLogEntry("Total New Data to Upload: " ~ to!string((totalDataToUpload / 1024)) ~ " KB", ["verbose"]);
|
|
} else {
|
|
// Display as MB to upload
|
|
addLogEntry("Total New Data to Upload: " ~ to!string((totalDataToUpload / 1024 / 1024)) ~ " MB", ["verbose"]);
|
|
}
|
|
}
|
|
|
|
// How much space is available (Account Drive ID)
|
|
// The file, could be uploaded to a shared folder, which, we are not tracking how much free space is available there ...
|
|
addLogEntry("Current Available Space Online (Account Drive ID): " ~ to!string((appConfig.remainingFreeSpace / 1024 / 1024)) ~ " MB", ["debug"]);
|
|
|
|
// Perform the upload
|
|
uploadNewLocalFileItems();
|
|
|
|
// Cleanup array memory
|
|
newLocalFilesToUploadToOneDrive = [];
|
|
}
|
|
}
|
|
|
|
// Scan this path for new data
|
|
void scanPathForNewData(string path) {
|
|
|
|
ulong maxPathLength;
|
|
ulong pathWalkLength;
|
|
|
|
// Add this logging break to assist with what was checked for each path
|
|
if (path != ".") {
|
|
addLogEntry("------------------------------------------------------------------", ["debug"]);
|
|
}
|
|
|
|
// https://support.microsoft.com/en-us/help/3125202/restrictions-and-limitations-when-you-sync-files-and-folders
|
|
// If the path is greater than allowed characters, then one drive will return a '400 - Bad Request'
|
|
// Need to ensure that the URI is encoded before the check is made:
|
|
// - 400 Character Limit for OneDrive Business / Office 365
|
|
// - 430 Character Limit for OneDrive Personal
|
|
|
|
// Configure maxPathLength based on account type
|
|
if (appConfig.accountType == "personal") {
|
|
// Personal Account
|
|
maxPathLength = 430;
|
|
} else {
|
|
// Business Account / Office365 / SharePoint
|
|
maxPathLength = 400;
|
|
}
|
|
|
|
// A short lived item that has already disappeared will cause an error - is the path still valid?
|
|
if (!exists(path)) {
|
|
addLogEntry("Skipping item - path has disappeared: " ~ path);
|
|
return;
|
|
}
|
|
|
|
// Calculate the path length by walking the path and catch any UTF-8 sequence errors at the same time
|
|
// https://github.com/skilion/onedrive/issues/57
|
|
// https://github.com/abraunegg/onedrive/issues/487
|
|
// https://github.com/abraunegg/onedrive/issues/1192
|
|
try {
|
|
pathWalkLength = path.byGrapheme.walkLength;
|
|
} catch (std.utf.UTFException e) {
|
|
// Path contains characters which generate a UTF exception
|
|
addLogEntry("Skipping item - invalid UTF sequence: " ~ path, ["info", "notify"]);
|
|
addLogEntry(" Error Reason:" ~ e.msg, ["debug"]);
|
|
return;
|
|
}
|
|
|
|
// Is the path length is less than maxPathLength
|
|
if (pathWalkLength < maxPathLength) {
|
|
// Is this path unwanted
|
|
bool unwanted = false;
|
|
|
|
// First check of this item - if we are in a --dry-run scenario, we may have 'fake deleted' this path
|
|
// thus, the entries are not in the dry-run DB copy, thus, at this point the client thinks that this is an item to upload
|
|
// Check this 'path' for an entry in pathFakeDeletedArray - if it is there, this is unwanted
|
|
if (dryRun) {
|
|
// Is this path in the array of fake deleted items? If yes, return early, nothing else to do, save processing
|
|
if (canFind(pathFakeDeletedArray, path)) return;
|
|
}
|
|
|
|
// This not a Client Side Filtering check, nor a Microsoft Check, but is a sanity check that the path provided is UTF encoded correctly
|
|
// Check the std.encoding of the path against: Unicode 5.0, ASCII, ISO-8859-1, ISO-8859-2, WINDOWS-1250, WINDOWS-1251, WINDOWS-1252
|
|
if (!unwanted) {
|
|
if(!isValid(path)) {
|
|
// Path is not valid according to https://dlang.org/phobos/std_encoding.html
|
|
addLogEntry("Skipping item - invalid character encoding sequence: " ~ path, ["info", "notify"]);
|
|
unwanted = true;
|
|
}
|
|
}
|
|
|
|
// Check this path against the Client Side Filtering Rules
|
|
// - check_nosync
|
|
// - skip_dotfiles
|
|
// - skip_symlinks
|
|
// - skip_file
|
|
// - skip_dir
|
|
// - sync_list
|
|
// - skip_size
|
|
if (!unwanted) {
|
|
unwanted = checkPathAgainstClientSideFiltering(path);
|
|
}
|
|
|
|
// Check this path against the Microsoft Naming Conventions & Restristions
|
|
// - Check path against Microsoft OneDrive restriction and limitations about Windows naming for files and folders
|
|
// - Check path for bad whitespace items
|
|
// - Check path for HTML ASCII Codes
|
|
// - Check path for ASCII Control Codes
|
|
if (!unwanted) {
|
|
unwanted = checkPathAgainstMicrosoftNamingRestrictions(path);
|
|
}
|
|
|
|
if (!unwanted) {
|
|
// At this point, this path, we want to scan for new data as it is not excluded
|
|
if (isDir(path)) {
|
|
// Check if this path in the database
|
|
bool directoryFoundInDB = pathFoundInDatabase(path);
|
|
|
|
// Was the path found in the database?
|
|
if (!directoryFoundInDB) {
|
|
// Path not found in database when searching all drive id's
|
|
if (!cleanupLocalFiles) {
|
|
// --download-only --cleanup-local-files not used
|
|
// Create this directory on OneDrive so that we can upload files to it
|
|
createDirectoryOnline(path);
|
|
} else {
|
|
// we need to clean up this directory
|
|
addLogEntry("Removing local directory as --download-only & --cleanup-local-files configured");
|
|
// Remove any children of this path if they still exist
|
|
// Resolve 'Directory not empty' error when deleting local files
|
|
try {
|
|
foreach (DirEntry child; dirEntries(path, SpanMode.depth, false)) {
|
|
// what sort of child is this?
|
|
if (isDir(child.name)) {
|
|
addLogEntry("Removing local directory: " ~ child.name);
|
|
} else {
|
|
addLogEntry("Removing local file: " ~ child.name);
|
|
}
|
|
|
|
// are we in a --dry-run scenario?
|
|
if (!dryRun) {
|
|
// No --dry-run ... process local delete
|
|
if (exists(child)) {
|
|
try {
|
|
attrIsDir(child.linkAttributes) ? rmdir(child.name) : remove(child.name);
|
|
} catch (FileException e) {
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Remove the path now that it is empty of children
|
|
addLogEntry("Removing local directory: " ~ path);
|
|
// are we in a --dry-run scenario?
|
|
if (!dryRun) {
|
|
// No --dry-run ... process local delete
|
|
try {
|
|
rmdirRecurse(path);
|
|
} catch (FileException e) {
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
}
|
|
} catch (FileException e) {
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
// flag for if we are going traverse this path
|
|
bool skipFolderTraverse = false;
|
|
|
|
// Before we traverse this 'path', we need to make a last check to see if this was just excluded
|
|
if (appConfig.accountType == "business") {
|
|
// search businessSharedFoldersOnlineToSkip for this path
|
|
if (canFind(businessSharedFoldersOnlineToSkip, path)) {
|
|
// This path was skipped - why?
|
|
addLogEntry("Skipping item '" ~ path ~ "' due to this path matching an existing online Business Shared Folder name", ["info", "notify"]);
|
|
skipFolderTraverse = true;
|
|
}
|
|
}
|
|
|
|
// Do we traverse this path?
|
|
if (!skipFolderTraverse) {
|
|
// Try and access this directory and any path below
|
|
try {
|
|
auto entries = dirEntries(path, SpanMode.shallow, false);
|
|
foreach (DirEntry entry; entries) {
|
|
string thisPath = entry.name;
|
|
scanPathForNewData(thisPath);
|
|
}
|
|
} catch (FileException e) {
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
return;
|
|
}
|
|
}
|
|
} else {
|
|
// https://github.com/abraunegg/onedrive/issues/984
|
|
// path is not a directory, is it a valid file?
|
|
// pipes - whilst technically valid files, are not valid for this client
|
|
// prw-rw-r--. 1 user user 0 Jul 7 05:55 my_pipe
|
|
if (isFile(path)) {
|
|
// Path is a valid file, not a pipe
|
|
bool fileFoundInDB = pathFoundInDatabase(path);
|
|
// Was the file found in the database?
|
|
if (!fileFoundInDB) {
|
|
// File not found in database when searching all drive id's
|
|
// Do we upload the file or clean up the file?
|
|
if (!cleanupLocalFiles) {
|
|
// --download-only --cleanup-local-files not used
|
|
// Add this path as a file we need to upload
|
|
addLogEntry("OneDrive Client flagging to upload this file to OneDrive: " ~ path, ["debug"]);
|
|
newLocalFilesToUploadToOneDrive ~= path;
|
|
} else {
|
|
// we need to clean up this file
|
|
addLogEntry("Removing local file as --download-only & --cleanup-local-files configured");
|
|
// are we in a --dry-run scenario?
|
|
addLogEntry("Removing local file: " ~ path);
|
|
if (!dryRun) {
|
|
// No --dry-run ... process local file delete
|
|
safeRemove(path);
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// path is not a valid file
|
|
addLogEntry("Skipping item - item is not a valid file: " ~ path, ["info", "notify"]);
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// This path was skipped - why?
|
|
addLogEntry("Skipping item '" ~ path ~ "' due to the full path exceeding " ~ to!string(maxPathLength) ~ " characters (Microsoft OneDrive limitation)", ["info", "notify"]);
|
|
}
|
|
}
|
|
|
|
// Handle a single file inotify trigger when using --monitor
|
|
void handleLocalFileTrigger(string localFilePath) {
|
|
// Is this path a new file or an existing one?
|
|
// Normally we would use pathFoundInDatabase() to calculate, but we need 'databaseItem' as well if the item is in the database
|
|
Item databaseItem;
|
|
bool fileFoundInDB = false;
|
|
string[3][] modifiedItemToUpload;
|
|
|
|
foreach (driveId; driveIDsArray) {
|
|
if (itemDB.selectByPath(localFilePath, driveId, databaseItem)) {
|
|
fileFoundInDB = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Was the file found in the database?
|
|
if (!fileFoundInDB) {
|
|
// This is a new file as it is not in the database
|
|
// Log that the file has been added locally
|
|
addLogEntry("[M] New local file added: " ~ localFilePath, ["verbose"]);
|
|
// Scan the parent path for any new data, not just this this item
|
|
scanLocalFilesystemPathForNewData(dirName(localFilePath));
|
|
} else {
|
|
// This is a potentially modified file, needs to be handled as such. Is the item truly modified?
|
|
if (!testFileHash(localFilePath, databaseItem)) {
|
|
// The local file failed the hash comparison test - there is a data difference
|
|
// Log that the file has changed locally
|
|
addLogEntry("[M] Local file changed: " ~ localFilePath, ["verbose"]);
|
|
// Add the modified item to the array to upload
|
|
modifiedItemToUpload ~= [databaseItem.driveId, databaseItem.id, localFilePath];
|
|
uploadChangedLocalFileToOneDrive(modifiedItemToUpload);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Query the database to determine if this path is within the existing database
|
|
bool pathFoundInDatabase(string searchPath) {
|
|
|
|
// Check if this path in the database
|
|
Item databaseItem;
|
|
bool pathFoundInDB = false;
|
|
foreach (driveId; driveIDsArray) {
|
|
if (itemDB.selectByPath(searchPath, driveId, databaseItem)) {
|
|
pathFoundInDB = true;
|
|
}
|
|
}
|
|
return pathFoundInDB;
|
|
}
|
|
|
|
// Create a new directory online on OneDrive
|
|
// - Test if we can get the parent path details from the database, otherwise we need to search online
|
|
// for the path flow and create the folder that way
|
|
void createDirectoryOnline(string thisNewPathToCreate) {
|
|
// Log what we are doing
|
|
addLogEntry("OneDrive Client requested to create this directory online: " ~ thisNewPathToCreate);
|
|
|
|
Item parentItem;
|
|
JSONValue onlinePathData;
|
|
|
|
// Create a new API Instance for this thread and initialise it
|
|
OneDriveApi createDirectoryOnlineOneDriveApiInstance;
|
|
createDirectoryOnlineOneDriveApiInstance = new OneDriveApi(appConfig);
|
|
createDirectoryOnlineOneDriveApiInstance.initialise();
|
|
|
|
// What parent path to use?
|
|
string parentPath = dirName(thisNewPathToCreate); // will be either . or something else
|
|
|
|
// Configure the parentItem by if this is the account 'root' use the root details, or search the database for the parent details
|
|
if (parentPath == ".") {
|
|
// Parent path is '.' which is the account root
|
|
// Use client defaults
|
|
parentItem.driveId = appConfig.defaultDriveId; // Should give something like 12345abcde1234a1
|
|
parentItem.id = appConfig.defaultRootId; // Should give something like 12345ABCDE1234A1!101
|
|
} else {
|
|
// Query the parent path online
|
|
addLogEntry("Attempting to query Local Database for this parent path: " ~ parentPath, ["debug"]);
|
|
|
|
// Attempt a 2 step process to work out where to create the directory
|
|
// Step 1: Query the DB first for the parent path, to try and avoid an API call
|
|
// Step 2: Query online as last resort
|
|
|
|
// Step 1: Check if this parent path in the database
|
|
Item databaseItem;
|
|
bool parentPathFoundInDB = false;
|
|
|
|
foreach (driveId; driveIDsArray) {
|
|
addLogEntry("Query DB with this driveID for the Parent Path: " ~ driveId, ["debug"]);
|
|
// Query the database for this parent path using each driveId that we know about
|
|
if (itemDB.selectByPath(parentPath, driveId, databaseItem)) {
|
|
parentPathFoundInDB = true;
|
|
addLogEntry("Parent databaseItem: " ~ to!string(databaseItem), ["debug"]);
|
|
addLogEntry("parentPathFoundInDB: " ~ to!string(parentPathFoundInDB), ["debug"]);
|
|
parentItem = databaseItem;
|
|
}
|
|
}
|
|
|
|
// After querying all DB entries for each driveID for the parent path, what are the details in parentItem?
|
|
addLogEntry("Parent parentItem after DB Query exhausted: " ~ to!string(parentItem), ["debug"]);
|
|
|
|
// Step 2: Query for the path online if not found in the local database
|
|
if (!parentPathFoundInDB) {
|
|
// parent path not found in database
|
|
try {
|
|
addLogEntry("Attempting to query OneDrive Online for this parent path as path not found in local database: " ~ parentPath, ["debug"]);
|
|
onlinePathData = createDirectoryOnlineOneDriveApiInstance.getPathDetails(parentPath);
|
|
addLogEntry("Online Parent Path Query Response: " ~ to!string(onlinePathData), ["debug"]);
|
|
|
|
// Save item to the database
|
|
saveItem(onlinePathData);
|
|
parentItem = makeItem(onlinePathData);
|
|
} catch (OneDriveException exception) {
|
|
if (exception.httpStatusCode == 404) {
|
|
// Parent does not exist ... need to create parent
|
|
addLogEntry("Parent path does not exist online: " ~ parentPath, ["debug"]);
|
|
createDirectoryOnline(parentPath);
|
|
// no return here as we need to continue, but need to re-query the OneDrive API to get the right parental details now that they exist
|
|
onlinePathData = createDirectoryOnlineOneDriveApiInstance.getPathDetails(parentPath);
|
|
parentItem = makeItem(onlinePathData);
|
|
} else {
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(createDirectoryOnlineOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to create a remote directory on OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
createDirectoryOnline(thisNewPathToCreate);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Make sure the full path does not exist online, this should generate a 404 response, to which then the folder will be created online
|
|
try {
|
|
// Try and query the OneDrive API for the path we need to create
|
|
addLogEntry("Attempting to query OneDrive API for this path: " ~ thisNewPathToCreate, ["debug"]);
|
|
addLogEntry("parentItem details: " ~ to!string(parentItem), ["debug"]);
|
|
|
|
if (parentItem.driveId == appConfig.defaultDriveId) {
|
|
// Use getPathDetailsByDriveId
|
|
onlinePathData = createDirectoryOnlineOneDriveApiInstance.getPathDetailsByDriveId(parentItem.driveId, thisNewPathToCreate);
|
|
} else {
|
|
// If the parentItem.driveId is not our driveId - the path we are looking for will not be at the logical location that getPathDetailsByDriveId
|
|
// can use - as it will always return a 404 .. even if the path actually exists (which is the whole point of this test)
|
|
// Search the parentItem.driveId for any folder name match that we are going to create, then compare response JSON items with parentItem.id
|
|
// If no match, the folder we want to create does not exist at the location we are seeking to create it at, thus generate a 404
|
|
onlinePathData = createDirectoryOnlineOneDriveApiInstance.searchDriveForPath(parentItem.driveId, baseName(thisNewPathToCreate));
|
|
|
|
// Process the response from searching the drive
|
|
ulong responseCount = count(onlinePathData["value"].array);
|
|
if (responseCount > 0) {
|
|
// Search 'name' matches were found .. need to match these against parentItem.id
|
|
bool foundDirectoryOnline = false;
|
|
JSONValue foundDirectoryJSONItem;
|
|
// Items were returned .. but is one of these what we are looking for?
|
|
foreach (childJSON; onlinePathData["value"].array) {
|
|
// Is this item not a file?
|
|
if (!isFileItem(childJSON)) {
|
|
Item thisChildItem = makeItem(childJSON);
|
|
// Direct Match Check
|
|
if ((parentItem.id == thisChildItem.parentId) && (baseName(thisNewPathToCreate) == thisChildItem.name)) {
|
|
// High confidence that this child folder is a direct match we are trying to create and it already exists online
|
|
addLogEntry("Path we are searching for exists online (Direct Match): " ~ baseName(thisNewPathToCreate), ["debug"]);
|
|
addLogEntry("childJSON: " ~ to!string(childJSON), ["debug"]);
|
|
foundDirectoryOnline = true;
|
|
foundDirectoryJSONItem = childJSON;
|
|
break;
|
|
}
|
|
// Full Lower Case POSIX Match Check
|
|
string childAsLower = toLower(childJSON["name"].str);
|
|
string thisFolderNameAsLower = toLower(baseName(thisNewPathToCreate));
|
|
|
|
if (childAsLower == thisFolderNameAsLower) {
|
|
// This is a POSIX 'case in-sensitive match' .....
|
|
// Local item name has a 'case-insensitive match' to an existing item on OneDrive
|
|
addLogEntry("Path we are searching for exists online (POSIX 'case in-sensitive match'): " ~ baseName(thisNewPathToCreate), ["debug"]);
|
|
addLogEntry("childJSON: " ~ to!string(childJSON), ["debug"]);
|
|
foundDirectoryOnline = true;
|
|
foundDirectoryJSONItem = childJSON;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (foundDirectoryOnline) {
|
|
// Directory we are seeking was found online ...
|
|
onlinePathData = foundDirectoryJSONItem;
|
|
} else {
|
|
// No 'search item matches found' - raise a 404 so that the exception handling will take over to create the folder
|
|
throw new OneDriveException(404, "Name not found via search");
|
|
}
|
|
} else {
|
|
// No 'search item matches found' - raise a 404 so that the exception handling will take over to create the folder
|
|
throw new OneDriveException(404, "Name not found via search");
|
|
}
|
|
}
|
|
} catch (OneDriveException exception) {
|
|
if (exception.httpStatusCode == 404) {
|
|
// This is a good error - it means that the directory to create 100% does not exist online
|
|
// The directory was not found on the drive id we queried
|
|
addLogEntry("The requested directory to create was not found on OneDrive - creating remote directory: " ~ thisNewPathToCreate, ["verbose"]);
|
|
|
|
// Build up the create directory request
|
|
JSONValue createDirectoryOnlineAPIResponse;
|
|
JSONValue newDriveItem = [
|
|
"name": JSONValue(baseName(thisNewPathToCreate)),
|
|
"folder": parseJSON("{}")
|
|
];
|
|
|
|
// Submit the creation request
|
|
// Fix for https://github.com/skilion/onedrive/issues/356
|
|
if (!dryRun) {
|
|
try {
|
|
// Attempt to create a new folder on the required driveId and parent item id
|
|
string requiredDriveId;
|
|
string requiredParentItemId;
|
|
|
|
// Is this a Personal Account and is the item a Remote Object (Shared Folder) ?
|
|
if ((appConfig.accountType == "personal") && (parentItem.type == ItemType.remote)) {
|
|
// Yes .. Shared Folder
|
|
addLogEntry("parentItem data: " ~ to!string(parentItem), ["debug"]);
|
|
requiredDriveId = parentItem.remoteDriveId;
|
|
requiredParentItemId = parentItem.remoteId;
|
|
} else {
|
|
// Not a personal account + Shared Folder
|
|
requiredDriveId = parentItem.driveId;
|
|
requiredParentItemId = parentItem.id;
|
|
}
|
|
|
|
// Where are we creating this new folder?
|
|
addLogEntry("requiredDriveId: " ~ requiredDriveId, ["debug"]);
|
|
addLogEntry("requiredParentItemId: " ~ requiredParentItemId, ["debug"]);
|
|
addLogEntry("newDriveItem JSON: " ~ to!string(newDriveItem), ["debug"]);
|
|
|
|
// Create the new folder
|
|
createDirectoryOnlineAPIResponse = createDirectoryOnlineOneDriveApiInstance.createById(requiredDriveId, requiredParentItemId, newDriveItem);
|
|
// Is the response a valid JSON object - validation checking done in saveItem
|
|
saveItem(createDirectoryOnlineAPIResponse);
|
|
// Log that the directory was created
|
|
addLogEntry("Successfully created the remote directory " ~ thisNewPathToCreate ~ " on Microsoft OneDrive");
|
|
} catch (OneDriveException exception) {
|
|
if (exception.httpStatusCode == 409) {
|
|
// OneDrive API returned a 404 (above) to say the directory did not exist
|
|
// but when we attempted to create it, OneDrive responded that it now already exists
|
|
addLogEntry("OneDrive reported that " ~ thisNewPathToCreate ~ " already exists .. OneDrive API race condition", ["verbose"]);
|
|
return;
|
|
} else {
|
|
// some other error from OneDrive was returned - display what it is
|
|
addLogEntry("OneDrive generated an error when creating this path: " ~ thisNewPathToCreate);
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
return;
|
|
}
|
|
}
|
|
} else {
|
|
// Simulate a successful 'directory create' & save it to the dryRun database copy
|
|
addLogEntry("Successfully created the remote directory " ~ thisNewPathToCreate ~ " on Microsoft OneDrive");
|
|
// The simulated response has to pass 'makeItem' as part of saveItem
|
|
auto fakeResponse = createFakeResponse(thisNewPathToCreate);
|
|
// Save item to the database
|
|
saveItem(fakeResponse);
|
|
}
|
|
|
|
// Shutdown API instance
|
|
createDirectoryOnlineOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(createDirectoryOnlineOneDriveApiInstance);
|
|
return;
|
|
|
|
} else {
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(createDirectoryOnlineOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to create a remote directory on OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
createDirectoryOnline(thisNewPathToCreate);
|
|
} else {
|
|
// If we get a 400 error, there is an issue creating this folder on Microsoft OneDrive for some reason
|
|
// If the error is not 400, re-try, else fail
|
|
if (exception.httpStatusCode != 400) {
|
|
// Attempt a re-try
|
|
createDirectoryOnline(thisNewPathToCreate);
|
|
} else {
|
|
// We cant create this directory online
|
|
addLogEntry("This folder cannot be created online: " ~ buildNormalizedPath(absolutePath(thisNewPathToCreate)), ["debug"]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// If we get to this point - onlinePathData = createDirectoryOnlineOneDriveApiInstance.getPathDetailsByDriveId(parentItem.driveId, thisNewPathToCreate) generated a 'valid' response ....
|
|
// This means that the folder potentially exists online .. which is odd .. as it should not have existed
|
|
if (onlinePathData.type() == JSONType.object) {
|
|
// A valid object was responded with
|
|
if (onlinePathData["name"].str == baseName(thisNewPathToCreate)) {
|
|
// OneDrive 'name' matches local path name
|
|
if (appConfig.accountType == "business") {
|
|
// We are a business account, this existing online folder, could be a Shared Online Folder and is the 'Add shortcut to My files' item
|
|
addLogEntry("onlinePathData: " ~ to!string(onlinePathData), ["debug"]);
|
|
|
|
if (isItemRemote(onlinePathData)) {
|
|
// The folder is a remote item ... we do not want to create this ...
|
|
addLogEntry("Remote Existing Online Folder is most likely a OneDrive Shared Business Folder Link added by 'Add shortcut to My files'", ["debug"]);
|
|
addLogEntry("We need to skip this path: " ~ thisNewPathToCreate, ["debug"]);
|
|
|
|
// Add this path to businessSharedFoldersOnlineToSkip
|
|
businessSharedFoldersOnlineToSkip ~= [thisNewPathToCreate];
|
|
// no save to database, no online create
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Path found online
|
|
addLogEntry("The requested directory to create was found on OneDrive - skipping creating the directory: " ~ thisNewPathToCreate, ["verbose"]);
|
|
|
|
// Is the response a valid JSON object - validation checking done in saveItem
|
|
saveItem(onlinePathData);
|
|
return;
|
|
} else {
|
|
// Normally this would throw an error, however we cant use throw new posixException()
|
|
string msg = format("POSIX 'case-insensitive match' between '%s' (local) and '%s' (online) which violates the Microsoft OneDrive API namespace convention", baseName(thisNewPathToCreate), onlinePathData["name"].str);
|
|
displayPosixErrorMessage(msg);
|
|
addLogEntry("ERROR: Requested directory to create has a 'case-insensitive match' to an existing directory on OneDrive online.");
|
|
addLogEntry("ERROR: To resolve, rename this local directory: " ~ buildNormalizedPath(absolutePath(thisNewPathToCreate)));
|
|
addLogEntry("Skipping creating this directory online due to 'case-insensitive match': " ~ thisNewPathToCreate);
|
|
// Add this path to posixViolationPaths
|
|
posixViolationPaths ~= [thisNewPathToCreate];
|
|
return;
|
|
}
|
|
} else {
|
|
// response is not valid JSON, an error was returned from OneDrive
|
|
addLogEntry("ERROR: There was an error performing this operation on Microsoft OneDrive");
|
|
addLogEntry("ERROR: Increase logging verbosity to assist determining why.");
|
|
addLogEntry("Skipping: " ~ buildNormalizedPath(absolutePath(thisNewPathToCreate)));
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Test that the online name actually matches the requested local name
|
|
void performPosixTest(string localNameToCheck, string onlineName) {
|
|
|
|
// https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file
|
|
// Do not assume case sensitivity. For example, consider the names OSCAR, Oscar, and oscar to be the same,
|
|
// even though some file systems (such as a POSIX-compliant file system) may consider them as different.
|
|
// Note that NTFS supports POSIX semantics for case sensitivity but this is not the default behavior.
|
|
if (localNameToCheck != onlineName) {
|
|
// POSIX Error
|
|
// Local item name has a 'case-insensitive match' to an existing item on OneDrive
|
|
throw new posixException(localNameToCheck, onlineName);
|
|
}
|
|
}
|
|
|
|
// Upload new file items as identified
|
|
void uploadNewLocalFileItems() {
|
|
// Lets deal with the new local items in a batch process
|
|
ulong batchSize = appConfig.concurrentThreads;
|
|
ulong batchCount = (newLocalFilesToUploadToOneDrive.length + batchSize - 1) / batchSize;
|
|
ulong batchesProcessed = 0;
|
|
|
|
foreach (chunk; newLocalFilesToUploadToOneDrive.chunks(batchSize)) {
|
|
uploadNewLocalFileItemsInParallel(chunk);
|
|
}
|
|
}
|
|
|
|
// Upload the file batches in parallel
|
|
void uploadNewLocalFileItemsInParallel(string[] array) {
|
|
foreach (i, fileToUpload; taskPool.parallel(array)) {
|
|
addLogEntry("Upload Thread " ~ to!string(i) ~ " Starting: " ~ to!string(Clock.currTime()), ["debug"]);
|
|
uploadNewFile(fileToUpload);
|
|
addLogEntry("Upload Thread " ~ to!string(i) ~ " Finished: " ~ to!string(Clock.currTime()), ["debug"]);
|
|
}
|
|
}
|
|
|
|
// Upload a new file to OneDrive
|
|
void uploadNewFile(string fileToUpload) {
|
|
// Debug for the moment
|
|
addLogEntry("fileToUpload: " ~ fileToUpload, ["debug"]);
|
|
|
|
// These are the details of the item we need to upload
|
|
// How much space is remaining on OneDrive
|
|
ulong remainingFreeSpaceOnline;
|
|
// Did the upload fail?
|
|
bool uploadFailed = false;
|
|
// Did we skip due to exceeding maximum allowed size?
|
|
bool skippedMaxSize = false;
|
|
// Did we skip to an exception error?
|
|
bool skippedExceptionError = false;
|
|
// Is the parent path in the item database?
|
|
bool parentPathFoundInDB = false;
|
|
// Get this file size
|
|
ulong thisFileSize;
|
|
// Is there space available online
|
|
bool spaceAvailableOnline = false;
|
|
|
|
// Check the database for the parent path of fileToUpload
|
|
Item parentItem;
|
|
// What parent path to use?
|
|
string parentPath = dirName(fileToUpload); // will be either . or something else
|
|
if (parentPath == "."){
|
|
// Assume this is a new file in the users configured sync_dir root
|
|
// Use client defaults
|
|
parentItem.id = appConfig.defaultRootId; // Should give something like 12345ABCDE1234A1!101
|
|
parentItem.driveId = appConfig.defaultDriveId; // Should give something like 12345abcde1234a1
|
|
parentPathFoundInDB = true;
|
|
} else {
|
|
// Query the database using each of the driveId's we are using
|
|
foreach (driveId; driveIDsArray) {
|
|
// Query the database for this parent path using each driveId
|
|
Item dbResponse;
|
|
if(itemDB.selectByPath(parentPath, driveId, dbResponse)){
|
|
// parent path was found in the database
|
|
parentItem = dbResponse;
|
|
parentPathFoundInDB = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// If the parent path was found in the DB, to ensure we are uploading the the right location 'parentItem.driveId' must not be empty
|
|
if ((parentPathFoundInDB) && (parentItem.driveId.empty)) {
|
|
// switch to using defaultDriveId
|
|
addLogEntry("parentItem.driveId is empty - using defaultDriveId for upload API calls");
|
|
parentItem.driveId = appConfig.defaultDriveId;
|
|
}
|
|
|
|
// Can we read the file - as a permissions issue or actual file corruption will cause a failure
|
|
// Resolves: https://github.com/abraunegg/onedrive/issues/113
|
|
if (readLocalFile(fileToUpload)) {
|
|
if (parentPathFoundInDB) {
|
|
// The local file can be read - so we can read it to attemtp to upload it in this thread
|
|
// Get the file size
|
|
thisFileSize = getSize(fileToUpload);
|
|
// Does this file exceed the maximum filesize for OneDrive
|
|
// Resolves: https://github.com/skilion/onedrive/issues/121 , https://github.com/skilion/onedrive/issues/294 , https://github.com/skilion/onedrive/issues/329
|
|
if (thisFileSize <= maxUploadFileSize) {
|
|
// Is there enough free space on OneDrive when we started this thread, to upload the file to OneDrive?
|
|
remainingFreeSpaceOnline = getRemainingFreeSpace(parentItem.driveId);
|
|
addLogEntry("Current Available Space Online (Upload Target Drive ID): " ~ to!string((remainingFreeSpaceOnline / 1024 / 1024)) ~ " MB", ["debug"]);
|
|
|
|
// When we compare the space online to the total we are trying to upload - is there space online?
|
|
ulong calculatedSpaceOnlinePostUpload = remainingFreeSpaceOnline - thisFileSize;
|
|
|
|
// If 'personal' accounts, if driveId == defaultDriveId, then we will have data - appConfig.quotaAvailable will be updated
|
|
// If 'personal' accounts, if driveId != defaultDriveId, then we will not have quota data - appConfig.quotaRestricted will be set as true
|
|
// If 'business' accounts, if driveId == defaultDriveId, then we will have data
|
|
// If 'business' accounts, if driveId != defaultDriveId, then we will have data, but it will be a 0 value - appConfig.quotaRestricted will be set as true
|
|
|
|
if (remainingFreeSpaceOnline > totalDataToUpload) {
|
|
// Space available
|
|
spaceAvailableOnline = true;
|
|
} else {
|
|
// we need to look more granular
|
|
// What was the latest getRemainingFreeSpace() value?
|
|
if (appConfig.quotaAvailable) {
|
|
// Our query told us we have free space online .. if we upload this file, will we exceed space online - thus upload will fail during upload?
|
|
if (calculatedSpaceOnlinePostUpload > 0) {
|
|
// Based on this thread action, we beleive that there is space available online to upload - proceed
|
|
spaceAvailableOnline = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Is quota being restricted?
|
|
if (appConfig.quotaRestricted) {
|
|
// If the upload target drive is not our drive id, then it is a shared folder .. we need to print a space warning message
|
|
if (parentItem.driveId != appConfig.defaultDriveId) {
|
|
// Different message depending on account type
|
|
if (appConfig.accountType == "personal") {
|
|
addLogEntry("WARNING: Shared Folder OneDrive quota information is being restricted or providing a zero value. Space available online cannot be guaranteed.", ["verbose"]);
|
|
} else {
|
|
addLogEntry("WARNING: Shared Folder OneDrive quota information is being restricted or providing a zero value. Please fix by speaking to your OneDrive / Office 365 Administrator.", ["verbose"]);
|
|
}
|
|
} else {
|
|
if (appConfig.accountType == "personal") {
|
|
addLogEntry("WARNING: OneDrive quota information is being restricted or providing a zero value. Space available online cannot be guaranteed.", ["verbose"]);
|
|
} else {
|
|
addLogEntry("WARNING: OneDrive quota information is being restricted or providing a zero value. Please fix by speaking to your OneDrive / Office 365 Administrator.", ["verbose"]);
|
|
}
|
|
}
|
|
// Space available online is being restricted - so we have no way to really know if there is space available online
|
|
spaceAvailableOnline = true;
|
|
}
|
|
|
|
// Do we have space available or is space available being restricted (so we make the blind assumption that there is space available)
|
|
if (spaceAvailableOnline) {
|
|
// We need to check that this new local file does not exist on OneDrive
|
|
|
|
// Create a new API Instance for this thread and initialise it
|
|
OneDriveApi checkFileOneDriveApiInstance;
|
|
checkFileOneDriveApiInstance = new OneDriveApi(appConfig);
|
|
checkFileOneDriveApiInstance.initialise();
|
|
|
|
JSONValue fileDetailsFromOneDrive;
|
|
|
|
// https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file
|
|
// Do not assume case sensitivity. For example, consider the names OSCAR, Oscar, and oscar to be the same,
|
|
// even though some file systems (such as a POSIX-compliant file systems that Linux use) may consider them as different.
|
|
// Note that NTFS supports POSIX semantics for case sensitivity but this is not the default behavior, OneDrive does not use this.
|
|
|
|
// In order to upload this file - this query HAS to respond as a 404 - Not Found
|
|
|
|
// Does this 'file' already exist on OneDrive?
|
|
try {
|
|
fileDetailsFromOneDrive = checkFileOneDriveApiInstance.getPathDetailsByDriveId(parentItem.driveId, fileToUpload);
|
|
// Portable Operating System Interface (POSIX) testing of JSON response from OneDrive API
|
|
if (hasName(fileDetailsFromOneDrive)) {
|
|
performPosixTest(baseName(fileToUpload), fileDetailsFromOneDrive["name"].str);
|
|
} else {
|
|
throw new jsonResponseException("Unable to perform POSIX test as the OneDrive API request generated an invalid JSON response");
|
|
}
|
|
|
|
// No 404 or otherwise was triggered, meaning that the file already exists online and passes the POSIX test ...
|
|
addLogEntry("fileDetailsFromOneDrive after exist online check: " ~ to!string(fileDetailsFromOneDrive), ["debug"]);
|
|
|
|
// Does the data from online match our local file?
|
|
if (performUploadIntegrityValidationChecks(fileDetailsFromOneDrive, fileToUpload, thisFileSize)) {
|
|
// Save item to the database
|
|
saveItem(fileDetailsFromOneDrive);
|
|
}
|
|
} catch (OneDriveException exception) {
|
|
// If we get a 404 .. the file is not online .. this is what we want .. file does not exist online
|
|
if (exception.httpStatusCode == 404) {
|
|
// The file has been checked, client side filtering checked, does not exist online - we need to upload it
|
|
addLogEntry("fileDetailsFromOneDrive = checkFileOneDriveApiInstance.getPathDetailsByDriveId(parentItem.driveId, fileToUpload); generated a 404 - file does not exist online - must upload it", ["debug"]);
|
|
uploadFailed = performNewFileUpload(parentItem, fileToUpload, thisFileSize);
|
|
} else {
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(checkFileOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to validate file details on OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
uploadNewFile(fileToUpload);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
|
|
}
|
|
}
|
|
} catch (posixException e) {
|
|
displayPosixErrorMessage(e.msg);
|
|
uploadFailed = true;
|
|
} catch (jsonResponseException e) {
|
|
addLogEntry(e.msg, ["debug"]);
|
|
uploadFailed = true;
|
|
}
|
|
|
|
// Operations in this thread are done / complete - either upload was done or it failed
|
|
checkFileOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(checkFileOneDriveApiInstance);
|
|
} else {
|
|
// skip file upload - insufficent space to upload
|
|
addLogEntry("Skipping uploading this new file as it exceeds the available free space on OneDrive: " ~ fileToUpload);
|
|
uploadFailed = true;
|
|
}
|
|
} else {
|
|
// Skip file upload - too large
|
|
addLogEntry("Skipping uploading this new file as it exceeds the maximum size allowed by OneDrive: " ~ fileToUpload);
|
|
uploadFailed = true;
|
|
}
|
|
} else {
|
|
// why was the parent path not in the database?
|
|
if (canFind(posixViolationPaths, parentPath)) {
|
|
addLogEntry("ERROR: POSIX 'case-insensitive match' for the parent path which violates the Microsoft OneDrive API namespace convention.");
|
|
} else {
|
|
addLogEntry("ERROR: Parent path is not in the database or online.");
|
|
}
|
|
addLogEntry("ERROR: Unable to upload this file: " ~ fileToUpload);
|
|
uploadFailed = true;
|
|
}
|
|
} else {
|
|
// Unable to read local file
|
|
addLogEntry("Skipping uploading this file as it cannot be read (file permissions or file corruption): " ~ fileToUpload);
|
|
uploadFailed = true;
|
|
}
|
|
|
|
// Upload success or failure?
|
|
if (uploadFailed) {
|
|
// Need to add this to fileUploadFailures to capture at the end
|
|
fileUploadFailures ~= fileToUpload;
|
|
}
|
|
}
|
|
|
|
// Perform the actual upload to OneDrive
|
|
bool performNewFileUpload(Item parentItem, string fileToUpload, ulong thisFileSize) {
|
|
|
|
// Assume that by default the upload fails
|
|
bool uploadFailed = true;
|
|
|
|
// OneDrive API Upload Response
|
|
JSONValue uploadResponse;
|
|
|
|
// Create the OneDriveAPI Upload Instance
|
|
OneDriveApi uploadFileOneDriveApiInstance;
|
|
uploadFileOneDriveApiInstance = new OneDriveApi(appConfig);
|
|
uploadFileOneDriveApiInstance.initialise();
|
|
|
|
// Calculate upload speed
|
|
auto uploadStartTime = Clock.currTime();
|
|
|
|
// Is this a dry-run scenario?
|
|
if (!dryRun) {
|
|
// Not a dry-run situation
|
|
// Do we use simpleUpload or create an upload session?
|
|
bool useSimpleUpload = false;
|
|
if (thisFileSize <= sessionThresholdFileSize) {
|
|
useSimpleUpload = true;
|
|
}
|
|
|
|
// We can only upload zero size files via simpleFileUpload regardless of account type
|
|
// Reference: https://github.com/OneDrive/onedrive-api-docs/issues/53
|
|
// Additionally, only where file size is < 4MB should be uploaded by simpleUpload - everything else should use a session to upload
|
|
|
|
if ((thisFileSize == 0) || (useSimpleUpload)) {
|
|
try {
|
|
// Attempt to upload the zero byte file using simpleUpload for all account types
|
|
uploadResponse = uploadFileOneDriveApiInstance.simpleUpload(fileToUpload, parentItem.driveId, parentItem.id, baseName(fileToUpload));
|
|
uploadFailed = false;
|
|
addLogEntry("Uploading new file " ~ fileToUpload ~ " ... done.");
|
|
// Shutdown the API
|
|
uploadFileOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(uploadFileOneDriveApiInstance);
|
|
} catch (OneDriveException exception) {
|
|
// An error was responded with - what was it
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to upload a new file to OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
performNewFileUpload(parentItem, fileToUpload, thisFileSize);
|
|
// Return upload status
|
|
return uploadFailed;
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
addLogEntry("Uploading new file " ~ fileToUpload ~ " ... failed.");
|
|
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
|
|
}
|
|
|
|
} catch (FileException e) {
|
|
// display the error message
|
|
addLogEntry("Uploading new file " ~ fileToUpload ~ " ... failed.");
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
} else {
|
|
// Session Upload for this criteria:
|
|
// - Personal Account and file size > 4MB
|
|
// - All Business | Office365 | SharePoint files > 0 bytes
|
|
JSONValue uploadSessionData;
|
|
// As this is a unique thread, the sessionFilePath for where we save the data needs to be unique
|
|
// The best way to do this is generate a 10 digit alphanumeric string, and use this as the file extention
|
|
string threadUploadSessionFilePath = appConfig.uploadSessionFilePath ~ "." ~ generateAlphanumericString();
|
|
|
|
// Attempt to upload the > 4MB file using an upload session for all account types
|
|
try {
|
|
// Create the Upload Session
|
|
uploadSessionData = createSessionFileUpload(uploadFileOneDriveApiInstance, fileToUpload, parentItem.driveId, parentItem.id, baseName(fileToUpload), null, threadUploadSessionFilePath);
|
|
} catch (OneDriveException exception) {
|
|
// An error was responded with - what was it
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to create an upload session on OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
performNewFileUpload(parentItem, fileToUpload, thisFileSize);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
addLogEntry("Uploading new file " ~ fileToUpload ~ " ... failed.");
|
|
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
|
|
}
|
|
|
|
} catch (FileException e) {
|
|
// display the error message
|
|
addLogEntry("Uploading new file " ~ fileToUpload ~ " ... failed.");
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
|
|
// Do we have a valid session URL that we can use ?
|
|
if (uploadSessionData.type() == JSONType.object) {
|
|
// This is a valid JSON object
|
|
bool sessionDataValid = true;
|
|
|
|
// Validate that we have the following items which we need
|
|
if (!hasUploadURL(uploadSessionData)) {
|
|
sessionDataValid = false;
|
|
addLogEntry("Session data missing 'uploadUrl'", ["debug"]);
|
|
}
|
|
|
|
if (!hasNextExpectedRanges(uploadSessionData)) {
|
|
sessionDataValid = false;
|
|
addLogEntry("Session data missing 'nextExpectedRanges'", ["debug"]);
|
|
}
|
|
|
|
if (!hasLocalPath(uploadSessionData)) {
|
|
sessionDataValid = false;
|
|
addLogEntry("Session data missing 'localPath'", ["debug"]);
|
|
}
|
|
|
|
if (sessionDataValid) {
|
|
// We have a valid Upload Session Data we can use
|
|
|
|
try {
|
|
// Try and perform the upload session
|
|
uploadResponse = performSessionFileUpload(uploadFileOneDriveApiInstance, thisFileSize, uploadSessionData, threadUploadSessionFilePath);
|
|
|
|
if (uploadResponse.type() == JSONType.object) {
|
|
uploadFailed = false;
|
|
addLogEntry("Uploading new file " ~ fileToUpload ~ " ... done.");
|
|
} else {
|
|
addLogEntry("Uploading new file " ~ fileToUpload ~ " ... failed.");
|
|
uploadFailed = true;
|
|
}
|
|
} catch (OneDriveException exception) {
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to upload a new file via a session to OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
performNewFileUpload(parentItem, fileToUpload, thisFileSize);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
addLogEntry("Uploading new file " ~ fileToUpload ~ " ... failed.");
|
|
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
|
|
}
|
|
}
|
|
} else {
|
|
// No Upload URL or nextExpectedRanges or localPath .. not a valid JSON we can use
|
|
addLogEntry("Session data is missing required elements to perform a session upload.", ["verbose"]);
|
|
addLogEntry("Uploading new file " ~ fileToUpload ~ " ... failed.");
|
|
}
|
|
} else {
|
|
// Create session Upload URL failed
|
|
addLogEntry("Uploading new file " ~ fileToUpload ~ " ... failed.");
|
|
}
|
|
}
|
|
} else {
|
|
// We are in a --dry-run scenario
|
|
uploadResponse = createFakeResponse(fileToUpload);
|
|
uploadFailed = false;
|
|
addLogEntry("Uploading new file " ~ fileToUpload ~ " ... done.", ["info", "notify"]);
|
|
}
|
|
|
|
// Upload has finished
|
|
auto uploadFinishTime = Clock.currTime();
|
|
// If no upload failure, calculate metrics, perform integrity validation
|
|
if (!uploadFailed) {
|
|
// Upload did not fail ...
|
|
auto uploadDuration = uploadFinishTime - uploadStartTime;
|
|
addLogEntry("File Size: " ~ to!string(thisFileSize) ~ " Bytes", ["debug"]);
|
|
addLogEntry("Upload Duration: " ~ to!string((uploadDuration.total!"msecs"/1e3)) ~ " Seconds", ["debug"]);
|
|
auto uploadSpeed = (thisFileSize / (uploadDuration.total!"msecs"/1e3)/ 1024 / 1024);
|
|
addLogEntry("Upload Speed: " ~ to!string(uploadSpeed) ~ " Mbps (approx)", ["debug"]);
|
|
|
|
// OK as the upload did not fail, we need to save the response from OneDrive, but it has to be a valid JSON response
|
|
if (uploadResponse.type() == JSONType.object) {
|
|
// check if the path still exists locally before we try to set the file times online - as short lived files, whilst we uploaded it - it may not exist locally aready
|
|
if (exists(fileToUpload)) {
|
|
if (!dryRun) {
|
|
// Check the integrity of the uploaded file, if the local file still exists
|
|
performUploadIntegrityValidationChecks(uploadResponse, fileToUpload, thisFileSize);
|
|
|
|
// Update the file modified time on OneDrive and save item details to database
|
|
// Update the item's metadata on OneDrive
|
|
SysTime mtime = timeLastModified(fileToUpload).toUTC();
|
|
mtime.fracSecs = Duration.zero;
|
|
string newFileId = uploadResponse["id"].str;
|
|
string newFileETag = uploadResponse["eTag"].str;
|
|
// Attempt to update the online date time stamp based on our local data
|
|
uploadLastModifiedTime(parentItem.driveId, newFileId, mtime, newFileETag);
|
|
}
|
|
} else {
|
|
// will be removed in different event!
|
|
addLogEntry("File disappeared locally after upload: " ~ fileToUpload);
|
|
}
|
|
} else {
|
|
// Log that an invalid JSON object was returned
|
|
addLogEntry("uploadFileOneDriveApiInstance.simpleUpload or session.upload call returned an invalid JSON Object from the OneDrive API", ["debug"]);
|
|
}
|
|
}
|
|
|
|
// Return upload status
|
|
return uploadFailed;
|
|
}
|
|
|
|
// Create the OneDrive Upload Session
|
|
JSONValue createSessionFileUpload(OneDriveApi activeOneDriveApiInstance, string fileToUpload, string parentDriveId, string parentId, string filename, string eTag, string threadUploadSessionFilePath) {
|
|
|
|
// Upload file via a OneDrive API session
|
|
JSONValue uploadSession;
|
|
|
|
// Calculate modification time
|
|
SysTime localFileLastModifiedTime = timeLastModified(fileToUpload).toUTC();
|
|
localFileLastModifiedTime.fracSecs = Duration.zero;
|
|
|
|
// Construct the fileSystemInfo JSON component needed to create the Upload Session
|
|
JSONValue fileSystemInfo = [
|
|
"item": JSONValue([
|
|
"@microsoft.graph.conflictBehavior": JSONValue("replace"),
|
|
"fileSystemInfo": JSONValue([
|
|
"lastModifiedDateTime": localFileLastModifiedTime.toISOExtString()
|
|
])
|
|
])
|
|
];
|
|
|
|
// Try to create the upload session for this file
|
|
uploadSession = activeOneDriveApiInstance.createUploadSession(parentDriveId, parentId, filename, eTag, fileSystemInfo);
|
|
|
|
if (uploadSession.type() == JSONType.object) {
|
|
// a valid session object was created
|
|
if ("uploadUrl" in uploadSession) {
|
|
// Add the file path we are uploading to this JSON Session Data
|
|
uploadSession["localPath"] = fileToUpload;
|
|
// Save this session
|
|
saveSessionFile(threadUploadSessionFilePath, uploadSession);
|
|
}
|
|
} else {
|
|
// no valid session was created
|
|
addLogEntry("Creation of OneDrive API Upload Session failed.", ["verbose"]);
|
|
// return upload() will return a JSONValue response, create an empty JSONValue response to return
|
|
uploadSession = null;
|
|
}
|
|
// Return the JSON
|
|
return uploadSession;
|
|
}
|
|
|
|
// Save the session upload data
|
|
void saveSessionFile(string threadUploadSessionFilePath, JSONValue uploadSessionData) {
|
|
|
|
try {
|
|
std.file.write(threadUploadSessionFilePath, uploadSessionData.toString());
|
|
} catch (FileException e) {
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
}
|
|
|
|
// Perform the upload of file via the Upload Session that was created
|
|
JSONValue performSessionFileUpload(OneDriveApi activeOneDriveApiInstance, ulong thisFileSize, JSONValue uploadSessionData, string threadUploadSessionFilePath) {
|
|
|
|
// Response for upload
|
|
JSONValue uploadResponse;
|
|
|
|
// Session JSON needs to contain valid elements
|
|
// Get the offset details
|
|
ulong fragmentSize = 10 * 2^^20; // 10 MiB
|
|
ulong fragmentCount = 0;
|
|
ulong fragSize = 0;
|
|
ulong offset = uploadSessionData["nextExpectedRanges"][0].str.splitter('-').front.to!ulong;
|
|
size_t expected_total_fragments = cast(ulong) ceil(double(thisFileSize) / double(fragmentSize));
|
|
ulong start_unix_time = Clock.currTime.toUnixTime();
|
|
int h, m, s;
|
|
string etaString;
|
|
string uploadLogEntry = "Uploading: " ~ uploadSessionData["localPath"].str ~ " ... ";
|
|
|
|
// Start the session upload using the active API instance for this thread
|
|
while (true) {
|
|
fragmentCount++;
|
|
addLogEntry("Fragment: " ~ to!string(fragmentCount) ~ " of " ~ to!string(expected_total_fragments), ["debug"]);
|
|
|
|
// What ETA string do we use?
|
|
auto eta = calc_eta((fragmentCount -1), expected_total_fragments, start_unix_time);
|
|
if (eta == 0) {
|
|
// Initial calculation ...
|
|
etaString = format!"| ETA --:--:--";
|
|
} else {
|
|
// we have at least an ETA provided
|
|
dur!"seconds"(eta).split!("hours", "minutes", "seconds")(h, m, s);
|
|
etaString = format!"| ETA %02d:%02d:%02d"( h, m, s);
|
|
}
|
|
|
|
// Calculate this progress output
|
|
auto ratio = cast(double)(fragmentCount -1) / expected_total_fragments;
|
|
// Convert the ratio to a percentage and format it to two decimal places
|
|
string percentage = leftJustify(format("%d%%", cast(int)(ratio * 100)), 5, ' ');
|
|
addLogEntry(uploadLogEntry ~ percentage ~ etaString, ["consoleOnly"]);
|
|
|
|
// What fragment size will be used?
|
|
addLogEntry("fragmentSize: " ~ to!string(fragmentSize) ~ " offset: " ~ to!string(offset) ~ " thisFileSize: " ~ to!string(thisFileSize), ["debug"]);
|
|
fragSize = fragmentSize < thisFileSize - offset ? fragmentSize : thisFileSize - offset;
|
|
addLogEntry("Using fragSize: " ~ to!string(fragSize), ["debug"]);
|
|
|
|
// fragSize must not be a negative value
|
|
if (fragSize < 0) {
|
|
// Session upload will fail
|
|
// not a JSON object - fragment upload failed
|
|
addLogEntry("File upload session failed - invalid calculation of fragment size", ["verbose"]);
|
|
if (exists(threadUploadSessionFilePath)) {
|
|
remove(threadUploadSessionFilePath);
|
|
}
|
|
// set uploadResponse to null as error
|
|
uploadResponse = null;
|
|
return uploadResponse;
|
|
}
|
|
|
|
// If the resume upload fails, we need to check for a return code here
|
|
try {
|
|
uploadResponse = activeOneDriveApiInstance.uploadFragment(
|
|
uploadSessionData["uploadUrl"].str,
|
|
uploadSessionData["localPath"].str,
|
|
offset,
|
|
fragSize,
|
|
thisFileSize
|
|
);
|
|
} catch (OneDriveException exception) {
|
|
// if a 100 uploadResponse is generated, continue
|
|
if (exception.httpStatusCode == 100) {
|
|
continue;
|
|
}
|
|
// There was an error uploadResponse from OneDrive when uploading the file fragment
|
|
|
|
// Handle transient errors:
|
|
// 408 - Request Time Out
|
|
// 429 - Too Many Requests
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle 'HTTP request returned status code 429 (Too Many Requests)' first
|
|
addLogEntry("Fragment upload failed - received throttle request uploadResponse from OneDrive", ["debug"]);
|
|
|
|
if (exception.httpStatusCode == 429) {
|
|
auto retryAfterValue = activeOneDriveApiInstance.getRetryAfterValue();
|
|
addLogEntry("Using Retry-After Value = " ~ to!string(retryAfterValue), ["debug"]);
|
|
|
|
// Sleep thread as per request
|
|
addLogEntry();
|
|
addLogEntry("Thread sleeping due to 'HTTP request returned status code 429' - The request has been throttled");
|
|
addLogEntry("Sleeping for " ~ to!string(retryAfterValue) ~ " seconds");
|
|
Thread.sleep(dur!"seconds"(retryAfterValue));
|
|
addLogEntry("Retrying fragment upload");
|
|
} else {
|
|
// Handle 408, 503 and 504
|
|
auto errorArray = splitLines(exception.msg);
|
|
auto retryAfterValue = 30;
|
|
addLogEntry();
|
|
addLogEntry("Thread sleeping due to '" ~ to!string(errorArray[0]) ~ "' - retrying applicable request in 30 seconds");
|
|
addLogEntry("Sleeping for " ~ to!string(retryAfterValue) ~ " seconds");
|
|
Thread.sleep(dur!"seconds"(retryAfterValue));
|
|
addLogEntry("Retrying fragment upload");
|
|
}
|
|
} else {
|
|
// insert a new line as well, so that the below error is inserted on the console in the right location
|
|
addLogEntry("Fragment upload failed - received an exception response from OneDrive API", ["verbose"]);
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
// retry fragment upload in case error is transient
|
|
addLogEntry("Retrying fragment upload", ["verbose"]);
|
|
}
|
|
|
|
try {
|
|
uploadResponse = activeOneDriveApiInstance.uploadFragment(
|
|
uploadSessionData["uploadUrl"].str,
|
|
uploadSessionData["localPath"].str,
|
|
offset,
|
|
fragSize,
|
|
thisFileSize
|
|
);
|
|
} catch (OneDriveException e) {
|
|
// OneDrive threw another error on retry
|
|
addLogEntry("Retry to upload fragment failed", ["verbose"]);
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
|
// set uploadResponse to null as the fragment upload was in error twice
|
|
uploadResponse = null;
|
|
} catch (std.exception.ErrnoException e) {
|
|
// There was a file system error - display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
return uploadResponse;
|
|
}
|
|
} catch (ErrnoException e) {
|
|
// There was a file system error
|
|
// display the error message
|
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
|
uploadResponse = null;
|
|
return uploadResponse;
|
|
}
|
|
|
|
// was the fragment uploaded without issue?
|
|
if (uploadResponse.type() == JSONType.object){
|
|
offset += fragmentSize;
|
|
if (offset >= thisFileSize) {
|
|
break;
|
|
}
|
|
// update the uploadSessionData details
|
|
uploadSessionData["expirationDateTime"] = uploadResponse["expirationDateTime"];
|
|
uploadSessionData["nextExpectedRanges"] = uploadResponse["nextExpectedRanges"];
|
|
saveSessionFile(threadUploadSessionFilePath, uploadSessionData);
|
|
} else {
|
|
// not a JSON object - fragment upload failed
|
|
addLogEntry("File upload session failed - invalid response from OneDrive API", ["verbose"]);
|
|
|
|
// cleanup session data
|
|
if (exists(threadUploadSessionFilePath)) {
|
|
remove(threadUploadSessionFilePath);
|
|
}
|
|
// set uploadResponse to null as error
|
|
uploadResponse = null;
|
|
return uploadResponse;
|
|
}
|
|
}
|
|
|
|
// upload complete
|
|
ulong end_unix_time = Clock.currTime.toUnixTime();
|
|
auto upload_duration = cast(int)(end_unix_time - start_unix_time);
|
|
dur!"seconds"(upload_duration).split!("hours", "minutes", "seconds")(h, m, s);
|
|
etaString = format!"| DONE in %02d:%02d:%02d"( h, m, s);
|
|
addLogEntry(uploadLogEntry ~ "100% " ~ etaString, ["consoleOnly"]);
|
|
|
|
// Remove session file if it exists
|
|
if (exists(threadUploadSessionFilePath)) {
|
|
remove(threadUploadSessionFilePath);
|
|
}
|
|
|
|
// Return the session upload response
|
|
return uploadResponse;
|
|
}
|
|
|
|
// Delete an item on OneDrive
|
|
void uploadDeletedItem(Item itemToDelete, string path) {
|
|
|
|
// Are we in a situation where we HAVE to keep the data online - do not delete the remote object
|
|
if (noRemoteDelete) {
|
|
if ((itemToDelete.type == ItemType.dir)) {
|
|
// Do not process remote directory delete
|
|
addLogEntry("Skipping remote directory delete as --upload-only & --no-remote-delete configured", ["verbose"]);
|
|
} else {
|
|
// Do not process remote file delete
|
|
addLogEntry("Skipping remote file delete as --upload-only & --no-remote-delete configured", ["verbose"]);
|
|
}
|
|
} else {
|
|
|
|
// Is this a --download-only operation?
|
|
if (!appConfig.getValueBool("download_only")) {
|
|
// Process the delete - delete the object online
|
|
addLogEntry("Deleting item from OneDrive: " ~ path);
|
|
bool flagAsBigDelete = false;
|
|
|
|
Item[] children;
|
|
ulong itemsToDelete;
|
|
|
|
if ((itemToDelete.type == ItemType.dir)) {
|
|
// Query the database - how many objects will this remove?
|
|
children = getChildren(itemToDelete.driveId, itemToDelete.id);
|
|
// Count the returned items + the original item (1)
|
|
itemsToDelete = count(children) + 1;
|
|
addLogEntry("Number of items online to delete: " ~ to!string(itemsToDelete), ["debug"]);
|
|
} else {
|
|
itemsToDelete = 1;
|
|
}
|
|
|
|
// A local delete of a file|folder when using --monitor will issue a inotify event, which will trigger the local & remote data immediately be deleted
|
|
// The user may also be --sync process, so we are checking if something was deleted between application use
|
|
if (itemsToDelete >= appConfig.getValueLong("classify_as_big_delete")) {
|
|
// A big delete has been detected
|
|
flagAsBigDelete = true;
|
|
if (!appConfig.getValueBool("force")) {
|
|
addLogEntry("ERROR: An attempt to remove a large volume of data from OneDrive has been detected. Exiting client to preserve data on Microsoft OneDrive");
|
|
addLogEntry("ERROR: To delete a large volume of data use --force or increase the config value 'classify_as_big_delete' to a larger value");
|
|
// Must exit here to preserve data on online , allow logging to be done
|
|
forceExit();
|
|
}
|
|
}
|
|
|
|
// Are we in a --dry-run scenario?
|
|
if (!dryRun) {
|
|
// We are not in a dry run scenario
|
|
addLogEntry("itemToDelete: " ~ to!string(itemToDelete), ["debug"]);
|
|
|
|
// Create new OneDrive API Instance
|
|
OneDriveApi uploadDeletedItemOneDriveApiInstance;
|
|
uploadDeletedItemOneDriveApiInstance = new OneDriveApi(appConfig);
|
|
uploadDeletedItemOneDriveApiInstance.initialise();
|
|
|
|
// what item are we trying to delete?
|
|
addLogEntry("Attempting to delete this single item id: " ~ itemToDelete.id ~ " from drive: " ~ itemToDelete.driveId, ["debug"]);
|
|
|
|
try {
|
|
// perform the delete via the default OneDrive API instance
|
|
uploadDeletedItemOneDriveApiInstance.deleteById(itemToDelete.driveId, itemToDelete.id);
|
|
// Shutdown API
|
|
uploadDeletedItemOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(uploadDeletedItemOneDriveApiInstance);
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 404) {
|
|
// item.id, item.eTag could not be found on the specified driveId
|
|
addLogEntry("OneDrive reported: The resource could not be found to be deleted.", ["verbose"]);
|
|
}
|
|
}
|
|
|
|
// Delete the reference in the local database
|
|
itemDB.deleteById(itemToDelete.driveId, itemToDelete.id);
|
|
if (itemToDelete.remoteId != null) {
|
|
// If the item is a remote item, delete the reference in the local database
|
|
itemDB.deleteById(itemToDelete.remoteDriveId, itemToDelete.remoteId);
|
|
}
|
|
} else {
|
|
// log that this is a dry-run activity
|
|
addLogEntry("dry run - no delete activity");
|
|
}
|
|
} else {
|
|
// --download-only operation, we are not uploading any delete event to OneDrive
|
|
addLogEntry("Not pushing local delete to Microsoft OneDrive due to --download-only being used", ["debug"]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Get the children of an item id from the database
|
|
Item[] getChildren(string driveId, string id) {
|
|
|
|
Item[] children;
|
|
children ~= itemDB.selectChildren(driveId, id);
|
|
foreach (Item child; children) {
|
|
if (child.type != ItemType.file) {
|
|
// recursively get the children of this child
|
|
children ~= getChildren(child.driveId, child.id);
|
|
}
|
|
}
|
|
return children;
|
|
}
|
|
|
|
// Perform a 'reverse' delete of all child objects on OneDrive
|
|
void performReverseDeletionOfOneDriveItems(Item[] children, Item itemToDelete) {
|
|
|
|
// Log what is happening
|
|
addLogEntry("Attempting a reverse delete of all child objects from OneDrive", ["debug"]);
|
|
|
|
// Create a new API Instance for this thread and initialise it
|
|
OneDriveApi performReverseDeletionOneDriveApiInstance;
|
|
performReverseDeletionOneDriveApiInstance = new OneDriveApi(appConfig);
|
|
performReverseDeletionOneDriveApiInstance.initialise();
|
|
|
|
foreach_reverse (Item child; children) {
|
|
// Log the action
|
|
addLogEntry("Attempting to delete this child item id: " ~ child.id ~ " from drive: " ~ child.driveId, ["debug"]);
|
|
|
|
// perform the delete via the default OneDrive API instance
|
|
performReverseDeletionOneDriveApiInstance.deleteById(child.driveId, child.id, child.eTag);
|
|
// delete the child reference in the local database
|
|
itemDB.deleteById(child.driveId, child.id);
|
|
}
|
|
// Log the action
|
|
addLogEntry("Attempting to delete this parent item id: " ~ itemToDelete.id ~ " from drive: " ~ itemToDelete.driveId, ["debug"]);
|
|
|
|
// Perform the delete via the default OneDrive API instance
|
|
performReverseDeletionOneDriveApiInstance.deleteById(itemToDelete.driveId, itemToDelete.id, itemToDelete.eTag);
|
|
// Shutdown API instance
|
|
performReverseDeletionOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(performReverseDeletionOneDriveApiInstance);
|
|
}
|
|
|
|
// Create a fake OneDrive response suitable for use with saveItem
|
|
JSONValue createFakeResponse(const(string) path) {
|
|
|
|
import std.digest.sha;
|
|
// Generate a simulated JSON response which can be used
|
|
// At a minimum we need:
|
|
// 1. eTag
|
|
// 2. cTag
|
|
// 3. fileSystemInfo
|
|
// 4. file or folder. if file, hash of file
|
|
// 5. id
|
|
// 6. name
|
|
// 7. parent reference
|
|
|
|
string fakeDriveId = appConfig.defaultDriveId;
|
|
string fakeRootId = appConfig.defaultRootId;
|
|
SysTime mtime = timeLastModified(path).toUTC();
|
|
|
|
// Need to update the 'fakeDriveId' & 'fakeRootId' with elements from the --dry-run database
|
|
// Otherwise some calls to validate objects will fail as the actual driveId being used is invalid
|
|
string parentPath = dirName(path);
|
|
Item databaseItem;
|
|
|
|
if (parentPath != ".") {
|
|
// Not a 'root' parent
|
|
// For each driveid in the existing driveIDsArray
|
|
foreach (searchDriveId; driveIDsArray) {
|
|
addLogEntry("FakeResponse: searching database for: " ~ searchDriveId ~ " " ~ parentPath, ["debug"]);
|
|
|
|
if (itemDB.selectByPath(parentPath, searchDriveId, databaseItem)) {
|
|
addLogEntry("FakeResponse: Found Database Item: " ~ to!string(databaseItem), ["debug"]);
|
|
fakeDriveId = databaseItem.driveId;
|
|
fakeRootId = databaseItem.id;
|
|
}
|
|
}
|
|
}
|
|
|
|
// real id / eTag / cTag are different format for personal / business account
|
|
auto sha1 = new SHA1Digest();
|
|
ubyte[] fakedOneDriveItemValues = sha1.digest(path);
|
|
|
|
JSONValue fakeResponse;
|
|
|
|
if (isDir(path)) {
|
|
// path is a directory
|
|
fakeResponse = [
|
|
"id": JSONValue(toHexString(fakedOneDriveItemValues)),
|
|
"cTag": JSONValue(toHexString(fakedOneDriveItemValues)),
|
|
"eTag": JSONValue(toHexString(fakedOneDriveItemValues)),
|
|
"fileSystemInfo": JSONValue([
|
|
"createdDateTime": mtime.toISOExtString(),
|
|
"lastModifiedDateTime": mtime.toISOExtString()
|
|
]),
|
|
"name": JSONValue(baseName(path)),
|
|
"parentReference": JSONValue([
|
|
"driveId": JSONValue(fakeDriveId),
|
|
"driveType": JSONValue(appConfig.accountType),
|
|
"id": JSONValue(fakeRootId)
|
|
]),
|
|
"folder": JSONValue("")
|
|
];
|
|
} else {
|
|
// path is a file
|
|
// compute file hash - both business and personal responses use quickXorHash
|
|
string quickXorHash = computeQuickXorHash(path);
|
|
|
|
fakeResponse = [
|
|
"id": JSONValue(toHexString(fakedOneDriveItemValues)),
|
|
"cTag": JSONValue(toHexString(fakedOneDriveItemValues)),
|
|
"eTag": JSONValue(toHexString(fakedOneDriveItemValues)),
|
|
"fileSystemInfo": JSONValue([
|
|
"createdDateTime": mtime.toISOExtString(),
|
|
"lastModifiedDateTime": mtime.toISOExtString()
|
|
]),
|
|
"name": JSONValue(baseName(path)),
|
|
"parentReference": JSONValue([
|
|
"driveId": JSONValue(fakeDriveId),
|
|
"driveType": JSONValue(appConfig.accountType),
|
|
"id": JSONValue(fakeRootId)
|
|
]),
|
|
"file": JSONValue([
|
|
"hashes":JSONValue([
|
|
"quickXorHash": JSONValue(quickXorHash)
|
|
])
|
|
|
|
])
|
|
];
|
|
}
|
|
|
|
addLogEntry("Generated Fake OneDrive Response: " ~ to!string(fakeResponse), ["debug"]);
|
|
return fakeResponse;
|
|
}
|
|
|
|
// Save JSON item details into the item database
|
|
void saveItem(JSONValue jsonItem) {
|
|
|
|
// jsonItem has to be a valid object
|
|
if (jsonItem.type() == JSONType.object) {
|
|
// Check if the response JSON has an 'id', otherwise makeItem() fails with 'Key not found: id'
|
|
if (hasId(jsonItem)) {
|
|
// Are we in a --upload-only & --remove-source-files scenario?
|
|
// We do not want to add the item to the database in this situation as there is no local reference to the file post file deletion
|
|
// If the item is a directory, we need to add this to the DB, if this is a file, we dont add this, the parent path is not in DB, thus any new files in this directory are not added
|
|
if ((uploadOnly) && (localDeleteAfterUpload) && (isItemFile(jsonItem))) {
|
|
// Log that we skipping adding item to the local DB and the reason why
|
|
addLogEntry("Skipping adding to database as --upload-only & --remove-source-files configured", ["debug"]);
|
|
} else {
|
|
// What is the JSON item we are trying to create a DB record with?
|
|
addLogEntry("saveItem - creating DB item from this JSON: " ~ to!string(jsonItem), ["debug"]);
|
|
|
|
// Takes a JSON input and formats to an item which can be used by the database
|
|
Item item = makeItem(jsonItem);
|
|
|
|
// Is this JSON item a 'root' item?
|
|
if ((isItemRoot(jsonItem)) && (item.name == "root")) {
|
|
addLogEntry("Updating DB Item object with correct values as this is a 'root' object", ["debug"]);
|
|
item.parentId = null; // ensures that this database entry has no parent
|
|
// Check for parentReference
|
|
if (hasParentReference(jsonItem)) {
|
|
// Set the correct item.driveId
|
|
addLogEntry("ROOT JSON Item HAS parentReference .... setting item.driveId = jsonItem['parentReference']['driveId'].str", ["debug"]);
|
|
item.driveId = jsonItem["parentReference"]["driveId"].str;
|
|
}
|
|
|
|
// We only should be adding our account 'root' to the database, not shared folder 'root' items
|
|
if (item.driveId != appConfig.defaultDriveId) {
|
|
// Shared Folder drive 'root' object .. we dont want this item
|
|
addLogEntry("NOT adding 'remote root' object to database: " ~ to!string(item), ["debug"]);
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Add to the local database
|
|
addLogEntry("Adding to database: " ~ to!string(item), ["debug"]);
|
|
itemDB.upsert(item);
|
|
|
|
// If we have a remote drive ID, add this to our list of known drive id's
|
|
if (!item.remoteDriveId.empty) {
|
|
// Keep the driveIDsArray with unique entries only
|
|
if (!canFind(driveIDsArray, item.remoteDriveId)) {
|
|
// Add this drive id to the array to search with
|
|
driveIDsArray ~= item.remoteDriveId;
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// log error
|
|
addLogEntry("ERROR: OneDrive response missing required 'id' element");
|
|
addLogEntry("ERROR: " ~ to!string(jsonItem));
|
|
}
|
|
} else {
|
|
// log error
|
|
addLogEntry("ERROR: An error was returned from OneDrive and the resulting response is not a valid JSON object");
|
|
addLogEntry("ERROR: Increase logging verbosity to assist determining why.");
|
|
}
|
|
}
|
|
|
|
// Wrapper function for makeDatabaseItem so we can check to ensure that the item has the required hashes
|
|
Item makeItem(JSONValue onedriveJSONItem) {
|
|
|
|
// Make the DB Item from the JSON data provided
|
|
Item newDatabaseItem = makeDatabaseItem(onedriveJSONItem);
|
|
|
|
// Is this a 'file' item that has not been deleted? Deleted items have no hash
|
|
if ((newDatabaseItem.type == ItemType.file) && (!isItemDeleted(onedriveJSONItem))) {
|
|
// Does this item have a file size attribute?
|
|
if (hasFileSize(onedriveJSONItem)) {
|
|
// Is the file size greater than 0?
|
|
if (onedriveJSONItem["size"].integer > 0) {
|
|
// Does the DB item have any hashes as per the API provided JSON data?
|
|
if ((newDatabaseItem.quickXorHash.empty) && (newDatabaseItem.sha256Hash.empty)) {
|
|
// Odd .. there is no hash for this item .. why is that?
|
|
// Is there a 'file' JSON element?
|
|
if ("file" in onedriveJSONItem) {
|
|
// Microsoft OneDrive OneNote objects will report as files but have 'application/msonenote' and 'application/octet-stream' as mime types
|
|
if ((isMicrosoftOneNoteMimeType1(onedriveJSONItem)) || (isMicrosoftOneNoteMimeType2(onedriveJSONItem))) {
|
|
// Debug log output that this is a potential OneNote object
|
|
addLogEntry("This item is potentially an associated Microsoft OneNote Object Item", ["debug"]);
|
|
} else {
|
|
// Not a Microsoft OneNote Mime Type Object ..
|
|
string apiWarningMessage = "WARNING: OneDrive API inconsistency - this file does not have any hash: ";
|
|
// This is computationally expensive .. but we are only doing this if there are no hashses provided
|
|
bool parentInDatabase = itemDB.idInLocalDatabase(newDatabaseItem.driveId, newDatabaseItem.parentId);
|
|
// Is the parent id in the database?
|
|
if (parentInDatabase) {
|
|
// This is again computationally expensive .. calculate this item path to advise the user the actual path of this item that has no hash
|
|
string newItemPath = computeItemPath(newDatabaseItem.driveId, newDatabaseItem.parentId) ~ "/" ~ newDatabaseItem.name;
|
|
addLogEntry(apiWarningMessage ~ newItemPath);
|
|
} else {
|
|
// Parent is not in the database .. why?
|
|
// Check if the parent item had been skipped ..
|
|
if (newDatabaseItem.parentId in skippedItems) {
|
|
addLogEntry(apiWarningMessage ~ "newDatabaseItem.parentId listed within skippedItems", ["debug"]);
|
|
} else {
|
|
// Use the item ID .. there is no other reference available, parent is not being skipped, so we should have been able to calculate this - but we could not
|
|
addLogEntry(apiWarningMessage ~ newDatabaseItem.id);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// zero file size
|
|
addLogEntry("This item file is zero size - potentially no hash provided by the OneDrive API", ["debug"]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Return the new database item
|
|
return newDatabaseItem;
|
|
}
|
|
|
|
// Print the fileDownloadFailures and fileUploadFailures arrays if they are not empty
|
|
void displaySyncFailures() {
|
|
|
|
// Were there any file download failures?
|
|
if (!fileDownloadFailures.empty) {
|
|
// There are download failures ...
|
|
addLogEntry();
|
|
addLogEntry("Failed items to download from OneDrive: " ~ to!string(fileDownloadFailures.length));
|
|
foreach(failedFileToDownload; fileDownloadFailures) {
|
|
// List the detail of the item that failed to download
|
|
addLogEntry("Failed to download: " ~ failedFileToDownload, ["info", "notify"]);
|
|
|
|
// Is this failed item in the DB? It should not be ..
|
|
Item downloadDBItem;
|
|
// Need to check all driveid's we know about, not just the defaultDriveId
|
|
foreach (searchDriveId; driveIDsArray) {
|
|
if (itemDB.selectByPath(failedFileToDownload, searchDriveId, downloadDBItem)) {
|
|
// item was found in the DB
|
|
addLogEntry("ERROR: Failed Download Path found in database, must delete this item from the database .. it should not be in there if it failed to download");
|
|
// Process the database entry removal. In a --dry-run scenario, this is being done against a DB copy
|
|
itemDB.deleteById(downloadDBItem.driveId, downloadDBItem.id);
|
|
if (downloadDBItem.remoteDriveId != null) {
|
|
// delete the linked remote folder
|
|
itemDB.deleteById(downloadDBItem.remoteDriveId, downloadDBItem.remoteId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Set the flag
|
|
syncFailures = true;
|
|
}
|
|
|
|
// Were there any file upload failures?
|
|
if (!fileUploadFailures.empty) {
|
|
// There are download failures ...
|
|
addLogEntry();
|
|
addLogEntry("Failed items to upload to OneDrive: " ~ to!string(fileUploadFailures.length));
|
|
foreach(failedFileToUpload; fileUploadFailures) {
|
|
// List the path of the item that failed to upload
|
|
addLogEntry("Failed to upload: " ~ failedFileToUpload, ["info", "notify"]);
|
|
|
|
// Is this failed item in the DB? It should not be ..
|
|
Item uploadDBItem;
|
|
// Need to check all driveid's we know about, not just the defaultDriveId
|
|
foreach (searchDriveId; driveIDsArray) {
|
|
if (itemDB.selectByPath(failedFileToUpload, searchDriveId, uploadDBItem)) {
|
|
// item was found in the DB
|
|
addLogEntry("ERROR: Failed Upload Path found in database, must delete this item from the database .. it should not be in there if it failed to upload");
|
|
// Process the database entry removal. In a --dry-run scenario, this is being done against a DB copy
|
|
itemDB.deleteById(uploadDBItem.driveId, uploadDBItem.id);
|
|
if (uploadDBItem.remoteDriveId != null) {
|
|
// delete the linked remote folder
|
|
itemDB.deleteById(uploadDBItem.remoteDriveId, uploadDBItem.remoteId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Set the flag
|
|
syncFailures = true;
|
|
}
|
|
}
|
|
|
|
// Generate a /delta compatible response - for use when we cant actually use /delta
|
|
// This is required when the application is configured to use National Azure AD deployments as these do not support /delta queries
|
|
// The same technique can also be used when we are using --single-directory. The parent objects up to the single directory target can be added,
|
|
// then once the target of the --single-directory request is hit, all of the children of that path can be queried, giving a much more focused
|
|
// JSON response which can then be processed, negating the need to continuously traverse the tree and 'exclude' items
|
|
JSONValue generateDeltaResponse(string pathToQuery = null) {
|
|
|
|
// JSON value which will be responded with
|
|
JSONValue selfGeneratedDeltaResponse;
|
|
|
|
// Function variables
|
|
Item searchItem;
|
|
JSONValue rootData;
|
|
JSONValue driveData;
|
|
JSONValue pathData;
|
|
JSONValue topLevelChildren;
|
|
JSONValue[] childrenData;
|
|
string nextLink;
|
|
|
|
// Was a path to query passed in?
|
|
if (pathToQuery.empty) {
|
|
// Will query for the 'root'
|
|
pathToQuery = ".";
|
|
}
|
|
|
|
// Create new OneDrive API Instance
|
|
OneDriveApi generateDeltaResponseOneDriveApiInstance;
|
|
generateDeltaResponseOneDriveApiInstance = new OneDriveApi(appConfig);
|
|
generateDeltaResponseOneDriveApiInstance.initialise();
|
|
|
|
if (!singleDirectoryScope) {
|
|
// In a --resync scenario, there is no DB data to query, so we have to query the OneDrive API here to get relevant details
|
|
try {
|
|
// Query the OneDrive API
|
|
pathData = generateDeltaResponseOneDriveApiInstance.getPathDetails(pathToQuery);
|
|
// Is the path on OneDrive local or remote to our account drive id?
|
|
if (isItemRemote(pathData)) {
|
|
// The path we are seeking is remote to our account drive id
|
|
searchItem.driveId = pathData["remoteItem"]["parentReference"]["driveId"].str;
|
|
searchItem.id = pathData["remoteItem"]["id"].str;
|
|
} else {
|
|
// The path we are seeking is local to our account drive id
|
|
searchItem.driveId = pathData["parentReference"]["driveId"].str;
|
|
searchItem.id = pathData["id"].str;
|
|
}
|
|
} catch (OneDriveException e) {
|
|
// Display error message
|
|
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
|
// Must exit here
|
|
generateDeltaResponseOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(generateDeltaResponseOneDriveApiInstance);
|
|
// Must force exit here, allow logging to be done
|
|
forceExit();
|
|
}
|
|
} else {
|
|
// When setSingleDirectoryScope() was called, the following were set to the correct items, even if the path was remote:
|
|
// - singleDirectoryScopeDriveId
|
|
// - singleDirectoryScopeItemId
|
|
// Reuse these prior set values
|
|
searchItem.driveId = singleDirectoryScopeDriveId;
|
|
searchItem.id = singleDirectoryScopeItemId;
|
|
}
|
|
|
|
// Before we get any data from the OneDrive API, flag any child object in the database as out-of-sync for this driveId & and object id
|
|
// Downgrade ONLY files associated with this driveId and idToQuery
|
|
addLogEntry("Downgrading all children for this searchItem.driveId (" ~ searchItem.driveId ~ ") and searchItem.id (" ~ searchItem.id ~ ") to an out-of-sync state", ["debug"]);
|
|
|
|
auto drivePathChildren = getChildren(searchItem.driveId, searchItem.id);
|
|
if (count(drivePathChildren) > 0) {
|
|
// Children to process and flag as out-of-sync
|
|
foreach (drivePathChild; drivePathChildren) {
|
|
// Flag any object in the database as out-of-sync for this driveId & and object id
|
|
addLogEntry("Downgrading item as out-of-sync: " ~ drivePathChild.id, ["debug"]);
|
|
itemDB.downgradeSyncStatusFlag(drivePathChild.driveId, drivePathChild.id);
|
|
}
|
|
}
|
|
|
|
// Get drive details for the provided driveId
|
|
try {
|
|
driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id);
|
|
} catch (OneDriveException exception) {
|
|
addLogEntry("driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id) generated a OneDriveException", ["debug"]);
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to query path details on OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
generateDeltaResponse(pathToQuery);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
|
|
}
|
|
}
|
|
|
|
// Was a valid JSON response for 'driveData' provided?
|
|
if (driveData.type() == JSONType.object) {
|
|
|
|
// Dynamic output for a non-verbose run so that the user knows something is happening
|
|
if (appConfig.verbosityCount == 0) {
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
addLogEntry("Fetching items from the OneDrive API for Drive ID: " ~ searchItem.driveId, ["logFileOnly"]);
|
|
|
|
// Use the dots to show the application is 'doing something'
|
|
addLogEntry("Fetching items from the OneDrive API for Drive ID: " ~ searchItem.driveId ~ " .", ["consoleOnlyNoNewLine"]);
|
|
}
|
|
} else {
|
|
addLogEntry("Generating a /delta response from the OneDrive API for Drive ID: " ~ searchItem.driveId, ["verbose"]);
|
|
}
|
|
|
|
// Process this initial JSON response
|
|
if (!isItemRoot(driveData)) {
|
|
// Get root details for the provided driveId
|
|
try {
|
|
rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId);
|
|
} catch (OneDriveException exception) {
|
|
addLogEntry("rootData = onedrive.getDriveIdRoot(searchItem.driveId) generated a OneDriveException", ["debug"]);
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to query drive root details on OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Query: rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId)");
|
|
rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
|
|
}
|
|
}
|
|
// Add driveData JSON data to array
|
|
addLogEntry("Adding OneDrive root details for processing", ["verbose"]);
|
|
childrenData ~= rootData;
|
|
}
|
|
|
|
// Add driveData JSON data to array
|
|
addLogEntry("Adding OneDrive folder details for processing", ["verbose"]);
|
|
childrenData ~= driveData;
|
|
} else {
|
|
// driveData is an invalid JSON object
|
|
writeln("CODING TO DO: The query of OneDrive API to getPathDetailsById generated an invalid JSON response - thus we cant build our own /delta simulated response ... how to handle?");
|
|
// Must exit here
|
|
generateDeltaResponseOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(generateDeltaResponseOneDriveApiInstance);
|
|
// Must force exit here, allow logging to be done
|
|
forceExit();
|
|
}
|
|
|
|
// For each child object, query the OneDrive API
|
|
for (;;) {
|
|
// query top level children
|
|
try {
|
|
topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink);
|
|
} catch (OneDriveException exception) {
|
|
// OneDrive threw an error
|
|
addLogEntry("------------------------------------------------------------------", ["debug"]);
|
|
addLogEntry("Query Error: topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)", ["debug"]);
|
|
addLogEntry("driveId: " ~ searchItem.driveId, ["debug"]);
|
|
addLogEntry("idToQuery: " ~ searchItem.id, ["debug"]);
|
|
addLogEntry("nextLink: " ~ nextLink, ["debug"]);
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)", ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to query OneDrive top level drive children on OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry("generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink) previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
generateDeltaResponse(pathToQuery);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
|
|
}
|
|
}
|
|
|
|
// process top level children
|
|
addLogEntry("Adding " ~ to!string(count(topLevelChildren["value"].array)) ~ " OneDrive items for processing from the OneDrive 'root' folder", ["verbose"]);
|
|
|
|
foreach (child; topLevelChildren["value"].array) {
|
|
// Check for any Client Side Filtering here ... we should skip querying the OneDrive API for 'folders' that we are going to just process and skip anyway.
|
|
// This avoids needless calls to the OneDrive API, and potentially speeds up this process.
|
|
if (!checkJSONAgainstClientSideFiltering(child)) {
|
|
// add this child to the array of objects
|
|
childrenData ~= child;
|
|
// is this child a folder?
|
|
if (isItemFolder(child)) {
|
|
// We have to query this folders children if childCount > 0
|
|
if (child["folder"]["childCount"].integer > 0){
|
|
// This child folder has children
|
|
string childIdToQuery = child["id"].str;
|
|
string childDriveToQuery = child["parentReference"]["driveId"].str;
|
|
auto childParentPath = child["parentReference"]["path"].str.split(":");
|
|
string folderPathToScan = childParentPath[1] ~ "/" ~ child["name"].str;
|
|
|
|
string pathForLogging;
|
|
// Are we in a --single-directory situation? If we are, the path we are using for logging needs to use the input path as a base
|
|
if (singleDirectoryScope) {
|
|
pathForLogging = appConfig.getValueString("single_directory") ~ "/" ~ child["name"].str;
|
|
} else {
|
|
pathForLogging = child["name"].str;
|
|
}
|
|
|
|
// Query the children of this item
|
|
JSONValue[] grandChildrenData = queryForChildren(childDriveToQuery, childIdToQuery, folderPathToScan, pathForLogging);
|
|
foreach (grandChild; grandChildrenData.array) {
|
|
// add the grandchild to the array
|
|
childrenData ~= grandChild;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// If a collection exceeds the default page size (200 items), the @odata.nextLink property is returned in the response
|
|
// to indicate more items are available and provide the request URL for the next page of items.
|
|
if ("@odata.nextLink" in topLevelChildren) {
|
|
// Update nextLink to next changeSet bundle
|
|
addLogEntry("Setting nextLink to (@odata.nextLink): " ~ nextLink, ["debug"]);
|
|
nextLink = topLevelChildren["@odata.nextLink"].str;
|
|
} else break;
|
|
}
|
|
|
|
if (appConfig.verbosityCount == 0) {
|
|
// Dynamic output for a non-verbose run so that the user knows something is happening
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
addLogEntry("\n", ["consoleOnlyNoNewLine"]);
|
|
}
|
|
}
|
|
|
|
// Craft response from all returned JSON elements
|
|
selfGeneratedDeltaResponse = [
|
|
"@odata.context": JSONValue("https://graph.microsoft.com/v1.0/$metadata#Collection(driveItem)"),
|
|
"value": JSONValue(childrenData.array)
|
|
];
|
|
|
|
// Shutdown API
|
|
generateDeltaResponseOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(generateDeltaResponseOneDriveApiInstance);
|
|
|
|
// Return the generated JSON response
|
|
return selfGeneratedDeltaResponse;
|
|
}
|
|
|
|
// Query the OneDrive API for the specified child id for any children objects
|
|
JSONValue[] queryForChildren(string driveId, string idToQuery, string childParentPath, string pathForLogging) {
|
|
|
|
// function variables
|
|
JSONValue thisLevelChildren;
|
|
JSONValue[] thisLevelChildrenData;
|
|
string nextLink;
|
|
|
|
// Create new OneDrive API Instance
|
|
OneDriveApi queryChildrenOneDriveApiInstance;
|
|
queryChildrenOneDriveApiInstance = new OneDriveApi(appConfig);
|
|
queryChildrenOneDriveApiInstance.initialise();
|
|
|
|
for (;;) {
|
|
// query this level children
|
|
try {
|
|
thisLevelChildren = queryThisLevelChildren(driveId, idToQuery, nextLink, queryChildrenOneDriveApiInstance);
|
|
} catch (OneDriveException exception) {
|
|
|
|
writeln("CODING TO DO: EXCEPTION HANDLING NEEDED: thisLevelChildren = queryThisLevelChildren(driveId, idToQuery, nextLink, queryChildrenOneDriveApiInstance)");
|
|
|
|
}
|
|
|
|
if (appConfig.verbosityCount == 0) {
|
|
// Dynamic output for a non-verbose run so that the user knows something is happening
|
|
if (!appConfig.surpressLoggingOutput) {
|
|
addLogEntry(".", ["consoleOnlyNoNewLine"]);
|
|
}
|
|
}
|
|
|
|
// Was a valid JSON response for 'thisLevelChildren' provided?
|
|
if (thisLevelChildren.type() == JSONType.object) {
|
|
// process this level children
|
|
if (!childParentPath.empty) {
|
|
// We dont use childParentPath to log, as this poses an information leak risk.
|
|
// The full parent path of the child, as per the JSON might be:
|
|
// /Level 1/Level 2/Level 3/Child Shared Folder/some folder/another folder
|
|
// But 'Child Shared Folder' is what is shared, thus '/Level 1/Level 2/Level 3/' is a potential information leak if logged.
|
|
// Plus, the application output now shows accuratly what is being shared - so that is a good thing.
|
|
addLogEntry("Adding " ~ to!string(count(thisLevelChildren["value"].array)) ~ " OneDrive items for processing from " ~ pathForLogging, ["verbose"]);
|
|
}
|
|
foreach (child; thisLevelChildren["value"].array) {
|
|
// Check for any Client Side Filtering here ... we should skip querying the OneDrive API for 'folders' that we are going to just process and skip anyway.
|
|
// This avoids needless calls to the OneDrive API, and potentially speeds up this process.
|
|
if (!checkJSONAgainstClientSideFiltering(child)) {
|
|
// add this child to the array of objects
|
|
thisLevelChildrenData ~= child;
|
|
// is this child a folder?
|
|
if (isItemFolder(child)){
|
|
// We have to query this folders children if childCount > 0
|
|
if (child["folder"]["childCount"].integer > 0){
|
|
// This child folder has children
|
|
string childIdToQuery = child["id"].str;
|
|
string childDriveToQuery = child["parentReference"]["driveId"].str;
|
|
auto grandchildParentPath = child["parentReference"]["path"].str.split(":");
|
|
string folderPathToScan = grandchildParentPath[1] ~ "/" ~ child["name"].str;
|
|
string newLoggingPath = pathForLogging ~ "/" ~ child["name"].str;
|
|
JSONValue[] grandChildrenData = queryForChildren(childDriveToQuery, childIdToQuery, folderPathToScan, newLoggingPath);
|
|
foreach (grandChild; grandChildrenData.array) {
|
|
// add the grandchild to the array
|
|
thisLevelChildrenData ~= grandChild;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// If a collection exceeds the default page size (200 items), the @odata.nextLink property is returned in the response
|
|
// to indicate more items are available and provide the request URL for the next page of items.
|
|
if ("@odata.nextLink" in thisLevelChildren) {
|
|
// Update nextLink to next changeSet bundle
|
|
nextLink = thisLevelChildren["@odata.nextLink"].str;
|
|
addLogEntry("Setting nextLink to (@odata.nextLink): " ~ nextLink, ["debug"]);
|
|
} else break;
|
|
|
|
} else {
|
|
// Invalid JSON response when querying this level children
|
|
addLogEntry("INVALID JSON response when attempting a retry of parent function - queryForChildren(driveId, idToQuery, childParentPath, pathForLogging)", ["debug"]);
|
|
|
|
// retry thisLevelChildren = queryThisLevelChildren
|
|
addLogEntry("Thread sleeping for an additional 30 seconds", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
addLogEntry("Retry this call thisLevelChildren = queryThisLevelChildren(driveId, idToQuery, nextLink, queryChildrenOneDriveApiInstance)", ["debug"]);
|
|
thisLevelChildren = queryThisLevelChildren(driveId, idToQuery, nextLink, queryChildrenOneDriveApiInstance);
|
|
}
|
|
}
|
|
|
|
// Shutdown API instance
|
|
queryChildrenOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(queryChildrenOneDriveApiInstance);
|
|
|
|
// return response
|
|
return thisLevelChildrenData;
|
|
}
|
|
|
|
// Query the OneDrive API for the child objects for this element
|
|
JSONValue queryThisLevelChildren(string driveId, string idToQuery, string nextLink, OneDriveApi queryChildrenOneDriveApiInstance) {
|
|
|
|
// function variables
|
|
JSONValue thisLevelChildren;
|
|
|
|
// query children
|
|
try {
|
|
// attempt API call
|
|
addLogEntry("Attempting Query: thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)", ["debug"]);
|
|
thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink);
|
|
addLogEntry("Query 'thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)' performed successfully", ["debug"]);
|
|
} catch (OneDriveException exception) {
|
|
// OneDrive threw an error
|
|
addLogEntry("------------------------------------------------------------------", ["debug"]);
|
|
addLogEntry("Query Error: thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink)", ["debug"]);
|
|
addLogEntry("driveId: " ~ driveId, ["debug"]);
|
|
addLogEntry("idToQuery: " ~ idToQuery, ["debug"]);
|
|
addLogEntry("nextLink: " ~ nextLink, ["debug"]);
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(queryChildrenOneDriveApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to query OneDrive drive item children - retrying applicable request in 30 seconds");
|
|
addLogEntry("thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink) previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
queryThisLevelChildren(driveId, idToQuery, nextLink, queryChildrenOneDriveApiInstance);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
|
|
}
|
|
}
|
|
|
|
// return response
|
|
return thisLevelChildren;
|
|
}
|
|
|
|
// Traverses the provided path online, via the OneDrive API, following correct parent driveId and itemId elements across the account
|
|
// to find if this full path exists. If this path exists online, the last item in the object path will be returned as a full JSON item.
|
|
//
|
|
// If the createPathIfMissing = false + no path exists online, a null invalid JSON item will be returned.
|
|
// If the createPathIfMissing = true + no path exists online, the requested path will be created in the correct location online. The resulting
|
|
// response to the directory creation will then be returned.
|
|
//
|
|
// This function also ensures that each path in the requested path actually matches the requested element to ensure that the OneDrive API response
|
|
// is not falsely matching a 'case insensitive' match to the actual request which is a POSIX compliance issue.
|
|
JSONValue queryOneDriveForSpecificPathAndCreateIfMissing(string thisNewPathToSearch, bool createPathIfMissing) {
|
|
|
|
// function variables
|
|
JSONValue getPathDetailsAPIResponse;
|
|
string currentPathTree;
|
|
Item parentDetails;
|
|
JSONValue topLevelChildren;
|
|
string nextLink;
|
|
bool directoryFoundOnline = false;
|
|
bool posixIssue = false;
|
|
|
|
// Create a new API Instance for this thread and initialise it
|
|
OneDriveApi queryOneDriveForSpecificPath;
|
|
queryOneDriveForSpecificPath = new OneDriveApi(appConfig);
|
|
queryOneDriveForSpecificPath.initialise();
|
|
|
|
foreach (thisFolderName; pathSplitter(thisNewPathToSearch)) {
|
|
addLogEntry("Testing for the existance online of this folder path: " ~ thisFolderName, ["debug"]);
|
|
directoryFoundOnline = false;
|
|
|
|
// If this is '.' this is the account root
|
|
if (thisFolderName == ".") {
|
|
currentPathTree = thisFolderName;
|
|
} else {
|
|
currentPathTree = currentPathTree ~ "/" ~ thisFolderName;
|
|
}
|
|
|
|
addLogEntry("Attempting to query OneDrive for this path: " ~ currentPathTree, ["debug"]);
|
|
|
|
// What query do we use?
|
|
if (thisFolderName == ".") {
|
|
// Query the root, set the right details
|
|
try {
|
|
getPathDetailsAPIResponse = queryOneDriveForSpecificPath.getPathDetails(currentPathTree);
|
|
parentDetails = makeItem(getPathDetailsAPIResponse);
|
|
// Save item to the database
|
|
saveItem(getPathDetailsAPIResponse);
|
|
directoryFoundOnline = true;
|
|
} catch (OneDriveException exception) {
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(queryOneDriveForSpecificPath);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to query path on OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
queryOneDriveForSpecificPathAndCreateIfMissing(thisNewPathToSearch, createPathIfMissing);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
|
|
}
|
|
}
|
|
} else {
|
|
// Ensure we have a valid driveId to search here
|
|
if (parentDetails.driveId.empty) {
|
|
parentDetails.driveId = appConfig.defaultDriveId;
|
|
}
|
|
|
|
// If the prior JSON 'getPathDetailsAPIResponse' is on this account driveId .. then continue to use getPathDetails
|
|
if (parentDetails.driveId == appConfig.defaultDriveId) {
|
|
|
|
try {
|
|
// Query OneDrive API for this path
|
|
getPathDetailsAPIResponse = queryOneDriveForSpecificPath.getPathDetails(currentPathTree);
|
|
|
|
// Portable Operating System Interface (POSIX) testing of JSON response from OneDrive API
|
|
if (hasName(getPathDetailsAPIResponse)) {
|
|
performPosixTest(thisFolderName, getPathDetailsAPIResponse["name"].str);
|
|
} else {
|
|
throw new jsonResponseException("Unable to perform POSIX test as the OneDrive API request generated an invalid JSON response");
|
|
}
|
|
|
|
// No POSIX issue with requested path element
|
|
parentDetails = makeItem(getPathDetailsAPIResponse);
|
|
// Save item to the database
|
|
saveItem(getPathDetailsAPIResponse);
|
|
directoryFoundOnline = true;
|
|
|
|
// Is this JSON a remote object
|
|
addLogEntry("Testing if this is a remote Shared Folder", ["debug"]);
|
|
if (isItemRemote(getPathDetailsAPIResponse)) {
|
|
// Remote Directory .. need a DB Tie Item
|
|
addLogEntry("Creating a DB Tie for this Shared Folder", ["debug"]);
|
|
|
|
// New DB Tie Item to bind the 'remote' path to our parent path
|
|
Item tieDBItem;
|
|
// Set the name
|
|
tieDBItem.name = parentDetails.name;
|
|
// Set the correct item type
|
|
tieDBItem.type = ItemType.dir;
|
|
// Set the right elements using the 'remote' of the parent as the 'actual' for this DB Tie
|
|
tieDBItem.driveId = parentDetails.remoteDriveId;
|
|
tieDBItem.id = parentDetails.remoteId;
|
|
// Set the correct mtime
|
|
tieDBItem.mtime = parentDetails.mtime;
|
|
// Add tie DB record to the local database
|
|
addLogEntry("Adding DB Tie record to database: " ~ to!string(tieDBItem), ["debug"]);
|
|
itemDB.upsert(tieDBItem);
|
|
// Update parentDetails to use the DB Tie record
|
|
parentDetails = tieDBItem;
|
|
}
|
|
} catch (OneDriveException exception) {
|
|
if (exception.httpStatusCode == 404) {
|
|
directoryFoundOnline = false;
|
|
} else {
|
|
|
|
string thisFunctionName = getFunctionName!({});
|
|
// HTTP request returned status code 408,429,503,504
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// Handle the 429
|
|
if (exception.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(queryOneDriveForSpecificPath);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]);
|
|
}
|
|
// re-try the specific changes queries
|
|
if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) {
|
|
// 408 - Request Time Out
|
|
// 503 - Service Unavailable
|
|
// 504 - Gateway Timeout
|
|
// Transient error - try again in 30 seconds
|
|
auto errorArray = splitLines(exception.msg);
|
|
addLogEntry(to!string(errorArray[0]) ~ " when attempting to query path on OneDrive - retrying applicable request in 30 seconds");
|
|
addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429, 503, 504 - but loop back calling this function
|
|
addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]);
|
|
queryOneDriveForSpecificPathAndCreateIfMissing(thisNewPathToSearch, createPathIfMissing);
|
|
} else {
|
|
// Default operation if not 408,429,503,504 errors
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, thisFunctionName);
|
|
}
|
|
}
|
|
} catch (jsonResponseException e) {
|
|
addLogEntry(e.msg, ["debug"]);
|
|
}
|
|
} else {
|
|
// parentDetails.driveId is not the account drive id - thus will be a remote shared item
|
|
addLogEntry("This parent directory is a remote object this next path will be on a remote drive", ["debug"]);
|
|
|
|
// For this parentDetails.driveId, parentDetails.id object, query the OneDrive API for it's children
|
|
for (;;) {
|
|
// Query this remote object for its children
|
|
topLevelChildren = queryOneDriveForSpecificPath.listChildren(parentDetails.driveId, parentDetails.id, nextLink);
|
|
// Process each child
|
|
foreach (child; topLevelChildren["value"].array) {
|
|
// Is this child a folder?
|
|
if (isItemFolder(child)) {
|
|
// Is this the child folder we are looking for, and is a POSIX match?
|
|
if (child["name"].str == thisFolderName) {
|
|
// EXACT MATCH including case sensitivity: Flag that we found the folder online
|
|
directoryFoundOnline = true;
|
|
// Use these details for the next entry path
|
|
getPathDetailsAPIResponse = child;
|
|
parentDetails = makeItem(getPathDetailsAPIResponse);
|
|
// Save item to the database
|
|
saveItem(getPathDetailsAPIResponse);
|
|
// No need to continue searching
|
|
break;
|
|
} else {
|
|
string childAsLower = toLower(child["name"].str);
|
|
string thisFolderNameAsLower = toLower(thisFolderName);
|
|
if (childAsLower == thisFolderNameAsLower) {
|
|
// This is a POSIX 'case in-sensitive match' .....
|
|
// Local item name has a 'case-insensitive match' to an existing item on OneDrive
|
|
posixIssue = true;
|
|
throw new posixException(thisFolderName, child["name"].str);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (directoryFoundOnline) {
|
|
// We found the folder, no need to continue searching nextLink data
|
|
break;
|
|
}
|
|
|
|
// If a collection exceeds the default page size (200 items), the @odata.nextLink property is returned in the response
|
|
// to indicate more items are available and provide the request URL for the next page of items.
|
|
if ("@odata.nextLink" in topLevelChildren) {
|
|
// Update nextLink to next changeSet bundle
|
|
addLogEntry("Setting nextLink to (@odata.nextLink): " ~ nextLink, ["debug"]);
|
|
nextLink = topLevelChildren["@odata.nextLink"].str;
|
|
} else break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// If we did not find the folder, we need to create this folder
|
|
if (!directoryFoundOnline) {
|
|
// Folder not found online
|
|
// Set any response to be an invalid JSON item
|
|
getPathDetailsAPIResponse = null;
|
|
// Was there a POSIX issue?
|
|
if (!posixIssue) {
|
|
// No POSIX issue
|
|
if (createPathIfMissing) {
|
|
// Create this path as it is missing on OneDrive online and there is no POSIX issue with a 'case-insensitive match'
|
|
addLogEntry("FOLDER NOT FOUND ONLINE AND WE ARE REQUESTED TO CREATE IT", ["debug"]);
|
|
addLogEntry("Create folder on this drive: " ~ parentDetails.driveId, ["debug"]);
|
|
addLogEntry("Create folder as a child on this object: " ~ parentDetails.id, ["debug"]);
|
|
addLogEntry("Create this folder name: " ~ thisFolderName, ["debug"]);
|
|
|
|
// Generate the JSON needed to create the folder online
|
|
JSONValue newDriveItem = [
|
|
"name": JSONValue(thisFolderName),
|
|
"folder": parseJSON("{}")
|
|
];
|
|
|
|
JSONValue createByIdAPIResponse;
|
|
// Submit the creation request
|
|
// Fix for https://github.com/skilion/onedrive/issues/356
|
|
if (!dryRun) {
|
|
try {
|
|
// Attempt to create a new folder on the configured parent driveId & parent id
|
|
createByIdAPIResponse = queryOneDriveForSpecificPath.createById(parentDetails.driveId, parentDetails.id, newDriveItem);
|
|
// Is the response a valid JSON object - validation checking done in saveItem
|
|
saveItem(createByIdAPIResponse);
|
|
// Set getPathDetailsAPIResponse to createByIdAPIResponse
|
|
getPathDetailsAPIResponse = createByIdAPIResponse;
|
|
} catch (OneDriveException e) {
|
|
// 409 - API Race Condition
|
|
if (e.httpStatusCode == 409) {
|
|
// When we attempted to create it, OneDrive responded that it now already exists
|
|
addLogEntry("OneDrive reported that " ~ thisFolderName ~ " already exists .. OneDrive API race condition", ["verbose"]);
|
|
} else {
|
|
// some other error from OneDrive was returned - display what it is
|
|
addLogEntry("OneDrive generated an error when creating this path: " ~ thisFolderName);
|
|
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
}
|
|
} else {
|
|
// Simulate a successful 'directory create' & save it to the dryRun database copy
|
|
// The simulated response has to pass 'makeItem' as part of saveItem
|
|
auto fakeResponse = createFakeResponse(thisNewPathToSearch);
|
|
// Save item to the database
|
|
saveItem(fakeResponse);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Shutdown API instance
|
|
queryOneDriveForSpecificPath.shutdown();
|
|
// Free object and memory
|
|
object.destroy(queryOneDriveForSpecificPath);
|
|
|
|
// Output our search results
|
|
addLogEntry("queryOneDriveForSpecificPathAndCreateIfMissing.getPathDetailsAPIResponse = " ~ to!string(getPathDetailsAPIResponse), ["debug"]);
|
|
return getPathDetailsAPIResponse;
|
|
}
|
|
|
|
// Delete an item by it's path
|
|
// This function is only used in --monitor mode and --remove-directory directive
|
|
void deleteByPath(string path) {
|
|
|
|
// function variables
|
|
Item dbItem;
|
|
|
|
// Need to check all driveid's we know about, not just the defaultDriveId
|
|
bool itemInDB = false;
|
|
foreach (searchDriveId; driveIDsArray) {
|
|
if (itemDB.selectByPath(path, searchDriveId, dbItem)) {
|
|
// item was found in the DB
|
|
itemInDB = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Was the item found in the database?
|
|
if (!itemInDB) {
|
|
// path to delete is not in the local database ..
|
|
// was this a --remove-directory attempt?
|
|
if (!appConfig.getValueBool("monitor")) {
|
|
// --remove-directory deletion attempt
|
|
addLogEntry("The item to delete is not in the local database - unable to delete online");
|
|
return;
|
|
} else {
|
|
// normal use .. --monitor being used
|
|
throw new SyncException("The item to delete is not in the local database");
|
|
}
|
|
}
|
|
|
|
// This needs to be enforced as we have to know the parent id of the object being deleted
|
|
if (dbItem.parentId == null) {
|
|
// the item is a remote folder, need to do the operation on the parent
|
|
enforce(itemDB.selectByPathIncludingRemoteItems(path, appConfig.defaultDriveId, dbItem));
|
|
}
|
|
|
|
try {
|
|
if (noRemoteDelete) {
|
|
// do not process remote delete
|
|
addLogEntry("Skipping remote delete as --upload-only & --no-remote-delete configured", ["verbose"]);
|
|
} else {
|
|
uploadDeletedItem(dbItem, path);
|
|
}
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 404) {
|
|
addLogEntry(e.msg);
|
|
} else {
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
|
}
|
|
}
|
|
}
|
|
|
|
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_move
|
|
// This function is only called in monitor mode when an move event is coming from
|
|
// inotify and we try to move the item.
|
|
void uploadMoveItem(string oldPath, string newPath) {
|
|
// Log that we are doing a move
|
|
addLogEntry("Moving " ~ oldPath ~ " to " ~ newPath);
|
|
// Is this move unwanted?
|
|
bool unwanted = false;
|
|
// Item variables
|
|
Item oldItem, newItem, parentItem;
|
|
|
|
// This not a Client Side Filtering check, nor a Microsoft Check, but is a sanity check that the path provided is UTF encoded correctly
|
|
// Check the std.encoding of the path against: Unicode 5.0, ASCII, ISO-8859-1, ISO-8859-2, WINDOWS-1250, WINDOWS-1251, WINDOWS-1252
|
|
if (!unwanted) {
|
|
if(!isValid(newPath)) {
|
|
// Path is not valid according to https://dlang.org/phobos/std_encoding.html
|
|
addLogEntry("Skipping item - invalid character encoding sequence: " ~ newPath, ["info", "notify"]);
|
|
unwanted = true;
|
|
}
|
|
}
|
|
|
|
// Check this path against the Client Side Filtering Rules
|
|
// - check_nosync
|
|
// - skip_dotfiles
|
|
// - skip_symlinks
|
|
// - skip_file
|
|
// - skip_dir
|
|
// - sync_list
|
|
// - skip_size
|
|
if (!unwanted) {
|
|
unwanted = checkPathAgainstClientSideFiltering(newPath);
|
|
}
|
|
|
|
// Check this path against the Microsoft Naming Conventions & Restristions
|
|
// - Check path against Microsoft OneDrive restriction and limitations about Windows naming for files and folders
|
|
// - Check path for bad whitespace items
|
|
// - Check path for HTML ASCII Codes
|
|
// - Check path for ASCII Control Codes
|
|
if (!unwanted) {
|
|
unwanted = checkPathAgainstMicrosoftNamingRestrictions(newPath);
|
|
}
|
|
|
|
// 'newPath' has passed client side filtering validation
|
|
if (!unwanted) {
|
|
|
|
if (!itemDB.selectByPath(oldPath, appConfig.defaultDriveId, oldItem)) {
|
|
// The old path|item is not synced with the database, upload as a new file
|
|
addLogEntry("Moved local item was not in-sync with local databse - uploading as new item");
|
|
uploadNewFile(newPath);
|
|
return;
|
|
}
|
|
|
|
if (oldItem.parentId == null) {
|
|
// the item is a remote folder, need to do the operation on the parent
|
|
enforce(itemDB.selectByPathIncludingRemoteItems(oldPath, appConfig.defaultDriveId, oldItem));
|
|
}
|
|
|
|
if (itemDB.selectByPath(newPath, appConfig.defaultDriveId, newItem)) {
|
|
// the destination has been overwritten
|
|
addLogEntry("Moved local item overwrote an existing item - deleting old online item");
|
|
uploadDeletedItem(newItem, newPath);
|
|
}
|
|
|
|
if (!itemDB.selectByPath(dirName(newPath), appConfig.defaultDriveId, parentItem)) {
|
|
// the parent item is not in the database
|
|
throw new SyncException("Can't move an item to an unsynced directory");
|
|
}
|
|
|
|
if (oldItem.driveId != parentItem.driveId) {
|
|
// items cannot be moved between drives
|
|
uploadDeletedItem(oldItem, oldPath);
|
|
|
|
// what sort of move is this?
|
|
if (isFile(newPath)) {
|
|
// newPath is a file
|
|
uploadNewFile(newPath);
|
|
} else {
|
|
// newPath is a directory
|
|
scanLocalFilesystemPathForNewData(newPath);
|
|
}
|
|
} else {
|
|
if (!exists(newPath)) {
|
|
// is this --monitor use?
|
|
if (appConfig.getValueBool("monitor")) {
|
|
addLogEntry("uploadMoveItem target has disappeared: " ~ newPath, ["verbose"]);
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Configure the modification JSON item
|
|
SysTime mtime;
|
|
if (appConfig.getValueBool("monitor")) {
|
|
// Use the newPath modified timestamp
|
|
mtime = timeLastModified(newPath).toUTC();
|
|
} else {
|
|
// Use the current system time
|
|
mtime = Clock.currTime().toUTC();
|
|
}
|
|
|
|
JSONValue data = [
|
|
"name": JSONValue(baseName(newPath)),
|
|
"parentReference": JSONValue([
|
|
"id": parentItem.id
|
|
]),
|
|
"fileSystemInfo": JSONValue([
|
|
"lastModifiedDateTime": mtime.toISOExtString()
|
|
])
|
|
];
|
|
|
|
// Perform the move operation on OneDrive
|
|
JSONValue response;
|
|
|
|
// Create a new API Instance for this thread and initialise it
|
|
OneDriveApi movePathOnlineApiInstance;
|
|
movePathOnlineApiInstance = new OneDriveApi(appConfig);
|
|
movePathOnlineApiInstance.initialise();
|
|
|
|
try {
|
|
response = movePathOnlineApiInstance.updateById(oldItem.driveId, oldItem.id, data, oldItem.eTag);
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 412) {
|
|
// OneDrive threw a 412 error, most likely: ETag does not match current item's value
|
|
// Retry without eTag
|
|
addLogEntry("File Move Failed - OneDrive eTag / cTag match issue", ["debug"]);
|
|
addLogEntry("OneDrive returned a 'HTTP 412 - Precondition Failed' when attempting to move the file - gracefully handling error", ["verbose"]);
|
|
string nullTag = null;
|
|
// move the file but without the eTag
|
|
response = movePathOnlineApiInstance.updateById(oldItem.driveId, oldItem.id, data, nullTag);
|
|
}
|
|
}
|
|
// Shutdown API instance
|
|
movePathOnlineApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(movePathOnlineApiInstance);
|
|
|
|
// save the move response from OneDrive in the database
|
|
// Is the response a valid JSON object - validation checking done in saveItem
|
|
saveItem(response);
|
|
}
|
|
} else {
|
|
// Moved item is unwanted
|
|
addLogEntry("Item has been moved to a location that is excluded from sync operations. Removing item from OneDrive");
|
|
uploadDeletedItem(oldItem, oldPath);
|
|
}
|
|
}
|
|
|
|
// Perform integrity validation of the file that was uploaded
|
|
bool performUploadIntegrityValidationChecks(JSONValue uploadResponse, string localFilePath, ulong localFileSize) {
|
|
|
|
bool integrityValid = false;
|
|
|
|
if (!disableUploadValidation) {
|
|
// Integrity validation has not been disabled (this is the default so we are always integrity checking our uploads)
|
|
if (uploadResponse.type() == JSONType.object) {
|
|
// Provided JSON is a valid JSON
|
|
ulong uploadFileSize = uploadResponse["size"].integer;
|
|
string uploadFileHash = uploadResponse["file"]["hashes"]["quickXorHash"].str;
|
|
string localFileHash = computeQuickXorHash(localFilePath);
|
|
|
|
if ((localFileSize == uploadFileSize) && (localFileHash == uploadFileHash)) {
|
|
// Uploaded file integrity intact
|
|
addLogEntry("Uploaded local file matches reported online size and hash values", ["debug"]);
|
|
integrityValid = true;
|
|
} else {
|
|
// Upload integrity failure .. what failed?
|
|
// There are 2 scenarios where this happens:
|
|
// 1. Failed Transfer
|
|
// 2. Upload file is going to a SharePoint Site, where Microsoft enriches the file with additional metadata with no way to disable
|
|
addLogEntry("WARNING: Uploaded file integrity failure for: " ~ localFilePath, ["info", "notify"]);
|
|
|
|
// What integrity failed - size?
|
|
if (localFileSize != uploadFileSize) {
|
|
addLogEntry("WARNING: Uploaded file integrity failure - Size Mismatch", ["verbose"]);
|
|
}
|
|
// What integrity failed - hash?
|
|
if (localFileHash != uploadFileHash) {
|
|
addLogEntry("WARNING: Uploaded file integrity failure - Hash Mismatch", ["verbose"]);
|
|
}
|
|
|
|
// What account type is this?
|
|
if (appConfig.accountType != "personal") {
|
|
// Not a personal account, thus the integrity failure is most likely due to SharePoint
|
|
addLogEntry("CAUTION: Microsoft OneDrive when using SharePoint as a backend enhances files after you upload them, which means this file may now have technical differences from your local copy, resulting in a data integrity issue.", ["verbose"]);
|
|
addLogEntry("See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details", ["verbose"]);
|
|
}
|
|
// How can this be disabled?
|
|
addLogEntry("To disable the integrity checking of uploaded files use --disable-upload-validation");
|
|
}
|
|
} else {
|
|
addLogEntry("Upload file validation unable to be performed: input JSON was invalid");
|
|
addLogEntry("WARNING: Skipping upload integrity check for: " ~ localFilePath);
|
|
}
|
|
} else {
|
|
// We are bypassing integrity checks due to --disable-upload-validation
|
|
addLogEntry("Upload file validation disabled due to --disable-upload-validation", ["debug"]);
|
|
addLogEntry("WARNING: Skipping upload integrity check for: " ~ localFilePath, ["info", "notify"]);
|
|
}
|
|
|
|
// Is the file integrity online valid?
|
|
return integrityValid;
|
|
}
|
|
|
|
// Query Office 365 SharePoint Shared Library site name to obtain it's Drive ID
|
|
void querySiteCollectionForDriveID(string sharepointLibraryNameToQuery) {
|
|
// Steps to get the ID:
|
|
// 1. Query https://graph.microsoft.com/v1.0/sites?search= with the name entered
|
|
// 2. Evaluate the response. A valid response will contain the description and the id. If the response comes back with nothing, the site name cannot be found or no access
|
|
// 3. If valid, use the returned ID and query the site drives
|
|
// https://graph.microsoft.com/v1.0/sites/<site_id>/drives
|
|
// 4. Display Shared Library Name & Drive ID
|
|
|
|
string site_id;
|
|
string drive_id;
|
|
bool found = false;
|
|
JSONValue siteQuery;
|
|
string nextLink;
|
|
string[] siteSearchResults;
|
|
|
|
// Create a new API Instance for this thread and initialise it
|
|
OneDriveApi querySharePointLibraryNameApiInstance;
|
|
querySharePointLibraryNameApiInstance = new OneDriveApi(appConfig);
|
|
querySharePointLibraryNameApiInstance.initialise();
|
|
|
|
// The account type must not be a personal account type
|
|
if (appConfig.accountType == "personal") {
|
|
addLogEntry("ERROR: A OneDrive Personal Account cannot be used with --get-sharepoint-drive-id. Please re-authenticate your client using a OneDrive Business Account.");
|
|
return;
|
|
}
|
|
|
|
// What query are we performing?
|
|
addLogEntry();
|
|
addLogEntry("Office 365 Library Name Query: " ~ sharepointLibraryNameToQuery);
|
|
|
|
for (;;) {
|
|
try {
|
|
siteQuery = querySharePointLibraryNameApiInstance.o365SiteSearch(nextLink);
|
|
} catch (OneDriveException e) {
|
|
addLogEntry("ERROR: Query of OneDrive for Office 365 Library Name failed");
|
|
// Forbidden - most likely authentication scope needs to be updated
|
|
if (e.httpStatusCode == 403) {
|
|
addLogEntry("ERROR: Authentication scope needs to be updated. Use --reauth and re-authenticate client.");
|
|
return;
|
|
}
|
|
// Requested resource cannot be found
|
|
if (e.httpStatusCode == 404) {
|
|
string siteSearchUrl;
|
|
if (nextLink.empty) {
|
|
siteSearchUrl = querySharePointLibraryNameApiInstance.getSiteSearchUrl();
|
|
} else {
|
|
siteSearchUrl = nextLink;
|
|
}
|
|
// log the error
|
|
addLogEntry("ERROR: Your OneDrive Account and Authentication Scope cannot access this OneDrive API: " ~ siteSearchUrl);
|
|
addLogEntry("ERROR: To resolve, please discuss this issue with whomever supports your OneDrive and SharePoint environment.");
|
|
return;
|
|
}
|
|
// HTTP request returned status code 429 (Too Many Requests)
|
|
if (e.httpStatusCode == 429) {
|
|
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
|
handleOneDriveThrottleRequest(querySharePointLibraryNameApiInstance);
|
|
addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children", ["debug"]);
|
|
}
|
|
// HTTP request returned status code 504 (Gateway Timeout) or 429 retry
|
|
if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) {
|
|
// re-try the specific changes queries
|
|
if (e.httpStatusCode == 504) {
|
|
addLogEntry("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query Sharepoint Sites - retrying applicable request");
|
|
addLogEntry("siteQuery = onedrive.o365SiteSearch(nextLink) previously threw an error - retrying", ["debug"]);
|
|
|
|
// The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request.
|
|
addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]);
|
|
Thread.sleep(dur!"seconds"(30));
|
|
}
|
|
// re-try original request - retried for 429 and 504
|
|
try {
|
|
addLogEntry("Retrying Query: siteQuery = onedrive.o365SiteSearch(nextLink)", ["debug"]);
|
|
siteQuery = querySharePointLibraryNameApiInstance.o365SiteSearch(nextLink);
|
|
addLogEntry("Query 'siteQuery = onedrive.o365SiteSearch(nextLink)' performed successfully on re-try", ["debug"]);
|
|
} catch (OneDriveException e) {
|
|
// display what the error is
|
|
addLogEntry("Query Error: siteQuery = onedrive.o365SiteSearch(nextLink) on re-try after delay", ["debug"]);
|
|
// error was not a 504 this time
|
|
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
|
return;
|
|
}
|
|
} else {
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
|
return;
|
|
}
|
|
}
|
|
|
|
// is siteQuery a valid JSON object & contain data we can use?
|
|
if ((siteQuery.type() == JSONType.object) && ("value" in siteQuery)) {
|
|
// valid JSON object
|
|
addLogEntry("O365 Query Response: " ~ to!string(siteQuery), ["debug"]);
|
|
|
|
foreach (searchResult; siteQuery["value"].array) {
|
|
// Need an 'exclusive' match here with sharepointLibraryNameToQuery as entered
|
|
addLogEntry("Found O365 Site: " ~ to!string(searchResult), ["debug"]);
|
|
|
|
// 'displayName' and 'id' have to be present in the search result record in order to query the site
|
|
if (("displayName" in searchResult) && ("id" in searchResult)) {
|
|
if (sharepointLibraryNameToQuery == searchResult["displayName"].str){
|
|
// 'displayName' matches search request
|
|
site_id = searchResult["id"].str;
|
|
JSONValue siteDriveQuery;
|
|
|
|
try {
|
|
siteDriveQuery = querySharePointLibraryNameApiInstance.o365SiteDrives(site_id);
|
|
} catch (OneDriveException e) {
|
|
addLogEntry("ERROR: Query of OneDrive for Office Site ID failed");
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
|
|
return;
|
|
}
|
|
|
|
// is siteDriveQuery a valid JSON object & contain data we can use?
|
|
if ((siteDriveQuery.type() == JSONType.object) && ("value" in siteDriveQuery)) {
|
|
// valid JSON object
|
|
foreach (driveResult; siteDriveQuery["value"].array) {
|
|
// Display results
|
|
writeln("-----------------------------------------------");
|
|
addLogEntry("Site Details: " ~ to!string(driveResult), ["debug"]);
|
|
found = true;
|
|
writeln("Site Name: ", searchResult["displayName"].str);
|
|
writeln("Library Name: ", driveResult["name"].str);
|
|
writeln("drive_id: ", driveResult["id"].str);
|
|
writeln("Library URL: ", driveResult["webUrl"].str);
|
|
}
|
|
// closeout
|
|
writeln("-----------------------------------------------");
|
|
} else {
|
|
// not a valid JSON object
|
|
addLogEntry("ERROR: There was an error performing this operation on Microsoft OneDrive");
|
|
addLogEntry("ERROR: Increase logging verbosity to assist determining why.");
|
|
return;
|
|
}
|
|
}
|
|
} else {
|
|
// 'displayName', 'id' or ''webUrl' not present in JSON results for a specific site
|
|
string siteNameAvailable = "Site 'name' was restricted by OneDrive API permissions";
|
|
bool displayNameAvailable = false;
|
|
bool idAvailable = false;
|
|
if ("name" in searchResult) siteNameAvailable = searchResult["name"].str;
|
|
if ("displayName" in searchResult) displayNameAvailable = true;
|
|
if ("id" in searchResult) idAvailable = true;
|
|
|
|
// Display error details for this site data
|
|
addLogEntry();
|
|
addLogEntry("ERROR: SharePoint Site details not provided for: " ~ siteNameAvailable);
|
|
addLogEntry("ERROR: The SharePoint Site results returned from OneDrive API do not contain the required items to match. Please check your permissions with your site administrator.");
|
|
addLogEntry("ERROR: Your site security settings is preventing the following details from being accessed: 'displayName' or 'id'");
|
|
addLogEntry(" - Is 'displayName' available = " ~ to!string(displayNameAvailable), ["verbose"]);
|
|
addLogEntry(" - Is 'id' available = " ~ to!string(idAvailable), ["verbose"]);
|
|
addLogEntry("ERROR: To debug this further, please increase verbosity (--verbose or --verbose --verbose) to provide further insight as to what details are actually being returned.");
|
|
}
|
|
}
|
|
|
|
if(!found) {
|
|
// The SharePoint site we are searching for was not found in this bundle set
|
|
// Add to siteSearchResults so we can display what we did find
|
|
string siteSearchResultsEntry;
|
|
foreach (searchResult; siteQuery["value"].array) {
|
|
// We can only add the displayName if it is available
|
|
if ("displayName" in searchResult) {
|
|
// Use the displayName
|
|
siteSearchResultsEntry = " * " ~ searchResult["displayName"].str;
|
|
siteSearchResults ~= siteSearchResultsEntry;
|
|
} else {
|
|
// Add, but indicate displayName unavailable, use id
|
|
if ("id" in searchResult) {
|
|
siteSearchResultsEntry = " * " ~ "Unknown displayName (Data not provided by API), Site ID: " ~ searchResult["id"].str;
|
|
siteSearchResults ~= siteSearchResultsEntry;
|
|
} else {
|
|
// displayName and id unavailable, display in debug log the entry
|
|
addLogEntry("Bad SharePoint Data for site: " ~ to!string(searchResult), ["debug"]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// not a valid JSON object
|
|
addLogEntry("ERROR: There was an error performing this operation on Microsoft OneDrive");
|
|
addLogEntry("ERROR: Increase logging verbosity to assist determining why.");
|
|
return;
|
|
}
|
|
|
|
// If a collection exceeds the default page size (200 items), the @odata.nextLink property is returned in the response
|
|
// to indicate more items are available and provide the request URL for the next page of items.
|
|
if ("@odata.nextLink" in siteQuery) {
|
|
// Update nextLink to next set of SharePoint library names
|
|
nextLink = siteQuery["@odata.nextLink"].str;
|
|
addLogEntry("Setting nextLink to (@odata.nextLink): " ~ nextLink, ["debug"]);
|
|
} else break;
|
|
}
|
|
|
|
// Was the intended target found?
|
|
if(!found) {
|
|
|
|
// Was the search a wildcard?
|
|
if (sharepointLibraryNameToQuery != "*") {
|
|
// Only print this out if the search was not a wildcard
|
|
addLogEntry();
|
|
addLogEntry("ERROR: The requested SharePoint site could not be found. Please check it's name and your permissions to access the site.");
|
|
}
|
|
// List all sites returned to assist user
|
|
addLogEntry();
|
|
addLogEntry("The following SharePoint site names were returned:");
|
|
foreach (searchResultEntry; siteSearchResults) {
|
|
// list the display name that we use to match against the user query
|
|
addLogEntry(searchResultEntry);
|
|
}
|
|
}
|
|
|
|
// Shutdown API instance
|
|
querySharePointLibraryNameApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(querySharePointLibraryNameApiInstance);
|
|
}
|
|
|
|
// Query the sync status of the client and the local system
|
|
void queryOneDriveForSyncStatus(string pathToQueryStatusOn) {
|
|
|
|
// Query the account driveId and rootId to get the /delta JSON information
|
|
// Process that JSON data for relevancy
|
|
|
|
// Function variables
|
|
ulong downloadSize = 0;
|
|
string deltaLink = null;
|
|
string driveIdToQuery = appConfig.defaultDriveId;
|
|
string itemIdToQuery = appConfig.defaultRootId;
|
|
JSONValue deltaChanges;
|
|
|
|
// Array of JSON items
|
|
JSONValue[] jsonItemsArray;
|
|
|
|
// Query Database for a potential deltaLink starting point
|
|
deltaLink = itemDB.getDeltaLink(driveIdToQuery, itemIdToQuery);
|
|
|
|
// Log what we are doing
|
|
addLogEntry("Querying the change status of Drive ID: " ~ driveIdToQuery ~ " .", ["consoleOnlyNoNewLine"]);
|
|
|
|
// Query the OenDrive API using the applicable details, following nextLink if applicable
|
|
// Create a new API Instance for querying /delta and initialise it
|
|
OneDriveApi getDeltaQueryOneDriveApiInstance;
|
|
getDeltaQueryOneDriveApiInstance = new OneDriveApi(appConfig);
|
|
getDeltaQueryOneDriveApiInstance.initialise();
|
|
|
|
for (;;) {
|
|
// Add a processing '.'
|
|
addLogEntry(".", ["consoleOnlyNoNewLine"]);
|
|
|
|
// Get the /delta changes via the OneDrive API
|
|
// getDeltaChangesByItemId has the re-try logic for transient errors
|
|
deltaChanges = getDeltaChangesByItemId(driveIdToQuery, itemIdToQuery, deltaLink, getDeltaQueryOneDriveApiInstance);
|
|
|
|
// If the initial deltaChanges response is an invalid JSON object, keep trying ..
|
|
if (deltaChanges.type() != JSONType.object) {
|
|
while (deltaChanges.type() != JSONType.object) {
|
|
// Handle the invalid JSON response adn retry
|
|
addLogEntry("ERROR: Query of the OneDrive API via deltaChanges = getDeltaChangesByItemId() returned an invalid JSON response", ["debug"]);
|
|
deltaChanges = getDeltaChangesByItemId(driveIdToQuery, itemIdToQuery, deltaLink, getDeltaQueryOneDriveApiInstance);
|
|
}
|
|
}
|
|
|
|
// We have a valid deltaChanges JSON array. This means we have at least 200+ JSON items to process.
|
|
// The API response however cannot be run in parallel as the OneDrive API sends the JSON items in the order in which they must be processed
|
|
foreach (onedriveJSONItem; deltaChanges["value"].array) {
|
|
// is the JSON a root object - we dont want to count this
|
|
if (!isItemRoot(onedriveJSONItem)) {
|
|
// Files are the only item that we want to calculate
|
|
if (isItemFile(onedriveJSONItem)) {
|
|
// JSON item is a file
|
|
// Is the item filtered out due to client side filtering rules?
|
|
if (!checkJSONAgainstClientSideFiltering(onedriveJSONItem)) {
|
|
// Is the path of this JSON item 'in-scope' or 'out-of-scope' ?
|
|
if (pathToQueryStatusOn != "/") {
|
|
// We need to check the path of this item against pathToQueryStatusOn
|
|
string thisItemPath = "";
|
|
if (("path" in onedriveJSONItem["parentReference"]) != null) {
|
|
// If there is a parent reference path, try and use it
|
|
string selfBuiltPath = onedriveJSONItem["parentReference"]["path"].str ~ "/" ~ onedriveJSONItem["name"].str;
|
|
|
|
// Check for ':' and split if present
|
|
auto splitIndex = selfBuiltPath.indexOf(":");
|
|
if (splitIndex != -1) {
|
|
// Keep only the part after ':'
|
|
selfBuiltPath = selfBuiltPath[splitIndex + 1 .. $];
|
|
}
|
|
|
|
// Set thisItemPath to the self built path
|
|
thisItemPath = selfBuiltPath;
|
|
} else {
|
|
// no parent reference path available
|
|
thisItemPath = onedriveJSONItem["name"].str;
|
|
}
|
|
// can we find 'pathToQueryStatusOn' in 'thisItemPath' ?
|
|
if (canFind(thisItemPath, pathToQueryStatusOn)) {
|
|
// Add this to the array for processing
|
|
jsonItemsArray ~= onedriveJSONItem;
|
|
}
|
|
} else {
|
|
// We are not doing a --single-directory check
|
|
// Add this to the array for processing
|
|
jsonItemsArray ~= onedriveJSONItem;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// The response may contain either @odata.deltaLink or @odata.nextLink
|
|
if ("@odata.deltaLink" in deltaChanges) {
|
|
deltaLink = deltaChanges["@odata.deltaLink"].str;
|
|
addLogEntry("Setting next deltaLink to (@odata.deltaLink): " ~ deltaLink, ["debug"]);
|
|
}
|
|
|
|
// Update deltaLink to next changeSet bundle
|
|
if ("@odata.nextLink" in deltaChanges) {
|
|
deltaLink = deltaChanges["@odata.nextLink"].str;
|
|
addLogEntry("Setting next deltaLink to (@odata.nextLink): " ~ deltaLink, ["debug"]);
|
|
}
|
|
else break;
|
|
}
|
|
// Needed after printing out '....' when fetching changes from OneDrive API
|
|
addLogEntry("\n", ["consoleOnlyNoNewLine"]);
|
|
|
|
// Are there any JSON items to process?
|
|
if (count(jsonItemsArray) != 0) {
|
|
// There are items to process
|
|
foreach (onedriveJSONItem; jsonItemsArray.array) {
|
|
|
|
// variables we need
|
|
string thisItemParentDriveId;
|
|
string thisItemId;
|
|
string thisItemHash;
|
|
bool existingDBEntry = false;
|
|
|
|
// Is this file a remote item (on a shared folder) ?
|
|
if (isItemRemote(onedriveJSONItem)) {
|
|
// remote drive item
|
|
thisItemParentDriveId = onedriveJSONItem["remoteItem"]["parentReference"]["driveId"].str;
|
|
thisItemId = onedriveJSONItem["id"].str;
|
|
} else {
|
|
// standard drive item
|
|
thisItemParentDriveId = onedriveJSONItem["parentReference"]["driveId"].str;
|
|
thisItemId = onedriveJSONItem["id"].str;
|
|
}
|
|
|
|
// Get the file hash
|
|
if (hasHashes(onedriveJSONItem)) {
|
|
thisItemHash = onedriveJSONItem["file"]["hashes"]["quickXorHash"].str;
|
|
|
|
// Check if the item has been seen before
|
|
Item existingDatabaseItem;
|
|
existingDBEntry = itemDB.selectById(thisItemParentDriveId, thisItemId, existingDatabaseItem);
|
|
|
|
if (existingDBEntry) {
|
|
// item exists in database .. do the database details match the JSON record?
|
|
if (existingDatabaseItem.quickXorHash != thisItemHash) {
|
|
// file hash is different, this will trigger a download event
|
|
downloadSize = downloadSize + onedriveJSONItem["size"].integer;
|
|
}
|
|
} else {
|
|
// item does not exist in the database
|
|
// this item has already passed client side filtering rules (skip_dir, skip_file, sync_list)
|
|
// this will trigger a download event
|
|
downloadSize = downloadSize + onedriveJSONItem["size"].integer;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Was anything detected that would constitute a download?
|
|
if (downloadSize > 0) {
|
|
// we have something to download
|
|
if (pathToQueryStatusOn != "/") {
|
|
writeln("The selected local directory via --single-directory is out of sync with Microsoft OneDrive");
|
|
} else {
|
|
writeln("The configured local 'sync_dir' directory is out of sync with Microsoft OneDrive");
|
|
}
|
|
writeln("Approximate data to download from Microsoft OneDrive: ", (downloadSize/1024), " KB");
|
|
} else {
|
|
// No changes were returned
|
|
writeln("There are no pending changes from Microsoft OneDrive; your local directory matches the data online.");
|
|
}
|
|
}
|
|
|
|
// Query OneDrive for file details of a given path, returning either the 'webURL' or 'lastModifiedBy' JSON facet
|
|
void queryOneDriveForFileDetails(string inputFilePath, string runtimePath, string outputType) {
|
|
|
|
// Calculate the full local file path
|
|
string fullLocalFilePath = buildNormalizedPath(buildPath(runtimePath, inputFilePath));
|
|
|
|
// Query if file is valid locally
|
|
if (exists(fullLocalFilePath)) {
|
|
// search drive_id list
|
|
string[] distinctDriveIds = itemDB.selectDistinctDriveIds();
|
|
bool pathInDB = false;
|
|
Item dbItem;
|
|
|
|
foreach (searchDriveId; distinctDriveIds) {
|
|
// Does this path exist in the database, use the 'inputFilePath'
|
|
if (itemDB.selectByPath(inputFilePath, searchDriveId, dbItem)) {
|
|
// item is in the database
|
|
pathInDB = true;
|
|
JSONValue fileDetailsFromOneDrive;
|
|
|
|
// Create a new API Instance for this thread and initialise it
|
|
OneDriveApi queryOneDriveForFileDetailsApiInstance;
|
|
queryOneDriveForFileDetailsApiInstance = new OneDriveApi(appConfig);
|
|
queryOneDriveForFileDetailsApiInstance.initialise();
|
|
|
|
try {
|
|
fileDetailsFromOneDrive = queryOneDriveForFileDetailsApiInstance.getPathDetailsById(dbItem.driveId, dbItem.id);
|
|
} catch (OneDriveException exception) {
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
return;
|
|
}
|
|
|
|
// Is the API response a valid JSON file?
|
|
if (fileDetailsFromOneDrive.type() == JSONType.object) {
|
|
|
|
// debug output of response
|
|
addLogEntry("API Response: " ~ to!string(fileDetailsFromOneDrive), ["debug"]);
|
|
|
|
// What sort of response to we generate
|
|
// --get-file-link response
|
|
if (outputType == "URL") {
|
|
if ((fileDetailsFromOneDrive.type() == JSONType.object) && ("webUrl" in fileDetailsFromOneDrive)) {
|
|
// Valid JSON object
|
|
addLogEntry();
|
|
writeln("WebURL: ", fileDetailsFromOneDrive["webUrl"].str);
|
|
}
|
|
}
|
|
|
|
// --modified-by response
|
|
if (outputType == "ModifiedBy") {
|
|
if ((fileDetailsFromOneDrive.type() == JSONType.object) && ("lastModifiedBy" in fileDetailsFromOneDrive)) {
|
|
// Valid JSON object
|
|
writeln();
|
|
writeln("Last modified: ", fileDetailsFromOneDrive["lastModifiedDateTime"].str);
|
|
writeln("Last modified by: ", fileDetailsFromOneDrive["lastModifiedBy"]["user"]["displayName"].str);
|
|
// if 'email' provided, add this to the output
|
|
if ("email" in fileDetailsFromOneDrive["lastModifiedBy"]["user"]) {
|
|
writeln("Email Address: ", fileDetailsFromOneDrive["lastModifiedBy"]["user"]["email"].str);
|
|
}
|
|
}
|
|
}
|
|
|
|
// --create-share-link response
|
|
if (outputType == "ShareableLink") {
|
|
|
|
JSONValue accessScope;
|
|
JSONValue createShareableLinkResponse;
|
|
string thisDriveId = fileDetailsFromOneDrive["parentReference"]["driveId"].str;
|
|
string thisItemId = fileDetailsFromOneDrive["id"].str;
|
|
string fileShareLink;
|
|
bool writeablePermissions = appConfig.getValueBool("with_editing_perms");
|
|
|
|
// What sort of shareable link is required?
|
|
if (writeablePermissions) {
|
|
// configure the read-write access scope
|
|
accessScope = [
|
|
"type": "edit",
|
|
"scope": "anonymous"
|
|
];
|
|
} else {
|
|
// configure the read-only access scope (default)
|
|
accessScope = [
|
|
"type": "view",
|
|
"scope": "anonymous"
|
|
];
|
|
}
|
|
|
|
// Try and create the shareable file link
|
|
try {
|
|
createShareableLinkResponse = queryOneDriveForFileDetailsApiInstance.createShareableLink(thisDriveId, thisItemId, accessScope);
|
|
} catch (OneDriveException exception) {
|
|
// display what the error is
|
|
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
|
|
return;
|
|
}
|
|
|
|
// Is the API response a valid JSON file?
|
|
if ((createShareableLinkResponse.type() == JSONType.object) && ("link" in createShareableLinkResponse)) {
|
|
// Extract the file share link from the JSON response
|
|
fileShareLink = createShareableLinkResponse["link"]["webUrl"].str;
|
|
writeln("File Shareable Link: ", fileShareLink);
|
|
if (writeablePermissions) {
|
|
writeln("Shareable Link has read-write permissions - use and provide with caution");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Shutdown the API access
|
|
queryOneDriveForFileDetailsApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(queryOneDriveForFileDetailsApiInstance);
|
|
}
|
|
}
|
|
|
|
// was path found?
|
|
if (!pathInDB) {
|
|
// File has not been synced with OneDrive
|
|
addLogEntry("Selected path has not been synced with OneDrive: " ~ inputFilePath);
|
|
}
|
|
} else {
|
|
// File does not exist locally
|
|
addLogEntry("Selected path not found on local system: " ~ inputFilePath);
|
|
}
|
|
}
|
|
|
|
// Query OneDrive for the quota details
|
|
void queryOneDriveForQuotaDetails() {
|
|
// This function is similar to getRemainingFreeSpace() but is different in data being analysed and output method
|
|
JSONValue currentDriveQuota;
|
|
string driveId;
|
|
|
|
if (appConfig.getValueString("drive_id").length) {
|
|
driveId = appConfig.getValueString("drive_id");
|
|
} else {
|
|
driveId = appConfig.defaultDriveId;
|
|
}
|
|
|
|
try {
|
|
// Create a new OneDrive API instance
|
|
OneDriveApi getCurrentDriveQuotaApiInstance;
|
|
getCurrentDriveQuotaApiInstance = new OneDriveApi(appConfig);
|
|
getCurrentDriveQuotaApiInstance.initialise();
|
|
addLogEntry("Seeking available quota for this drive id: " ~ driveId, ["debug"]);
|
|
currentDriveQuota = getCurrentDriveQuotaApiInstance.getDriveQuota(driveId);
|
|
// Shut this API instance down
|
|
getCurrentDriveQuotaApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(getCurrentDriveQuotaApiInstance);
|
|
} catch (OneDriveException e) {
|
|
addLogEntry("currentDriveQuota = onedrive.getDriveQuota(driveId) generated a OneDriveException", ["debug"]);
|
|
}
|
|
|
|
// validate that currentDriveQuota is a JSON value
|
|
if (currentDriveQuota.type() == JSONType.object) {
|
|
// was 'quota' in response?
|
|
if ("quota" in currentDriveQuota) {
|
|
|
|
// debug output of response
|
|
addLogEntry("currentDriveQuota: " ~ to!string(currentDriveQuota), ["debug"]);
|
|
|
|
// human readable output of response
|
|
string deletedValue = "Not Provided";
|
|
string remainingValue = "Not Provided";
|
|
string stateValue = "Not Provided";
|
|
string totalValue = "Not Provided";
|
|
string usedValue = "Not Provided";
|
|
|
|
// Update values
|
|
if ("deleted" in currentDriveQuota["quota"]) {
|
|
deletedValue = byteToGibiByte(currentDriveQuota["quota"]["deleted"].integer);
|
|
}
|
|
|
|
if ("remaining" in currentDriveQuota["quota"]) {
|
|
remainingValue = byteToGibiByte(currentDriveQuota["quota"]["remaining"].integer);
|
|
}
|
|
|
|
if ("state" in currentDriveQuota["quota"]) {
|
|
stateValue = currentDriveQuota["quota"]["state"].str;
|
|
}
|
|
|
|
if ("total" in currentDriveQuota["quota"]) {
|
|
totalValue = byteToGibiByte(currentDriveQuota["quota"]["total"].integer);
|
|
}
|
|
|
|
if ("used" in currentDriveQuota["quota"]) {
|
|
usedValue = byteToGibiByte(currentDriveQuota["quota"]["used"].integer);
|
|
}
|
|
|
|
writeln("Microsoft OneDrive quota information as reported for this Drive ID: ", driveId);
|
|
writeln();
|
|
writeln("Deleted: ", deletedValue, " GB (", currentDriveQuota["quota"]["deleted"].integer, " bytes)");
|
|
writeln("Remaining: ", remainingValue, " GB (", currentDriveQuota["quota"]["remaining"].integer, " bytes)");
|
|
writeln("State: ", stateValue);
|
|
writeln("Total: ", totalValue, " GB (", currentDriveQuota["quota"]["total"].integer, " bytes)");
|
|
writeln("Used: ", usedValue, " GB (", currentDriveQuota["quota"]["used"].integer, " bytes)");
|
|
writeln();
|
|
} else {
|
|
writeln("Microsoft OneDrive quota information is being restricted for this Drive ID: ", driveId);
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
// Query the system for session_upload.* files
|
|
bool checkForInterruptedSessionUploads() {
|
|
|
|
bool interruptedUploads = false;
|
|
ulong interruptedUploadsCount;
|
|
|
|
// Scan the filesystem for the files we are interested in, build up interruptedUploadsSessionFiles array
|
|
foreach (sessionFile; dirEntries(appConfig.configDirName, "session_upload.*", SpanMode.shallow)) {
|
|
// calculate the full path
|
|
string tempPath = buildNormalizedPath(buildPath(appConfig.configDirName, sessionFile));
|
|
// add to array
|
|
interruptedUploadsSessionFiles ~= [tempPath];
|
|
}
|
|
|
|
// Count all 'session_upload' files in appConfig.configDirName
|
|
//interruptedUploadsCount = count(dirEntries(appConfig.configDirName, "session_upload.*", SpanMode.shallow));
|
|
interruptedUploadsCount = count(interruptedUploadsSessionFiles);
|
|
if (interruptedUploadsCount != 0) {
|
|
interruptedUploads = true;
|
|
}
|
|
|
|
// return if there are interrupted uploads to process
|
|
return interruptedUploads;
|
|
}
|
|
|
|
// Process interrupted 'session_upload' files
|
|
void processForInterruptedSessionUploads() {
|
|
// For each upload_session file that has been found, process the data to ensure it is still valid
|
|
foreach (sessionFilePath; interruptedUploadsSessionFiles) {
|
|
if (!validateUploadSessionFileData(sessionFilePath)) {
|
|
// Remove upload_session file as it is invalid
|
|
// upload_session file file contains an error - cant resume this session
|
|
addLogEntry("Restore file upload session failed - cleaning up resumable session data file: " ~ sessionFilePath, ["verbose"]);
|
|
|
|
// cleanup session path
|
|
if (exists(sessionFilePath)) {
|
|
if (!dryRun) {
|
|
remove(sessionFilePath);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// At this point we should have an array of JSON items to resume uploading
|
|
if (count(jsonItemsToResumeUpload) > 0) {
|
|
// there are valid items to resume upload
|
|
|
|
// Lets deal with all the JSON items that need to be reumed for upload in a batch process
|
|
ulong batchSize = appConfig.concurrentThreads;
|
|
ulong batchCount = (jsonItemsToResumeUpload.length + batchSize - 1) / batchSize;
|
|
ulong batchesProcessed = 0;
|
|
|
|
foreach (chunk; jsonItemsToResumeUpload.chunks(batchSize)) {
|
|
// send an array containing 'appConfig.concurrentThreads' (16) JSON items to resume upload
|
|
resumeSessionUploadsInParallel(chunk);
|
|
}
|
|
}
|
|
}
|
|
|
|
bool validateUploadSessionFileData(string sessionFilePath) {
|
|
|
|
JSONValue sessionFileData;
|
|
|
|
// Try and read the text from the session file as a JSON array
|
|
try {
|
|
sessionFileData = readText(sessionFilePath).parseJSON();
|
|
} catch (JSONException e) {
|
|
addLogEntry("SESSION-RESUME: Invalid JSON data in: " ~ sessionFilePath, ["debug"]);
|
|
return false;
|
|
}
|
|
|
|
// Does the file we wish to resume uploading exist locally still?
|
|
if ("localPath" in sessionFileData) {
|
|
string sessionLocalFilePath = sessionFileData["localPath"].str;
|
|
addLogEntry("SESSION-RESUME: sessionLocalFilePath: " ~ sessionLocalFilePath, ["debug"]);
|
|
|
|
// Does the file exist?
|
|
if (!exists(sessionLocalFilePath)) {
|
|
addLogEntry("The local file to upload does not exist locally anymore", ["verbose"]);
|
|
return false;
|
|
}
|
|
|
|
// Can we read the file?
|
|
if (!readLocalFile(sessionLocalFilePath)) {
|
|
// filesystem error already returned if unable to read
|
|
return false;
|
|
}
|
|
|
|
} else {
|
|
addLogEntry("SESSION-RESUME: No localPath data in: " ~ sessionFilePath, ["debug"]);
|
|
return false;
|
|
}
|
|
|
|
// Check the session data for expirationDateTime
|
|
if ("expirationDateTime" in sessionFileData) {
|
|
auto expiration = SysTime.fromISOExtString(sessionFileData["expirationDateTime"].str);
|
|
if (expiration < Clock.currTime()) {
|
|
addLogEntry("The upload session has expired for: " ~ sessionFilePath, ["verbose"]);
|
|
return false;
|
|
}
|
|
} else {
|
|
addLogEntry("SESSION-RESUME: No expirationDateTime data in: " ~ sessionFilePath, ["debug"]);
|
|
return false;
|
|
}
|
|
|
|
// Check the online upload status, using the uloadURL in sessionFileData
|
|
if ("uploadUrl" in sessionFileData) {
|
|
JSONValue response;
|
|
|
|
// Create a new OneDrive API instance
|
|
OneDriveApi validateUploadSessionFileDataApiInstance;
|
|
validateUploadSessionFileDataApiInstance = new OneDriveApi(appConfig);
|
|
validateUploadSessionFileDataApiInstance.initialise();
|
|
|
|
try {
|
|
response = validateUploadSessionFileDataApiInstance.requestUploadStatus(sessionFileData["uploadUrl"].str);
|
|
} catch (OneDriveException e) {
|
|
// handle any onedrive error response as invalid
|
|
addLogEntry("SESSION-RESUME: Invalid response when using uploadUrl in: " ~ sessionFilePath, ["debug"]);
|
|
return false;
|
|
}
|
|
|
|
// Shutdown API instance
|
|
validateUploadSessionFileDataApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(validateUploadSessionFileDataApiInstance);
|
|
|
|
// Do we have a valid response from OneDrive?
|
|
if (response.type() == JSONType.object) {
|
|
// Valid JSON object was returned
|
|
if (("expirationDateTime" in response) && ("nextExpectedRanges" in response)) {
|
|
// The 'uploadUrl' is valid, and the response contains elements we need
|
|
sessionFileData["expirationDateTime"] = response["expirationDateTime"];
|
|
sessionFileData["nextExpectedRanges"] = response["nextExpectedRanges"];
|
|
|
|
if (sessionFileData["nextExpectedRanges"].array.length == 0) {
|
|
addLogEntry("The upload session was already completed", ["verbose"]);
|
|
return false;
|
|
}
|
|
} else {
|
|
addLogEntry("SESSION-RESUME: No expirationDateTime & nextExpectedRanges data in Microsoft OneDrive API response: " ~ to!string(response), ["debug"]);
|
|
return false;
|
|
}
|
|
} else {
|
|
// not a JSON object
|
|
addLogEntry("Restore file upload session failed - invalid response from Microsoft OneDrive", ["verbose"]);
|
|
return false;
|
|
}
|
|
} else {
|
|
addLogEntry("SESSION-RESUME: No uploadUrl data in: " ~ sessionFilePath, ["debug"]);
|
|
return false;
|
|
}
|
|
|
|
// Add 'sessionFilePath' to 'sessionFileData' so that it can be used when we re-use the JSON data to resume the upload
|
|
sessionFileData["sessionFilePath"] = sessionFilePath;
|
|
|
|
// Add sessionFileData to jsonItemsToResumeUpload as it is now valid
|
|
jsonItemsToResumeUpload ~= sessionFileData;
|
|
return true;
|
|
}
|
|
|
|
void resumeSessionUploadsInParallel(JSONValue[] array) {
|
|
// This function recieved an array of 16 JSON items to resume upload
|
|
foreach (i, jsonItemToResume; taskPool.parallel(array)) {
|
|
// Take each JSON item and resume upload using the JSON data
|
|
|
|
JSONValue uploadResponse;
|
|
OneDriveApi uploadFileOneDriveApiInstance;
|
|
uploadFileOneDriveApiInstance = new OneDriveApi(appConfig);
|
|
uploadFileOneDriveApiInstance.initialise();
|
|
|
|
// Pull out data from this JSON element
|
|
string threadUploadSessionFilePath = jsonItemToResume["sessionFilePath"].str;
|
|
ulong thisFileSizeLocal = getSize(jsonItemToResume["localPath"].str);
|
|
|
|
// Try to resume the session upload using the provided data
|
|
try {
|
|
uploadResponse = performSessionFileUpload(uploadFileOneDriveApiInstance, thisFileSizeLocal, jsonItemToResume, threadUploadSessionFilePath);
|
|
} catch (OneDriveException exception) {
|
|
writeln("CODING TO DO: Handle an exception when performing a resume session upload");
|
|
}
|
|
|
|
// Was the response from the OneDrive API a valid JSON item?
|
|
if (uploadResponse.type() == JSONType.object) {
|
|
// A valid JSON object was returned - session resumption upload sucessful
|
|
|
|
// Are we in an --upload-only & --remove-source-files scenario?
|
|
// Use actual config values as we are doing an upload session recovery
|
|
if (localDeleteAfterUpload) {
|
|
// Log that we are deleting a local item
|
|
addLogEntry("Removing local file as --upload-only & --remove-source-files configured");
|
|
// are we in a --dry-run scenario?
|
|
if (!dryRun) {
|
|
// No --dry-run ... process local file delete
|
|
// Only perform the delete if we have a valid file path
|
|
if (exists(jsonItemToResume["localPath"].str)) {
|
|
// file exists
|
|
addLogEntry("Removing local file: " ~ jsonItemToResume["localPath"].str, ["debug"]);
|
|
safeRemove(jsonItemToResume["localPath"].str);
|
|
}
|
|
}
|
|
// as file is removed, we have nothing to add to the local database
|
|
addLogEntry("Skipping adding to database as --upload-only & --remove-source-files configured", ["debug"]);
|
|
} else {
|
|
// Save JSON item in database
|
|
saveItem(uploadResponse);
|
|
}
|
|
} else {
|
|
// No valid response was returned
|
|
addLogEntry("CODING TO DO: what to do when session upload resumption JSON data is not valid ... nothing ? error message ?");
|
|
}
|
|
|
|
// Shutdown API instance
|
|
uploadFileOneDriveApiInstance.shutdown();
|
|
// Free object and memory
|
|
object.destroy(uploadFileOneDriveApiInstance);
|
|
}
|
|
}
|
|
} |