deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-06-09 01:12:29 +02:00
Code Issues Projects Releases Wiki Activity

Update PR

Browse source 
* Update PR
This commit is contained in:
abraunegg 2023-09-24 11:07:26 +10:00
parent 22eb96e3f1
commit 7b3051f449
3 changed files with 95 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/itemdb.d
View file

@ -45,8 +45,6 @@ struct Item {
// Construct an Item struct from a JSON driveItem // Construct an Item struct from a JSON driveItem
Item makeDatabaseItem(JSONValue driveItem) { Item makeDatabaseItem(JSONValue driveItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
Item item = { Item item = {
id: driveItem["id"].str, id: driveItem["id"].str,
name: "name" in driveItem ? driveItem["name"].str : null, // name may be missing for deleted files in OneDrive Business name: "name" in driveItem ? driveItem["name"].str : null, // name may be missing for deleted files in OneDrive Business
@ -587,6 +585,10 @@ final class ItemDatabase {
} }
string getDeltaLink(const(char)[] driveId, const(char)[] id) { string getDeltaLink(const(char)[] driveId, const(char)[] id) {
// Log what we received
log.vdebug("DeltaLink Query (driveId): ", driveId);
log.vdebug("DeltaLink Query (id): ", id);
assert(driveId && id); assert(driveId && id);
auto stmt = db.prepare("SELECT deltaLink FROM item WHERE driveId = ?1 AND id = ?2"); auto stmt = db.prepare("SELECT deltaLink FROM item WHERE driveId = ?1 AND id = ?2");
stmt.bind(1, driveId); stmt.bind(1, driveId);

3
src/main.d
View file

@ -770,6 +770,9 @@ void performStandardSyncProcess(string localPath, Monitor filesystemMonitor = nu
filesystemMonitor.update(true); filesystemMonitor.update(true);
} }
// Make sure we sync any DB data to this point
itemDB.performVacuum();
// Perform the final true up scan to ensure we have correctly replicated the current online state locally // Perform the final true up scan to ensure we have correctly replicated the current online state locally
if (!appConfig.surpressLoggingOutput) { if (!appConfig.surpressLoggingOutput) {
log.log("Perfoming final true up scan of online data from OneDrive"); log.log("Perfoming final true up scan of online data from OneDrive");

196
src/sync.d
View file

@ -223,7 +223,7 @@ class SyncEngine {
// Initialise the Sync Engine class // Initialise the Sync Engine class
bool initialise() { bool initialise() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// create a new instance of the OneDrive API // create a new instance of the OneDrive API
oneDriveApiInstance = new OneDriveApi(appConfig); oneDriveApiInstance = new OneDriveApi(appConfig);
if (oneDriveApiInstance.initialise()) { if (oneDriveApiInstance.initialise()) {
@ -242,7 +242,7 @@ class SyncEngine {
// Get Default Drive Details for this Account // Get Default Drive Details for this Account
void getDefaultDriveDetails() { void getDefaultDriveDetails() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Function variables // Function variables
JSONValue defaultOneDriveDriveDetails; JSONValue defaultOneDriveDriveDetails;
@ -357,7 +357,7 @@ class SyncEngine {
// Get Default Root Details for this Account // Get Default Root Details for this Account
void getDefaultRootDetails() { void getDefaultRootDetails() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Function variables // Function variables
JSONValue defaultOneDriveRootDetails; JSONValue defaultOneDriveRootDetails;
@ -427,7 +427,7 @@ class SyncEngine {
// - Process any deletes (remove local data) // - Process any deletes (remove local data)
// - Walk local file system for any differences (new files / data to upload to OneDrive) // - Walk local file system for any differences (new files / data to upload to OneDrive)
void syncOneDriveAccountToLocalDisk(bool performFullScanTrueUp = false) { void syncOneDriveAccountToLocalDisk(bool performFullScanTrueUp = false) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// performFullScanTrueUp value // performFullScanTrueUp value
log.vdebug("performFullScanTrueUp: ", performFullScanTrueUp); log.vdebug("performFullScanTrueUp: ", performFullScanTrueUp);
// Fetch the API response of /delta to track changes on OneDrive // Fetch the API response of /delta to track changes on OneDrive
@ -504,9 +504,6 @@ class SyncEngine {
// Process any download activities or cleanup actions for this OneDrive Personal Shared Folder // Process any download activities or cleanup actions for this OneDrive Personal Shared Folder
processDownloadActivities(); processDownloadActivities();
} }
} }
} }
@ -516,7 +513,6 @@ class SyncEngine {
// Configure singleDirectoryScope = true if this function is called // Configure singleDirectoryScope = true if this function is called
// By default, singleDirectoryScope = false // By default, singleDirectoryScope = false
void setSingleDirectoryScope(string normalisedSingleDirectoryPath) { void setSingleDirectoryScope(string normalisedSingleDirectoryPath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Function variables // Function variables
Item searchItem; Item searchItem;
@ -567,8 +563,7 @@ class SyncEngine {
// Query OneDrive API for /delta changes and iterate through items online // Query OneDrive API for /delta changes and iterate through items online
void fetchOneDriveDeltaAPIResponse(string driveIdToQuery = null, string itemIdToQuery = null, string sharedFolderName = null, bool performFullScanTrueUp = false) { void fetchOneDriveDeltaAPIResponse(string driveIdToQuery = null, string itemIdToQuery = null, string sharedFolderName = null, bool performFullScanTrueUp = false) {
log.vdebug("Starting this function: ", getFunctionName!({}));
string deltaLink = null; string deltaLink = null;
string deltaLinkAvailable; string deltaLinkAvailable;
JSONValue deltaChanges; JSONValue deltaChanges;
@ -695,13 +690,16 @@ class SyncEngine {
else break; else break;
} }
// To finish off the JSON processing items, this is needed to reflect this in the log
log.vdebug("------------------------------------------------------------------");
// Log that we have finished querying the /delta API // Log that we have finished querying the /delta API
if (log.verbose <= 1) { if (log.verbose <= 1) {
if (!appConfig.surpressLoggingOutput) { if (!appConfig.surpressLoggingOutput) {
write("\n"); write("\n");
} }
} else { } else {
log.vdebug("Finished getting /delta from the OneDrive API"); log.vdebug("Finished processing /delta JSON response from the OneDrive API");
} }
} else { } else {
// We have to generate our own /delta response // We have to generate our own /delta response
@ -747,6 +745,10 @@ class SyncEngine {
// Process the OneDrive object item JSON // Process the OneDrive object item JSON
processDeltaJSONItem(onedriveJSONItem, nrChanges, changeCount, responseBundleCount, singleDirectoryScope); processDeltaJSONItem(onedriveJSONItem, nrChanges, changeCount, responseBundleCount, singleDirectoryScope);
} }
// To finish off the JSON processing items, this is needed to reflect this in the log
log.vdebug("------------------------------------------------------------------");
// Log that we have finished generating our self generated /delta response // Log that we have finished generating our self generated /delta response
if (!appConfig.surpressLoggingOutput) { if (!appConfig.surpressLoggingOutput) {
log.log("Finished processing self generated /delta JSON response from the OneDrive API"); log.log("Finished processing self generated /delta JSON response from the OneDrive API");
@ -777,7 +779,9 @@ class SyncEngine {
// Chunk the total items to process into 500 lot items // Chunk the total items to process into 500 lot items
batchesProcessed++; batchesProcessed++;
log.vlog("Processing OneDrive JSON item batch [", batchesProcessed,"/", batchCount, "] to ensure consistent local state"); log.vlog("Processing OneDrive JSON item batch [", batchesProcessed,"/", batchCount, "] to ensure consistent local state");
processJSONItemsInBatch(batchOfJSONItems); processJSONItemsInBatch(batchOfJSONItems, batchesProcessed);
// To finish off the JSON processing items, this is needed to reflect this in the log
log.vdebug("------------------------------------------------------------------");
} }
log.vdebug("Number of JSON items to process is: ", jsonItemsToProcess.length); log.vdebug("Number of JSON items to process is: ", jsonItemsToProcess.length);
@ -795,7 +799,7 @@ class SyncEngine {
// Process the /delta API JSON response items // Process the /delta API JSON response items
void processDeltaJSONItem(JSONValue onedriveJSONItem, ulong nrChanges, int changeCount, ulong responseBundleCount, bool singleDirectoryScope) { void processDeltaJSONItem(JSONValue onedriveJSONItem, ulong nrChanges, int changeCount, ulong responseBundleCount, bool singleDirectoryScope) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Variables for this foreach loop // Variables for this foreach loop
string thisItemId; string thisItemId;
bool itemIsRoot = false; bool itemIsRoot = false;
@ -894,7 +898,7 @@ class SyncEngine {
// Process 'root' and 'deleted' OneDrive JSON items // Process 'root' and 'deleted' OneDrive JSON items
void processRootAndDeletedJSONItems(JSONValue onedriveJSONItem, string driveId, bool handleItemAsRootObject, bool itemIsDeletedOnline, bool itemHasParentReferenceId) { void processRootAndDeletedJSONItems(JSONValue onedriveJSONItem, string driveId, bool handleItemAsRootObject, bool itemIsDeletedOnline, bool itemHasParentReferenceId) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Is the item deleted online? // Is the item deleted online?
if(!itemIsDeletedOnline) { if(!itemIsDeletedOnline) {
@ -938,11 +942,18 @@ class SyncEngine {
} }
// Process each of the elements contained in jsonItemsToProcess[] // Process each of the elements contained in jsonItemsToProcess[]
void processJSONItemsInBatch(JSONValue[] array) { void processJSONItemsInBatch(JSONValue[] array, ulong batchGroup) {
log.vdebug("Starting this function: ", getFunctionName!({}));
ulong batchElementCount = array.length;
foreach (i, onedriveJSONItem; array.enumerate) { foreach (i, onedriveJSONItem; array.enumerate) {
// Use the JSON elements rather can computing a DB struct via makeItem() // Use the JSON elements rather can computing a DB struct via makeItem()
ulong elementCount = i +1;
// To show this is the processing for this particular item, start off with this breaker line
log.vdebug("------------------------------------------------------------------");
log.vdebug("Processing OneDrive JSON item ", elementCount, " of ", batchElementCount, " as part of JSON item batch ", batchGroup);
string thisItemId = onedriveJSONItem["id"].str; string thisItemId = onedriveJSONItem["id"].str;
string thisItemDriveId = onedriveJSONItem["parentReference"]["driveId"].str; string thisItemDriveId = onedriveJSONItem["parentReference"]["driveId"].str;
string thisItemParentId = onedriveJSONItem["parentReference"]["id"].str; string thisItemParentId = onedriveJSONItem["parentReference"]["id"].str;
@ -1346,7 +1357,6 @@ class SyncEngine {
} else { } else {
// This JSON item is wanted - we need to process this JSON item further // This JSON item is wanted - we need to process this JSON item further
// Take the JSON item and create a consumable object for eventual database insertion // Take the JSON item and create a consumable object for eventual database insertion
log.vdebug("Making newDatabaseItem from this JSON: ", onedriveJSONItem);
Item newDatabaseItem = makeItem(onedriveJSONItem); Item newDatabaseItem = makeItem(onedriveJSONItem);
if (existingDBEntry) { if (existingDBEntry) {
@ -1380,12 +1390,6 @@ class SyncEngine {
log.vdebug("OneDrive change is potentially a new local item"); log.vdebug("OneDrive change is potentially a new local item");
// Attempt to apply this potentially new item // Attempt to apply this potentially new item
//writeln("newDatabaseItem: ", newDatabaseItem);
//writeln("onedriveJSONItem: ", onedriveJSONItem);
//writeln("newItemPath: ", newItemPath);
applyPotentiallyNewLocalItem(newDatabaseItem, onedriveJSONItem, newItemPath); applyPotentiallyNewLocalItem(newDatabaseItem, onedriveJSONItem, newItemPath);
} }
} }
@ -1397,8 +1401,7 @@ class SyncEngine {
// Perform the download of any required objects in parallel // Perform the download of any required objects in parallel
void processDownloadActivities() { void processDownloadActivities() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Are there any items to delete locally? Cleanup space locally first // Are there any items to delete locally? Cleanup space locally first
if (!idsToDelete.empty) { if (!idsToDelete.empty) {
// There are elements that potentially need to be deleted locally // There are elements that potentially need to be deleted locally
@ -1439,8 +1442,7 @@ class SyncEngine {
// If the JSON item is not in the database, it is potentially a new item that we need to action // If the JSON item is not in the database, it is potentially a new item that we need to action
void applyPotentiallyNewLocalItem(Item newDatabaseItem, JSONValue onedriveJSONItem, string newItemPath) { void applyPotentiallyNewLocalItem(Item newDatabaseItem, JSONValue onedriveJSONItem, string newItemPath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// The JSON and Database items being passed in here have passed the following checks: // The JSON and Database items being passed in here have passed the following checks:
// - skip_file // - skip_file
// - skip_dir // - skip_dir
@ -1601,8 +1603,7 @@ class SyncEngine {
// If the JSON item IS in the database, this will be an update to an existing in-sync item // If the JSON item IS in the database, this will be an update to an existing in-sync item
void applyPotentiallyChangedItem(Item existingDatabaseItem, string existingItemPath, Item changedOneDriveItem, string changedItemPath, JSONValue onedriveJSONItem) { void applyPotentiallyChangedItem(Item existingDatabaseItem, string existingItemPath, Item changedOneDriveItem, string changedItemPath, JSONValue onedriveJSONItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// If we are moving the item, we do not need to download it again // If we are moving the item, we do not need to download it again
bool itemWasMoved = false; bool itemWasMoved = false;
@ -1698,7 +1699,7 @@ class SyncEngine {
// Download new file items as identified // Download new file items as identified
void downloadOneDriveItems() { void downloadOneDriveItems() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Lets deal with the JSON items in a batch process // Lets deal with the JSON items in a batch process
ulong batchSize = appConfig.concurrentThreads; ulong batchSize = appConfig.concurrentThreads;
ulong batchCount = (fileJSONItemsToDownload.length + batchSize - 1) / batchSize; ulong batchCount = (fileJSONItemsToDownload.length + batchSize - 1) / batchSize;
@ -1711,7 +1712,7 @@ class SyncEngine {
// Download items in parallel // Download items in parallel
void downloadOneDriveItemsInParallel(JSONValue[] array) { void downloadOneDriveItemsInParallel(JSONValue[] array) {
log.vdebug("Starting this function: ", getFunctionName!({}));
foreach (i, onedriveJSONItem; taskPool.parallel(array)) { foreach (i, onedriveJSONItem; taskPool.parallel(array)) {
// Take the JSON item and create a consumable object for eventual database insertion // Take the JSON item and create a consumable object for eventual database insertion
Item newDatabaseItem = makeItem(onedriveJSONItem); Item newDatabaseItem = makeItem(onedriveJSONItem);
@ -1721,8 +1722,7 @@ class SyncEngine {
// Perform the actual download of an object from OneDrive // Perform the actual download of an object from OneDrive
void downloadFileItem(Item newDatabaseItem, JSONValue onedriveJSONItem) { void downloadFileItem(Item newDatabaseItem, JSONValue onedriveJSONItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
bool downloadFailed = false; bool downloadFailed = false;
string OneDriveFileXORHash; string OneDriveFileXORHash;
string OneDriveFileSHA256Hash; string OneDriveFileSHA256Hash;
@ -1960,7 +1960,7 @@ class SyncEngine {
// Test if the given item is in-sync. Returns true if the given item corresponds to the local one // Test if the given item is in-sync. Returns true if the given item corresponds to the local one
bool isItemSynced(Item item, string path, string itemSource) { bool isItemSynced(Item item, string path, string itemSource) {
log.vdebug("Starting this function: ", getFunctionName!({}));
if (!exists(path)) return false; if (!exists(path)) return false;
final switch (item.type) { final switch (item.type) {
case ItemType.file: case ItemType.file:
@ -2033,7 +2033,7 @@ class SyncEngine {
// Get the /delta data using the provided details // Get the /delta data using the provided details
JSONValue getDeltaChangesByItemId(string selectedDriveId, string selectedItemId, string providedDeltaLink) { JSONValue getDeltaChangesByItemId(string selectedDriveId, string selectedItemId, string providedDeltaLink) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Function variables // Function variables
JSONValue deltaChangesBundle; JSONValue deltaChangesBundle;
// Get the /delta data for this account | driveId | deltaLink combination // Get the /delta data for this account | driveId | deltaLink combination
@ -2102,7 +2102,7 @@ class SyncEngine {
// Common code to handle a 408 or 429 response from the OneDrive API // Common code to handle a 408 or 429 response from the OneDrive API
void handleOneDriveThrottleRequest(OneDriveApi activeOneDriveApiInstance) { void handleOneDriveThrottleRequest(OneDriveApi activeOneDriveApiInstance) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// If OneDrive sends a status code 429 then this function will be used to process the Retry-After response header which contains the value by which we need to wait // If OneDrive sends a status code 429 then this function will be used to process the Retry-After response header which contains the value by which we need to wait
log.vdebug("Handling a OneDrive HTTP 429 Response Code (Too Many Requests)"); log.vdebug("Handling a OneDrive HTTP 429 Response Code (Too Many Requests)");
// Read in the Retry-After HTTP header as set and delay as per this value before retrying the request // Read in the Retry-After HTTP header as set and delay as per this value before retrying the request
@ -2136,7 +2136,7 @@ class SyncEngine {
// If the JSON response is not correct JSON object, exit // If the JSON response is not correct JSON object, exit
void invalidJSONResponseFromOneDriveAPI() { void invalidJSONResponseFromOneDriveAPI() {
log.vdebug("Starting this function: ", getFunctionName!({}));
log.error("ERROR: Query of the OneDrive API returned an invalid JSON response"); log.error("ERROR: Query of the OneDrive API returned an invalid JSON response");
// Must exit // Must exit
exit(-1); exit(-1);
@ -2144,7 +2144,7 @@ class SyncEngine {
// Handle an unhandled API error // Handle an unhandled API error
void defaultUnhandledHTTPErrorCode(OneDriveException exception) { void defaultUnhandledHTTPErrorCode(OneDriveException exception) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// display error // display error
displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
// Must exit here // Must exit here
@ -2153,7 +2153,7 @@ class SyncEngine {
// Display the pertinant details of the sync engine // Display the pertinant details of the sync engine
void displaySyncEngineDetails() { void displaySyncEngineDetails() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Display accountType, defaultDriveId, defaultRootId & remainingFreeSpace for verbose logging purposes // Display accountType, defaultDriveId, defaultRootId & remainingFreeSpace for verbose logging purposes
//log.vlog("Application version: ", strip(import("version"))); //log.vlog("Application version: ", strip(import("version")));
@ -2180,7 +2180,7 @@ class SyncEngine {
// Query itemdb.computePath() and catch potential assert when DB consistency issue occurs // Query itemdb.computePath() and catch potential assert when DB consistency issue occurs
string computeItemPath(string thisDriveId, string thisItemId) { string computeItemPath(string thisDriveId, string thisItemId) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// static declare this for this function // static declare this for this function
static import core.exception; static import core.exception;
string calculatedPath; string calculatedPath;
@ -2200,7 +2200,7 @@ class SyncEngine {
// Try and compute the file hash for the given item // Try and compute the file hash for the given item
bool testFileHash(string path, Item item) { bool testFileHash(string path, Item item) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Generate QuickXORHash first before attempting to generate any other type of hash // Generate QuickXORHash first before attempting to generate any other type of hash
if (item.quickXorHash) { if (item.quickXorHash) {
if (item.quickXorHash == computeQuickXorHash(path)) return true; if (item.quickXorHash == computeQuickXorHash(path)) return true;
@ -2212,7 +2212,7 @@ class SyncEngine {
// Process items that need to be removed // Process items that need to be removed
void processDeleteItems() { void processDeleteItems() {
log.vdebug("Starting this function: ", getFunctionName!({}));
foreach_reverse (i; idsToDelete) { foreach_reverse (i; idsToDelete) {
Item item; Item item;
string path; string path;
@ -2298,7 +2298,7 @@ class SyncEngine {
// Update the timestamp of an object online // Update the timestamp of an object online
void uploadLastModifiedTime(string driveId, string id, SysTime mtime, string eTag) { void uploadLastModifiedTime(string driveId, string id, SysTime mtime, string eTag) {
log.vdebug("Starting this function: ", getFunctionName!({}));
string itemModifiedTime; string itemModifiedTime;
itemModifiedTime = mtime.toISOExtString(); itemModifiedTime = mtime.toISOExtString();
JSONValue data = [ JSONValue data = [
@ -2366,8 +2366,6 @@ class SyncEngine {
} }
} }
} }
// save the updated response from OneDrive in the database
log.vdebug("uploadLastModifiedTime response: ", response);
// Is the response a valid JSON object - validation checking done in saveItem // Is the response a valid JSON object - validation checking done in saveItem
saveItem(response); saveItem(response);
@ -2375,7 +2373,7 @@ class SyncEngine {
// Perform a database integrity check - checking all the items that are in-sync at the moment, validating what we know should be on disk, to what is actually on disk // Perform a database integrity check - checking all the items that are in-sync at the moment, validating what we know should be on disk, to what is actually on disk
void performDatabaseConsistencyAndIntegrityCheck() { void performDatabaseConsistencyAndIntegrityCheck() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Log what we are doing // Log what we are doing
if (!appConfig.surpressLoggingOutput) { if (!appConfig.surpressLoggingOutput) {
log.log("Performing a database consistency and integrity check on locally stored data ... "); log.log("Performing a database consistency and integrity check on locally stored data ... ");
@ -2478,8 +2476,7 @@ class SyncEngine {
// Check this Database Item for its consistency on disk // Check this Database Item for its consistency on disk
void checkDatabaseItemForConsistency(Item dbItem) { void checkDatabaseItemForConsistency(Item dbItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// What is the local path item // What is the local path item
string localFilePath; string localFilePath;
// Do we want to onward process this item? // Do we want to onward process this item?
@ -2522,7 +2519,7 @@ class SyncEngine {
// Perform the database consistency check on this file item // Perform the database consistency check on this file item
void checkFileDatabaseItemForConsistency(Item dbItem, string localFilePath) { void checkFileDatabaseItemForConsistency(Item dbItem, string localFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// What is the source of this item data? // What is the source of this item data?
string itemSource = "database"; string itemSource = "database";
@ -2616,8 +2613,7 @@ class SyncEngine {
// Perform the database consistency check on this directory item // Perform the database consistency check on this directory item
void checkDirectoryDatabaseItemForConsistency(Item dbItem, string localFilePath) { void checkDirectoryDatabaseItemForConsistency(Item dbItem, string localFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// What is the source of this item data? // What is the source of this item data?
string itemSource = "database"; string itemSource = "database";
@ -2701,7 +2697,7 @@ class SyncEngine {
// Does this Database Item (directory or file) get excluded from any operation based on any client side filtering rules? // Does this Database Item (directory or file) get excluded from any operation based on any client side filtering rules?
bool checkDBItemAndPathAgainstClientSideFiltering(Item dbItem, string localFilePath) { bool checkDBItemAndPathAgainstClientSideFiltering(Item dbItem, string localFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Check the item and path against client side filtering rules // Check the item and path against client side filtering rules
// Return a true|false response // Return a true|false response
bool clientSideRuleExcludesItem = false; bool clientSideRuleExcludesItem = false;
@ -2760,8 +2756,7 @@ class SyncEngine {
// Does this local path (directory or file) conform with the Microsoft Naming Restrictions? // Does this local path (directory or file) conform with the Microsoft Naming Restrictions?
bool checkPathAgainstMicrosoftNamingRestrictions(string localFilePath) { bool checkPathAgainstMicrosoftNamingRestrictions(string localFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Check if the given path violates certain Microsoft restrictions and limitations // Check if the given path violates certain Microsoft restrictions and limitations
// Return a true|false response // Return a true|false response
bool invalidPath = false; bool invalidPath = false;
@ -2795,7 +2790,7 @@ class SyncEngine {
// Does this local path (directory or file) get excluded from any operation based on any client side filtering rules? // Does this local path (directory or file) get excluded from any operation based on any client side filtering rules?
bool checkPathAgainstClientSideFiltering(string localFilePath) { bool checkPathAgainstClientSideFiltering(string localFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Unlike checkDBItemAndPathAgainstClientSideFiltering - we need to check the path only // Unlike checkDBItemAndPathAgainstClientSideFiltering - we need to check the path only
// Check the path against client side filtering rules // Check the path against client side filtering rules
@ -2964,8 +2959,7 @@ class SyncEngine {
// Does this JSON item (as received from OneDrive API) get excluded from any operation based on any client side filtering rules? // Does this JSON item (as received from OneDrive API) get excluded from any operation based on any client side filtering rules?
// This function is only used when we are fetching objects from the OneDrive API using a /children query to help speed up what object we query // This function is only used when we are fetching objects from the OneDrive API using a /children query to help speed up what object we query
bool checkJSONAgainstClientSideFiltering(JSONValue onedriveJSONItem) { bool checkJSONAgainstClientSideFiltering(JSONValue onedriveJSONItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
bool clientSideRuleExcludesPath = false; bool clientSideRuleExcludesPath = false;
// Check the path against client side filtering rules // Check the path against client side filtering rules
@ -3111,7 +3105,7 @@ class SyncEngine {
// Process the list of local changes to upload to OneDrive // Process the list of local changes to upload to OneDrive
void processChangedLocalItemsToUpload() { void processChangedLocalItemsToUpload() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Each element in this array 'databaseItemsWhereContentHasChanged' is an Database Item ID that has been modified locally // Each element in this array 'databaseItemsWhereContentHasChanged' is an Database Item ID that has been modified locally
ulong batchSize = appConfig.concurrentThreads; ulong batchSize = appConfig.concurrentThreads;
ulong batchCount = (databaseItemsWhereContentHasChanged.length + batchSize - 1) / batchSize; ulong batchCount = (databaseItemsWhereContentHasChanged.length + batchSize - 1) / batchSize;
@ -3125,8 +3119,7 @@ class SyncEngine {
// Upload changed local files to OneDrive in parallel // Upload changed local files to OneDrive in parallel
void uploadChangedLocalFileToOneDrive(string[3][] array) { void uploadChangedLocalFileToOneDrive(string[3][] array) {
log.vdebug("Starting this function: ", getFunctionName!({}));
foreach (i, localItemDetails; taskPool.parallel(array)) { foreach (i, localItemDetails; taskPool.parallel(array)) {
log.vdebug("Thread ", i, " Starting: ", Clock.currTime()); log.vdebug("Thread ", i, " Starting: ", Clock.currTime());
@ -3254,8 +3247,7 @@ class SyncEngine {
// Perform the upload of a locally modified file to OneDrive // Perform the upload of a locally modified file to OneDrive
JSONValue performModifiedFileUpload(Item dbItem, string localFilePath, ulong thisFileSizeLocal) { JSONValue performModifiedFileUpload(Item dbItem, string localFilePath, ulong thisFileSizeLocal) {
log.vdebug("Starting this function: ", getFunctionName!({}));
JSONValue uploadResponse; JSONValue uploadResponse;
OneDriveApi uploadFileOneDriveApiInstance; OneDriveApi uploadFileOneDriveApiInstance;
uploadFileOneDriveApiInstance = new OneDriveApi(appConfig); uploadFileOneDriveApiInstance = new OneDriveApi(appConfig);
@ -3475,8 +3467,7 @@ class SyncEngine {
// Query the OneDrive API using the provided driveId to get the latest quota details // Query the OneDrive API using the provided driveId to get the latest quota details
ulong getRemainingFreeSpace(string driveId) { ulong getRemainingFreeSpace(string driveId) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Get the quota details for this driveId, as this could have changed since we started the application - the user could have added / deleted data online, or purchased additional storage // Get the quota details for this driveId, as this could have changed since we started the application - the user could have added / deleted data online, or purchased additional storage
// Quota details are ONLY available for the main default driveId, as the OneDrive API does not provide quota details for shared folders // Quota details are ONLY available for the main default driveId, as the OneDrive API does not provide quota details for shared folders
@ -3488,6 +3479,7 @@ class SyncEngine {
OneDriveApi getCurrentDriveQuotaApiInstance; OneDriveApi getCurrentDriveQuotaApiInstance;
getCurrentDriveQuotaApiInstance = new OneDriveApi(appConfig); getCurrentDriveQuotaApiInstance = new OneDriveApi(appConfig);
getCurrentDriveQuotaApiInstance.initialise(); getCurrentDriveQuotaApiInstance.initialise();
log.vdebug("Seeking available quota for this drive id: ", driveId);
currentDriveQuota = getCurrentDriveQuotaApiInstance.getDriveQuota(driveId); currentDriveQuota = getCurrentDriveQuotaApiInstance.getDriveQuota(driveId);
// Shut this API instance down // Shut this API instance down
getCurrentDriveQuotaApiInstance.shutdown(); getCurrentDriveQuotaApiInstance.shutdown();
@ -3553,12 +3545,14 @@ class SyncEngine {
} }
} }
// what was the determined available quota?
log.vdebug("Available quota: ", remainingQuota);
return remainingQuota; return remainingQuota;
} }
// Perform a filesystem walk to uncover new data to upload to OneDrive // Perform a filesystem walk to uncover new data to upload to OneDrive
void scanLocalFilesystemPathForNewData(string path) { void scanLocalFilesystemPathForNewData(string path) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// To improve logging output for this function, what is the 'logical path' we are scanning for file & folder differences? // To improve logging output for this function, what is the 'logical path' we are scanning for file & folder differences?
string logPath; string logPath;
if (path == ".") { if (path == ".") {
@ -3636,8 +3630,7 @@ class SyncEngine {
// Scan this path for new data // Scan this path for new data
void scanPathForNewData(string path) { void scanPathForNewData(string path) {
log.vdebug("Starting this function: ", getFunctionName!({}));
ulong maxPathLength; ulong maxPathLength;
ulong pathWalkLength; ulong pathWalkLength;
@ -3848,15 +3841,13 @@ class SyncEngine {
// Query the database to determine if this path is within the existing database // Query the database to determine if this path is within the existing database
bool pathFoundInDatabase(string searchPath) { bool pathFoundInDatabase(string searchPath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Check if this path in the database // Check if this path in the database
Item databaseItem; Item databaseItem;
bool pathFoundInDB = false; bool pathFoundInDB = false;
foreach (driveId; driveIDsArray) { foreach (driveId; driveIDsArray) {
if (itemDB.selectByPath(searchPath, driveId, databaseItem)) { if (itemDB.selectByPath(searchPath, driveId, databaseItem)) {
pathFoundInDB = true; pathFoundInDB = true;
log.vdebug("databaseItem: ", databaseItem);
log.vdebug("pathFoundInDB: ", pathFoundInDB);
} }
} }
return pathFoundInDB; return pathFoundInDB;
@ -3866,7 +3857,7 @@ class SyncEngine {
// - Test if we can get the parent path details from the database, otherwise we need to search online // - Test if we can get the parent path details from the database, otherwise we need to search online
// for the path flow and create the folder that way // for the path flow and create the folder that way
void createDirectoryOnline(string thisNewPathToCreate) { void createDirectoryOnline(string thisNewPathToCreate) {
log.vdebug("Starting this function: ", getFunctionName!({}));
log.log("OneDrive Client requested to create this directory online: ", thisNewPathToCreate); log.log("OneDrive Client requested to create this directory online: ", thisNewPathToCreate);
Item parentItem; Item parentItem;
@ -4149,8 +4140,7 @@ class SyncEngine {
// Test that the online name actually matches the requested local name // Test that the online name actually matches the requested local name
void performPosixTest(string localNameToCheck, string onlineName) { void performPosixTest(string localNameToCheck, string onlineName) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file // https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file
// Do not assume case sensitivity. For example, consider the names OSCAR, Oscar, and oscar to be the same, // Do not assume case sensitivity. For example, consider the names OSCAR, Oscar, and oscar to be the same,
// even though some file systems (such as a POSIX-compliant file system) may consider them as different. // even though some file systems (such as a POSIX-compliant file system) may consider them as different.
@ -4164,7 +4154,7 @@ class SyncEngine {
// Upload new file items as identified // Upload new file items as identified
void uploadNewLocalFileItems() { void uploadNewLocalFileItems() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Lets deal with the new local items in a batch process // Lets deal with the new local items in a batch process
ulong batchSize = appConfig.concurrentThreads; ulong batchSize = appConfig.concurrentThreads;
ulong batchCount = (newLocalFilesToUploadToOneDrive.length + batchSize - 1) / batchSize; ulong batchCount = (newLocalFilesToUploadToOneDrive.length + batchSize - 1) / batchSize;
@ -4177,7 +4167,7 @@ class SyncEngine {
// Upload the file batches in parallel // Upload the file batches in parallel
void uploadNewLocalFileItemsInParallel(string[] array) { void uploadNewLocalFileItemsInParallel(string[] array) {
log.vdebug("Starting this function: ", getFunctionName!({}));
foreach (i, fileToUpload; taskPool.parallel(array)) { foreach (i, fileToUpload; taskPool.parallel(array)) {
log.vdebug("Upload Thread ", i, " Starting: ", Clock.currTime()); log.vdebug("Upload Thread ", i, " Starting: ", Clock.currTime());
uploadNewFile(fileToUpload); uploadNewFile(fileToUpload);
@ -4187,8 +4177,7 @@ class SyncEngine {
// Upload a new file to OneDrive // Upload a new file to OneDrive
void uploadNewFile(string fileToUpload) { void uploadNewFile(string fileToUpload) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Debug for the moment // Debug for the moment
log.vdebug("fileToUpload: ", fileToUpload); log.vdebug("fileToUpload: ", fileToUpload);
@ -4401,8 +4390,7 @@ class SyncEngine {
// Perform the actual upload to OneDrive // Perform the actual upload to OneDrive
bool performNewFileUpload(Item parentItem, string fileToUpload, ulong thisFileSize) { bool performNewFileUpload(Item parentItem, string fileToUpload, ulong thisFileSize) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Assume that by default the upload fails // Assume that by default the upload fails
bool uploadFailed = true; bool uploadFailed = true;
@ -4662,7 +4650,7 @@ class SyncEngine {
// Create the OneDrive Upload Session // Create the OneDrive Upload Session
JSONValue createSessionFileUpload(OneDriveApi activeOneDriveApiInstance, string fileToUpload, string parentDriveId, string parentId, string filename, string eTag, string threadUploadSessionFilePath) { JSONValue createSessionFileUpload(OneDriveApi activeOneDriveApiInstance, string fileToUpload, string parentDriveId, string parentId, string filename, string eTag, string threadUploadSessionFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Upload file via a OneDrive API session // Upload file via a OneDrive API session
JSONValue uploadSession; JSONValue uploadSession;
@ -4703,7 +4691,7 @@ class SyncEngine {
// Save the session upload data // Save the session upload data
void saveSessionFile(string threadUploadSessionFilePath, JSONValue uploadSessionData) { void saveSessionFile(string threadUploadSessionFilePath, JSONValue uploadSessionData) {
log.vdebug("Starting this function: ", getFunctionName!({}));
try { try {
std.file.write(threadUploadSessionFilePath, uploadSessionData.toString()); std.file.write(threadUploadSessionFilePath, uploadSessionData.toString());
} catch (FileException e) { } catch (FileException e) {
@ -4714,8 +4702,7 @@ class SyncEngine {
// Perform the upload of file via the Upload Session that was created // Perform the upload of file via the Upload Session that was created
JSONValue performSessionFileUpload(OneDriveApi activeOneDriveApiInstance, ulong thisFileSize, JSONValue uploadSessionData, string threadUploadSessionFilePath) { JSONValue performSessionFileUpload(OneDriveApi activeOneDriveApiInstance, ulong thisFileSize, JSONValue uploadSessionData, string threadUploadSessionFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Response for upload // Response for upload
JSONValue uploadResponse; JSONValue uploadResponse;
@ -4861,8 +4848,7 @@ class SyncEngine {
// Delete an item on OneDrive // Delete an item on OneDrive
void uploadDeletedItem(Item itemToDelete, string path) { void uploadDeletedItem(Item itemToDelete, string path) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Are we in a situation where we HAVE to keep the data online - do not delete the remote object // Are we in a situation where we HAVE to keep the data online - do not delete the remote object
if (noRemoteDelete) { if (noRemoteDelete) {
if ((itemToDelete.type == ItemType.dir)) { if ((itemToDelete.type == ItemType.dir)) {
@ -4943,8 +4929,7 @@ class SyncEngine {
// Get the children of an item id from the database // Get the children of an item id from the database
Item[] getChildren(string driveId, string id) { Item[] getChildren(string driveId, string id) {
log.vdebug("Starting this function: ", getFunctionName!({}));
Item[] children; Item[] children;
children ~= itemDB.selectChildren(driveId, id); children ~= itemDB.selectChildren(driveId, id);
foreach (Item child; children) { foreach (Item child; children) {
@ -4958,8 +4943,8 @@ class SyncEngine {
// Perform a 'reverse' delete of all child objects on OneDrive // Perform a 'reverse' delete of all child objects on OneDrive
void performReverseDeletionOfOneDriveItems(Item[] children, Item itemToDelete) { void performReverseDeletionOfOneDriveItems(Item[] children, Item itemToDelete) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Log what is happening
log.vdebug("Attempting a reverse delete of all child objects from OneDrive"); log.vdebug("Attempting a reverse delete of all child objects from OneDrive");
// Create a new API Instance for this thread and initialise it // Create a new API Instance for this thread and initialise it
@ -4985,8 +4970,7 @@ class SyncEngine {
// Create a fake OneDrive response suitable for use with saveItem // Create a fake OneDrive response suitable for use with saveItem
JSONValue createFakeResponse(const(string) path) { JSONValue createFakeResponse(const(string) path) {
log.vdebug("Starting this function: ", getFunctionName!({}));
import std.digest.sha; import std.digest.sha;
// Generate a simulated JSON response which can be used // Generate a simulated JSON response which can be used
// At a minimum we need: // At a minimum we need:
@ -5078,8 +5062,7 @@ class SyncEngine {
// Save JSON item details into the item database // Save JSON item details into the item database
void saveItem(JSONValue jsonItem) { void saveItem(JSONValue jsonItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// jsonItem has to be a valid object // jsonItem has to be a valid object
if (jsonItem.type() == JSONType.object){ if (jsonItem.type() == JSONType.object){
// Check if the response JSON has an 'id', otherwise makeItem() fails with 'Key not found: id' // Check if the response JSON has an 'id', otherwise makeItem() fails with 'Key not found: id'
@ -5092,7 +5075,7 @@ class SyncEngine {
log.vdebug("Skipping adding to database as --upload-only & --remove-source-files configured"); log.vdebug("Skipping adding to database as --upload-only & --remove-source-files configured");
} else { } else {
// What is the JSON item we are trying to create a DB record with? // What is the JSON item we are trying to create a DB record with?
log.vdebug("Creating DB item from this JSON: ", jsonItem); log.vdebug("saveItem - creating DB item from this JSON: ", jsonItem);
// Takes a JSON input and formats to an item which can be used by the database // Takes a JSON input and formats to an item which can be used by the database
Item item = makeItem(jsonItem); Item item = makeItem(jsonItem);
@ -5142,8 +5125,7 @@ class SyncEngine {
// Wrapper function for makeDatabaseItem so we can check to ensure that the item has the required hashes // Wrapper function for makeDatabaseItem so we can check to ensure that the item has the required hashes
Item makeItem(JSONValue onedriveJSONItem) { Item makeItem(JSONValue onedriveJSONItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Make the DB Item from the JSON data provided // Make the DB Item from the JSON data provided
Item newDatabaseItem = makeDatabaseItem(onedriveJSONItem); Item newDatabaseItem = makeDatabaseItem(onedriveJSONItem);
@ -5198,8 +5180,7 @@ class SyncEngine {
// Print the fileDownloadFailures and fileUploadFailures arrays if they are not empty // Print the fileDownloadFailures and fileUploadFailures arrays if they are not empty
void displaySyncFailures() { void displaySyncFailures() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Were there any file download failures? // Were there any file download failures?
if (!fileDownloadFailures.empty) { if (!fileDownloadFailures.empty) {
// There are download failures ... // There are download failures ...
@ -5263,8 +5244,7 @@ class SyncEngine {
// then once the target of the --single-directory request is hit, all of the children of that path can be queried, giving a much more focused // then once the target of the --single-directory request is hit, all of the children of that path can be queried, giving a much more focused
// JSON response which can then be processed, negating the need to continuously traverse the tree and 'exclude' items // JSON response which can then be processed, negating the need to continuously traverse the tree and 'exclude' items
JSONValue generateDeltaResponse(string pathToQuery = null) { JSONValue generateDeltaResponse(string pathToQuery = null) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// JSON value which will be responded with // JSON value which will be responded with
JSONValue selfGeneratedDeltaResponse; JSONValue selfGeneratedDeltaResponse;
@ -5541,8 +5521,7 @@ class SyncEngine {
// Query the OneDrive API for the specified child id for any children objects // Query the OneDrive API for the specified child id for any children objects
JSONValue[] queryForChildren(string driveId, string idToQuery, string childParentPath, string pathForLogging) { JSONValue[] queryForChildren(string driveId, string idToQuery, string childParentPath, string pathForLogging) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// function variables // function variables
JSONValue thisLevelChildren; JSONValue thisLevelChildren;
JSONValue[] thisLevelChildrenData; JSONValue[] thisLevelChildrenData;
@ -5619,8 +5598,8 @@ class SyncEngine {
// Query the OneDrive API for the child objects for this element // Query the OneDrive API for the child objects for this element
JSONValue queryThisLevelChildren(string driveId, string idToQuery, string nextLink) { JSONValue queryThisLevelChildren(string driveId, string idToQuery, string nextLink) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// function variables
JSONValue thisLevelChildren; JSONValue thisLevelChildren;
// Create new OneDrive API Instance // Create new OneDrive API Instance
@ -5692,8 +5671,8 @@ class SyncEngine {
// This function also ensures that each path in the requested path actually matches the requested element to ensure that the OneDrive API response // This function also ensures that each path in the requested path actually matches the requested element to ensure that the OneDrive API response
// is not falsely matching a 'case insensitive' match to the actual request which is a POSIX compliance issue. // is not falsely matching a 'case insensitive' match to the actual request which is a POSIX compliance issue.
JSONValue queryOneDriveForSpecificPathAndCreateIfMissing(string thisNewPathToSearch, bool createPathIfMissing) { JSONValue queryOneDriveForSpecificPathAndCreateIfMissing(string thisNewPathToSearch, bool createPathIfMissing) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// function variables
JSONValue getPathDetailsAPIResponse; JSONValue getPathDetailsAPIResponse;
string currentPathTree; string currentPathTree;
Item parentDetails; Item parentDetails;
@ -5954,9 +5933,10 @@ class SyncEngine {
// Delete an item by it's path // Delete an item by it's path
// This function is only used in --monitor mode // This function is only used in --monitor mode
void deleteByPath(const(string) path) { void deleteByPath(const(string) path) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// function variables
Item dbItem; Item dbItem;
// Need to check all driveid's we know about, not just the defaultDriveId // Need to check all driveid's we know about, not just the defaultDriveId
bool itemInDB = false; bool itemInDB = false;
foreach (searchDriveId; driveIDsArray) { foreach (searchDriveId; driveIDsArray) {