Initial code commit for onedrive client version v2.5.0-alpha-1

Add initial code for onedrive client version v2.5.0-alpha-1, adding in support for OneDrive Business Shared Folders
This commit is contained in:
abraunegg 2023-09-22 05:34:42 +10:00
parent 3502e0cee4
commit 4bd9ae5092
7 changed files with 548 additions and 285 deletions

View file

@ -41,7 +41,7 @@ class ClientSideFiltering {
// Load the Business Shared Items file if it exists // Load the Business Shared Items file if it exists
if (exists(appConfig.businessSharedItemsFilePath)){ if (exists(appConfig.businessSharedItemsFilePath)){
loadSyncList(appConfig.businessSharedItemsFilePath); loadBusinessSharedItems(appConfig.businessSharedItemsFilePath);
} }
// Configure skip_dir, skip_file, skip-dir-strict-match & skip_dotfiles from config entries // Configure skip_dir, skip_file, skip-dir-strict-match & skip_dotfiles from config entries

View file

@ -55,7 +55,7 @@ class ApplicationConfig {
// - Identify as ISV and include Company Name, App Name separated by a pipe character and then adding Version number separated with a slash character // - Identify as ISV and include Company Name, App Name separated by a pipe character and then adding Version number separated with a slash character
//immutable string defaultUserAgent = isvTag ~ "|" ~ companyName ~ "|" ~ appTitle ~ "/" ~ strip(import("version")); //immutable string defaultUserAgent = isvTag ~ "|" ~ companyName ~ "|" ~ appTitle ~ "/" ~ strip(import("version"));
immutable string defaultUserAgent = isvTag ~ "|" ~ companyName ~ "|" ~ appTitle ~ "/" ~ "v2.5.0-alpha-0"; immutable string defaultUserAgent = isvTag ~ "|" ~ companyName ~ "|" ~ appTitle ~ "/" ~ "v2.5.0-alpha-1";
// HTTP Struct items, used for configuring HTTP() // HTTP Struct items, used for configuring HTTP()
// Curl Timeout Handling // Curl Timeout Handling
@ -146,20 +146,18 @@ class ApplicationConfig {
private string userConfigFilePath = ""; private string userConfigFilePath = "";
// - Store the system 'config' file path // - Store the system 'config' file path
private string systemConfigFilePath = ""; private string systemConfigFilePath = "";
// - What is the 'config' file path that will be used?
private string applicableConfigFilePath = "";
// - Store the 'sync_list' file path // - Store the 'sync_list' file path
string syncListFilePath = ""; string syncListFilePath = "";
// - Store the 'business_shared_items' file path // - Store the 'business_shared_items' file path
string businessSharedItemsFilePath = ""; string businessSharedItemsFilePath = "";
// - What is the 'config' file path that will be used?
private string applicableConfigFilePath = "";
// Hash files so that we can detect when the configuration has changed, in items that will require a --resync // Hash files so that we can detect when the configuration has changed, in items that will require a --resync
private string configHashFile = ""; private string configHashFile = "";
private string configBackupFile = ""; private string configBackupFile = "";
private string syncListHashFile = ""; private string syncListHashFile = "";
private string businessSharedItemsHashFile = ""; private string businessSharedItemsHashFile = "";
// hash file permission values (set via initialize function)
private int convertedPermissionValue;
// Store the actual 'runtime' hash // Store the actual 'runtime' hash
private string currentConfigHash = ""; private string currentConfigHash = "";
@ -178,6 +176,10 @@ class ApplicationConfig {
private string configFileDriveId = ""; // Default here is that no drive id is specified private string configFileDriveId = ""; // Default here is that no drive id is specified
private bool configFileSkipDotfiles = false; private bool configFileSkipDotfiles = false;
private bool configFileSkipSymbolicLinks = false; private bool configFileSkipSymbolicLinks = false;
private bool configFileSyncBusinessSharedItems = false;
// File permission values (set via initialize function)
private int convertedPermissionValue;
// Array of values that are the actual application runtime configuration // Array of values that are the actual application runtime configuration
// The values stored in these array's are the actual application configuration which can then be accessed by getValue & setValue // The values stored in these array's are the actual application configuration which can then be accessed by getValue & setValue
@ -323,7 +325,7 @@ class ApplicationConfig {
// Print in debug the application version as soon as possible // Print in debug the application version as soon as possible
//log.vdebug("Application Version: ", strip(import("version"))); //log.vdebug("Application Version: ", strip(import("version")));
string tempVersion = "v2.5.0-alpha-0" ~ " GitHub version: " ~ strip(import("version")); string tempVersion = "v2.5.0-alpha-1" ~ " GitHub version: " ~ strip(import("version"));
log.vdebug("Application Version: ", tempVersion); log.vdebug("Application Version: ", tempVersion);
// EXPAND USERS HOME DIRECTORY // EXPAND USERS HOME DIRECTORY
@ -448,9 +450,14 @@ class ApplicationConfig {
userConfigFilePath = buildNormalizedPath(configDirName ~ "/config"); userConfigFilePath = buildNormalizedPath(configDirName ~ "/config");
// - What is the full path for the system 'config' file if it is required // - What is the full path for the system 'config' file if it is required
systemConfigFilePath = buildNormalizedPath(systemConfigDirName ~ "/config"); systemConfigFilePath = buildNormalizedPath(systemConfigDirName ~ "/config");
// - What is the full path for the 'business_shared_items' // - What is the full path for the 'business_shared_items'
businessSharedItemsFilePath = buildNormalizedPath(configDirName ~ "/business_shared_items"); businessSharedItemsFilePath = buildNormalizedPath(configDirName ~ "/business_shared_items");
// To determine if any configuration items has changed, where a --resync would be required, we need to have a hash file for the following items // To determine if any configuration items has changed, where a --resync would be required, we need to have a hash file for the following items
// - 'config.backup' file // - 'config.backup' file
// - applicable 'config' file // - applicable 'config' file
@ -707,6 +714,12 @@ class ApplicationConfig {
if (key == "skip_symlinks") { if (key == "skip_symlinks") {
configFileSkipSymbolicLinks = true; configFileSkipSymbolicLinks = true;
} }
// sync_business_shared_items tracking for change
if (key == "sync_business_shared_items") {
configFileSyncBusinessSharedItems = true;
}
} else { } else {
auto pp = key in stringValues; auto pp = key in stringValues;
if (pp) { if (pp) {
@ -1084,12 +1097,6 @@ class ApplicationConfig {
"version", "version",
"Print the version and exit", "Print the version and exit",
&tmpBol, &tmpBol,
"list-shared-folders",
"List OneDrive Business Shared Items",
&boolValues["list_business_shared_items"],
"sync-shared-folders",
"Sync OneDrive Business Shared Items",
&boolValues["sync_business_shared_items"],
"with-editing-perms", "with-editing-perms",
"Create a read-write shareable link for an existing file on OneDrive when used with --create-share-link <file>", "Create a read-write shareable link for an existing file on OneDrive when used with --create-share-link <file>",
&boolValues["with_editing_perms"] &boolValues["with_editing_perms"]
@ -1119,7 +1126,7 @@ class ApplicationConfig {
// Display application version // Display application version
//writeln("onedrive version = ", strip(import("version"))); //writeln("onedrive version = ", strip(import("version")));
string tempVersion = "v2.5.0-alpha-0" ~ " GitHub version: " ~ strip(import("version")); string tempVersion = "v2.5.0-alpha-1" ~ " GitHub version: " ~ strip(import("version"));
writeln("onedrive version = ", tempVersion); writeln("onedrive version = ", tempVersion);
// Display all of the pertinent configuration options // Display all of the pertinent configuration options
@ -1188,7 +1195,7 @@ class ApplicationConfig {
writeln("Config option 'ip_protocol_version' = ", getValueLong("ip_protocol_version")); writeln("Config option 'ip_protocol_version' = ", getValueLong("ip_protocol_version"));
// Is sync_list configured ? // Is sync_list configured ?
writeln("Config option 'sync_root_files' = ", getValueBool("sync_root_files")); writeln("\nConfig option 'sync_root_files' = ", getValueBool("sync_root_files"));
if (exists(syncListFilePath)){ if (exists(syncListFilePath)){
writeln("Selective sync 'sync_list' configured = true"); writeln("Selective sync 'sync_list' configured = true");
@ -1206,9 +1213,10 @@ class ApplicationConfig {
} }
// Is sync_business_shared_items enabled and configured ? // Is sync_business_shared_items enabled and configured ?
writeln("Config option 'sync_business_shared_items' = ", getValueBool("sync_business_shared_items")); writeln("\nConfig option 'sync_business_shared_items' = ", getValueBool("sync_business_shared_items"));
if (exists(businessSharedItemsFilePath)){ if (exists(businessSharedItemsFilePath)){
writeln("Business Shared Items configured = true"); writeln("Selective Business Shared Items configured = true");
writeln("sync_business_shared_items contents:"); writeln("sync_business_shared_items contents:");
// Output the sync_business_shared_items contents // Output the sync_business_shared_items contents
auto businessSharedItemsFileList = File(businessSharedItemsFilePath, "r"); auto businessSharedItemsFileList = File(businessSharedItemsFilePath, "r");
@ -1218,11 +1226,13 @@ class ApplicationConfig {
writeln(line); writeln(line);
} }
} else { } else {
writeln("Business Shared Items configured = false"); writeln("Selective Business Shared Items configured = false");
} }
// Are webhooks enabled? // Are webhooks enabled?
writeln("Config option 'webhook_enabled' = ", getValueBool("webhook_enabled")); writeln("\nConfig option 'webhook_enabled' = ", getValueBool("webhook_enabled"));
if (getValueBool("webhook_enabled")) { if (getValueBool("webhook_enabled")) {
writeln("Config option 'webhook_public_url' = ", getValueString("webhook_public_url")); writeln("Config option 'webhook_public_url' = ", getValueString("webhook_public_url"));
writeln("Config option 'webhook_listening_host' = ", getValueString("webhook_listening_host")); writeln("Config option 'webhook_listening_host' = ", getValueString("webhook_listening_host"));
@ -1292,13 +1302,14 @@ class ApplicationConfig {
// Configuration File Flags // Configuration File Flags
bool configFileOptionsDifferent = false; bool configFileOptionsDifferent = false;
bool syncListFileDifferent = false; bool syncListFileDifferent = false;
bool businessSharedItemsFileDifferent = false;
bool syncDirDifferent = false; bool syncDirDifferent = false;
bool skipFileDifferent = false; bool skipFileDifferent = false;
bool skipDirDifferent = false; bool skipDirDifferent = false;
bool skipDotFilesDifferent = false; bool skipDotFilesDifferent = false;
bool skipSymbolicLinksDifferent = false; bool skipSymbolicLinksDifferent = false;
bool driveIdDifferent = false; bool driveIdDifferent = false;
bool syncBusinessSharedItemsDifferent = false;
bool businessSharedItemsFileDifferent = false;
// Create the required initial hash files // Create the required initial hash files
createRequiredInitialConfigurationHashFiles(); createRequiredInitialConfigurationHashFiles();
@ -1334,6 +1345,7 @@ class ApplicationConfig {
// # skip_dir = "" // # skip_dir = ""
// # skip_dotfiles = "" // # skip_dotfiles = ""
// # skip_symlinks = "" // # skip_symlinks = ""
// # sync_business_shared_items = ""
string[string] backupConfigStringValues; string[string] backupConfigStringValues;
backupConfigStringValues["drive_id"] = ""; backupConfigStringValues["drive_id"] = "";
backupConfigStringValues["sync_dir"] = ""; backupConfigStringValues["sync_dir"] = "";
@ -1341,6 +1353,7 @@ class ApplicationConfig {
backupConfigStringValues["skip_dir"] = ""; backupConfigStringValues["skip_dir"] = "";
backupConfigStringValues["skip_dotfiles"] = ""; backupConfigStringValues["skip_dotfiles"] = "";
backupConfigStringValues["skip_symlinks"] = ""; backupConfigStringValues["skip_symlinks"] = "";
backupConfigStringValues["sync_business_shared_items"] = "";
// bool flags to trigger if the entries that trigger a --resync were found in the backup config file // bool flags to trigger if the entries that trigger a --resync were found in the backup config file
// if these were not in the backup file, they may have been added ... thus new, thus we need to double check the existing // if these were not in the backup file, they may have been added ... thus new, thus we need to double check the existing
@ -1351,6 +1364,7 @@ class ApplicationConfig {
bool skip_dir_present = false; bool skip_dir_present = false;
bool skip_dotfiles_present = false; bool skip_dotfiles_present = false;
bool skip_symlinks_present = false; bool skip_symlinks_present = false;
bool sync_business_shared_items_present = false;
// Common debug message if an element is different // Common debug message if an element is different
string configOptionModifiedMessage = " was modified since the last time the application was successfully run, --resync required"; string configOptionModifiedMessage = " was modified since the last time the application was successfully run, --resync required";
@ -1421,6 +1435,14 @@ class ApplicationConfig {
configFileOptionsDifferent = true; configFileOptionsDifferent = true;
} }
} }
if (key == "sync_business_shared_items") {
sync_business_shared_items_present = true;
if (c.front.dup != to!string(getValueBool("sync_business_shared_items"))) {
log.vdebug(key, configOptionModifiedMessage);
configFileOptionsDifferent = true;
}
}
} }
} }
} }
@ -1439,6 +1461,7 @@ class ApplicationConfig {
log.vdebug("skip_dir present in config backup: ", skip_dir_present); log.vdebug("skip_dir present in config backup: ", skip_dir_present);
log.vdebug("skip_dotfiles present in config backup: ", skip_dotfiles_present); log.vdebug("skip_dotfiles present in config backup: ", skip_dotfiles_present);
log.vdebug("skip_symlinks present in config backup: ", skip_symlinks_present); log.vdebug("skip_symlinks present in config backup: ", skip_symlinks_present);
log.vdebug("sync_business_shared_items present in config backup: ", sync_business_shared_items_present);
if ((!drive_id_present) && (configFileDriveId != "")) { if ((!drive_id_present) && (configFileDriveId != "")) {
writeln("drive_id newly added ... --resync needed"); writeln("drive_id newly added ... --resync needed");
@ -1475,6 +1498,12 @@ class ApplicationConfig {
configFileOptionsDifferent = true; configFileOptionsDifferent = true;
skipSymbolicLinksDifferent = true; skipSymbolicLinksDifferent = true;
} }
if ((!sync_business_shared_items_present) && (configFileSyncBusinessSharedItems)) {
writeln("sync_business_shared_items newly added ... --resync needed");
configFileOptionsDifferent = true;
syncBusinessSharedItemsDifferent = true;
}
} }
} else { } else {
// no backup to check // no backup to check
@ -1544,15 +1573,19 @@ class ApplicationConfig {
// Did any of the config files or CLI options trigger a --resync requirement? // Did any of the config files or CLI options trigger a --resync requirement?
log.vdebug("configFileOptionsDifferent: ", configFileOptionsDifferent); log.vdebug("configFileOptionsDifferent: ", configFileOptionsDifferent);
log.vdebug("syncListFileDifferent: ", syncListFileDifferent); // Options
log.vdebug("businessSharedItemsFileDifferent: ", businessSharedItemsFileDifferent); log.vdebug("driveIdDifferent: ", driveIdDifferent);
log.vdebug("syncDirDifferent: ", syncDirDifferent); log.vdebug("syncDirDifferent: ", syncDirDifferent);
log.vdebug("skipFileDifferent: ", skipFileDifferent); log.vdebug("skipFileDifferent: ", skipFileDifferent);
log.vdebug("skipDirDifferent: ", skipDirDifferent); log.vdebug("skipDirDifferent: ", skipDirDifferent);
log.vdebug("driveIdDifferent: ", driveIdDifferent);
log.vdebug("skipDotFilesDifferent: ", skipDotFilesDifferent); log.vdebug("skipDotFilesDifferent: ", skipDotFilesDifferent);
log.vdebug("skipSymbolicLinksDifferent: ", skipSymbolicLinksDifferent);
log.vdebug("syncBusinessSharedItemsDifferent: ", syncBusinessSharedItemsDifferent);
// Files
log.vdebug("syncListFileDifferent: ", syncListFileDifferent);
log.vdebug("businessSharedItemsFileDifferent: ", businessSharedItemsFileDifferent);
if ((configFileOptionsDifferent) || (syncListFileDifferent) || (businessSharedItemsFileDifferent) || (syncDirDifferent) || (skipFileDifferent) || (skipDirDifferent) || (driveIdDifferent) || (skipDotFilesDifferent) || (skipSymbolicLinksDifferent) ) { if ((configFileOptionsDifferent) || (syncListFileDifferent) || (businessSharedItemsFileDifferent) || (syncDirDifferent) || (skipFileDifferent) || (skipDirDifferent) || (driveIdDifferent) || (skipDotFilesDifferent) || (skipSymbolicLinksDifferent) || (syncBusinessSharedItemsDifferent) ) {
// set the flag // set the flag
resyncRequired = true; resyncRequired = true;
} }
@ -1594,6 +1627,8 @@ class ApplicationConfig {
// Hash file should only be readable by the user who created it - 0600 permissions needed // Hash file should only be readable by the user who created it - 0600 permissions needed
syncListHashFile.setAttributes(convertedPermissionValue); syncListHashFile.setAttributes(convertedPermissionValue);
} }
// Update 'update business_shared_items' files // Update 'update business_shared_items' files
if (exists(businessSharedItemsFilePath)) { if (exists(businessSharedItemsFilePath)) {
// update business_shared_folders hash // update business_shared_folders hash
@ -1602,6 +1637,7 @@ class ApplicationConfig {
// Hash file should only be readable by the user who created it - 0600 permissions needed // Hash file should only be readable by the user who created it - 0600 permissions needed
businessSharedItemsHashFile.setAttributes(convertedPermissionValue); businessSharedItemsHashFile.setAttributes(convertedPermissionValue);
} }
} else { } else {
// --dry-run scenario ... technically we should not be making any local file changes ....... // --dry-run scenario ... technically we should not be making any local file changes .......
log.log("DRY RUN: Not updating hash files as --dry-run has been used"); log.log("DRY RUN: Not updating hash files as --dry-run has been used");

View file

@ -28,6 +28,7 @@ struct Item {
string driveId; string driveId;
string id; string id;
string name; string name;
string remoteName;
ItemType type; ItemType type;
string eTag; string eTag;
string cTag; string cTag;
@ -43,11 +44,15 @@ struct Item {
// Construct an Item struct from a JSON driveItem // Construct an Item struct from a JSON driveItem
Item makeDatabaseItem(JSONValue driveItem) { Item makeDatabaseItem(JSONValue driveItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
Item item = { Item item = {
id: driveItem["id"].str, id: driveItem["id"].str,
name: "name" in driveItem ? driveItem["name"].str : null, // name may be missing for deleted files in OneDrive Biz name: "name" in driveItem ? driveItem["name"].str : null, // name may be missing for deleted files in OneDrive Business
eTag: "eTag" in driveItem ? driveItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Biz eTag: "eTag" in driveItem ? driveItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Business
cTag: "cTag" in driveItem ? driveItem["cTag"].str : null, // cTag is missing in old files (and all folders in OneDrive Biz) cTag: "cTag" in driveItem ? driveItem["cTag"].str : null, // cTag is missing in old files (and all folders in OneDrive Business)
remoteName: "actualOnlineName" in driveItem ? driveItem["actualOnlineName"].str : null, // actualOnlineName is only used with OneDrive Business Shared Folders
}; };
// OneDrive API Change: https://github.com/OneDrive/onedrive-api-docs/issues/834 // OneDrive API Change: https://github.com/OneDrive/onedrive-api-docs/issues/834
@ -164,6 +169,7 @@ final class ItemDatabase {
string insertItemStmt; string insertItemStmt;
string updateItemStmt; string updateItemStmt;
string selectItemByIdStmt; string selectItemByIdStmt;
string selectItemByRemoteIdStmt;
string selectItemByParentIdStmt; string selectItemByParentIdStmt;
string deleteItemByIdStmt; string deleteItemByIdStmt;
bool databaseInitialised = false; bool databaseInitialised = false;
@ -205,7 +211,11 @@ final class ItemDatabase {
db.exec("PRAGMA recursive_triggers = TRUE"); db.exec("PRAGMA recursive_triggers = TRUE");
// Set the journal mode for databases associated with the current connection // Set the journal mode for databases associated with the current connection
// https://www.sqlite.org/pragma.html#pragma_journal_mode // https://www.sqlite.org/pragma.html#pragma_journal_mode
db.exec("PRAGMA journal_mode = WAL");
//db.exec("PRAGMA journal_mode = WAL");
// Automatic indexing is enabled by default as of version 3.7.17 // Automatic indexing is enabled by default as of version 3.7.17
// https://www.sqlite.org/pragma.html#pragma_automatic_index // https://www.sqlite.org/pragma.html#pragma_automatic_index
// PRAGMA automatic_index = boolean; // PRAGMA automatic_index = boolean;
@ -223,12 +233,12 @@ final class ItemDatabase {
db.exec("PRAGMA locking_mode = EXCLUSIVE"); db.exec("PRAGMA locking_mode = EXCLUSIVE");
insertItemStmt = " insertItemStmt = "
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, quickXorHash, sha256Hash, remoteDriveId, remoteId, syncStatus, size) INSERT OR REPLACE INTO item (driveId, id, name, remoteName, type, eTag, cTag, mtime, parentId, quickXorHash, sha256Hash, remoteDriveId, remoteId, syncStatus, size)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)
"; ";
updateItemStmt = " updateItemStmt = "
UPDATE item UPDATE item
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, quickXorHash = ?9, sha256Hash = ?10, remoteDriveId = ?11, remoteId = ?12, syncStatus = ?13, size = ?14 SET name = ?3, remoteName = ?4, type = ?5, eTag = ?6, cTag = ?7, mtime = ?8, parentId = ?9, quickXorHash = ?10, sha256Hash = ?11, remoteDriveId = ?12, remoteId = ?13, syncStatus = ?14, size = ?15
WHERE driveId = ?1 AND id = ?2 WHERE driveId = ?1 AND id = ?2
"; ";
selectItemByIdStmt = " selectItemByIdStmt = "
@ -236,6 +246,11 @@ final class ItemDatabase {
FROM item FROM item
WHERE driveId = ?1 AND id = ?2 WHERE driveId = ?1 AND id = ?2
"; ";
selectItemByRemoteIdStmt = "
SELECT *
FROM item
WHERE remoteDriveId = ?1 AND remoteId = ?2
";
selectItemByParentIdStmt = "SELECT * FROM item WHERE driveId = ? AND parentId = ?"; selectItemByParentIdStmt = "SELECT * FROM item WHERE driveId = ? AND parentId = ?";
deleteItemByIdStmt = "DELETE FROM item WHERE driveId = ? AND id = ?"; deleteItemByIdStmt = "DELETE FROM item WHERE driveId = ? AND id = ?";
@ -252,6 +267,7 @@ final class ItemDatabase {
driveId TEXT NOT NULL, driveId TEXT NOT NULL,
id TEXT NOT NULL, id TEXT NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
remoteName TEXT,
type TEXT NOT NULL, type TEXT NOT NULL,
eTag TEXT, eTag TEXT,
cTag TEXT, cTag TEXT,
@ -334,6 +350,18 @@ final class ItemDatabase {
return false; return false;
} }
bool selectByRemoteId(const(char)[] remoteDriveId, const(char)[] remoteId, out Item item) {
auto p = db.prepare(selectItemByRemoteIdStmt);
p.bind(1, remoteDriveId);
p.bind(2, remoteId);
auto r = p.exec();
if (!r.empty) {
item = buildItem(r);
return true;
}
return false;
}
// returns true if an item id is in the database // returns true if an item id is in the database
bool idInLocalDatabase(const(string) driveId, const(string)id) { bool idInLocalDatabase(const(string) driveId, const(string)id) {
auto p = db.prepare(selectItemByIdStmt); auto p = db.prepare(selectItemByIdStmt);
@ -420,6 +448,7 @@ final class ItemDatabase {
bind(1, driveId); bind(1, driveId);
bind(2, id); bind(2, id);
bind(3, name); bind(3, name);
bind(4, remoteName);
string typeStr = null; string typeStr = null;
final switch (type) with (ItemType) { final switch (type) with (ItemType) {
case file: typeStr = "file"; break; case file: typeStr = "file"; break;
@ -427,41 +456,60 @@ final class ItemDatabase {
case remote: typeStr = "remote"; break; case remote: typeStr = "remote"; break;
case unknown: typeStr = "unknown"; break; case unknown: typeStr = "unknown"; break;
} }
bind(4, typeStr); bind(5, typeStr);
bind(5, eTag); bind(6, eTag);
bind(6, cTag); bind(7, cTag);
bind(7, mtime.toISOExtString()); bind(8, mtime.toISOExtString());
bind(8, parentId); bind(9, parentId);
bind(9, quickXorHash); bind(10, quickXorHash);
bind(10, sha256Hash); bind(11, sha256Hash);
bind(11, remoteDriveId); bind(12, remoteDriveId);
bind(12, remoteId); bind(13, remoteId);
bind(13, syncStatus); bind(14, syncStatus);
bind(14, size); bind(15, size);
} }
} }
private Item buildItem(Statement.Result result) { private Item buildItem(Statement.Result result) {
assert(!result.empty, "The result must not be empty"); assert(!result.empty, "The result must not be empty");
assert(result.front.length == 15, "The result must have 15 columns"); assert(result.front.length == 16, "The result must have 16 columns");
Item item = { Item item = {
// column 0: driveId
// column 1: id
// column 2: name
// column 3: remoteName - only used when there is a difference in the local name & remote shared folder name
// column 4: type
// column 5: eTag
// column 6: cTag
// column 7: mtime
// column 8: parentId
// column 9: quickXorHash
// column 10: sha256Hash
// column 11: remoteDriveId
// column 12: remoteId
// column 13: deltaLink
// column 14: syncStatus
// column 15: size
driveId: result.front[0].dup, driveId: result.front[0].dup,
id: result.front[1].dup, id: result.front[1].dup,
name: result.front[2].dup, name: result.front[2].dup,
// Column 3 is type - not set here remoteName: result.front[3].dup,
eTag: result.front[4].dup, // Column 4 is type - not set here
cTag: result.front[5].dup, eTag: result.front[5].dup,
mtime: SysTime.fromISOExtString(result.front[6]), cTag: result.front[6].dup,
parentId: result.front[7].dup, mtime: SysTime.fromISOExtString(result.front[7]),
quickXorHash: result.front[8].dup, parentId: result.front[8].dup,
sha256Hash: result.front[9].dup, quickXorHash: result.front[9].dup,
remoteDriveId: result.front[10].dup, sha256Hash: result.front[10].dup,
remoteId: result.front[11].dup, remoteDriveId: result.front[11].dup,
// Column 12 is deltaLink - not set here remoteId: result.front[12].dup,
syncStatus: result.front[13].dup, // Column 13 is deltaLink - not set here
size: result.front[14].dup syncStatus: result.front[14].dup,
size: result.front[15].dup
}; };
switch (result.front[3]) { switch (result.front[4]) {
case "file": item.type = ItemType.file; break; case "file": item.type = ItemType.file; break;
case "dir": item.type = ItemType.dir; break; case "dir": item.type = ItemType.dir; break;
case "remote": item.type = ItemType.remote; break; case "remote": item.type = ItemType.remote; break;

View file

@ -112,7 +112,7 @@ int main(string[] cliArgs) {
// Print the version and exit // Print the version and exit
if (printVersion) { if (printVersion) {
//writeln("onedrive ", strip(import("version"))); //writeln("onedrive ", strip(import("version")));
string tempVersion = "v2.5.0-alpha-0" ~ " GitHub version: " ~ strip(import("version")); string tempVersion = "v2.5.0-alpha-1" ~ " GitHub version: " ~ strip(import("version"));
writeln(tempVersion); writeln(tempVersion);
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }

View file

@ -284,7 +284,7 @@ class OneDriveApi {
} else { } else {
// Try and read the value from the appConfig if it is set, rather than trying to read the value from disk // Try and read the value from the appConfig if it is set, rather than trying to read the value from disk
if (!appConfig.refreshToken.empty) { if (!appConfig.refreshToken.empty) {
log.vdebug("read token from appConfig"); log.vdebug("Read token from appConfig");
refreshToken = strip(appConfig.refreshToken); refreshToken = strip(appConfig.refreshToken);
authorised = true; authorised = true;
} else { } else {
@ -494,6 +494,12 @@ class OneDriveApi {
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta
JSONValue viewChangesByItemId(string driveId, string id, string deltaLink) { JSONValue viewChangesByItemId(string driveId, string id, string deltaLink) {
checkAccessTokenExpired(); checkAccessTokenExpired();
// If Business Account add addIncludeFeatureRequestHeader() which should add Prefer: Include-Feature=AddToOneDrive
if ((appConfig.accountType != "personal") && ( appConfig.getValueBool("sync_business_shared_items"))) {
addIncludeFeatureRequestHeader();
}
string url; string url;
// configure deltaLink to query // configure deltaLink to query
if (deltaLink.empty) { if (deltaLink.empty) {
@ -507,6 +513,12 @@ class OneDriveApi {
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_list_children // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_list_children
JSONValue listChildren(string driveId, string id, string nextLink) { JSONValue listChildren(string driveId, string id, string nextLink) {
checkAccessTokenExpired(); checkAccessTokenExpired();
// If Business Account add addIncludeFeatureRequestHeader() which should add Prefer: Include-Feature=AddToOneDrive
if ((appConfig.accountType != "personal") && ( appConfig.getValueBool("sync_business_shared_items"))) {
addIncludeFeatureRequestHeader();
}
string url; string url;
// configure URL to query // configure URL to query
if (nextLink.empty) { if (nextLink.empty) {
@ -677,6 +689,11 @@ class OneDriveApi {
curlEngine.http.addRequestHeader("Authorization", accessToken); curlEngine.http.addRequestHeader("Authorization", accessToken);
} }
private void addIncludeFeatureRequestHeader() {
log.vdebug("Adding 'Include-Feature=AddToOneDrive' API request header as 'sync_business_shared_items' config option is enabled");
curlEngine.http.addRequestHeader("Prefer", "Include-Feature=AddToOneDrive");
}
private void acquireToken(char[] postData) { private void acquireToken(char[] postData) {
JSONValue response; JSONValue response;
@ -684,6 +701,10 @@ class OneDriveApi {
response = post(tokenUrl, postData); response = post(tokenUrl, postData);
} catch (OneDriveException e) { } catch (OneDriveException e) {
// an error was generated // an error was generated
if ((e.httpStatusCode == 400) || (e.httpStatusCode == 401)) {
// Handle an unauthorised client
handleClientUnauthorised(e.httpStatusCode, e.msg);
} else {
if (e.httpStatusCode >= 500) { if (e.httpStatusCode >= 500) {
// There was a HTTP 5xx Server Side Error - retry // There was a HTTP 5xx Server Side Error - retry
acquireToken(postData); acquireToken(postData);
@ -691,6 +712,7 @@ class OneDriveApi {
displayOneDriveErrorMessage(e.msg, getFunctionName!({})); displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
} }
} }
}
if (response.type() == JSONType.object) { if (response.type() == JSONType.object) {
// Has the client been configured to use read_only_auth_scope // Has the client been configured to use read_only_auth_scope

View file

@ -124,8 +124,6 @@ class SyncEngine {
// VARIABLES NEEDED BUT STILL TO BE TESTED WITH AND USED CORRECTLY // VARIABLES NEEDED BUT STILL TO BE TESTED WITH AND USED CORRECTLY
bool syncBusinessFolders = false; // this one will change as we will not be just doing business folders bool syncBusinessFolders = false; // this one will change as we will not be just doing business folders
// Configure this class instance // Configure this class instance
this(ApplicationConfig appConfig, ItemDatabase itemDB, ClientSideFiltering selectiveSync) { this(ApplicationConfig appConfig, ItemDatabase itemDB, ClientSideFiltering selectiveSync) {
// Configure the class varaible to consume the application configuration // Configure the class varaible to consume the application configuration
@ -221,6 +219,7 @@ class SyncEngine {
// Initialise the Sync Engine class // Initialise the Sync Engine class
bool initialise() { bool initialise() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// create a new instance of the OneDrive API // create a new instance of the OneDrive API
oneDriveApiInstance = new OneDriveApi(appConfig); oneDriveApiInstance = new OneDriveApi(appConfig);
if (oneDriveApiInstance.initialise()) { if (oneDriveApiInstance.initialise()) {
@ -239,6 +238,7 @@ class SyncEngine {
// Get Default Drive Details for this Account // Get Default Drive Details for this Account
void getDefaultDriveDetails() { void getDefaultDriveDetails() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Function variables // Function variables
JSONValue defaultOneDriveDriveDetails; JSONValue defaultOneDriveDriveDetails;
@ -251,23 +251,9 @@ class SyncEngine {
string thisFunctionName = getFunctionName!({}); string thisFunctionName = getFunctionName!({});
if (exception.httpStatusCode == 400) { if ((exception.httpStatusCode == 400) || (exception.httpStatusCode == 401)) {
displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); // Handle the 400 | 401 error
// Check this handleClientUnauthorised(exception.httpStatusCode, exception.msg);
if (appConfig.getValueString("drive_id").length) {
writeln();
log.error("ERROR: Check your 'drive_id' entry in your configuration file as it may be incorrect");
writeln();
}
// Must exit here
oneDriveApiInstance.shutdown();
exit(-1);
}
if (exception.httpStatusCode == 401) {
// HTTP request returned status code 401 (Unauthorized)
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
handleClientUnauthorised();
} }
// HTTP request returned status code 408,429,503,504 // HTTP request returned status code 408,429,503,504
@ -367,6 +353,7 @@ class SyncEngine {
// Get Default Root Details for this Account // Get Default Root Details for this Account
void getDefaultRootDetails() { void getDefaultRootDetails() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Function variables // Function variables
JSONValue defaultOneDriveRootDetails; JSONValue defaultOneDriveRootDetails;
@ -379,23 +366,9 @@ class SyncEngine {
string thisFunctionName = getFunctionName!({}); string thisFunctionName = getFunctionName!({});
if (exception.httpStatusCode == 400) { if ((exception.httpStatusCode == 400) || (exception.httpStatusCode == 401)) {
displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); // Handle the 400 | 401 error
// Check this handleClientUnauthorised(exception.httpStatusCode, exception.msg);
if (appConfig.getValueString("drive_id").length) {
writeln();
log.error("ERROR: Check your 'drive_id' entry in your configuration file as it may be incorrect");
writeln();
}
// Must exit here
oneDriveApiInstance.shutdown();
exit(-1);
}
if (exception.httpStatusCode == 401) {
// HTTP request returned status code 401 (Unauthorized)
displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
handleClientUnauthorised();
} }
// HTTP request returned status code 408,429,503,504 // HTTP request returned status code 408,429,503,504
@ -450,6 +423,7 @@ class SyncEngine {
// - Process any deletes (remove local data) // - Process any deletes (remove local data)
// - Walk local file system for any differences (new files / data to upload to OneDrive) // - Walk local file system for any differences (new files / data to upload to OneDrive)
void syncOneDriveAccountToLocalDisk(bool performFullScanTrueUp = false) { void syncOneDriveAccountToLocalDisk(bool performFullScanTrueUp = false) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// performFullScanTrueUp value // performFullScanTrueUp value
log.vdebug("performFullScanTrueUp: ", performFullScanTrueUp); log.vdebug("performFullScanTrueUp: ", performFullScanTrueUp);
// Fetch the API response of /delta to track changes on OneDrive // Fetch the API response of /delta to track changes on OneDrive
@ -489,10 +463,48 @@ class SyncEngine {
processDownloadActivities(); processDownloadActivities();
} }
} else { } else {
// Not a 'Personal' Account Type - so will either be Business or SharePoint Library, and these need to follow a different process // Is this a Business Account with Sync Business Shared Items enabled?
// - OneDrive Business Shared Folder Handling if ((appConfig.accountType == "business") && ( appConfig.getValueBool("sync_business_shared_items"))) {
// - SharePoint Links ?
// Business Account Shared Items Handling
// - OneDrive Business Shared Folder
// - OneDrive Business Shared Files ??
// - SharePoint Links
// Get the Remote Items from the Database
Item[] remoteItems = itemDB.selectRemoteItems();
foreach (remoteItem; remoteItems) {
// Check if this path is specifically excluded by 'skip_dir', but only if 'skip_dir' is not empty
if (appConfig.getValueString("skip_dir") != "") {
// The path that needs to be checked needs to include the '/'
// This due to if the user has specified in skip_dir an exclusive path: '/path' - that is what must be matched
if (selectiveSync.isDirNameExcluded(remoteItem.name)) {
// This directory name is excluded
log.vlog("Skipping item - excluded by skip_dir config: ", remoteItem.name);
continue;
}
}
// Directory name is not excluded or skip_dir is not populated
if (!appConfig.surpressLoggingOutput) {
log.log("Syncing this OneDrive Business Shared Folder: ", remoteItem.name);
}
log.vdebug("Fetching /delta API response for:");
log.vdebug(" remoteItem.remoteDriveId: ", remoteItem.remoteDriveId);
log.vdebug(" remoteItem.remoteId: ", remoteItem.remoteId);
// Check this OneDrive Personal Shared Folder for changes
fetchOneDriveDeltaAPIResponse(remoteItem.remoteDriveId, remoteItem.remoteId, remoteItem.name, performFullScanTrueUp);
// Process any download activities or cleanup actions for this OneDrive Personal Shared Folder
processDownloadActivities();
}
}
} }
} }
} }
@ -500,6 +512,7 @@ class SyncEngine {
// Configure singleDirectoryScope = true if this function is called // Configure singleDirectoryScope = true if this function is called
// By default, singleDirectoryScope = false // By default, singleDirectoryScope = false
void setSingleDirectoryScope(string normalisedSingleDirectoryPath) { void setSingleDirectoryScope(string normalisedSingleDirectoryPath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Function variables // Function variables
Item searchItem; Item searchItem;
@ -550,6 +563,8 @@ class SyncEngine {
// Query OneDrive API for /delta changes and iterate through items online // Query OneDrive API for /delta changes and iterate through items online
void fetchOneDriveDeltaAPIResponse(string driveIdToQuery = null, string itemIdToQuery = null, string sharedFolderName = null, bool performFullScanTrueUp = false) { void fetchOneDriveDeltaAPIResponse(string driveIdToQuery = null, string itemIdToQuery = null, string sharedFolderName = null, bool performFullScanTrueUp = false) {
log.vdebug("Starting this function: ", getFunctionName!({}));
string deltaLink = null; string deltaLink = null;
string deltaLinkAvailable; string deltaLinkAvailable;
JSONValue deltaChanges; JSONValue deltaChanges;
@ -776,6 +791,7 @@ class SyncEngine {
// Process the /delta API JSON response items // Process the /delta API JSON response items
void processDeltaJSONItem(JSONValue onedriveJSONItem, ulong nrChanges, int changeCount, ulong responseBundleCount, bool singleDirectoryScope) { void processDeltaJSONItem(JSONValue onedriveJSONItem, ulong nrChanges, int changeCount, ulong responseBundleCount, bool singleDirectoryScope) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Variables for this foreach loop // Variables for this foreach loop
string thisItemId; string thisItemId;
bool itemIsRoot = false; bool itemIsRoot = false;
@ -833,16 +849,48 @@ class SyncEngine {
log.vdebug("handleItemAsRootObject = ", handleItemAsRootObject); log.vdebug("handleItemAsRootObject = ", handleItemAsRootObject);
log.vdebug("itemHasParentReferenceId = ", itemHasParentReferenceId); log.vdebug("itemHasParentReferenceId = ", itemHasParentReferenceId);
log.vdebug("itemIsDeletedOnline = ", itemIsDeletedOnline); log.vdebug("itemIsDeletedOnline = ", itemIsDeletedOnline);
log.vdebug("Handling change as 'root item', or has no parent reference id or is a deleted item"); log.vdebug("Handling change immediately as 'root item', or has no parent reference id or is a deleted item");
// OK ... do something with this JSON post here .... // OK ... do something with this JSON post here ....
processRootAndDeletedJSONItems(onedriveJSONItem, objectParentDriveId, handleItemAsRootObject, itemIsDeletedOnline, itemHasParentReferenceId); processRootAndDeletedJSONItems(onedriveJSONItem, objectParentDriveId, handleItemAsRootObject, itemIsDeletedOnline, itemHasParentReferenceId);
} else { } else {
// Do we need to update this RAW JSON from OneDrive?
if ( (objectParentDriveId != appConfig.defaultDriveId) && (appConfig.accountType == "business") && (appConfig.getValueBool("sync_business_shared_items")) ) {
// Potentially need to update this JSON data
log.vdebug("Potentially need to update this source JSON .... need to check the database");
// Check the DB for 'remote' objects, searching 'remoteDriveId' and 'remoteId' items for this remoteItem.driveId and remoteItem.id
Item remoteDBItem;
itemDB.selectByRemoteId(objectParentDriveId, thisItemId, remoteDBItem);
// Is the data that was returned from the database what we are looking for?
if ((remoteDBItem.remoteDriveId == objectParentDriveId) && (remoteDBItem.remoteId == thisItemId)) {
// Yes, this is the record we are looking for
log.vdebug("DB Item response for remoteDBItem: ", remoteDBItem);
// Must compare remoteDBItem.name with remoteItem.name
if (remoteDBItem.name != onedriveJSONItem["name"].str) {
// Update JSON Item
string actualOnlineName = onedriveJSONItem["name"].str;
log.vdebug("Updating source JSON 'name' to that which is the actual local directory");
log.vdebug("onedriveJSONItem['name'] was: ", onedriveJSONItem["name"].str);
log.vdebug("Updating onedriveJSONItem['name'] to: ", remoteDBItem.name);
onedriveJSONItem["name"] = remoteDBItem.name;
log.vdebug("onedriveJSONItem['name'] now: ", onedriveJSONItem["name"].str);
// Add the original name to the JSON
onedriveJSONItem["actualOnlineName"] = actualOnlineName;
}
}
}
// Add this JSON item for further processing
log.vdebug("Adding this Raw JSON OneDrive Item to jsonItemsToProcess array for further processing");
jsonItemsToProcess ~= onedriveJSONItem; jsonItemsToProcess ~= onedriveJSONItem;
} }
} }
// Process 'root' and 'deleted' OneDrive JSON items // Process 'root' and 'deleted' OneDrive JSON items
void processRootAndDeletedJSONItems(JSONValue onedriveJSONItem, string driveId, bool handleItemAsRootObject, bool itemIsDeletedOnline, bool itemHasParentReferenceId) { void processRootAndDeletedJSONItems(JSONValue onedriveJSONItem, string driveId, bool handleItemAsRootObject, bool itemIsDeletedOnline, bool itemHasParentReferenceId) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Is the item deleted online? // Is the item deleted online?
if(!itemIsDeletedOnline) { if(!itemIsDeletedOnline) {
@ -887,6 +935,7 @@ class SyncEngine {
// Process each of the elements contained in jsonItemsToProcess[] // Process each of the elements contained in jsonItemsToProcess[]
void processJSONItemsInBatch(JSONValue[] array) { void processJSONItemsInBatch(JSONValue[] array) {
log.vdebug("Starting this function: ", getFunctionName!({}));
foreach (i, onedriveJSONItem; array.enumerate) { foreach (i, onedriveJSONItem; array.enumerate) {
// Use the JSON elements rather can computing a DB struct via makeItem() // Use the JSON elements rather can computing a DB struct via makeItem()
@ -905,6 +954,9 @@ class SyncEngine {
// What is the path of the new item // What is the path of the new item
string newItemPath; string newItemPath;
// Configure the remoteItem - so if it is used, it can be utilised later
Item remoteItem;
// Check the database for an existing entry for this JSON item // Check the database for an existing entry for this JSON item
bool existingDBEntry = itemDB.selectById(thisItemDriveId, thisItemId, existingDatabaseItem); bool existingDBEntry = itemDB.selectById(thisItemDriveId, thisItemId, existingDatabaseItem);
@ -930,34 +982,73 @@ class SyncEngine {
unwanted = true; unwanted = true;
} else { } else {
// Edge case as the parent (from another users OneDrive account) will never be in the database - potentially a shared object? // Edge case as the parent (from another users OneDrive account) will never be in the database - potentially a shared object?
log.log("Potential Shared Object Item: ", onedriveJSONItem); log.vdebug("Potential Shared Object Item: ", onedriveJSONItem);
// Format the OneDrive change into a consumable object for the database // Format the OneDrive change into a consumable object for the database
Item remoteItem = makeItem(onedriveJSONItem); remoteItem = makeItem(onedriveJSONItem);
log.log("The reported parentId is not in the database. This potentially is a shared folder as 'remoteItem.driveId' != 'appConfig.defaultDriveId'. Relevant Details: remoteItem.driveId (", remoteItem.driveId,"), remoteItem.parentId (", remoteItem.parentId,")"); log.vdebug("The reported parentId is not in the database. This potentially is a shared folder as 'remoteItem.driveId' != 'appConfig.defaultDriveId'. Relevant Details: remoteItem.driveId (", remoteItem.driveId,"), remoteItem.parentId (", remoteItem.parentId,")");
// If we are syncing OneDrive Business Shared Folders, a 'folder' shared with us, has a 'parent' that is not shared with us hence the above message
// What we need to do is query the DB for this 'remoteItem.driveId' and use the response from the DB to set the 'remoteItem.parentId' for this new item we are trying to add to the database
if (appConfig.accountType == "personal") { if (appConfig.accountType == "personal") {
// Personal Account Type // Personal Account Handling
// - Ensure that this item has no parent // Ensure that this item has no parent
log.log("Setting remoteItem.parentId to be null"); log.vdebug("Setting remoteItem.parentId to be null");
remoteItem.parentId = null; remoteItem.parentId = null;
} else {
// This is a Business or SharePoint Account Type
// Has the user configured Business Shared Folders to sync ?
if (syncBusinessFolders) {
foreach(dbItem; itemDB.selectByDriveId(remoteItem.driveId)) {
if (dbItem.name == "root") {
// Ensure that this item uses the root id as parent
log.vdebug("Falsifying remoteItem.parentId to be ", dbItem.id);
remoteItem.parentId = dbItem.id;
}
}
}
}
// Add this record to the local database // Add this record to the local database
log.log("Update/Insert local database with remoteItem details: ", remoteItem); log.vdebug("Update/Insert local database with remoteItem details with remoteItem.parentId as null: ", remoteItem);
itemDB.upsert(remoteItem); itemDB.upsert(remoteItem);
} else {
// Business or SharePoint Account Handling
log.vdebug("Handling a Business or SharePoint Shared Item JSON object");
if (appConfig.accountType == "business") {
// Create a DB Tie Record for this parent object
Item parentItem;
parentItem.driveId = onedriveJSONItem["parentReference"]["driveId"].str;
parentItem.id = onedriveJSONItem["parentReference"]["id"].str;
parentItem.name = "root";
parentItem.type = ItemType.dir;
parentItem.mtime = remoteItem.mtime;
parentItem.parentId = null;
// Add this parent record to the local database
log.vdebug("Insert local database with remoteItem parent details: ", parentItem);
itemDB.upsert(parentItem);
// Ensure that this item has no parent
log.vdebug("Setting remoteItem.parentId to be null");
remoteItem.parentId = null;
// Check the DB for 'remote' objects, searching 'remoteDriveId' and 'remoteId' items for this remoteItem.driveId and remoteItem.id
Item remoteDBItem;
itemDB.selectByRemoteId(remoteItem.driveId, remoteItem.id, remoteDBItem);
// Must compare remoteDBItem.name with remoteItem.name
if ((!remoteDBItem.name.empty) && (remoteDBItem.name != remoteItem.name)) {
// Update DB Item
log.vdebug("The shared item stored in OneDrive, has a different name to the actual name on the remote drive");
log.vdebug("Updating remoteItem.name JSON data with the actual name being used on account drive and local folder");
log.vdebug("remoteItem.name was: ", remoteItem.name);
log.vdebug("Updating remoteItem.name to: ", remoteDBItem.name);
remoteItem.name = remoteDBItem.name;
log.vdebug("Setting remoteItem.remoteName to: ", onedriveJSONItem["name"].str);
// Update JSON Item
remoteItem.remoteName = onedriveJSONItem["name"].str;
log.vdebug("Updating source JSON 'name' to that which is the actual local directory");
log.vdebug("onedriveJSONItem['name'] was: ", onedriveJSONItem["name"].str);
log.vdebug("Updating onedriveJSONItem['name'] to: ", remoteDBItem.name);
onedriveJSONItem["name"] = remoteDBItem.name;
log.vdebug("onedriveJSONItem['name'] now: ", onedriveJSONItem["name"].str);
// Update newItemPath value
newItemPath = computeItemPath(thisItemDriveId, thisItemParentId) ~ "/" ~ remoteDBItem.name;
log.vdebug("New Item updated calculated full path is: ", newItemPath);
}
// Add this record to the local database
log.vdebug("Update/Insert local database with remoteItem details: ", remoteItem);
itemDB.upsert(remoteItem);
}
}
} }
} }
@ -999,7 +1090,9 @@ class SyncEngine {
log.vdebug("The item we are syncing is a folder"); log.vdebug("The item we are syncing is a folder");
} else if (isItemRemote(onedriveJSONItem)) { } else if (isItemRemote(onedriveJSONItem)) {
log.vdebug("The item we are syncing is a remote item"); log.vdebug("The item we are syncing is a remote item");
/**
assert(isItemFolder(onedriveJSONItem["remoteItem"]), "The remote item is not a folder"); assert(isItemFolder(onedriveJSONItem["remoteItem"]), "The remote item is not a folder");
**/
} else { } else {
// Why was this unwanted? // Why was this unwanted?
if (newItemPath.empty) { if (newItemPath.empty) {
@ -1048,7 +1141,9 @@ class SyncEngine {
} else { } else {
log.vdebug("Parent details not in database - unable to compute complex path to check"); log.vdebug("Parent details not in database - unable to compute complex path to check");
} }
if (!complexPathToCheck.empty) {
log.vdebug("skip_dir path to check (complex): ", complexPathToCheck); log.vdebug("skip_dir path to check (complex): ", complexPathToCheck);
}
} else { } else {
simplePathToCheck = onedriveJSONItem["name"].str; simplePathToCheck = onedriveJSONItem["name"].str;
} }
@ -1247,6 +1342,7 @@ class SyncEngine {
} else { } else {
// This JSON item is wanted - we need to process this JSON item further // This JSON item is wanted - we need to process this JSON item further
// Take the JSON item and create a consumable object for eventual database insertion // Take the JSON item and create a consumable object for eventual database insertion
log.vdebug("Making newDatabaseItem from this JSON: ", onedriveJSONItem);
Item newDatabaseItem = makeItem(onedriveJSONItem); Item newDatabaseItem = makeItem(onedriveJSONItem);
if (existingDBEntry) { if (existingDBEntry) {
@ -1278,7 +1374,14 @@ class SyncEngine {
// The actual item may actually exist locally already, meaning that just the database is out-of-date or missing the data due to --resync // The actual item may actually exist locally already, meaning that just the database is out-of-date or missing the data due to --resync
// But we also cannot compute the newItemPath as the parental objects may not exist as well // But we also cannot compute the newItemPath as the parental objects may not exist as well
log.vdebug("OneDrive change is potentially a new local item"); log.vdebug("OneDrive change is potentially a new local item");
// Attempt to apply this new item
// Attempt to apply this potentially new item
//writeln("newDatabaseItem: ", newDatabaseItem);
//writeln("onedriveJSONItem: ", onedriveJSONItem);
//writeln("newItemPath: ", newItemPath);
applyPotentiallyNewLocalItem(newDatabaseItem, onedriveJSONItem, newItemPath); applyPotentiallyNewLocalItem(newDatabaseItem, onedriveJSONItem, newItemPath);
} }
} }
@ -1290,6 +1393,7 @@ class SyncEngine {
// Perform the download of any required objects in parallel // Perform the download of any required objects in parallel
void processDownloadActivities() { void processDownloadActivities() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Are there any items to delete locally? Cleanup space locally first // Are there any items to delete locally? Cleanup space locally first
if (!idsToDelete.empty) { if (!idsToDelete.empty) {
@ -1331,6 +1435,7 @@ class SyncEngine {
// If the JSON item is not in the database, it is potentially a new item that we need to action // If the JSON item is not in the database, it is potentially a new item that we need to action
void applyPotentiallyNewLocalItem(Item newDatabaseItem, JSONValue onedriveJSONItem, string newItemPath) { void applyPotentiallyNewLocalItem(Item newDatabaseItem, JSONValue onedriveJSONItem, string newItemPath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// The JSON and Database items being passed in here have passed the following checks: // The JSON and Database items being passed in here have passed the following checks:
// - skip_file // - skip_file
@ -1362,8 +1467,8 @@ class SyncEngine {
// Item details from OneDrive and local item details in database are in-sync // Item details from OneDrive and local item details in database are in-sync
log.vdebug("The item to sync is already present on the local filesystem and is in-sync with the local database"); log.vdebug("The item to sync is already present on the local filesystem and is in-sync with the local database");
log.vdebug("Update/Insert local database with item details"); log.vdebug("Update/Insert local database with item details");
log.vdebug("item details to update/insert: ", newDatabaseItem);
itemDB.upsert(newDatabaseItem); itemDB.upsert(newDatabaseItem);
log.vdebug("item details: ", newDatabaseItem);
return; return;
} else { } else {
// Item details from OneDrive and local item details in database are NOT in-sync // Item details from OneDrive and local item details in database are NOT in-sync
@ -1492,6 +1597,7 @@ class SyncEngine {
// If the JSON item IS in the database, this will be an update to an existing in-sync item // If the JSON item IS in the database, this will be an update to an existing in-sync item
void applyPotentiallyChangedItem(Item existingDatabaseItem, string existingItemPath, Item changedOneDriveItem, string changedItemPath, JSONValue onedriveJSONItem) { void applyPotentiallyChangedItem(Item existingDatabaseItem, string existingItemPath, Item changedOneDriveItem, string changedItemPath, JSONValue onedriveJSONItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// If we are moving the item, we do not need to download it again // If we are moving the item, we do not need to download it again
bool itemWasMoved = false; bool itemWasMoved = false;
@ -1503,7 +1609,7 @@ class SyncEngine {
log.log("Moving ", existingItemPath, " to ", changedItemPath); log.log("Moving ", existingItemPath, " to ", changedItemPath);
// Is the destination path empty .. or does something exist at that location? // Is the destination path empty .. or does something exist at that location?
if (exists(changedItemPath)) { if (exists(changedItemPath)) {
// Destination exists ... does this destination exist in the database? // Destination we are moving to exists ...
Item changedLocalItem; Item changedLocalItem;
// Query DB for this changed item in specified path that exists and see if it is in-sync // Query DB for this changed item in specified path that exists and see if it is in-sync
if (itemDB.selectByPath(changedItemPath, changedOneDriveItem.driveId, changedLocalItem)) { if (itemDB.selectByPath(changedItemPath, changedOneDriveItem.driveId, changedLocalItem)) {
@ -1568,6 +1674,17 @@ class SyncEngine {
} else { } else {
// Save this item in the database // Save this item in the database
saveItem(onedriveJSONItem); saveItem(onedriveJSONItem);
// If the 'Add shortcut to My files' link was the item that was actually renamed .. we have to update our DB records
if (changedOneDriveItem.type == ItemType.remote) {
// Select remote item data from the database
Item existingRemoteDbItem;
itemDB.selectById(changedOneDriveItem.remoteDriveId, changedOneDriveItem.remoteId, existingRemoteDbItem);
// Update the 'name' in existingRemoteDbItem and save it back to the database
// This is the local name stored on disk that was just 'moved'
existingRemoteDbItem.name = changedOneDriveItem.name;
itemDB.upsert(existingRemoteDbItem);
}
} }
} else { } else {
// The existingDatabaseItem.eTag == changedOneDriveItem.eTag .. nothing has changed, so save this item // The existingDatabaseItem.eTag == changedOneDriveItem.eTag .. nothing has changed, so save this item
@ -1577,6 +1694,7 @@ class SyncEngine {
// Download new file items as identified // Download new file items as identified
void downloadOneDriveItems() { void downloadOneDriveItems() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Lets deal with the JSON items in a batch process // Lets deal with the JSON items in a batch process
ulong batchSize = appConfig.concurrentThreads; ulong batchSize = appConfig.concurrentThreads;
ulong batchCount = (fileJSONItemsToDownload.length + batchSize - 1) / batchSize; ulong batchCount = (fileJSONItemsToDownload.length + batchSize - 1) / batchSize;
@ -1589,6 +1707,7 @@ class SyncEngine {
// Download items in parallel // Download items in parallel
void downloadOneDriveItemsInParallel(JSONValue[] array) { void downloadOneDriveItemsInParallel(JSONValue[] array) {
log.vdebug("Starting this function: ", getFunctionName!({}));
foreach (i, onedriveJSONItem; taskPool.parallel(array)) { foreach (i, onedriveJSONItem; taskPool.parallel(array)) {
// Take the JSON item and create a consumable object for eventual database insertion // Take the JSON item and create a consumable object for eventual database insertion
Item newDatabaseItem = makeItem(onedriveJSONItem); Item newDatabaseItem = makeItem(onedriveJSONItem);
@ -1598,6 +1717,7 @@ class SyncEngine {
// Perform the actual download of an object from OneDrive // Perform the actual download of an object from OneDrive
void downloadFileItem(Item newDatabaseItem, JSONValue onedriveJSONItem) { void downloadFileItem(Item newDatabaseItem, JSONValue onedriveJSONItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
bool downloadFailed = false; bool downloadFailed = false;
string OneDriveFileXORHash; string OneDriveFileXORHash;
@ -1836,6 +1956,7 @@ class SyncEngine {
// Test if the given item is in-sync. Returns true if the given item corresponds to the local one // Test if the given item is in-sync. Returns true if the given item corresponds to the local one
bool isItemSynced(Item item, string path, string itemSource) { bool isItemSynced(Item item, string path, string itemSource) {
log.vdebug("Starting this function: ", getFunctionName!({}));
if (!exists(path)) return false; if (!exists(path)) return false;
final switch (item.type) { final switch (item.type) {
case ItemType.file: case ItemType.file:
@ -1908,6 +2029,7 @@ class SyncEngine {
// Get the /delta data using the provided details // Get the /delta data using the provided details
JSONValue getDeltaChangesByItemId(string selectedDriveId, string selectedItemId, string providedDeltaLink) { JSONValue getDeltaChangesByItemId(string selectedDriveId, string selectedItemId, string providedDeltaLink) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Function variables // Function variables
JSONValue deltaChangesBundle; JSONValue deltaChangesBundle;
// Get the /delta data for this account | driveId | deltaLink combination // Get the /delta data for this account | driveId | deltaLink combination
@ -1974,17 +2096,9 @@ class SyncEngine {
return deltaChangesBundle; return deltaChangesBundle;
} }
// Common code for handling when a client is unauthorised
void handleClientUnauthorised() {
writeln();
log.errorAndNotify("ERROR: Check your configuration as your refresh_token may be empty or invalid. You may need to issue a --reauth and re-authorise this client.");
writeln();
// Must exit here
exit(-1);
}
// Common code to handle a 408 or 429 response from the OneDrive API // Common code to handle a 408 or 429 response from the OneDrive API
void handleOneDriveThrottleRequest(OneDriveApi activeOneDriveApiInstance) { void handleOneDriveThrottleRequest(OneDriveApi activeOneDriveApiInstance) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// If OneDrive sends a status code 429 then this function will be used to process the Retry-After response header which contains the value by which we need to wait // If OneDrive sends a status code 429 then this function will be used to process the Retry-After response header which contains the value by which we need to wait
log.vdebug("Handling a OneDrive HTTP 429 Response Code (Too Many Requests)"); log.vdebug("Handling a OneDrive HTTP 429 Response Code (Too Many Requests)");
// Read in the Retry-After HTTP header as set and delay as per this value before retrying the request // Read in the Retry-After HTTP header as set and delay as per this value before retrying the request
@ -2018,6 +2132,7 @@ class SyncEngine {
// If the JSON response is not correct JSON object, exit // If the JSON response is not correct JSON object, exit
void invalidJSONResponseFromOneDriveAPI() { void invalidJSONResponseFromOneDriveAPI() {
log.vdebug("Starting this function: ", getFunctionName!({}));
log.error("ERROR: Query of the OneDrive API returned an invalid JSON response"); log.error("ERROR: Query of the OneDrive API returned an invalid JSON response");
// Must exit // Must exit
exit(-1); exit(-1);
@ -2025,6 +2140,7 @@ class SyncEngine {
// Handle an unhandled API error // Handle an unhandled API error
void defaultUnhandledHTTPErrorCode(OneDriveException exception) { void defaultUnhandledHTTPErrorCode(OneDriveException exception) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// display error // display error
displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); displayOneDriveErrorMessage(exception.msg, getFunctionName!({}));
// Must exit here // Must exit here
@ -2033,10 +2149,11 @@ class SyncEngine {
// Display the pertinant details of the sync engine // Display the pertinant details of the sync engine
void displaySyncEngineDetails() { void displaySyncEngineDetails() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Display accountType, defaultDriveId, defaultRootId & remainingFreeSpace for verbose logging purposes // Display accountType, defaultDriveId, defaultRootId & remainingFreeSpace for verbose logging purposes
//log.vlog("Application version: ", strip(import("version"))); //log.vlog("Application version: ", strip(import("version")));
string tempVersion = "v2.5.0-alpha-0" ~ " GitHub version: " ~ strip(import("version")); string tempVersion = "v2.5.0-alpha-1" ~ " GitHub version: " ~ strip(import("version"));
log.vlog("Application version: ", tempVersion); log.vlog("Application version: ", tempVersion);
log.vlog("Account Type: ", appConfig.accountType); log.vlog("Account Type: ", appConfig.accountType);
@ -2059,6 +2176,7 @@ class SyncEngine {
// Query itemdb.computePath() and catch potential assert when DB consistency issue occurs // Query itemdb.computePath() and catch potential assert when DB consistency issue occurs
string computeItemPath(string thisDriveId, string thisItemId) { string computeItemPath(string thisDriveId, string thisItemId) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// static declare this for this function // static declare this for this function
static import core.exception; static import core.exception;
string calculatedPath; string calculatedPath;
@ -2078,6 +2196,7 @@ class SyncEngine {
// Try and compute the file hash for the given item // Try and compute the file hash for the given item
bool testFileHash(string path, Item item) { bool testFileHash(string path, Item item) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Generate QuickXORHash first before attempting to generate any other type of hash // Generate QuickXORHash first before attempting to generate any other type of hash
if (item.quickXorHash) { if (item.quickXorHash) {
if (item.quickXorHash == computeQuickXorHash(path)) return true; if (item.quickXorHash == computeQuickXorHash(path)) return true;
@ -2089,6 +2208,7 @@ class SyncEngine {
// Process items that need to be removed // Process items that need to be removed
void processDeleteItems() { void processDeleteItems() {
log.vdebug("Starting this function: ", getFunctionName!({}));
foreach_reverse (i; idsToDelete) { foreach_reverse (i; idsToDelete) {
Item item; Item item;
string path; string path;
@ -2174,6 +2294,7 @@ class SyncEngine {
// Update the timestamp of an object online // Update the timestamp of an object online
void uploadLastModifiedTime(string driveId, string id, SysTime mtime, string eTag) { void uploadLastModifiedTime(string driveId, string id, SysTime mtime, string eTag) {
log.vdebug("Starting this function: ", getFunctionName!({}));
string itemModifiedTime; string itemModifiedTime;
itemModifiedTime = mtime.toISOExtString(); itemModifiedTime = mtime.toISOExtString();
JSONValue data = [ JSONValue data = [
@ -2250,6 +2371,7 @@ class SyncEngine {
// Perform a database integrity check - checking all the items that are in-sync at the moment, validating what we know should be on disk, to what is actually on disk // Perform a database integrity check - checking all the items that are in-sync at the moment, validating what we know should be on disk, to what is actually on disk
void performDatabaseConsistencyAndIntegrityCheck() { void performDatabaseConsistencyAndIntegrityCheck() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Log what we are doing // Log what we are doing
if (!appConfig.surpressLoggingOutput) { if (!appConfig.surpressLoggingOutput) {
log.log("Performing a database consistency and integrity check on locally stored data ... "); log.log("Performing a database consistency and integrity check on locally stored data ... ");
@ -2352,6 +2474,7 @@ class SyncEngine {
// Check this Database Item for its consistency on disk // Check this Database Item for its consistency on disk
void checkDatabaseItemForConsistency(Item dbItem) { void checkDatabaseItemForConsistency(Item dbItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// What is the local path item // What is the local path item
string localFilePath; string localFilePath;
@ -2371,16 +2494,17 @@ class SyncEngine {
logOutputPath = localFilePath; logOutputPath = localFilePath;
} }
// Log what we are doing
log.vlog("Processing ", logOutputPath);
// Determine which action to take // Determine which action to take
final switch (dbItem.type) { final switch (dbItem.type) {
case ItemType.file: case ItemType.file:
// Logging output // Logging output
log.vlog("Processing ", logOutputPath);
checkFileDatabaseItemForConsistency(dbItem, localFilePath); checkFileDatabaseItemForConsistency(dbItem, localFilePath);
break; break;
case ItemType.dir: case ItemType.dir:
// Logging output // Logging output
log.vlog("Processing ", logOutputPath);
checkDirectoryDatabaseItemForConsistency(dbItem, localFilePath); checkDirectoryDatabaseItemForConsistency(dbItem, localFilePath);
break; break;
case ItemType.remote: case ItemType.remote:
@ -2394,6 +2518,7 @@ class SyncEngine {
// Perform the database consistency check on this file item // Perform the database consistency check on this file item
void checkFileDatabaseItemForConsistency(Item dbItem, string localFilePath) { void checkFileDatabaseItemForConsistency(Item dbItem, string localFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// What is the source of this item data? // What is the source of this item data?
string itemSource = "database"; string itemSource = "database";
@ -2487,6 +2612,7 @@ class SyncEngine {
// Perform the database consistency check on this directory item // Perform the database consistency check on this directory item
void checkDirectoryDatabaseItemForConsistency(Item dbItem, string localFilePath) { void checkDirectoryDatabaseItemForConsistency(Item dbItem, string localFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// What is the source of this item data? // What is the source of this item data?
string itemSource = "database"; string itemSource = "database";
@ -2571,6 +2697,7 @@ class SyncEngine {
// Does this Database Item (directory or file) get excluded from any operation based on any client side filtering rules? // Does this Database Item (directory or file) get excluded from any operation based on any client side filtering rules?
bool checkDBItemAndPathAgainstClientSideFiltering(Item dbItem, string localFilePath) { bool checkDBItemAndPathAgainstClientSideFiltering(Item dbItem, string localFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Check the item and path against client side filtering rules // Check the item and path against client side filtering rules
// Return a true|false response // Return a true|false response
bool clientSideRuleExcludesItem = false; bool clientSideRuleExcludesItem = false;
@ -2629,6 +2756,7 @@ class SyncEngine {
// Does this local path (directory or file) conform with the Microsoft Naming Restrictions? // Does this local path (directory or file) conform with the Microsoft Naming Restrictions?
bool checkPathAgainstMicrosoftNamingRestrictions(string localFilePath) { bool checkPathAgainstMicrosoftNamingRestrictions(string localFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Check if the given path violates certain Microsoft restrictions and limitations // Check if the given path violates certain Microsoft restrictions and limitations
// Return a true|false response // Return a true|false response
@ -2663,6 +2791,7 @@ class SyncEngine {
// Does this local path (directory or file) get excluded from any operation based on any client side filtering rules? // Does this local path (directory or file) get excluded from any operation based on any client side filtering rules?
bool checkPathAgainstClientSideFiltering(string localFilePath) { bool checkPathAgainstClientSideFiltering(string localFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Unlike checkDBItemAndPathAgainstClientSideFiltering - we need to check the path only // Unlike checkDBItemAndPathAgainstClientSideFiltering - we need to check the path only
// Check the path against client side filtering rules // Check the path against client side filtering rules
@ -2831,6 +2960,7 @@ class SyncEngine {
// Does this JSON item (as received from OneDrive API) get excluded from any operation based on any client side filtering rules? // Does this JSON item (as received from OneDrive API) get excluded from any operation based on any client side filtering rules?
// This function is only used when we are fetching objects from the OneDrive API using a /children query to help speed up what object we query // This function is only used when we are fetching objects from the OneDrive API using a /children query to help speed up what object we query
bool checkJSONAgainstClientSideFiltering(JSONValue onedriveJSONItem) { bool checkJSONAgainstClientSideFiltering(JSONValue onedriveJSONItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
bool clientSideRuleExcludesPath = false; bool clientSideRuleExcludesPath = false;
@ -2885,7 +3015,9 @@ class SyncEngine {
} else { } else {
log.vdebug("Parent details not in database - unable to compute complex path to check"); log.vdebug("Parent details not in database - unable to compute complex path to check");
} }
if (!complexPathToCheck.empty) {
log.vdebug("skip_dir path to check (complex): ", complexPathToCheck); log.vdebug("skip_dir path to check (complex): ", complexPathToCheck);
}
} else { } else {
simplePathToCheck = onedriveJSONItem["name"].str; simplePathToCheck = onedriveJSONItem["name"].str;
} }
@ -2975,6 +3107,7 @@ class SyncEngine {
// Process the list of local changes to upload to OneDrive // Process the list of local changes to upload to OneDrive
void processChangedLocalItemsToUpload() { void processChangedLocalItemsToUpload() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Each element in this array 'databaseItemsWhereContentHasChanged' is an Database Item ID that has been modified locally // Each element in this array 'databaseItemsWhereContentHasChanged' is an Database Item ID that has been modified locally
ulong batchSize = appConfig.concurrentThreads; ulong batchSize = appConfig.concurrentThreads;
ulong batchCount = (databaseItemsWhereContentHasChanged.length + batchSize - 1) / batchSize; ulong batchCount = (databaseItemsWhereContentHasChanged.length + batchSize - 1) / batchSize;
@ -2988,6 +3121,7 @@ class SyncEngine {
// Upload changed local files to OneDrive in parallel // Upload changed local files to OneDrive in parallel
void uploadChangedLocalFileToOneDrive(string[3][] array) { void uploadChangedLocalFileToOneDrive(string[3][] array) {
log.vdebug("Starting this function: ", getFunctionName!({}));
foreach (i, localItemDetails; taskPool.parallel(array)) { foreach (i, localItemDetails; taskPool.parallel(array)) {
@ -3116,6 +3250,7 @@ class SyncEngine {
// Perform the upload of a locally modified file to OneDrive // Perform the upload of a locally modified file to OneDrive
JSONValue performModifiedFileUpload(Item dbItem, string localFilePath, ulong thisFileSizeLocal) { JSONValue performModifiedFileUpload(Item dbItem, string localFilePath, ulong thisFileSizeLocal) {
log.vdebug("Starting this function: ", getFunctionName!({}));
JSONValue uploadResponse; JSONValue uploadResponse;
OneDriveApi uploadFileOneDriveApiInstance; OneDriveApi uploadFileOneDriveApiInstance;
@ -3336,6 +3471,7 @@ class SyncEngine {
// Query the OneDrive API using the provided driveId to get the latest quota details // Query the OneDrive API using the provided driveId to get the latest quota details
ulong getRemainingFreeSpace(string driveId) { ulong getRemainingFreeSpace(string driveId) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Get the quota details for this driveId, as this could have changed since we started the application - the user could have added / deleted data online, or purchased additional storage // Get the quota details for this driveId, as this could have changed since we started the application - the user could have added / deleted data online, or purchased additional storage
// Quota details are ONLY available for the main default driveId, as the OneDrive API does not provide quota details for shared folders // Quota details are ONLY available for the main default driveId, as the OneDrive API does not provide quota details for shared folders
@ -3418,6 +3554,7 @@ class SyncEngine {
// Perform a filesystem walk to uncover new data to upload to OneDrive // Perform a filesystem walk to uncover new data to upload to OneDrive
void scanLocalFilesystemPathForNewData(string path) { void scanLocalFilesystemPathForNewData(string path) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// To improve logging output for this function, what is the 'logical path' we are scanning for file & folder differences? // To improve logging output for this function, what is the 'logical path' we are scanning for file & folder differences?
string logPath; string logPath;
if (path == ".") { if (path == ".") {
@ -3495,6 +3632,7 @@ class SyncEngine {
// Scan this path for new data // Scan this path for new data
void scanPathForNewData(string path) { void scanPathForNewData(string path) {
log.vdebug("Starting this function: ", getFunctionName!({}));
ulong maxPathLength; ulong maxPathLength;
ulong pathWalkLength; ulong pathWalkLength;
@ -3691,12 +3829,15 @@ class SyncEngine {
// Query the database to determine if this path is within the existing database // Query the database to determine if this path is within the existing database
bool pathFoundInDatabase(string searchPath) { bool pathFoundInDatabase(string searchPath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Check if this path in the database // Check if this path in the database
Item databaseItem; Item databaseItem;
bool pathFoundInDB = false; bool pathFoundInDB = false;
foreach (driveId; driveIDsArray) { foreach (driveId; driveIDsArray) {
if (itemDB.selectByPath(searchPath, driveId, databaseItem)) { if (itemDB.selectByPath(searchPath, driveId, databaseItem)) {
pathFoundInDB = true; pathFoundInDB = true;
log.vdebug("databaseItem: ", databaseItem);
log.vdebug("pathFoundInDB: ", pathFoundInDB);
} }
} }
return pathFoundInDB; return pathFoundInDB;
@ -3706,6 +3847,7 @@ class SyncEngine {
// - Test if we can get the parent path details from the database, otherwise we need to search online // - Test if we can get the parent path details from the database, otherwise we need to search online
// for the path flow and create the folder that way // for the path flow and create the folder that way
void createDirectoryOnline(string thisNewPathToCreate) { void createDirectoryOnline(string thisNewPathToCreate) {
log.vdebug("Starting this function: ", getFunctionName!({}));
log.log("OneDrive Client requested to create this directory online: ", thisNewPathToCreate); log.log("OneDrive Client requested to create this directory online: ", thisNewPathToCreate);
Item parentItem; Item parentItem;
@ -3727,8 +3869,28 @@ class SyncEngine {
parentItem.id = appConfig.defaultRootId; // Should give something like 12345ABCDE1234A1!101 parentItem.id = appConfig.defaultRootId; // Should give something like 12345ABCDE1234A1!101
} else { } else {
// Query the parent path online // Query the parent path online
log.vlog("Attempting to query Local Database for this parent path: ", parentPath);
// Attempt a 2 step process to work out where to create the directory
// Step 1: Query the DB first
// Step 2: Query online as last resort
// Step 1: Check if this path in the database
Item databaseItem;
bool pathFoundInDB = false;
foreach (driveId; driveIDsArray) {
if (itemDB.selectByPath(parentPath, driveId, databaseItem)) {
pathFoundInDB = true;
log.vdebug("databaseItem: ", databaseItem);
log.vdebug("pathFoundInDB: ", pathFoundInDB);
}
}
// Step 2: Query for the path online
if (!pathFoundInDB) {
try { try {
log.vdebug("Attempting to query OneDrive for this parent path: ", parentPath); log.vlog("Attempting to query OneDrive Online for this parent path: ", parentPath);
onlinePathData = createDirectoryOnlineOneDriveApiInstance.getPathDetails(parentPath); onlinePathData = createDirectoryOnlineOneDriveApiInstance.getPathDetails(parentPath);
saveItem(onlinePathData); saveItem(onlinePathData);
parentItem = makeItem(onlinePathData); parentItem = makeItem(onlinePathData);
@ -3772,6 +3934,12 @@ class SyncEngine {
} }
} }
} }
} else {
// parent path found in database ... use those details ...
parentItem = databaseItem;
}
} }
// Make sure the full path does not exist online, this should generate a 404 response, to which then the folder will be created online // Make sure the full path does not exist online, this should generate a 404 response, to which then the folder will be created online
@ -3779,7 +3947,6 @@ class SyncEngine {
// Try and query the OneDrive API for the path we need to create // Try and query the OneDrive API for the path we need to create
log.vlog("Attempting to query OneDrive for this path: ", thisNewPathToCreate); log.vlog("Attempting to query OneDrive for this path: ", thisNewPathToCreate);
// What query & method should be used to query if this path exists online?
if (parentItem.driveId == appConfig.defaultDriveId) { if (parentItem.driveId == appConfig.defaultDriveId) {
// Use getPathDetailsByDriveId // Use getPathDetailsByDriveId
onlinePathData = createDirectoryOnlineOneDriveApiInstance.getPathDetailsByDriveId(parentItem.driveId, thisNewPathToCreate); onlinePathData = createDirectoryOnlineOneDriveApiInstance.getPathDetailsByDriveId(parentItem.driveId, thisNewPathToCreate);
@ -3790,6 +3957,7 @@ class SyncEngine {
// If no match, the folder we want to create does not exist at the location we are seeking to create it at, thus generate a 404 // If no match, the folder we want to create does not exist at the location we are seeking to create it at, thus generate a 404
onlinePathData = createDirectoryOnlineOneDriveApiInstance.searchDriveForPath(parentItem.driveId, baseName(thisNewPathToCreate)); onlinePathData = createDirectoryOnlineOneDriveApiInstance.searchDriveForPath(parentItem.driveId, baseName(thisNewPathToCreate));
// Process the response from searching the drive
ulong responseCount = count(onlinePathData["value"].array); ulong responseCount = count(onlinePathData["value"].array);
if (responseCount > 0) { if (responseCount > 0) {
// Search 'name' matches were found .. need to match these against parentItem.id // Search 'name' matches were found .. need to match these against parentItem.id
@ -3948,6 +4116,7 @@ class SyncEngine {
// Test that the online name actually matches the requested local name // Test that the online name actually matches the requested local name
void performPosixTest(string localNameToCheck, string onlineName) { void performPosixTest(string localNameToCheck, string onlineName) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file // https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file
// Do not assume case sensitivity. For example, consider the names OSCAR, Oscar, and oscar to be the same, // Do not assume case sensitivity. For example, consider the names OSCAR, Oscar, and oscar to be the same,
@ -3962,6 +4131,7 @@ class SyncEngine {
// Upload new file items as identified // Upload new file items as identified
void uploadNewLocalFileItems() { void uploadNewLocalFileItems() {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Lets deal with the new local items in a batch process // Lets deal with the new local items in a batch process
ulong batchSize = appConfig.concurrentThreads; ulong batchSize = appConfig.concurrentThreads;
ulong batchCount = (newLocalFilesToUploadToOneDrive.length + batchSize - 1) / batchSize; ulong batchCount = (newLocalFilesToUploadToOneDrive.length + batchSize - 1) / batchSize;
@ -3974,6 +4144,7 @@ class SyncEngine {
// Upload the file batches in parallel // Upload the file batches in parallel
void uploadNewLocalFileItemsInParallel(string[] array) { void uploadNewLocalFileItemsInParallel(string[] array) {
log.vdebug("Starting this function: ", getFunctionName!({}));
foreach (i, fileToUpload; taskPool.parallel(array)) { foreach (i, fileToUpload; taskPool.parallel(array)) {
log.vdebug("Upload Thread ", i, " Starting: ", Clock.currTime()); log.vdebug("Upload Thread ", i, " Starting: ", Clock.currTime());
uploadNewFile(fileToUpload); uploadNewFile(fileToUpload);
@ -3983,6 +4154,7 @@ class SyncEngine {
// Upload a new file to OneDrive // Upload a new file to OneDrive
void uploadNewFile(string fileToUpload) { void uploadNewFile(string fileToUpload) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Debug for the moment // Debug for the moment
log.vdebug("fileToUpload: ", fileToUpload); log.vdebug("fileToUpload: ", fileToUpload);
@ -4196,6 +4368,7 @@ class SyncEngine {
// Perform the actual upload to OneDrive // Perform the actual upload to OneDrive
bool performNewFileUpload(Item parentItem, string fileToUpload, ulong thisFileSize) { bool performNewFileUpload(Item parentItem, string fileToUpload, ulong thisFileSize) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Assume that by default the upload fails // Assume that by default the upload fails
bool uploadFailed = true; bool uploadFailed = true;
@ -4456,6 +4629,7 @@ class SyncEngine {
// Create the OneDrive Upload Session // Create the OneDrive Upload Session
JSONValue createSessionFileUpload(OneDriveApi activeOneDriveApiInstance, string fileToUpload, string parentDriveId, string parentId, string filename, string eTag, string threadUploadSessionFilePath) { JSONValue createSessionFileUpload(OneDriveApi activeOneDriveApiInstance, string fileToUpload, string parentDriveId, string parentId, string filename, string eTag, string threadUploadSessionFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Upload file via a OneDrive API session // Upload file via a OneDrive API session
JSONValue uploadSession; JSONValue uploadSession;
@ -4496,6 +4670,7 @@ class SyncEngine {
// Save the session upload data // Save the session upload data
void saveSessionFile(string threadUploadSessionFilePath, JSONValue uploadSessionData) { void saveSessionFile(string threadUploadSessionFilePath, JSONValue uploadSessionData) {
log.vdebug("Starting this function: ", getFunctionName!({}));
try { try {
std.file.write(threadUploadSessionFilePath, uploadSessionData.toString()); std.file.write(threadUploadSessionFilePath, uploadSessionData.toString());
} catch (FileException e) { } catch (FileException e) {
@ -4506,6 +4681,7 @@ class SyncEngine {
// Perform the upload of file via the Upload Session that was created // Perform the upload of file via the Upload Session that was created
JSONValue performSessionFileUpload(OneDriveApi activeOneDriveApiInstance, ulong thisFileSize, JSONValue uploadSessionData, string threadUploadSessionFilePath) { JSONValue performSessionFileUpload(OneDriveApi activeOneDriveApiInstance, ulong thisFileSize, JSONValue uploadSessionData, string threadUploadSessionFilePath) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Response for upload // Response for upload
JSONValue uploadResponse; JSONValue uploadResponse;
@ -4652,6 +4828,7 @@ class SyncEngine {
// Delete an item on OneDrive // Delete an item on OneDrive
void uploadDeletedItem(Item itemToDelete, string path) { void uploadDeletedItem(Item itemToDelete, string path) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Are we in a situation where we HAVE to keep the data online - do not delete the remote object // Are we in a situation where we HAVE to keep the data online - do not delete the remote object
if (noRemoteDelete) { if (noRemoteDelete) {
@ -4668,29 +4845,30 @@ class SyncEngine {
bool flagAsBigDelete = false; bool flagAsBigDelete = false;
Item[] children; Item[] children;
ulong itemsToDelete;
if ((itemToDelete.type == ItemType.dir)) { if ((itemToDelete.type == ItemType.dir)) {
// Query the database - how many objects will this remove? // Query the database - how many objects will this remove?
children = getChildren(itemToDelete.driveId, itemToDelete.id); children = getChildren(itemToDelete.driveId, itemToDelete.id);
// Count the returned items + the original item (1) // Count the returned items + the original item (1)
ulong itemsToDelete = count(children) + 1; itemsToDelete = count(children) + 1;
log.vdebug("Number of items online to delete: ", itemsToDelete); log.vdebug("Number of items online to delete: ", itemsToDelete);
} else {
itemsToDelete = 1;
}
// Are we running in monitor mode? A local delete of a file|folder when using --monitor will issue a inotify event, which will trigger the local & remote data immediately be deleted // A local delete of a file|folder when using --monitor will issue a inotify event, which will trigger the local & remote data immediately be deleted
if (!appConfig.getValueBool("monitor")) { // The user may also be --sync process, so we are checking if something was deleted between application use
// not running in monitor mode if (itemsToDelete >= appConfig.getValueLong("classify_as_big_delete")) {
if (itemsToDelete > appConfig.getValueLong("classify_as_big_delete")) { // A big delete has been detected
// A big delete detected
flagAsBigDelete = true; flagAsBigDelete = true;
if (!appConfig.getValueBool("force")) { if (!appConfig.getValueBool("force")) {
log.error("ERROR: An attempt to remove a large volume of data from OneDrive has been detected. Exiting client to preserve data on OneDrive"); log.error("ERROR: An attempt to remove a large volume of data from OneDrive has been detected. Exiting client to preserve data on OneDrive");
log.error("ERROR: To delete a large volume of data use --force or increase the config value 'classify_as_big_delete' to a larger value"); log.error("ERROR: To delete a large volume of data use --force or increase the config value 'classify_as_big_delete' to a larger value");
// Must exit here to preserve data on OneDrive // Must exit here to preserve data on online
exit(-1); exit(-1);
} }
} }
}
}
// Are we in a --dry-run scenario? // Are we in a --dry-run scenario?
if (!dryRun) { if (!dryRun) {
@ -4732,6 +4910,8 @@ class SyncEngine {
// Get the children of an item id from the database // Get the children of an item id from the database
Item[] getChildren(string driveId, string id) { Item[] getChildren(string driveId, string id) {
log.vdebug("Starting this function: ", getFunctionName!({}));
Item[] children; Item[] children;
children ~= itemDB.selectChildren(driveId, id); children ~= itemDB.selectChildren(driveId, id);
foreach (Item child; children) { foreach (Item child; children) {
@ -4745,6 +4925,8 @@ class SyncEngine {
// Perform a 'reverse' delete of all child objects on OneDrive // Perform a 'reverse' delete of all child objects on OneDrive
void performReverseDeletionOfOneDriveItems(Item[] children, Item itemToDelete) { void performReverseDeletionOfOneDriveItems(Item[] children, Item itemToDelete) {
log.vdebug("Starting this function: ", getFunctionName!({}));
log.vdebug("Attempting a reverse delete of all child objects from OneDrive"); log.vdebug("Attempting a reverse delete of all child objects from OneDrive");
// Create a new API Instance for this thread and initialise it // Create a new API Instance for this thread and initialise it
@ -4770,6 +4952,8 @@ class SyncEngine {
// Create a fake OneDrive response suitable for use with saveItem // Create a fake OneDrive response suitable for use with saveItem
JSONValue createFakeResponse(const(string) path) { JSONValue createFakeResponse(const(string) path) {
log.vdebug("Starting this function: ", getFunctionName!({}));
import std.digest.sha; import std.digest.sha;
// Generate a simulated JSON response which can be used // Generate a simulated JSON response which can be used
// At a minimum we need: // At a minimum we need:
@ -4861,6 +5045,8 @@ class SyncEngine {
// Save JSON item details into the item database // Save JSON item details into the item database
void saveItem(JSONValue jsonItem) { void saveItem(JSONValue jsonItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// jsonItem has to be a valid object // jsonItem has to be a valid object
if (jsonItem.type() == JSONType.object){ if (jsonItem.type() == JSONType.object){
// Check if the response JSON has an 'id', otherwise makeItem() fails with 'Key not found: id' // Check if the response JSON has an 'id', otherwise makeItem() fails with 'Key not found: id'
@ -4884,10 +5070,10 @@ class SyncEngine {
// Check for parentReference // Check for parentReference
if (hasParentReference(jsonItem)) { if (hasParentReference(jsonItem)) {
// Set the correct item.driveId // Set the correct item.driveId
log.vdebug("ROOT JSON Item HAS parentReference .... setting item.driveId = jsonItem['parentReference']['driveId'].str");
item.driveId = jsonItem["parentReference"]["driveId"].str; item.driveId = jsonItem["parentReference"]["driveId"].str;
} else {
writeln("DEBUG TO REMOVE: saveItem ROOT JSON Item has no parentReference .... this may not even be needed .... ");
} }
// We only should be adding our account 'root' to the database, not shared folder 'root' items // We only should be adding our account 'root' to the database, not shared folder 'root' items
if (item.driveId != appConfig.defaultDriveId) { if (item.driveId != appConfig.defaultDriveId) {
// Shared Folder drive 'root' object .. we dont want this item // Shared Folder drive 'root' object .. we dont want this item
@ -4923,6 +5109,8 @@ class SyncEngine {
// Wrapper function for makeDatabaseItem so we can check to ensure that the item has the required hashes // Wrapper function for makeDatabaseItem so we can check to ensure that the item has the required hashes
Item makeItem(JSONValue onedriveJSONItem) { Item makeItem(JSONValue onedriveJSONItem) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// Make the DB Item from the JSON data provided // Make the DB Item from the JSON data provided
Item newDatabaseItem = makeDatabaseItem(onedriveJSONItem); Item newDatabaseItem = makeDatabaseItem(onedriveJSONItem);
@ -4977,8 +5165,9 @@ class SyncEngine {
// Print the fileDownloadFailures and fileUploadFailures arrays if they are not empty // Print the fileDownloadFailures and fileUploadFailures arrays if they are not empty
void displaySyncFailures() { void displaySyncFailures() {
// Were there any file download failures? log.vdebug("Starting this function: ", getFunctionName!({}));
// Were there any file download failures?
if (!fileDownloadFailures.empty) { if (!fileDownloadFailures.empty) {
// There are download failures ... // There are download failures ...
log.log("\nFailed items to download from OneDrive: ", fileDownloadFailures.length); log.log("\nFailed items to download from OneDrive: ", fileDownloadFailures.length);
@ -5041,6 +5230,7 @@ class SyncEngine {
// then once the target of the --single-directory request is hit, all of the children of that path can be queried, giving a much more focused // then once the target of the --single-directory request is hit, all of the children of that path can be queried, giving a much more focused
// JSON response which can then be processed, negating the need to continuously traverse the tree and 'exclude' items // JSON response which can then be processed, negating the need to continuously traverse the tree and 'exclude' items
JSONValue generateDeltaResponse(string pathToQuery = null) { JSONValue generateDeltaResponse(string pathToQuery = null) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// JSON value which will be responded with // JSON value which will be responded with
JSONValue selfGeneratedDeltaResponse; JSONValue selfGeneratedDeltaResponse;
@ -5318,6 +5508,8 @@ class SyncEngine {
// Query the OneDrive API for the specified child id for any children objects // Query the OneDrive API for the specified child id for any children objects
JSONValue[] queryForChildren(string driveId, string idToQuery, string childParentPath, string pathForLogging) { JSONValue[] queryForChildren(string driveId, string idToQuery, string childParentPath, string pathForLogging) {
log.vdebug("Starting this function: ", getFunctionName!({}));
// function variables // function variables
JSONValue thisLevelChildren; JSONValue thisLevelChildren;
JSONValue[] thisLevelChildrenData; JSONValue[] thisLevelChildrenData;
@ -5394,6 +5586,8 @@ class SyncEngine {
// Query the OneDrive API for the child objects for this element // Query the OneDrive API for the child objects for this element
JSONValue queryThisLevelChildren(string driveId, string idToQuery, string nextLink) { JSONValue queryThisLevelChildren(string driveId, string idToQuery, string nextLink) {
log.vdebug("Starting this function: ", getFunctionName!({}));
JSONValue thisLevelChildren; JSONValue thisLevelChildren;
// Create new OneDrive API Instance // Create new OneDrive API Instance
@ -5465,6 +5659,7 @@ class SyncEngine {
// This function also ensures that each path in the requested path actually matches the requested element to ensure that the OneDrive API response // This function also ensures that each path in the requested path actually matches the requested element to ensure that the OneDrive API response
// is not falsely matching a 'case insensitive' match to the actual request which is a POSIX compliance issue. // is not falsely matching a 'case insensitive' match to the actual request which is a POSIX compliance issue.
JSONValue queryOneDriveForSpecificPathAndCreateIfMissing(string thisNewPathToSearch, bool createPathIfMissing) { JSONValue queryOneDriveForSpecificPathAndCreateIfMissing(string thisNewPathToSearch, bool createPathIfMissing) {
log.vdebug("Starting this function: ", getFunctionName!({}));
JSONValue getPathDetailsAPIResponse; JSONValue getPathDetailsAPIResponse;
string currentPathTree; string currentPathTree;
@ -5726,6 +5921,8 @@ class SyncEngine {
// Delete an item by it's path // Delete an item by it's path
// This function is only used in --monitor mode // This function is only used in --monitor mode
void deleteByPath(const(string) path) { void deleteByPath(const(string) path) {
log.vdebug("Starting this function: ", getFunctionName!({}));
Item dbItem; Item dbItem;
// Need to check all driveid's we know about, not just the defaultDriveId // Need to check all driveid's we know about, not just the defaultDriveId
bool itemInDB = false; bool itemInDB = false;

View file

@ -2,6 +2,7 @@
module util; module util;
// What does this module require to function? // What does this module require to function?
import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit;
import std.base64; import std.base64;
import std.conv; import std.conv;
import std.digest.crc; import std.digest.crc;
@ -341,7 +342,7 @@ bool containsASCIIHTMLCodes(string path) {
return m.empty; return m.empty;
} }
/**
// Parse and display error message received from OneDrive // Parse and display error message received from OneDrive
void displayOneDriveErrorMessage(string message, string callingFunction) { void displayOneDriveErrorMessage(string message, string callingFunction) {
writeln(); writeln();
@ -412,85 +413,40 @@ void displayOneDriveErrorMessage(string message, string callingFunction) {
log.vdebug("Raw Error Data: ", message); log.vdebug("Raw Error Data: ", message);
log.vdebug("JSON Message: ", errorMessage); log.vdebug("JSON Message: ", errorMessage);
} }
**/
// Alpha-0 Testing ..... // Common code for handling when a client is unauthorised
void displayOneDriveErrorMessage(string message, string callingFunction) { void handleClientUnauthorised(int httpStatusCode, string message) {
writeln(); // Split the lines of the error message
log.log("ERROR: Microsoft OneDrive API returned an error with the following message:");
auto errorArray = splitLines(message); auto errorArray = splitLines(message);
log.log(" Error Message: ", errorArray[0]);
// Extract 'message' as the reason // Extract 'message' as the reason
JSONValue errorMessage = parseJSON(replace(message, errorArray[0], "")); JSONValue errorMessage = parseJSON(replace(message, errorArray[0], ""));
log.vdebug("errorMessage: ", errorMessage);
// What is the reason for the error if (httpStatusCode == 400) {
if (errorMessage.type() == JSONType.object) { // bad request or a new auth token is needed
// configure the error reason // configure the error reason
string errorReason; writeln();
string requestDate; string[] errorReason = splitLines(errorMessage["error_description"].str);
string requestId; log.errorAndNotify(errorReason[0]);
writeln();
// set the reason for the error log.errorAndNotify("ERROR: You will need to issue a --reauth and re-authorise this client to obtain a fresh auth token.");
try { writeln();
// Use error_description as reason
errorReason = errorMessage["error_description"].str;
} catch (JSONException e) {
// we dont want to do anything here
} }
// set the reason for the error if (httpStatusCode == 401) {
try {
// Use ["error"]["message"] as reason writeln("CODING TO DO: Triggered a 401 HTTP unauthorised response");
errorReason = errorMessage["error"]["message"].str;
} catch (JSONException e) { writeln();
// we dont want to do anything here log.errorAndNotify("ERROR: Check your configuration as your refresh_token may be empty or invalid. You may need to issue a --reauth and re-authorise this client.");
writeln();
} }
// Display the error reason // Must exit here
if (errorReason.startsWith("<!DOCTYPE")) { exit(EXIT_FAILURE);
// a HTML Error Reason was given
log.log(" Error Reason: A HTML Error response was provided. Use debug logging (--verbose --verbose) to view this error");
log.log(errorReason);
} else {
// a non HTML Error Reason was given
log.log(" Error Reason: ", errorReason);
}
// Get the date of request if available
try {
// Use ["error"]["innerError"]["date"] as date
requestDate = errorMessage["error"]["innerError"]["date"].str;
} catch (JSONException e) {
// we dont want to do anything here
}
// Get the request-id if available
try {
// Use ["error"]["innerError"]["request-id"] as request-id
requestId = errorMessage["error"]["innerError"]["request-id"].str;
} catch (JSONException e) {
// we dont want to do anything here
}
// Display the date and request id if available
if (requestDate != "") log.error(" Error Timestamp: ", requestDate);
if (requestId != "") log.error(" API Request ID: ", requestId);
}
// Where in the code was this error generated
log.log(" Calling Function: ", callingFunction);
// Extra Debug if we are using --verbose --verbose
log.log("Raw Error Data: ", message);
log.log("JSON Message: ", errorMessage);
} }
// Parse and display error message received from the local file system // Parse and display error message received from the local file system
void displayFileSystemErrorMessage(string message, string callingFunction) { void displayFileSystemErrorMessage(string message, string callingFunction) {
writeln(); writeln();
@ -504,7 +460,7 @@ void displayFileSystemErrorMessage(string message, string callingFunction) {
ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace(".")); ulong localActualFreeSpace = to!ulong(getAvailableDiskSpace("."));
if (localActualFreeSpace == 0) { if (localActualFreeSpace == 0) {
// force exit // force exit
exit(-1); exit(EXIT_FAILURE);
} }
} }
@ -821,3 +777,7 @@ bool hasLocalPath(const ref JSONValue item) {
bool hasETag(const ref JSONValue item) { bool hasETag(const ref JSONValue item) {
return ("eTag" in item) != null; return ("eTag" in item) != null;
} }
bool hasSharedElement(const ref JSONValue item) {
return ("eTag" in item) != null;
}