diff --git a/src/log.d b/src/log.d index 82a567b9..06f6a4ab 100644 --- a/src/log.d +++ b/src/log.d @@ -1,4 +1,6 @@ import std.stdio; +import std.file; +import std.datetime; // enable verbose logging bool verbose; @@ -6,14 +8,37 @@ bool verbose; void log(T...)(T args) { writeln(args); + // Write to log file + string logFileName = "/var/log/onedrive/onedrive.log"; + auto currentTime = Clock.currTime(); + auto timeString = currentTime.toString(); + File logFile = File(logFileName, "a"); + logFile.writeln(timeString, " ", args); + logFile.close(); } void vlog(T...)(T args) { - if (verbose) writeln(args); + if (verbose) { + writeln(args); + // Write to log file + string logFileName = "/var/log/onedrive/onedrive.log"; + auto currentTime = Clock.currTime(); + auto timeString = currentTime.toString(); + File logFile = File(logFileName, "a"); + logFile.writeln(timeString, " ", args); + logFile.close(); + } } void error(T...)(T args) { stderr.writeln(args); + // Write to log file + string logFileName = "/var/log/onedrive/onedrive.log"; + auto currentTime = Clock.currTime(); + auto timeString = currentTime.toString(); + File logFile = File(logFileName, "a"); + logFile.writeln(timeString, " ", args); + logFile.close(); } \ No newline at end of file diff --git a/src/main.d b/src/main.d index 86c64e5d..00386fd7 100644 --- a/src/main.d +++ b/src/main.d @@ -25,19 +25,46 @@ int main(string[] args) bool printAccessToken; // print the version and exit bool printVersion; - + + // Additional options added to support MyNAS Storage Appliance + // debug the HTTP(S) operations if required + bool debugHttp; + // This allows for selective directory syncing instead of everything under ~/OneDrive/ + string singleDirectory; + // Create a single root directory on OneDrive + string createDirectory; + // Remove a single directory on OneDrive + string removeDirectory; + // The source directory if we are using the OneDrive client to rename a directory + string sourceDirectory; + // The destination directory if we are using the OneDrive client to rename a directory + string destinationDirectory; + // Configure a flag to perform a sync + // This is beneficial so that if just running the client itself - without any options, or sync check, the client does not perform a sync + bool synchronize; + // Local sync - Upload local changes first before downloading changes from OneDrive + bool localFirst; + try { auto opt = getopt( args, std.getopt.config.bundling, std.getopt.config.caseSensitive, "confdir", "Set the directory used to store the configuration files", &configDirName, + "create-directory", "Create a directory on OneDrive - no sync will be performed.", &createDirectory, + "destination-directory", "Destination directory for renamed or move on OneDrive - no sync will be performed.", &destinationDirectory, + "debug-http", "Debug OneDrive HTTP communication.", &debugHttp, "download|d", "Only download remote changes", &downloadOnly, + "local-first", "Synchronize from the local directory source first, before downloading changes from OneDrive.", &localFirst, "logout", "Logout the current user", &logout, "monitor|m", "Keep monitoring for local and remote changes", &monitor, "print-token", "Print the access token, useful for debugging", &printAccessToken, "resync", "Forget the last saved state, perform a full sync", &resync, + "remove-directory", "Remove a directory on OneDrive - no sync will be performed.", &removeDirectory, + "single-directory", "Specify a single local directory within the OneDrive root to sync.", &singleDirectory, + "source-directory", "Source directory to rename or move on OneDrive - no sync will be performed.", &sourceDirectory, "syncdir", "Set the directory used to sync the files that are synced", &syncDirName, + "synchronize", "Perform a synchronization", &synchronize, "verbose|v", "Print more details, useful for debugging", &log.verbose, "version", "Print the version and exit", &printVersion ); @@ -63,6 +90,10 @@ int main(string[] args) return EXIT_SUCCESS; } + // Configure Logging + string logFilePath = "/var/log/onedrive/"; + if (!exists(logFilePath)) mkdirRecurse(logFilePath); + log.vlog("Loading config ..."); configDirName = configDirName.expandTilde().absolutePath(); if (!exists(configDirName)) mkdirRecurse(configDirName); @@ -95,7 +126,7 @@ int main(string[] args) log.error("No network connection"); return EXIT_FAILURE; } - auto onedrive = new OneDriveApi(cfg); + auto onedrive = new OneDriveApi(cfg, debugHttp); onedrive.printAccessToken = printAccessToken; if (!onedrive.init()) { log.error("Could not initialize the OneDrive API"); @@ -104,97 +135,168 @@ int main(string[] args) return EXIT_FAILURE; } + // initialize system log.vlog("Opening the item database ..."); auto itemdb = new ItemDatabase(cfg.databaseFilePath); - + + // Set the local path root string syncDir = expandTilde(cfg.getValue("sync_dir")); log.vlog("All operations will be performed in: ", syncDir); if (!exists(syncDir)) mkdirRecurse(syncDir); chdir(syncDir); - + + // Initialise the sync engine log.vlog("Initializing the Synchronization Engine ..."); auto selectiveSync = new SelectiveSync(); selectiveSync.load(cfg.syncListFilePath); selectiveSync.setMask(cfg.getValue("skip_file")); auto sync = new SyncEngine(cfg, onedrive, itemdb, selectiveSync); sync.init(); - if (online) performSync(sync); + + // Do we need to create or remove a directory? + if ((createDirectory != "") || (removeDirectory != "")) { + + if (createDirectory != "") { + // create a directory on OneDrive + sync.createDirectoryNoSync(createDirectory); + } + + if (removeDirectory != "") { + // remove a directory on OneDrive + sync.deleteDirectoryNoSync(removeDirectory); + } + } + + // Are we renaming or moving a directory? + if ((sourceDirectory != "") && (destinationDirectory != "")) { + // We are renaming or moving a directory + sync.renameDirectoryNoSync(sourceDirectory, destinationDirectory); + } + + // Are we performing a sync, resync or monitor operation? + if ((synchronize) || (resync) || (monitor)) { - if (monitor) { - log.vlog("Initializing monitor ..."); - Monitor m = new Monitor(selectiveSync); - m.onDirCreated = delegate(string path) { - log.vlog("[M] Directory created: ", path); - try { - sync.scanForDifferences(path); - } catch(Exception e) { - log.log(e.msg); - } - }; - m.onFileChanged = delegate(string path) { - log.vlog("[M] File changed: ", path); - try { - sync.scanForDifferences(path); - } catch(Exception e) { - log.log(e.msg); - } - }; - m.onDelete = delegate(string path) { - log.vlog("[M] Item deleted: ", path); - try { - sync.deleteByPath(path); - } catch(Exception e) { - log.log(e.msg); - } - }; - m.onMove = delegate(string from, string to) { - log.vlog("[M] Item moved: ", from, " -> ", to); - try { - sync.uploadMoveItem(from, to); - } catch(Exception e) { - log.log(e.msg); - } - }; - if (!downloadOnly) m.init(cfg, verbose); - // monitor loop - immutable auto checkInterval = dur!"seconds"(45); - auto lastCheckTime = MonoTime.currTime(); - while (true) { - if (!downloadOnly) m.update(online); - auto currTime = MonoTime.currTime(); - if (currTime - lastCheckTime > checkInterval) { - lastCheckTime = currTime; - online = testNetwork(); - if (online) { - performSync(sync); - if (!downloadOnly) { - // discard all events that may have been generated by the sync - m.update(false); + if ((synchronize) || (resync)) { + if (online) { + // Check user entry for local path - the above chdir means we are already in ~/OneDrive/ thus singleDirectory is local to this path + if (singleDirectory != ""){ + // Does the directory we want to sync actually exist? + if (!exists(singleDirectory)){ + // the requested directory does not exist .. + log.log("The requested local directory does not exist. Please check ~/OneDrive/ for requested path"); + onedrive.http.shutdown(); + return EXIT_FAILURE; } } - GC.collect(); - } else { - Thread.sleep(dur!"msecs"(100)); + + // Perform the sync + performSync(sync, singleDirectory, localFirst); } } + + if (monitor) { + log.vlog("Initializing monitor ..."); + Monitor m = new Monitor(selectiveSync); + m.onDirCreated = delegate(string path) { + log.vlog("[M] Directory created: ", path); + try { + sync.scanForDifferences(path); + } catch(Exception e) { + log.log(e.msg); + } + }; + m.onFileChanged = delegate(string path) { + log.vlog("[M] File changed: ", path); + try { + sync.scanForDifferences(path); + } catch(Exception e) { + log.log(e.msg); + } + }; + m.onDelete = delegate(string path) { + log.vlog("[M] Item deleted: ", path); + try { + sync.deleteByPath(path); + } catch(Exception e) { + log.log(e.msg); + } + }; + m.onMove = delegate(string from, string to) { + log.vlog("[M] Item moved: ", from, " -> ", to); + try { + sync.uploadMoveItem(from, to); + } catch(Exception e) { + log.log(e.msg); + } + }; + if (!downloadOnly) m.init(cfg, verbose); + // monitor loop + immutable auto checkInterval = dur!"seconds"(45); + auto lastCheckTime = MonoTime.currTime(); + while (true) { + if (!downloadOnly) m.update(online); + auto currTime = MonoTime.currTime(); + if (currTime - lastCheckTime > checkInterval) { + lastCheckTime = currTime; + online = testNetwork(); + if (online) { + performSync(sync, singleDirectory, localFirst); + if (!downloadOnly) { + // discard all events that may have been generated by the sync + m.update(false); + } + } + GC.collect(); + } else { + Thread.sleep(dur!"msecs"(100)); + } + } + } + } // workaround for segfault in std.net.curl.Curl.shutdown() on exit onedrive.http.shutdown(); return EXIT_SUCCESS; + } // try to synchronize the folder three times -void performSync(SyncEngine sync) +void performSync(SyncEngine sync, string singleDirectory, bool localFirst) { int count; + string remotePath = "/"; + string localPath = "."; + + // Are we doing a single directory sync? + if (singleDirectory != ""){ + // Need two different path strings here + remotePath = singleDirectory; + localPath = singleDirectory; + } + do { try { - sync.applyDifferences(); - if (!downloadOnly) { - sync.scanForDifferences(); - // ensure that the current state is updated + if (singleDirectory != ""){ + // we were requested to sync a single directory + log.vlog("Syncing changes from this selected path: ", singleDirectory); + if (localFirst) { + log.vlog("Syncing changes from selected local path first before downloading changes from OneDrive ..."); + sync.scanForDifferences(localPath); + sync.applyDifferencesSingleDirectory(remotePath); + } else { + log.vlog("Syncing changes from selected OneDrive path first before uploading local changes ..."); + sync.applyDifferencesSingleDirectory(remotePath); + sync.scanForDifferences(localPath); + } + } else { + // original onedrive client logic below sync.applyDifferences(); + if (!downloadOnly) { + sync.scanForDifferences(localPath); + // ensure that the current state is updated + sync.applyDifferences(); + } } count = -1; } catch (Exception e) { diff --git a/src/onedrive.d b/src/onedrive.d index d97c8c02..68daa74e 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -34,7 +34,7 @@ class OneDriveException: Exception { this.httpStatusCode = httpStatusCode; this.error = error; - string msg = format("HTTP request returned status code %d (%s)\n%s", httpStatusCode, reason, toJSON(error, true)); + string msg = format("HTTP request returned status code %d (%s)\n%s", httpStatusCode, reason, toJSON(&error, true)); super(msg, file, line); } } @@ -49,11 +49,13 @@ final class OneDriveApi // if true, every new access token is printed bool printAccessToken; - this(Config cfg) + this(Config cfg, bool debugHttp) { this.cfg = cfg; http = HTTP(); - //http.verbose = true; + if (debugHttp) { + http.verbose = true; + } } bool init() @@ -181,10 +183,32 @@ final class OneDriveApi { checkAccessTokenExpired(); const(char)[] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ "/children"; - http.addRequestHeader("Content-Type", "application/json"); + http.addRequestHeader("Content-Type", "application/json"); return post(url, item.toString()); } + // Return the details of the specified path + JSONValue getPathDetails(const(string) path) + { + checkAccessTokenExpired(); + const(char)[] url; + // string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; + if (path == ".") url = driveUrl ~ "/root/"; + else url = itemByPathUrl ~ encodeComponent(path) ~ ":/"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference"; + return get(url); + } + + // https://dev.onedrive.com/items/move.htm + JSONValue moveByPath(const(char)[] sourcePath, JSONValue moveData) + { + // Need to use itemByPathUrl + checkAccessTokenExpired(); + string url = itemByPathUrl ~ encodeComponent(sourcePath); + http.addRequestHeader("Content-Type", "application/json"); + return move(url, moveData.toString()); + } + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession JSONValue createUploadSession(const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null) { @@ -334,6 +358,17 @@ final class OneDriveApi return response; } + private auto move(T)(const(char)[] url, const(T)[] postData) + { + scope(exit) http.clearRequestHeaders(); + http.method = HTTP.Method.patch; + http.url = url; + addAccessTokenHeader(); + auto response = perform(postData); + checkHttpCode(); + return response; + } + private JSONValue upload(string filepath, string url) { scope(exit) { @@ -397,8 +432,100 @@ final class OneDriveApi private void checkHttpCode() { - if (http.statusLine.code / 100 != 2) { - throw new OneDriveException(http.statusLine.code, http.statusLine.reason); + // https://dev.onedrive.com/misc/errors.htm + // https://developer.overdrive.com/docs/reference-guide + + /* + Error response handling + + Errors in the OneDrive API are returned using standard HTTP status codes, as well as a JSON error response object. The following HTTP status codes should be expected. + + Status code Status message Description + + 200 OK Request was handled OK + 201 Created This means you've made a successful POST to checkout, lock in a format, or place a hold + 204 No Content This means you've made a successful DELETE to remove a hold or return a title + + 400 Bad Request Cannot process the request because it is malformed or incorrect. + 401 Unauthorized Required authentication information is either missing or not valid for the resource. + 403 Forbidden Access is denied to the requested resource. The user might not have enough permission. + 404 Not Found The requested resource doesn’t exist. + 405 Method Not Allowed The HTTP method in the request is not allowed on the resource. + 406 Not Acceptable This service doesn’t support the format requested in the Accept header. + 409 Conflict The current state conflicts with what the request expects. For example, the specified parent folder might not exist. + 410 Gone The requested resource is no longer available at the server. + 411 Length Required A Content-Length header is required on the request. + 412 Precondition Failed A precondition provided in the request (such as an if-match header) does not match the resource's current state. + 413 Request Entity Too Large The request size exceeds the maximum limit. + 415 Unsupported Media Type The content type of the request is a format that is not supported by the service. + 416 Requested Range Not Satisfiable The specified byte range is invalid or unavailable. + 422 Unprocessable Entity Cannot process the request because it is semantically incorrect. + 429 Too Many Requests Client application has been throttled and should not attempt to repeat the request until an amount of time has elapsed. + + 500 Internal Server Error There was an internal server error while processing the request. + 501 Not Implemented The requested feature isn’t implemented. + 502 Bad Gateway The service was unreachable + 503 Service Unavailable The service is temporarily unavailable. You may repeat the request after a delay. There may be a Retry-After header. + 507 Insufficient Storage The maximum storage quota has been reached. + 509 Bandwidth Limit Exceeded Your app has been throttled for exceeding the maximum bandwidth cap. Your app can retry the request again after more time has elapsed. + + */ + + switch(http.statusLine.code) + { + + // case 1,2,3,4: + + // 200 - OK + // 201 - Created OK + // 202 - Accepted + // 204 - Deleted OK + case 200,201,202,204: + // No actions, but log if verbose logging + log.vlog("OneDrive Response: '", http.statusLine.code, " - ", http.statusLine.reason, "'"); + break; + + // 400 - Bad Request + case 400: + // Bad Request .. how should we act? + log.vlog("OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error"); + break; + + // Item not found + case 404: + // Item was not found - do not throw an exception + log.vlog("OneDrive returned a 'HTTP 404 - Item not found' - gracefully handling error"); + break; + + // 409 - Conflict + case 409: + // Conflict handling .. how should we act? This only really gets triggered if we are using --local-first & we remove items.db as the DB thinks the file is not uploaded but it is + log.vlog("OneDrive returned a 'HTTP 409 - Conflict' - gracefully handling error"); + break; + + // 412 - Precondition Failed + case 412: + // A precondition provided in the request (such as an if-match header) does not match the resource's current state. + log.vlog("OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error"); + break; + + // 415 - Unsupported Media Type + case 415: + // Unsupported Media Type ... sometimes triggered on image files, especially PNG + log.vlog("OneDrive returned a 'HTTP 415 - Unsupported Media Type' - gracefully handling error"); + break; + + // 500 - Internal Server Error + // 502 - Bad Gateway + // 503 - Service Unavailable + case 500,502,503: + // No actions + break; + + // "else" + default: + throw new OneDriveException(http.statusLine.code, http.statusLine.reason); + break; } } diff --git a/src/sync.d b/src/sync.d index 775a7ba0..b258ff27 100644 --- a/src/sync.d +++ b/src/sync.d @@ -4,7 +4,7 @@ import std.datetime; import std.exception: enforce; import std.file, std.json, std.path; import std.regex; -import std.stdio, std.string; +import std.stdio, std.string, std.uni, std.uri; import config, itemdb, onedrive, selective, upload, util; static import log; @@ -141,7 +141,7 @@ final class SyncEngine // download all new changes from OneDrive void applyDifferences() { - // root folder + // Set defaults for the root folder string driveId = defaultDriveId = onedrive.getDefaultDrive()["id"].str; string rootId = onedrive.getDefaultRoot["id"].str; applyDifferences(driveId, rootId); @@ -152,14 +152,133 @@ final class SyncEngine foreach (item; items) applyDifferences(item.remoteDriveId, item.remoteId); } - + // download all new changes from a specified folder on OneDrive + void applyDifferencesSingleDirectory(string path) + { + // test if the path we are going to sync from actually exists on OneDrive + try { + onedrive.getPathDetails(path); + } catch (OneDriveException e) { + if (e.httpStatusCode == 404) { + // The directory was not found + log.vlog("ERROR: The requested single directory to sync was not found on OneDrive"); + return; + } + } + // OK - it should exist, get the driveId and rootId for this folder + log.vlog("Checking for differences from OneDrive ..."); + JSONValue onedrivePathDetails = onedrive.getPathDetails(path); // Returns a JSON String for the OneDrive Path + + // If the OneDrive Root is not in the local database, creating a remote folder will fail + checkDatabaseForOneDriveRoot(); + + // Configure the defaults + defaultDriveId = onedrive.getDefaultDrive()["id"].str; + string driveId = onedrivePathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1 + string folderId = onedrivePathDetails["id"].str; // Should give something like 12345ABCDE1234A1!101 + + // Apply any differences found on OneDrive for this path (download data) + applyDifferences(driveId, folderId); + } + + // make sure the OneDrive root is in our database + auto checkDatabaseForOneDriveRoot() + { + log.vlog("Fetching details for OneDrive Root"); + JSONValue rootPathDetails = onedrive.getDefaultRoot(); // Returns a JSON Value + Item rootPathItem = makeItem(rootPathDetails); + + // configure driveId and rootId for the OneDrive Root + + // Set defaults for the root folder + string driveId = rootPathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1 + string rootId = rootPathDetails["id"].str; // Should give something like 12345ABCDE1234A1!101 + + // Query the database + if (!itemdb.selectById(driveId, rootId, rootPathItem)) { + log.vlog("OneDrive Root does not exist in the database. We need to add it."); + applyDifference(rootPathDetails, driveId, true); + } else { + log.vlog("OneDrive Root exists in the database"); + } + } + + // create a directory on OneDrive without syncing + auto createDirectoryNoSync(string path) + { + // Attempt to create the requested path within OneDrive without performing a sync + log.vlog("Attempting to create the requested path within OneDrive"); + + // If the OneDrive Root is not in the local database, creating a remote folder will fail + checkDatabaseForOneDriveRoot(); + + // Handle the remote folder creation and updating of the local database without performing a sync + uploadCreateDir(path); + } + + // delete a directory on OneDrive without syncing + auto deleteDirectoryNoSync(string path) + { + // Set defaults for the root folder + defaultDriveId = onedrive.getDefaultDrive()["id"].str; + string rootId = onedrive.getDefaultRoot["id"].str; + + // Attempt to delete the requested path within OneDrive without performing a sync + log.vlog("Attempting to delete the requested path within OneDrive"); + + // test if the path we are going to exists on OneDrive + try { + onedrive.getPathDetails(path); + } catch (OneDriveException e) { + if (e.httpStatusCode == 404) { + // The directory was not found on OneDrive - no need to delete it + log.vlog("The requested directory to create was not found on OneDrive - skipping removing the remote directory as it doesnt exist"); + return; + } + } + + Item item; + if (!itemdb.selectByPath(path, defaultDriveId, item)) { + // this is odd .. this directory is not in the local database - just go delete it + log.vlog("The requested directory to delete was not found in the local database - pushing delete request direct to OneDrive"); + uploadDeleteItem(item, path); + } else { + // the folder was in the local database + // Handle the deletion and saving any update to the local database + log.vlog("The requested directory to delete was found in the local database. Processing the delection normally"); + deleteByPath(path); + } + } + + // rename a directory on OneDrive without syncing + auto renameDirectoryNoSync(string source, string destination) + { + try { + // test if the local path exists on OneDrive + onedrive.getPathDetails(source); + } catch (OneDriveException e) { + if (e.httpStatusCode == 404) { + // The directory was not found + log.vlog("The requested directory to rename was not found on OneDrive"); + return; + } + } + // The OneDrive API returned a 200 OK status, so the folder exists + // Rename the requested directory on OneDrive without performing a sync + moveByPath(source, destination); + } + // download the new changes of a specific item // id is the root of the drive or a shared folder private void applyDifferences(string driveId, const(char)[] id) { JSONValue changes; string deltaLink = itemdb.getDeltaLink(driveId, id); - log.vlog("Applying changes of " ~ id); + log.vlog("Applying changes of Path ID: " ~ id); + + // Get the OneDrive Root ID + string oneDriveRootId = onedrive.getDefaultRoot["id"].str; + for (;;) { try { changes = onedrive.viewChangesById(driveId, id, deltaLink); @@ -172,8 +291,14 @@ final class SyncEngine throw e; } } - foreach (item; changes["value"].array) { - bool isRoot = (id == item["id"].str); // fix for https://github.com/skilion/onedrive/issues/269 + foreach (item; changes["value"].array) { + // Test is this is the OneDrive Root - not say a single folder root sync + bool isRoot = false; + if ((id == oneDriveRootId) && (item["name"].str == "root")) { // fix for https://github.com/skilion/onedrive/issues/269 + // This IS the OneDrive Root + isRoot = true; + } + // Apply the change applyDifference(item, driveId, isRoot); } @@ -195,12 +320,16 @@ final class SyncEngine private void applyDifference(JSONValue driveItem, string driveId, bool isRoot) { Item item = makeItem(driveItem); - log.vlog("Processing ", item.id, " ", item.name); + //log.vlog("Processing item to apply differences"); if (isItemRoot(driveItem) || !item.parentId || isRoot) { - log.vlog("Root"); + log.vlog("Adding OneDrive Root to the local database"); item.parentId = null; // ensures that it has no parent item.driveId = driveId; // HACK: makeItem() cannot set the driveId propery of the root + + // What parent.driveId and parent.id are we using? + //log.vlog("Parent Drive ID: ", item.driveId); + //log.vlog("Parent ID: ", item.parentId); itemdb.upsert(item); return; } @@ -212,14 +341,14 @@ final class SyncEngine // check the item type if (!unwanted) { if (isItemFile(driveItem)) { - log.vlog("File"); + //log.vlog("The item we are syncing is a file"); } else if (isItemFolder(driveItem)) { - log.vlog("Folder"); + //log.vlog("The item we are syncing is a folder"); } else if (isItemRemote(driveItem)) { - log.vlog("Remote item"); + //log.vlog("The item we are syncing is a remote item"); assert(isItemFolder(driveItem["remoteItem"]), "The remote item is not a folder"); } else { - log.vlog("The item type is not supported"); + log.vlog("This item type (", item.name, ") is not supported"); unwanted = true; } } @@ -340,14 +469,14 @@ final class SyncEngine if (newItem.type == ItemType.file && oldItem.mtime != newItem.mtime && !testFileHash(newPath, newItem)) { downloadFileItem(newItem, newPath); } else { - log.vlog("The item content has not changed"); + //log.vlog("The item content has not changed"); } // handle changed time if (newItem.type == ItemType.file && oldItem.mtime != newItem.mtime) { setTimes(newPath, newItem.mtime, newItem.mtime); } } else { - log.vlog("The item has not changed"); + //log.vlog("", oldItem.name, " has not changed"); } } @@ -432,8 +561,18 @@ final class SyncEngine } // scan the given directory for differences and new items - void scanForDifferences(string path = ".") + void scanForDifferences(string path) { + // Make sure the OneDrive Root is in the database + checkDatabaseForOneDriveRoot(); + + // make sure defaultDriveId is set + if (defaultDriveId == ""){ + // defaultDriveId is not set ... odd .. + defaultDriveId = onedrive.getDefaultDrive()["id"].str; + } + + // scan for changes log.vlog("Uploading differences of ", path); Item item; if (itemdb.selectByPath(path, defaultDriveId, item)) { @@ -531,7 +670,7 @@ final class SyncEngine string eTag = item.eTag; if (!testFileHash(path, item)) { log.vlog("The file content has changed"); - write("Uploading ", path, "..."); + write("Uploading file ", path, "..."); JSONValue response; if (getSize(path) <= thresholdFileSize) { response = onedrive.simpleUploadReplace(path, item.driveId, item.id, item.eTag); @@ -540,6 +679,7 @@ final class SyncEngine writeln(""); response = session.upload(path, item.driveId, item.parentId, baseName(path), eTag); } + log.vlog("Uploading file ", path, "... done."); // saveItem(response); redundant // use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded eTag = response["cTag"].str; @@ -561,75 +701,233 @@ final class SyncEngine private void uploadNewItems(string path) { - // skip unexisting symbolic links - if (isSymlink(path) && !exists(readLink(path))) { - return; - } + // https://github.com/OneDrive/onedrive-api-docs/issues/443 + // If the path is greater than 430 characters, then one drive will return a '400 - Bad Request' + // Need to ensure that the URI is encoded before the check is made + if(encodeComponent(path).length < 430){ + // path is less than 430 characters - // skip filtered items - if (path != ".") { - if (selectiveSync.isNameExcluded(baseName(path))) { + if (defaultDriveId == ""){ + // defaultDriveId is not set ... odd .. + defaultDriveId = onedrive.getDefaultDrive()["id"].str; + } + + // skip unexisting symbolic links + if (isSymlink(path) && !exists(readLink(path))) { return; } - if (selectiveSync.isPathExcluded(path)) { - return; - } - } - if (isDir(path)) { - Item item; - if (!itemdb.selectByPath(path, defaultDriveId, item)) { - uploadCreateDir(path); + // skip filtered items + if (path != ".") { + if (selectiveSync.isNameExcluded(baseName(path))) { + return; + } + if (selectiveSync.isPathExcluded(path)) { + return; + } } - // recursively traverse children - auto entries = dirEntries(path, SpanMode.shallow, false); - foreach (DirEntry entry; entries) { - uploadNewItems(entry.name); + + if (isDir(path)) { + Item item; + if (!itemdb.selectByPath(path, defaultDriveId, item)) { + uploadCreateDir(path); + } + // recursively traverse children + auto entries = dirEntries(path, SpanMode.shallow, false); + foreach (DirEntry entry; entries) { + uploadNewItems(entry.name); + } + } else { + Item item; + if (!itemdb.selectByPath(path, defaultDriveId, item)) { + uploadNewFile(path); + } } } else { - Item item; - if (!itemdb.selectByPath(path, defaultDriveId, item)) { - uploadNewFile(path); - } + // This path was skipped - why? + log.log("Skipping item '", path, "' due to the full path exceeding 430 characters (Microsoft OneDrive limitation)"); } } - private void uploadCreateDir(const(char)[] path) + private void uploadCreateDir(const(string) path) { - log.log("Creating folder ", path); + log.vlog("OneDrive Client requested to create remote path: ", path); Item parent; - enforce(itemdb.selectByPath(dirName(path), defaultDriveId, parent), "The parent item is not in the database"); - JSONValue driveItem = [ - "name": JSONValue(baseName(path)), - "folder": parseJSON("{}") - ]; - auto res = onedrive.createById(parent.driveId, parent.id, driveItem); - saveItem(res); - } + + // Was the path entered the root path? + if (path == "."){ + // We cant create this directory, as this would essentially equal the users OneDrive root:/ + checkDatabaseForOneDriveRoot(); + } else { + // If this is null or empty - we cant query the database properly + if ((parent.driveId == "") && (parent.id == "")){ + // These are both empty .. not good + //log.vlog("WHOOPS: Well this is odd - parent.driveId & parent.id are empty - we have to query OneDrive for some values for the parent"); + + // What path to use? + string parentPath = dirName(path); // will be either . or something else + //log.vlog("WHOOPS FIX: Query OneDrive path details for parent: ", parentPath); + + if (parentPath == "."){ + // We cant create this directory, as this would essentially equal the users OneDrive root:/ + checkDatabaseForOneDriveRoot(); + } + + try { + onedrive.getPathDetails(parentPath); + } catch (OneDriveException e) { + if (e.httpStatusCode == 404) { + // Parent does not exist ... need to create parent + uploadCreateDir(parentPath); + } + } + + // Get the Parent Path Details + JSONValue onedrivePathDetails = onedrive.getPathDetails(parentPath); // Returns a JSON String for the OneDrive Path + + // JSON Response + //log.vlog("WHOOPS JSON Response: ", onedrivePathDetails); + + // configure the data + parent.driveId = onedrivePathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1 + parent.id = onedrivePathDetails["id"].str; // This item's ID. Should give something like 12345ABCDE1234A1!101 + + // What parent.driveId and parent.id did we find? + //log.vlog("Using Parent DriveID: ", parent.driveId); + //log.vlog("Using Parent ID: ", parent.id); + } + + // test if the path we are going to create already exists on OneDrive + try { + onedrive.getPathDetails(path); + } catch (OneDriveException e) { + if (e.httpStatusCode == 404) { + // The directory was not found + log.vlog("The requested directory to create was not found on OneDrive - creating remote directory: ", path); + // Perform the database lookup + enforce(itemdb.selectById(parent.driveId, parent.id, parent), "The parent item id is not in the database"); + JSONValue driveItem = [ + "name": JSONValue(baseName(path)), + "folder": parseJSON("{}") + ]; + + // Submit the creation request + auto res = onedrive.createById(parent.driveId, parent.id, driveItem); + // What is returned? + //log.vlog("Create Folder Response JSON: ", res); + saveItem(res); + log.vlog("Sucessfully created the remote directory ", path, " on OneDrive"); + return; + } + } + log.vlog("The requested directory to create was found on OneDrive - skipping creating the directory: ", path ); + + // Check that this path is in the database + if (!itemdb.selectById(parent.driveId, parent.id, parent)){ + // parent for 'path' is NOT in the database + log.vlog("The parent for this path is not in the local database - need to add parent to local database"); + string parentPath = dirName(path); + uploadCreateDir(parentPath); + } else { + // parent is in database + log.vlog("The parent for this path is in the local database - adding requested path (", path ,") to database"); + auto res = onedrive.getPathDetails(path); + saveItem(res); + } + } + } + private void uploadNewFile(string path) { - write("Uploading file ", path, "..."); Item parent; - enforce(itemdb.selectByPath(dirName(path), defaultDriveId, parent), "The parent item is not in the database"); - JSONValue response; - if (getSize(path) <= thresholdFileSize) { - response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path)); - writeln(" done."); - } else { - writeln(""); - response = session.upload(path, parent.driveId, parent.id, baseName(path)); + + if (defaultDriveId == ""){ + // defaultDriveId is not set ... odd .. + defaultDriveId = onedrive.getDefaultDrive()["id"].str; + } + + // Check the database for the parent + enforce(itemdb.selectByPath(dirName(path), defaultDriveId, parent), "The parent item is not in the local database"); + + // To avoid a 409 Conflict error - does the file actually exist on OneDrive already? + JSONValue fileDetailsFromOneDrive; + + // Does this 'file' already exist on OneDrive? + try { + // test if the local path exists on OneDrive + fileDetailsFromOneDrive = onedrive.getPathDetails(path); + } catch (OneDriveException e) { + if (e.httpStatusCode == 404) { + // The file was not found on OneDrive, need to upload it + write("Uploading file ", path, "..."); + JSONValue response; + if (getSize(path) <= thresholdFileSize) { + response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path)); + writeln(" done."); + } else { + writeln(""); + response = session.upload(path, parent.driveId, parent.id, baseName(path)); + } + log.vlog("Uploading file ", path, "... done."); + string id = response["id"].str; + string cTag = response["cTag"].str; + SysTime mtime = timeLastModified(path).toUTC(); + // use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded + uploadLastModifiedTime(parent.driveId, id, cTag, mtime); + return; + } + } + + log.vlog("Requested file to upload exists on OneDrive - local database is out of sync for this file: ", path); + + // Is the local file newer than the uploaded file? + SysTime localFileModifiedTime = timeLastModified(path).toUTC(); + SysTime remoteFileModifiedTime = SysTime.fromISOExtString(fileDetailsFromOneDrive["fileSystemInfo"]["lastModifiedDateTime"].str); + + if (localFileModifiedTime > remoteFileModifiedTime){ + // local file is newer + log.vlog("Requested file to upload is newer than existing file on OneDrive"); + + write("Uploading file ", path, "..."); + JSONValue response; + if (getSize(path) <= thresholdFileSize) { + response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path)); + writeln(" done."); + } else { + writeln(""); + response = session.upload(path, parent.driveId, parent.id, baseName(path)); + } + log.vlog("Uploading file ", path, "... done."); + string id = response["id"].str; + string cTag = response["cTag"].str; + SysTime mtime = timeLastModified(path).toUTC(); + // use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded + uploadLastModifiedTime(parent.driveId, id, cTag, mtime); + } else { + // Save the details of the file that we got from OneDrive + log.vlog("Updating the local database with details for this file: ", path); + saveItem(fileDetailsFromOneDrive); } - string id = response["id"].str; - string cTag = response["cTag"].str; - SysTime mtime = timeLastModified(path).toUTC(); - // use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded - uploadLastModifiedTime(parent.driveId, id, cTag, mtime); } - private void uploadDeleteItem(Item item, const(char)[] path) + private void uploadDeleteItem(Item item, string path) { - log.log("Deleting ", path); + log.log("Deleting directory from OneDrive: ", path); + + if ((item.driveId == "") && (item.id == "") && (item.eTag == "")){ + // These are empty ... we cannot delete if this is empty .... + JSONValue onedrivePathDetails = onedrive.getPathDetails(path); // Returns a JSON String for the OneDrive Path + //log.vlog("WHOOPS JSON Response: ", onedrivePathDetails); + item.driveId = onedrivePathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1 + item.id = onedrivePathDetails["id"].str; // This item's ID. Should give something like 12345ABCDE1234A1!101 + item.eTag = onedrivePathDetails["eTag"].str; // Should be something like aNjM2NjJFRUVGQjY2NjJFMSE5MzUuMA + + //log.vlog("item.driveId = ", item.driveId); + //log.vlog("item.id = ", item.id); + //log.vlog("item.eTag = ", item.eTag); + } + try { onedrive.deleteById(item.driveId, item.id, item.eTag); } catch (OneDriveException e) { @@ -655,7 +953,10 @@ final class SyncEngine private void saveItem(JSONValue jsonItem) { + // Takes a JSON input and formats to an item which can be used by the database Item item = makeItem(jsonItem); + + // Add to the local database itemdb.upsert(item); } @@ -699,11 +1000,11 @@ final class SyncEngine } } - void deleteByPath(const(char)[] path) + void deleteByPath(string path) { Item item; if (!itemdb.selectByPath(path, defaultDriveId, item)) { - throw new SyncException("Can't delete an unsynced item"); + throw new SyncException("The item to delete is not in the local database"); } if (item.parentId == null) { // the item is a remote folder, need to do the operation on the parent @@ -716,4 +1017,31 @@ final class SyncEngine else throw e; } } + + // move a OneDrive folder from one name to another + void moveByPath(const(string) source, const(string) destination) + { + log.vlog("Moving remote folder: ", source, " -> ", destination); + + // Source and Destination are relative to ~/OneDrive + string sourcePath = source; + string destinationBasePath = dirName(destination).idup; + + // if destinationBasePath == '.' then destinationBasePath needs to be "" + if (destinationBasePath == ".") { + destinationBasePath = ""; + } + + string newFolderName = baseName(destination).idup; + string destinationPathString = "/drive/root:/" ~ destinationBasePath; + + // Build up the JSON changes + JSONValue moveData = ["name": newFolderName]; + JSONValue destinationPath = ["path": destinationPathString]; + moveData["parentReference"] = destinationPath; + + // Make the change on OneDrive + auto res = onedrive.moveByPath(sourcePath, moveData); + } + }