OneDrive Client Changes

* Fix 4xx errors including (412 pre-condition)
* Add Logging - log to a file (/var/log/onedrive/onedrive.log)
* Add http(s) debugging as a flag
* Add dont sync when just blindly running the application
* Add individual folder sync - ie ~/OneDrive/blah/ vs. syncing
everything in ~/OneDrive/
* Add sync from local directory first rather than download first then
upload
* Add upload long path check (430 character limitation)
This commit is contained in:
abraunegg 2018-03-14 15:43:40 +11:00
parent c231b13ad4
commit dd73ae3c4b
4 changed files with 719 additions and 137 deletions

View file

@ -1,4 +1,6 @@
import std.stdio;
import std.file;
import std.datetime;
// enable verbose logging
bool verbose;
@ -6,14 +8,37 @@ bool verbose;
void log(T...)(T args)
{
writeln(args);
// Write to log file
string logFileName = "/var/log/onedrive/onedrive.log";
auto currentTime = Clock.currTime();
auto timeString = currentTime.toString();
File logFile = File(logFileName, "a");
logFile.writeln(timeString, " ", args);
logFile.close();
}
void vlog(T...)(T args)
{
if (verbose) writeln(args);
if (verbose) {
writeln(args);
// Write to log file
string logFileName = "/var/log/onedrive/onedrive.log";
auto currentTime = Clock.currTime();
auto timeString = currentTime.toString();
File logFile = File(logFileName, "a");
logFile.writeln(timeString, " ", args);
logFile.close();
}
}
void error(T...)(T args)
{
stderr.writeln(args);
// Write to log file
string logFileName = "/var/log/onedrive/onedrive.log";
auto currentTime = Clock.currTime();
auto timeString = currentTime.toString();
File logFile = File(logFileName, "a");
logFile.writeln(timeString, " ", args);
logFile.close();
}

View file

@ -25,19 +25,46 @@ int main(string[] args)
bool printAccessToken;
// print the version and exit
bool printVersion;
// Additional options added to support MyNAS Storage Appliance
// debug the HTTP(S) operations if required
bool debugHttp;
// This allows for selective directory syncing instead of everything under ~/OneDrive/
string singleDirectory;
// Create a single root directory on OneDrive
string createDirectory;
// Remove a single directory on OneDrive
string removeDirectory;
// The source directory if we are using the OneDrive client to rename a directory
string sourceDirectory;
// The destination directory if we are using the OneDrive client to rename a directory
string destinationDirectory;
// Configure a flag to perform a sync
// This is beneficial so that if just running the client itself - without any options, or sync check, the client does not perform a sync
bool synchronize;
// Local sync - Upload local changes first before downloading changes from OneDrive
bool localFirst;
try {
auto opt = getopt(
args,
std.getopt.config.bundling,
std.getopt.config.caseSensitive,
"confdir", "Set the directory used to store the configuration files", &configDirName,
"create-directory", "Create a directory on OneDrive - no sync will be performed.", &createDirectory,
"destination-directory", "Destination directory for renamed or move on OneDrive - no sync will be performed.", &destinationDirectory,
"debug-http", "Debug OneDrive HTTP communication.", &debugHttp,
"download|d", "Only download remote changes", &downloadOnly,
"local-first", "Synchronize from the local directory source first, before downloading changes from OneDrive.", &localFirst,
"logout", "Logout the current user", &logout,
"monitor|m", "Keep monitoring for local and remote changes", &monitor,
"print-token", "Print the access token, useful for debugging", &printAccessToken,
"resync", "Forget the last saved state, perform a full sync", &resync,
"remove-directory", "Remove a directory on OneDrive - no sync will be performed.", &removeDirectory,
"single-directory", "Specify a single local directory within the OneDrive root to sync.", &singleDirectory,
"source-directory", "Source directory to rename or move on OneDrive - no sync will be performed.", &sourceDirectory,
"syncdir", "Set the directory used to sync the files that are synced", &syncDirName,
"synchronize", "Perform a synchronization", &synchronize,
"verbose|v", "Print more details, useful for debugging", &log.verbose,
"version", "Print the version and exit", &printVersion
);
@ -63,6 +90,10 @@ int main(string[] args)
return EXIT_SUCCESS;
}
// Configure Logging
string logFilePath = "/var/log/onedrive/";
if (!exists(logFilePath)) mkdirRecurse(logFilePath);
log.vlog("Loading config ...");
configDirName = configDirName.expandTilde().absolutePath();
if (!exists(configDirName)) mkdirRecurse(configDirName);
@ -95,7 +126,7 @@ int main(string[] args)
log.error("No network connection");
return EXIT_FAILURE;
}
auto onedrive = new OneDriveApi(cfg);
auto onedrive = new OneDriveApi(cfg, debugHttp);
onedrive.printAccessToken = printAccessToken;
if (!onedrive.init()) {
log.error("Could not initialize the OneDrive API");
@ -104,97 +135,168 @@ int main(string[] args)
return EXIT_FAILURE;
}
// initialize system
log.vlog("Opening the item database ...");
auto itemdb = new ItemDatabase(cfg.databaseFilePath);
// Set the local path root
string syncDir = expandTilde(cfg.getValue("sync_dir"));
log.vlog("All operations will be performed in: ", syncDir);
if (!exists(syncDir)) mkdirRecurse(syncDir);
chdir(syncDir);
// Initialise the sync engine
log.vlog("Initializing the Synchronization Engine ...");
auto selectiveSync = new SelectiveSync();
selectiveSync.load(cfg.syncListFilePath);
selectiveSync.setMask(cfg.getValue("skip_file"));
auto sync = new SyncEngine(cfg, onedrive, itemdb, selectiveSync);
sync.init();
if (online) performSync(sync);
// Do we need to create or remove a directory?
if ((createDirectory != "") || (removeDirectory != "")) {
if (createDirectory != "") {
// create a directory on OneDrive
sync.createDirectoryNoSync(createDirectory);
}
if (removeDirectory != "") {
// remove a directory on OneDrive
sync.deleteDirectoryNoSync(removeDirectory);
}
}
// Are we renaming or moving a directory?
if ((sourceDirectory != "") && (destinationDirectory != "")) {
// We are renaming or moving a directory
sync.renameDirectoryNoSync(sourceDirectory, destinationDirectory);
}
// Are we performing a sync, resync or monitor operation?
if ((synchronize) || (resync) || (monitor)) {
if (monitor) {
log.vlog("Initializing monitor ...");
Monitor m = new Monitor(selectiveSync);
m.onDirCreated = delegate(string path) {
log.vlog("[M] Directory created: ", path);
try {
sync.scanForDifferences(path);
} catch(Exception e) {
log.log(e.msg);
}
};
m.onFileChanged = delegate(string path) {
log.vlog("[M] File changed: ", path);
try {
sync.scanForDifferences(path);
} catch(Exception e) {
log.log(e.msg);
}
};
m.onDelete = delegate(string path) {
log.vlog("[M] Item deleted: ", path);
try {
sync.deleteByPath(path);
} catch(Exception e) {
log.log(e.msg);
}
};
m.onMove = delegate(string from, string to) {
log.vlog("[M] Item moved: ", from, " -> ", to);
try {
sync.uploadMoveItem(from, to);
} catch(Exception e) {
log.log(e.msg);
}
};
if (!downloadOnly) m.init(cfg, verbose);
// monitor loop
immutable auto checkInterval = dur!"seconds"(45);
auto lastCheckTime = MonoTime.currTime();
while (true) {
if (!downloadOnly) m.update(online);
auto currTime = MonoTime.currTime();
if (currTime - lastCheckTime > checkInterval) {
lastCheckTime = currTime;
online = testNetwork();
if (online) {
performSync(sync);
if (!downloadOnly) {
// discard all events that may have been generated by the sync
m.update(false);
if ((synchronize) || (resync)) {
if (online) {
// Check user entry for local path - the above chdir means we are already in ~/OneDrive/ thus singleDirectory is local to this path
if (singleDirectory != ""){
// Does the directory we want to sync actually exist?
if (!exists(singleDirectory)){
// the requested directory does not exist ..
log.log("The requested local directory does not exist. Please check ~/OneDrive/ for requested path");
onedrive.http.shutdown();
return EXIT_FAILURE;
}
}
GC.collect();
} else {
Thread.sleep(dur!"msecs"(100));
// Perform the sync
performSync(sync, singleDirectory, localFirst);
}
}
if (monitor) {
log.vlog("Initializing monitor ...");
Monitor m = new Monitor(selectiveSync);
m.onDirCreated = delegate(string path) {
log.vlog("[M] Directory created: ", path);
try {
sync.scanForDifferences(path);
} catch(Exception e) {
log.log(e.msg);
}
};
m.onFileChanged = delegate(string path) {
log.vlog("[M] File changed: ", path);
try {
sync.scanForDifferences(path);
} catch(Exception e) {
log.log(e.msg);
}
};
m.onDelete = delegate(string path) {
log.vlog("[M] Item deleted: ", path);
try {
sync.deleteByPath(path);
} catch(Exception e) {
log.log(e.msg);
}
};
m.onMove = delegate(string from, string to) {
log.vlog("[M] Item moved: ", from, " -> ", to);
try {
sync.uploadMoveItem(from, to);
} catch(Exception e) {
log.log(e.msg);
}
};
if (!downloadOnly) m.init(cfg, verbose);
// monitor loop
immutable auto checkInterval = dur!"seconds"(45);
auto lastCheckTime = MonoTime.currTime();
while (true) {
if (!downloadOnly) m.update(online);
auto currTime = MonoTime.currTime();
if (currTime - lastCheckTime > checkInterval) {
lastCheckTime = currTime;
online = testNetwork();
if (online) {
performSync(sync, singleDirectory, localFirst);
if (!downloadOnly) {
// discard all events that may have been generated by the sync
m.update(false);
}
}
GC.collect();
} else {
Thread.sleep(dur!"msecs"(100));
}
}
}
}
// workaround for segfault in std.net.curl.Curl.shutdown() on exit
onedrive.http.shutdown();
return EXIT_SUCCESS;
}
// try to synchronize the folder three times
void performSync(SyncEngine sync)
void performSync(SyncEngine sync, string singleDirectory, bool localFirst)
{
int count;
string remotePath = "/";
string localPath = ".";
// Are we doing a single directory sync?
if (singleDirectory != ""){
// Need two different path strings here
remotePath = singleDirectory;
localPath = singleDirectory;
}
do {
try {
sync.applyDifferences();
if (!downloadOnly) {
sync.scanForDifferences();
// ensure that the current state is updated
if (singleDirectory != ""){
// we were requested to sync a single directory
log.vlog("Syncing changes from this selected path: ", singleDirectory);
if (localFirst) {
log.vlog("Syncing changes from selected local path first before downloading changes from OneDrive ...");
sync.scanForDifferences(localPath);
sync.applyDifferencesSingleDirectory(remotePath);
} else {
log.vlog("Syncing changes from selected OneDrive path first before uploading local changes ...");
sync.applyDifferencesSingleDirectory(remotePath);
sync.scanForDifferences(localPath);
}
} else {
// original onedrive client logic below
sync.applyDifferences();
if (!downloadOnly) {
sync.scanForDifferences(localPath);
// ensure that the current state is updated
sync.applyDifferences();
}
}
count = -1;
} catch (Exception e) {

View file

@ -34,7 +34,7 @@ class OneDriveException: Exception
{
this.httpStatusCode = httpStatusCode;
this.error = error;
string msg = format("HTTP request returned status code %d (%s)\n%s", httpStatusCode, reason, toJSON(error, true));
string msg = format("HTTP request returned status code %d (%s)\n%s", httpStatusCode, reason, toJSON(&error, true));
super(msg, file, line);
}
}
@ -49,11 +49,13 @@ final class OneDriveApi
// if true, every new access token is printed
bool printAccessToken;
this(Config cfg)
this(Config cfg, bool debugHttp)
{
this.cfg = cfg;
http = HTTP();
//http.verbose = true;
if (debugHttp) {
http.verbose = true;
}
}
bool init()
@ -181,10 +183,32 @@ final class OneDriveApi
{
checkAccessTokenExpired();
const(char)[] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ "/children";
http.addRequestHeader("Content-Type", "application/json");
http.addRequestHeader("Content-Type", "application/json");
return post(url, item.toString());
}
// Return the details of the specified path
JSONValue getPathDetails(const(string) path)
{
checkAccessTokenExpired();
const(char)[] url;
// string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
if (path == ".") url = driveUrl ~ "/root/";
else url = itemByPathUrl ~ encodeComponent(path) ~ ":/";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
return get(url);
}
// https://dev.onedrive.com/items/move.htm
JSONValue moveByPath(const(char)[] sourcePath, JSONValue moveData)
{
// Need to use itemByPathUrl
checkAccessTokenExpired();
string url = itemByPathUrl ~ encodeComponent(sourcePath);
http.addRequestHeader("Content-Type", "application/json");
return move(url, moveData.toString());
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
JSONValue createUploadSession(const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null)
{
@ -334,6 +358,17 @@ final class OneDriveApi
return response;
}
private auto move(T)(const(char)[] url, const(T)[] postData)
{
scope(exit) http.clearRequestHeaders();
http.method = HTTP.Method.patch;
http.url = url;
addAccessTokenHeader();
auto response = perform(postData);
checkHttpCode();
return response;
}
private JSONValue upload(string filepath, string url)
{
scope(exit) {
@ -397,8 +432,100 @@ final class OneDriveApi
private void checkHttpCode()
{
if (http.statusLine.code / 100 != 2) {
throw new OneDriveException(http.statusLine.code, http.statusLine.reason);
// https://dev.onedrive.com/misc/errors.htm
// https://developer.overdrive.com/docs/reference-guide
/*
Error response handling
Errors in the OneDrive API are returned using standard HTTP status codes, as well as a JSON error response object. The following HTTP status codes should be expected.
Status code Status message Description
200 OK Request was handled OK
201 Created This means you've made a successful POST to checkout, lock in a format, or place a hold
204 No Content This means you've made a successful DELETE to remove a hold or return a title
400 Bad Request Cannot process the request because it is malformed or incorrect.
401 Unauthorized Required authentication information is either missing or not valid for the resource.
403 Forbidden Access is denied to the requested resource. The user might not have enough permission.
404 Not Found The requested resource doesnt exist.
405 Method Not Allowed The HTTP method in the request is not allowed on the resource.
406 Not Acceptable This service doesnt support the format requested in the Accept header.
409 Conflict The current state conflicts with what the request expects. For example, the specified parent folder might not exist.
410 Gone The requested resource is no longer available at the server.
411 Length Required A Content-Length header is required on the request.
412 Precondition Failed A precondition provided in the request (such as an if-match header) does not match the resource's current state.
413 Request Entity Too Large The request size exceeds the maximum limit.
415 Unsupported Media Type The content type of the request is a format that is not supported by the service.
416 Requested Range Not Satisfiable The specified byte range is invalid or unavailable.
422 Unprocessable Entity Cannot process the request because it is semantically incorrect.
429 Too Many Requests Client application has been throttled and should not attempt to repeat the request until an amount of time has elapsed.
500 Internal Server Error There was an internal server error while processing the request.
501 Not Implemented The requested feature isnt implemented.
502 Bad Gateway The service was unreachable
503 Service Unavailable The service is temporarily unavailable. You may repeat the request after a delay. There may be a Retry-After header.
507 Insufficient Storage The maximum storage quota has been reached.
509 Bandwidth Limit Exceeded Your app has been throttled for exceeding the maximum bandwidth cap. Your app can retry the request again after more time has elapsed.
*/
switch(http.statusLine.code)
{
// case 1,2,3,4:
// 200 - OK
// 201 - Created OK
// 202 - Accepted
// 204 - Deleted OK
case 200,201,202,204:
// No actions, but log if verbose logging
log.vlog("OneDrive Response: '", http.statusLine.code, " - ", http.statusLine.reason, "'");
break;
// 400 - Bad Request
case 400:
// Bad Request .. how should we act?
log.vlog("OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error");
break;
// Item not found
case 404:
// Item was not found - do not throw an exception
log.vlog("OneDrive returned a 'HTTP 404 - Item not found' - gracefully handling error");
break;
// 409 - Conflict
case 409:
// Conflict handling .. how should we act? This only really gets triggered if we are using --local-first & we remove items.db as the DB thinks the file is not uploaded but it is
log.vlog("OneDrive returned a 'HTTP 409 - Conflict' - gracefully handling error");
break;
// 412 - Precondition Failed
case 412:
// A precondition provided in the request (such as an if-match header) does not match the resource's current state.
log.vlog("OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error");
break;
// 415 - Unsupported Media Type
case 415:
// Unsupported Media Type ... sometimes triggered on image files, especially PNG
log.vlog("OneDrive returned a 'HTTP 415 - Unsupported Media Type' - gracefully handling error");
break;
// 500 - Internal Server Error
// 502 - Bad Gateway
// 503 - Service Unavailable
case 500,502,503:
// No actions
break;
// "else"
default:
throw new OneDriveException(http.statusLine.code, http.statusLine.reason);
break;
}
}

View file

@ -4,7 +4,7 @@ import std.datetime;
import std.exception: enforce;
import std.file, std.json, std.path;
import std.regex;
import std.stdio, std.string;
import std.stdio, std.string, std.uni, std.uri;
import config, itemdb, onedrive, selective, upload, util;
static import log;
@ -141,7 +141,7 @@ final class SyncEngine
// download all new changes from OneDrive
void applyDifferences()
{
// root folder
// Set defaults for the root folder
string driveId = defaultDriveId = onedrive.getDefaultDrive()["id"].str;
string rootId = onedrive.getDefaultRoot["id"].str;
applyDifferences(driveId, rootId);
@ -152,14 +152,133 @@ final class SyncEngine
foreach (item; items) applyDifferences(item.remoteDriveId, item.remoteId);
}
// download all new changes from a specified folder on OneDrive
void applyDifferencesSingleDirectory(string path)
{
// test if the path we are going to sync from actually exists on OneDrive
try {
onedrive.getPathDetails(path);
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
// The directory was not found
log.vlog("ERROR: The requested single directory to sync was not found on OneDrive");
return;
}
}
// OK - it should exist, get the driveId and rootId for this folder
log.vlog("Checking for differences from OneDrive ...");
JSONValue onedrivePathDetails = onedrive.getPathDetails(path); // Returns a JSON String for the OneDrive Path
// If the OneDrive Root is not in the local database, creating a remote folder will fail
checkDatabaseForOneDriveRoot();
// Configure the defaults
defaultDriveId = onedrive.getDefaultDrive()["id"].str;
string driveId = onedrivePathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1
string folderId = onedrivePathDetails["id"].str; // Should give something like 12345ABCDE1234A1!101
// Apply any differences found on OneDrive for this path (download data)
applyDifferences(driveId, folderId);
}
// make sure the OneDrive root is in our database
auto checkDatabaseForOneDriveRoot()
{
log.vlog("Fetching details for OneDrive Root");
JSONValue rootPathDetails = onedrive.getDefaultRoot(); // Returns a JSON Value
Item rootPathItem = makeItem(rootPathDetails);
// configure driveId and rootId for the OneDrive Root
// Set defaults for the root folder
string driveId = rootPathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1
string rootId = rootPathDetails["id"].str; // Should give something like 12345ABCDE1234A1!101
// Query the database
if (!itemdb.selectById(driveId, rootId, rootPathItem)) {
log.vlog("OneDrive Root does not exist in the database. We need to add it.");
applyDifference(rootPathDetails, driveId, true);
} else {
log.vlog("OneDrive Root exists in the database");
}
}
// create a directory on OneDrive without syncing
auto createDirectoryNoSync(string path)
{
// Attempt to create the requested path within OneDrive without performing a sync
log.vlog("Attempting to create the requested path within OneDrive");
// If the OneDrive Root is not in the local database, creating a remote folder will fail
checkDatabaseForOneDriveRoot();
// Handle the remote folder creation and updating of the local database without performing a sync
uploadCreateDir(path);
}
// delete a directory on OneDrive without syncing
auto deleteDirectoryNoSync(string path)
{
// Set defaults for the root folder
defaultDriveId = onedrive.getDefaultDrive()["id"].str;
string rootId = onedrive.getDefaultRoot["id"].str;
// Attempt to delete the requested path within OneDrive without performing a sync
log.vlog("Attempting to delete the requested path within OneDrive");
// test if the path we are going to exists on OneDrive
try {
onedrive.getPathDetails(path);
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
// The directory was not found on OneDrive - no need to delete it
log.vlog("The requested directory to create was not found on OneDrive - skipping removing the remote directory as it doesnt exist");
return;
}
}
Item item;
if (!itemdb.selectByPath(path, defaultDriveId, item)) {
// this is odd .. this directory is not in the local database - just go delete it
log.vlog("The requested directory to delete was not found in the local database - pushing delete request direct to OneDrive");
uploadDeleteItem(item, path);
} else {
// the folder was in the local database
// Handle the deletion and saving any update to the local database
log.vlog("The requested directory to delete was found in the local database. Processing the delection normally");
deleteByPath(path);
}
}
// rename a directory on OneDrive without syncing
auto renameDirectoryNoSync(string source, string destination)
{
try {
// test if the local path exists on OneDrive
onedrive.getPathDetails(source);
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
// The directory was not found
log.vlog("The requested directory to rename was not found on OneDrive");
return;
}
}
// The OneDrive API returned a 200 OK status, so the folder exists
// Rename the requested directory on OneDrive without performing a sync
moveByPath(source, destination);
}
// download the new changes of a specific item
// id is the root of the drive or a shared folder
private void applyDifferences(string driveId, const(char)[] id)
{
JSONValue changes;
string deltaLink = itemdb.getDeltaLink(driveId, id);
log.vlog("Applying changes of " ~ id);
log.vlog("Applying changes of Path ID: " ~ id);
// Get the OneDrive Root ID
string oneDriveRootId = onedrive.getDefaultRoot["id"].str;
for (;;) {
try {
changes = onedrive.viewChangesById(driveId, id, deltaLink);
@ -172,8 +291,14 @@ final class SyncEngine
throw e;
}
}
foreach (item; changes["value"].array) {
bool isRoot = (id == item["id"].str); // fix for https://github.com/skilion/onedrive/issues/269
foreach (item; changes["value"].array) {
// Test is this is the OneDrive Root - not say a single folder root sync
bool isRoot = false;
if ((id == oneDriveRootId) && (item["name"].str == "root")) { // fix for https://github.com/skilion/onedrive/issues/269
// This IS the OneDrive Root
isRoot = true;
}
// Apply the change
applyDifference(item, driveId, isRoot);
}
@ -195,12 +320,16 @@ final class SyncEngine
private void applyDifference(JSONValue driveItem, string driveId, bool isRoot)
{
Item item = makeItem(driveItem);
log.vlog("Processing ", item.id, " ", item.name);
//log.vlog("Processing item to apply differences");
if (isItemRoot(driveItem) || !item.parentId || isRoot) {
log.vlog("Root");
log.vlog("Adding OneDrive Root to the local database");
item.parentId = null; // ensures that it has no parent
item.driveId = driveId; // HACK: makeItem() cannot set the driveId propery of the root
// What parent.driveId and parent.id are we using?
//log.vlog("Parent Drive ID: ", item.driveId);
//log.vlog("Parent ID: ", item.parentId);
itemdb.upsert(item);
return;
}
@ -212,14 +341,14 @@ final class SyncEngine
// check the item type
if (!unwanted) {
if (isItemFile(driveItem)) {
log.vlog("File");
//log.vlog("The item we are syncing is a file");
} else if (isItemFolder(driveItem)) {
log.vlog("Folder");
//log.vlog("The item we are syncing is a folder");
} else if (isItemRemote(driveItem)) {
log.vlog("Remote item");
//log.vlog("The item we are syncing is a remote item");
assert(isItemFolder(driveItem["remoteItem"]), "The remote item is not a folder");
} else {
log.vlog("The item type is not supported");
log.vlog("This item type (", item.name, ") is not supported");
unwanted = true;
}
}
@ -340,14 +469,14 @@ final class SyncEngine
if (newItem.type == ItemType.file && oldItem.mtime != newItem.mtime && !testFileHash(newPath, newItem)) {
downloadFileItem(newItem, newPath);
} else {
log.vlog("The item content has not changed");
//log.vlog("The item content has not changed");
}
// handle changed time
if (newItem.type == ItemType.file && oldItem.mtime != newItem.mtime) {
setTimes(newPath, newItem.mtime, newItem.mtime);
}
} else {
log.vlog("The item has not changed");
//log.vlog("", oldItem.name, " has not changed");
}
}
@ -432,8 +561,18 @@ final class SyncEngine
}
// scan the given directory for differences and new items
void scanForDifferences(string path = ".")
void scanForDifferences(string path)
{
// Make sure the OneDrive Root is in the database
checkDatabaseForOneDriveRoot();
// make sure defaultDriveId is set
if (defaultDriveId == ""){
// defaultDriveId is not set ... odd ..
defaultDriveId = onedrive.getDefaultDrive()["id"].str;
}
// scan for changes
log.vlog("Uploading differences of ", path);
Item item;
if (itemdb.selectByPath(path, defaultDriveId, item)) {
@ -531,7 +670,7 @@ final class SyncEngine
string eTag = item.eTag;
if (!testFileHash(path, item)) {
log.vlog("The file content has changed");
write("Uploading ", path, "...");
write("Uploading file ", path, "...");
JSONValue response;
if (getSize(path) <= thresholdFileSize) {
response = onedrive.simpleUploadReplace(path, item.driveId, item.id, item.eTag);
@ -540,6 +679,7 @@ final class SyncEngine
writeln("");
response = session.upload(path, item.driveId, item.parentId, baseName(path), eTag);
}
log.vlog("Uploading file ", path, "... done.");
// saveItem(response); redundant
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
eTag = response["cTag"].str;
@ -561,75 +701,233 @@ final class SyncEngine
private void uploadNewItems(string path)
{
// skip unexisting symbolic links
if (isSymlink(path) && !exists(readLink(path))) {
return;
}
// https://github.com/OneDrive/onedrive-api-docs/issues/443
// If the path is greater than 430 characters, then one drive will return a '400 - Bad Request'
// Need to ensure that the URI is encoded before the check is made
if(encodeComponent(path).length < 430){
// path is less than 430 characters
// skip filtered items
if (path != ".") {
if (selectiveSync.isNameExcluded(baseName(path))) {
if (defaultDriveId == ""){
// defaultDriveId is not set ... odd ..
defaultDriveId = onedrive.getDefaultDrive()["id"].str;
}
// skip unexisting symbolic links
if (isSymlink(path) && !exists(readLink(path))) {
return;
}
if (selectiveSync.isPathExcluded(path)) {
return;
}
}
if (isDir(path)) {
Item item;
if (!itemdb.selectByPath(path, defaultDriveId, item)) {
uploadCreateDir(path);
// skip filtered items
if (path != ".") {
if (selectiveSync.isNameExcluded(baseName(path))) {
return;
}
if (selectiveSync.isPathExcluded(path)) {
return;
}
}
// recursively traverse children
auto entries = dirEntries(path, SpanMode.shallow, false);
foreach (DirEntry entry; entries) {
uploadNewItems(entry.name);
if (isDir(path)) {
Item item;
if (!itemdb.selectByPath(path, defaultDriveId, item)) {
uploadCreateDir(path);
}
// recursively traverse children
auto entries = dirEntries(path, SpanMode.shallow, false);
foreach (DirEntry entry; entries) {
uploadNewItems(entry.name);
}
} else {
Item item;
if (!itemdb.selectByPath(path, defaultDriveId, item)) {
uploadNewFile(path);
}
}
} else {
Item item;
if (!itemdb.selectByPath(path, defaultDriveId, item)) {
uploadNewFile(path);
}
// This path was skipped - why?
log.log("Skipping item '", path, "' due to the full path exceeding 430 characters (Microsoft OneDrive limitation)");
}
}
private void uploadCreateDir(const(char)[] path)
private void uploadCreateDir(const(string) path)
{
log.log("Creating folder ", path);
log.vlog("OneDrive Client requested to create remote path: ", path);
Item parent;
enforce(itemdb.selectByPath(dirName(path), defaultDriveId, parent), "The parent item is not in the database");
JSONValue driveItem = [
"name": JSONValue(baseName(path)),
"folder": parseJSON("{}")
];
auto res = onedrive.createById(parent.driveId, parent.id, driveItem);
saveItem(res);
}
// Was the path entered the root path?
if (path == "."){
// We cant create this directory, as this would essentially equal the users OneDrive root:/
checkDatabaseForOneDriveRoot();
} else {
// If this is null or empty - we cant query the database properly
if ((parent.driveId == "") && (parent.id == "")){
// These are both empty .. not good
//log.vlog("WHOOPS: Well this is odd - parent.driveId & parent.id are empty - we have to query OneDrive for some values for the parent");
// What path to use?
string parentPath = dirName(path); // will be either . or something else
//log.vlog("WHOOPS FIX: Query OneDrive path details for parent: ", parentPath);
if (parentPath == "."){
// We cant create this directory, as this would essentially equal the users OneDrive root:/
checkDatabaseForOneDriveRoot();
}
try {
onedrive.getPathDetails(parentPath);
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
// Parent does not exist ... need to create parent
uploadCreateDir(parentPath);
}
}
// Get the Parent Path Details
JSONValue onedrivePathDetails = onedrive.getPathDetails(parentPath); // Returns a JSON String for the OneDrive Path
// JSON Response
//log.vlog("WHOOPS JSON Response: ", onedrivePathDetails);
// configure the data
parent.driveId = onedrivePathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1
parent.id = onedrivePathDetails["id"].str; // This item's ID. Should give something like 12345ABCDE1234A1!101
// What parent.driveId and parent.id did we find?
//log.vlog("Using Parent DriveID: ", parent.driveId);
//log.vlog("Using Parent ID: ", parent.id);
}
// test if the path we are going to create already exists on OneDrive
try {
onedrive.getPathDetails(path);
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
// The directory was not found
log.vlog("The requested directory to create was not found on OneDrive - creating remote directory: ", path);
// Perform the database lookup
enforce(itemdb.selectById(parent.driveId, parent.id, parent), "The parent item id is not in the database");
JSONValue driveItem = [
"name": JSONValue(baseName(path)),
"folder": parseJSON("{}")
];
// Submit the creation request
auto res = onedrive.createById(parent.driveId, parent.id, driveItem);
// What is returned?
//log.vlog("Create Folder Response JSON: ", res);
saveItem(res);
log.vlog("Sucessfully created the remote directory ", path, " on OneDrive");
return;
}
}
log.vlog("The requested directory to create was found on OneDrive - skipping creating the directory: ", path );
// Check that this path is in the database
if (!itemdb.selectById(parent.driveId, parent.id, parent)){
// parent for 'path' is NOT in the database
log.vlog("The parent for this path is not in the local database - need to add parent to local database");
string parentPath = dirName(path);
uploadCreateDir(parentPath);
} else {
// parent is in database
log.vlog("The parent for this path is in the local database - adding requested path (", path ,") to database");
auto res = onedrive.getPathDetails(path);
saveItem(res);
}
}
}
private void uploadNewFile(string path)
{
write("Uploading file ", path, "...");
Item parent;
enforce(itemdb.selectByPath(dirName(path), defaultDriveId, parent), "The parent item is not in the database");
JSONValue response;
if (getSize(path) <= thresholdFileSize) {
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
writeln(" done.");
} else {
writeln("");
response = session.upload(path, parent.driveId, parent.id, baseName(path));
if (defaultDriveId == ""){
// defaultDriveId is not set ... odd ..
defaultDriveId = onedrive.getDefaultDrive()["id"].str;
}
// Check the database for the parent
enforce(itemdb.selectByPath(dirName(path), defaultDriveId, parent), "The parent item is not in the local database");
// To avoid a 409 Conflict error - does the file actually exist on OneDrive already?
JSONValue fileDetailsFromOneDrive;
// Does this 'file' already exist on OneDrive?
try {
// test if the local path exists on OneDrive
fileDetailsFromOneDrive = onedrive.getPathDetails(path);
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
// The file was not found on OneDrive, need to upload it
write("Uploading file ", path, "...");
JSONValue response;
if (getSize(path) <= thresholdFileSize) {
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
writeln(" done.");
} else {
writeln("");
response = session.upload(path, parent.driveId, parent.id, baseName(path));
}
log.vlog("Uploading file ", path, "... done.");
string id = response["id"].str;
string cTag = response["cTag"].str;
SysTime mtime = timeLastModified(path).toUTC();
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
return;
}
}
log.vlog("Requested file to upload exists on OneDrive - local database is out of sync for this file: ", path);
// Is the local file newer than the uploaded file?
SysTime localFileModifiedTime = timeLastModified(path).toUTC();
SysTime remoteFileModifiedTime = SysTime.fromISOExtString(fileDetailsFromOneDrive["fileSystemInfo"]["lastModifiedDateTime"].str);
if (localFileModifiedTime > remoteFileModifiedTime){
// local file is newer
log.vlog("Requested file to upload is newer than existing file on OneDrive");
write("Uploading file ", path, "...");
JSONValue response;
if (getSize(path) <= thresholdFileSize) {
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
writeln(" done.");
} else {
writeln("");
response = session.upload(path, parent.driveId, parent.id, baseName(path));
}
log.vlog("Uploading file ", path, "... done.");
string id = response["id"].str;
string cTag = response["cTag"].str;
SysTime mtime = timeLastModified(path).toUTC();
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
} else {
// Save the details of the file that we got from OneDrive
log.vlog("Updating the local database with details for this file: ", path);
saveItem(fileDetailsFromOneDrive);
}
string id = response["id"].str;
string cTag = response["cTag"].str;
SysTime mtime = timeLastModified(path).toUTC();
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
}
private void uploadDeleteItem(Item item, const(char)[] path)
private void uploadDeleteItem(Item item, string path)
{
log.log("Deleting ", path);
log.log("Deleting directory from OneDrive: ", path);
if ((item.driveId == "") && (item.id == "") && (item.eTag == "")){
// These are empty ... we cannot delete if this is empty ....
JSONValue onedrivePathDetails = onedrive.getPathDetails(path); // Returns a JSON String for the OneDrive Path
//log.vlog("WHOOPS JSON Response: ", onedrivePathDetails);
item.driveId = onedrivePathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1
item.id = onedrivePathDetails["id"].str; // This item's ID. Should give something like 12345ABCDE1234A1!101
item.eTag = onedrivePathDetails["eTag"].str; // Should be something like aNjM2NjJFRUVGQjY2NjJFMSE5MzUuMA
//log.vlog("item.driveId = ", item.driveId);
//log.vlog("item.id = ", item.id);
//log.vlog("item.eTag = ", item.eTag);
}
try {
onedrive.deleteById(item.driveId, item.id, item.eTag);
} catch (OneDriveException e) {
@ -655,7 +953,10 @@ final class SyncEngine
private void saveItem(JSONValue jsonItem)
{
// Takes a JSON input and formats to an item which can be used by the database
Item item = makeItem(jsonItem);
// Add to the local database
itemdb.upsert(item);
}
@ -699,11 +1000,11 @@ final class SyncEngine
}
}
void deleteByPath(const(char)[] path)
void deleteByPath(string path)
{
Item item;
if (!itemdb.selectByPath(path, defaultDriveId, item)) {
throw new SyncException("Can't delete an unsynced item");
throw new SyncException("The item to delete is not in the local database");
}
if (item.parentId == null) {
// the item is a remote folder, need to do the operation on the parent
@ -716,4 +1017,31 @@ final class SyncEngine
else throw e;
}
}
// move a OneDrive folder from one name to another
void moveByPath(const(string) source, const(string) destination)
{
log.vlog("Moving remote folder: ", source, " -> ", destination);
// Source and Destination are relative to ~/OneDrive
string sourcePath = source;
string destinationBasePath = dirName(destination).idup;
// if destinationBasePath == '.' then destinationBasePath needs to be ""
if (destinationBasePath == ".") {
destinationBasePath = "";
}
string newFolderName = baseName(destination).idup;
string destinationPathString = "/drive/root:/" ~ destinationBasePath;
// Build up the JSON changes
JSONValue moveData = ["name": newFolderName];
JSONValue destinationPath = ["path": destinationPathString];
moveData["parentReference"] = destinationPath;
// Make the change on OneDrive
auto res = onedrive.moveByPath(sourcePath, moveData);
}
}