From f5fc13cbadbd2092cb09f5647a636c9c603eda32 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 18 Sep 2016 11:37:41 +0200 Subject: [PATCH 01/31] getValue() with default return value --- src/config.d | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/config.d b/src/config.d index 797c372a..23bcdb12 100644 --- a/src/config.d +++ b/src/config.d @@ -40,6 +40,16 @@ final class Config } } + string getValue(string key, string value) + { + auto p = key in values; + if (p) { + return *p; + } else { + return value; + } + } + private bool load(string filename) { scope(failure) return false; @@ -65,4 +75,5 @@ unittest auto cfg = new Config(""); cfg.load("onedrive.conf"); assert(cfg.getValue("sync_dir") == "~/OneDrive"); + assert(cfg.getValue("empty", "default") == "default"); } From 022ba09e4127d847d86aa5d91a36874428f61485 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 18 Sep 2016 11:50:10 +0200 Subject: [PATCH 02/31] added multiGlobMatch() --- src/util.d | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/util.d b/src/util.d index af51d0a2..6db7b0b7 100644 --- a/src/util.d +++ b/src/util.d @@ -89,3 +89,21 @@ bool testNetwork() return false; } } + +// call globMatch for each string in pattern separated by '|' +bool multiGlobMatch(const(char)[] path, const(char)[] pattern) +{ + foreach (glob; pattern.split('|')) { + if (globMatch!(std.path.CaseSensitive.yes)(path, glob)) { + return true; + } + } + return false; +} + +unittest +{ + assert(multiGlobMatch(".hidden", ".*")); + assert(multiGlobMatch(".hidden", "file|.*")); + assert(!multiGlobMatch("foo.bar", "foo|bar")); +} From 56af5796994d6285dda164542c6f0d10f493e5d5 Mon Sep 17 00:00:00 2001 From: hgdsraj Date: Sun, 4 Dec 2016 12:54:43 -0800 Subject: [PATCH 03/31] require => requires --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 70338d56..31e228b7 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Use `|` to separate multiple patterns. [3]: https://technet.microsoft.com/en-us/library/bb490639.aspx ### First run -The first time you run the program you will be asked to sign in. The procedure require a web browser. +The first time you run the program you will be asked to sign in. The procedure requires a web browser. ### Service If you want to sync your files automatically, enable and start the systemd service: From 947136cf629f175c76bc976ad5e958d9a99bf78c Mon Sep 17 00:00:00 2001 From: skilion Date: Tue, 13 Dec 2016 18:04:16 +0100 Subject: [PATCH 04/31] show onedrive error response --- src/onedrive.d | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/onedrive.d b/src/onedrive.d index 6e57294c..84b90107 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -17,6 +17,8 @@ class OneDriveException: Exception { // HTTP status code int code; + // error details + JSONValue error; @nogc @safe pure nothrow this(string msg, Throwable next, string file = __FILE__, size_t line = __LINE__) { @@ -26,9 +28,18 @@ class OneDriveException: Exception @safe pure this(int code, string reason, string file = __FILE__, size_t line = __LINE__) { this.code = code; + this.error = error; string msg = format("HTTP request returned status code %d (%s)", code, reason); super(msg, file, line, next); } + + this(int code, string reason, ref const JSONValue error, string file = __FILE__, size_t line = __LINE__) + { + this.code = code; + this.error = error; + string msg = format("HTTP request returned status code %d (%s)\n%s", code, reason, toJSON(&error, true)); + super(msg, file, line, next); + } } final class OneDriveApi @@ -234,7 +245,7 @@ final class OneDriveApi http.url = url; addAccessTokenHeader(); auto response = perform(); - checkHttpCode(); + checkHttpCode(response); return response; } @@ -244,8 +255,8 @@ final class OneDriveApi http.method = HTTP.Method.del; http.url = url; addAccessTokenHeader(); - perform(); - checkHttpCode(); + auto response = perform(); + checkHttpCode(response); } private void download(const(char)[] url, string filename) @@ -270,7 +281,7 @@ final class OneDriveApi http.url = url; addAccessTokenHeader(); auto response = perform(patchData); - checkHttpCode(); + checkHttpCode(response); return response; } @@ -281,7 +292,7 @@ final class OneDriveApi http.url = url; addAccessTokenHeader(); auto response = perform(postData); - checkHttpCode(); + checkHttpCode(response); return response; } @@ -300,7 +311,7 @@ final class OneDriveApi http.onSend = data => file.rawRead(data).length; http.contentLength = file.size; auto response = perform(); - checkHttpCode(); + checkHttpCode(response); return response; } @@ -348,4 +359,11 @@ final class OneDriveApi throw new OneDriveException(http.statusLine.code, http.statusLine.reason); } } + + private void checkHttpCode(ref const JSONValue response) + { + if (http.statusLine.code / 100 != 2) { + throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response); + } + } } From 7257c4c9bfb07efe4e333f562868e2ad80b739d3 Mon Sep 17 00:00:00 2001 From: skilion Date: Wed, 14 Dec 2016 15:17:20 +0100 Subject: [PATCH 05/31] do not throw on expired status token --- src/onedrive.d | 15 +++++++-------- src/sync.d | 18 +++++++++++++++--- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/onedrive.d b/src/onedrive.d index 84b90107..d07abd1b 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -15,8 +15,7 @@ private immutable { class OneDriveException: Exception { - // HTTP status code - int code; + int httpStatusCode; // error details JSONValue error; @@ -25,19 +24,19 @@ class OneDriveException: Exception super(msg, file, line, next); } - @safe pure this(int code, string reason, string file = __FILE__, size_t line = __LINE__) + @safe pure this(int httpStatusCode, string reason, string file = __FILE__, size_t line = __LINE__) { - this.code = code; + this.httpStatusCode = httpStatusCode; this.error = error; - string msg = format("HTTP request returned status code %d (%s)", code, reason); + string msg = format("HTTP request returned status code %d (%s)", httpStatusCode, reason); super(msg, file, line, next); } - this(int code, string reason, ref const JSONValue error, string file = __FILE__, size_t line = __LINE__) + this(int httpStatusCode, string reason, ref const JSONValue error, string file = __FILE__, size_t line = __LINE__) { - this.code = code; + this.httpStatusCode = httpStatusCode; this.error = error; - string msg = format("HTTP request returned status code %d (%s)\n%s", code, reason, toJSON(&error, true)); + string msg = format("HTTP request returned status code %d (%s)\n%s", httpStatusCode, reason, toJSON(&error, true)); super(msg, file, line, next); } } diff --git a/src/sync.d b/src/sync.d index 1789e777..994817c6 100644 --- a/src/sync.d +++ b/src/sync.d @@ -91,7 +91,19 @@ final class SyncEngine try { JSONValue changes; do { - changes = onedrive.viewChangesByPath("/", statusToken); + // get changes from the server + try { + changes = onedrive.viewChangesByPath("/", statusToken); + } catch (OneDriveException e) { + if (e.httpStatusCode == 410) { + log.log("Status token expired, resyncing"); + statusToken = null; + continue; + } + else { + throw e; + } + } foreach (item; changes["value"].array) { applyDifference(item); } @@ -497,7 +509,7 @@ final class SyncEngine try { onedrive.deleteById(item.id, item.eTag); } catch (OneDriveException e) { - if (e.code == 404) log.log(e.msg); + if (e.httpStatusCode == 404) log.log(e.msg); else throw e; } itemdb.deleteById(item.id); @@ -575,7 +587,7 @@ final class SyncEngine try { uploadDeleteItem(item, path); } catch (OneDriveException e) { - if (e.code == 404) log.log(e.msg); + if (e.httpStatusCode == 404) log.log(e.msg); else throw e; } } From e33f566ecf19c68e5d5b70775aaa17f9fefe253e Mon Sep 17 00:00:00 2001 From: skilion Date: Wed, 14 Dec 2016 19:51:23 +0100 Subject: [PATCH 06/31] bug fix --- src/sync.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sync.d b/src/sync.d index 994817c6..5d76fbf2 100644 --- a/src/sync.d +++ b/src/sync.d @@ -109,7 +109,7 @@ final class SyncEngine } statusToken = changes["@delta.token"].str; std.file.write(cfg.statusTokenFilePath, statusToken); - } while (("@odata.nextLink" in changes.object) !is null); + } while ((changes.type != JSON_TYPE.OBJECT) && (("@odata.nextLink" in changes) !is null)); } catch (ErrnoException e) { throw new SyncException(e.msg, e); } catch (FileException e) { From 9ad9394b9827ce3e2bd7ab290451b7959cfbad50 Mon Sep 17 00:00:00 2001 From: skilion Date: Sat, 24 Dec 2016 14:12:20 +0100 Subject: [PATCH 07/31] better handle the case when cTag is null --- src/itemdb.d | 112 +++++++++++++++++++++---------------------------- src/onedrive.d | 4 +- src/sync.d | 46 ++++++++------------ 3 files changed, 67 insertions(+), 95 deletions(-) diff --git a/src/itemdb.d b/src/itemdb.d index 1510a6d2..8db47ee2 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -1,4 +1,4 @@ -import std.datetime, std.path, std.string; +import std.datetime, std.path, std.exception, std.string; import sqlite; enum ItemType @@ -21,6 +21,9 @@ struct Item final class ItemDatabase { + // increment this for every change in the db schema + immutable int itemDatabaseVersion = 1; + Database db; Statement insertItemStmt; Statement updateItemStmt; @@ -35,7 +38,7 @@ final class ItemDatabase name TEXT NOT NULL, type TEXT NOT NULL, eTag TEXT NOT NULL, - cTag TEXT NOT NULL, + cTag TEXT, mtime TEXT NOT NULL, parentId TEXT, crc32 TEXT, @@ -44,6 +47,8 @@ final class ItemDatabase db.exec("CREATE INDEX IF NOT EXISTS name_idx ON item (name)"); db.exec("PRAGMA foreign_keys = ON"); db.exec("PRAGMA recursive_triggers = ON"); + db.setVersion(itemDatabaseVersion); + insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); updateItemStmt = db.prepare(" UPDATE item @@ -54,70 +59,28 @@ final class ItemDatabase selectItemByParentIdStmt = db.prepare("SELECT id FROM item WHERE parentId = ?"); } - void insert(const(char)[] id, const(char)[] name, ItemType type, const(char)[] eTag, const(char)[] cTag, const(char)[] mtime, const(char)[] parentId, const(char)[] crc32) + void insert(const ref Item item) { - with (insertItemStmt) { - bind(1, id); - bind(2, name); - string typeStr = void; - final switch (type) { - case ItemType.file: typeStr = "file"; break; - case ItemType.dir: typeStr = "dir"; break; - } - bind(3, typeStr); - bind(4, eTag); - bind(5, cTag); - bind(6, mtime); - bind(7, parentId); - bind(8, crc32); - exec(); - } + bindItem(item, insertItemStmt); + insertItemStmt.exec(); } - void update(const(char)[] id, const(char)[] name, ItemType type, const(char)[] eTag, const(char)[] cTag, const(char)[] mtime, const(char)[] parentId, const(char)[] crc32) + void update(const ref Item item) { - with (updateItemStmt) { - bind(1, id); - bind(2, name); - string typeStr = void; - final switch (type) { - case ItemType.file: typeStr = "file"; break; - case ItemType.dir: typeStr = "dir"; break; - } - bind(3, typeStr); - bind(4, eTag); - bind(5, cTag); - bind(6, mtime); - bind(7, parentId); - bind(8, crc32); - exec(); - } + bindItem(item, updateItemStmt); + updateItemStmt.exec(); } - void upsert(const(char)[] id, const(char)[] name, ItemType type, const(char)[] eTag, const(char)[] cTag, const(char)[] mtime, const(char)[] parentId, const(char)[] crc32) + void upsert(const ref Item item) { auto s = db.prepare("SELECT COUNT(*) FROM item WHERE id = ?"); - s.bind(1, id); + s.bind(1, item.id); auto r = s.exec(); - Statement* p; - if (r.front[0] == "0") p = &insertItemStmt; - else p = &updateItemStmt; - with (p) { - bind(1, id); - bind(2, name); - string typeStr = void; - final switch (type) { - case ItemType.file: typeStr = "file"; break; - case ItemType.dir: typeStr = "dir"; break; - } - bind(3, typeStr); - bind(4, eTag); - bind(5, cTag); - bind(6, mtime); - bind(7, parentId); - bind(8, crc32); - exec(); - } + Statement* stmt; + if (r.front[0] == "0") stmt = &insertItemStmt; + else stmt = &updateItemStmt; + bindItem(item, *stmt); + stmt.exec(); } Item[] selectChildren(const(char)[] id) @@ -218,6 +181,25 @@ final class ItemDatabase return false; } + private void bindItem(const ref Item item, ref Statement stmt) + { + with (stmt) with (item) { + bind(1, id); + bind(2, name); + string typeStr = null; + final switch (type) with (ItemType) { + case file: typeStr = "file"; break; + case dir: typeStr = "dir"; break; + } + bind(3, typeStr); + bind(4, eTag); + bind(5, cTag); + bind(6, mtime.toISOExtString()); + bind(7, parentId); + bind(8, crc32); + } + } + private Item buildItem(Statement.Result result) { assert(!result.empty && result.front.length == 8); @@ -231,25 +213,25 @@ final class ItemDatabase crc32: result.front[7].dup }; switch (result.front[2]) { - case "file": item.type = ItemType.file; break; - case "dir": item.type = ItemType.dir; break; - default: assert(0); + case "file": item.type = ItemType.file; break; + case "dir": item.type = ItemType.dir; break; + default: assert(0); } return item; } + // computes the path of the given item id + // the path is relative to the sync directory ex: "./Music/Turbo Killer.mp3" + // a trailing slash is never added string computePath(const(char)[] id) { - if (!id) return null; string path; auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?"); while (true) { s.bind(1, id); auto r = s.exec(); - if (r.empty) { - // no results - break; - } else if (r.front[1]) { + enforce(!r.empty, "Unknow item id"); + if (r.front[1]) { if (path) path = r.front[0].idup ~ "/" ~ path; else path = r.front[0].idup; } else { diff --git a/src/onedrive.d b/src/onedrive.d index d07abd1b..8ae6ccae 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -16,7 +16,7 @@ private immutable { class OneDriveException: Exception { int httpStatusCode; - // error details + // https://dev.onedrive.com/misc/errors.htm JSONValue error; @nogc @safe pure nothrow this(string msg, Throwable next, string file = __FILE__, size_t line = __LINE__) @@ -36,7 +36,7 @@ class OneDriveException: Exception { this.httpStatusCode = httpStatusCode; this.error = error; - string msg = format("HTTP request returned status code %d (%s)\n%s", httpStatusCode, reason, toJSON(&error, true)); + string msg = format("HTTP request returned status code %d (%s)\n%s", httpStatusCode, reason, toJSON(error, true)); super(msg, file, line, next); } } diff --git a/src/sync.d b/src/sync.d index 5d76fbf2..cb58566b 100644 --- a/src/sync.d +++ b/src/sync.d @@ -190,17 +190,6 @@ final class SyncEngine return; } - string cTag; - try { - cTag = item["cTag"].str; - } catch (JSONException e) { - // cTag is not returned if the Item is a folder - // https://dev.onedrive.com/resources/item.htm - cTag = ""; - } - - string mtime = item["fileSystemInfo"]["lastModifiedDateTime"].str; - string crc32; if (type == ItemType.file) { try { @@ -215,8 +204,8 @@ final class SyncEngine name: name, type: type, eTag: eTag, - cTag: cTag, - mtime: SysTime.fromISOExtString(mtime), + cTag: "cTag" in item ? item["cTag"].str : null, + mtime: SysTime.fromISOExtString(item["fileSystemInfo"]["lastModifiedDateTime"].str), parentId: parentId, crc32: crc32 }; @@ -229,9 +218,9 @@ final class SyncEngine // save the item in the db if (oldItem.id) { - itemdb.update(id, name, type, eTag, cTag, mtime, parentId, crc32); + itemdb.update(newItem); } else { - itemdb.insert(id, name, type, eTag, cTag, mtime, parentId, crc32); + itemdb.insert(newItem); } } @@ -526,31 +515,32 @@ final class SyncEngine saveItem(res); } - private void saveItem(JSONValue item) + private void saveItem(JSONValue jsonItem) { - string id = item["id"].str; + string id = jsonItem["id"].str; ItemType type; - if (isItemFile(item)) { + if (isItemFile(jsonItem)) { type = ItemType.file; - } else if (isItemFolder(item)) { + } else if (isItemFolder(jsonItem)) { type = ItemType.dir; } else { assert(0); } - string name = item["name"].str; - string eTag = item["eTag"].str; - string cTag = item["cTag"].str; - string mtime = item["fileSystemInfo"]["lastModifiedDateTime"].str; - string parentId = item["parentReference"]["id"].str; - string crc32; + Item item = { + name: jsonItem["name"].str, + eTag: jsonItem["eTag"].str, + cTag: "cTag" in jsonItem ? jsonItem["cTag"].str : null, + mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str), + parentId: jsonItem["parentReference"]["id"].str + }; if (type == ItemType.file) { try { - crc32 = item["file"]["hashes"]["crc32Hash"].str; + item.crc32 = jsonItem["file"]["hashes"]["crc32Hash"].str; } catch (JSONException e) { - // swallow exception + log.vlog("The hash is not available"); } } - itemdb.upsert(id, name, type, eTag, cTag, mtime, parentId, crc32); + itemdb.upsert(item); } void uploadMoveItem(string from, string to) From 82b2b63c759f8c38c9fca747525d5fc99f406c68 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 25 Dec 2016 00:15:33 +0100 Subject: [PATCH 08/31] added functions to get and set the db version --- src/sqlite.d | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/sqlite.d b/src/sqlite.d index 7df5c82d..e42b7dcd 100644 --- a/src/sqlite.d +++ b/src/sqlite.d @@ -64,6 +64,27 @@ struct Database } } + int getVersion() + { + int userVersion; + extern (C) int callback(void* user_version, int count, char** column_text, char** column_name) { + import std.c.stdlib: atoi; + *(cast(int*) user_version) = atoi(*column_text); + return 0; + } + int rc = sqlite3_exec(pDb, "PRAGMA user_version", &callback, &userVersion, null); + if (rc != SQLITE_OK) { + throw new SqliteException(ifromStringz(sqlite3_errmsg(pDb))); + } + return userVersion; + } + + void setVersion(int userVersion) + { + import std.conv: to; + exec("PRAGMA user_version=" ~ to!string(userVersion)); + } + Statement prepare(const(char)[] zSql) { Statement s; @@ -170,6 +191,10 @@ unittest value TEXT )"); + assert(db.getVersion() == 0); + db.setVersion(1); + assert(db.getVersion() == 1); + auto s = db.prepare("INSERT INTO test VALUES (?, ?)"); s.bind(1, "key1"); s.bind(2, "value1"); From ad4e910e5584f584e271e5ab6a52c9daacedb178 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 25 Dec 2016 16:38:55 +0100 Subject: [PATCH 09/31] check if the token is valid on init --- src/onedrive.d | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/onedrive.d b/src/onedrive.d index 8ae6ccae..9846bee3 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -9,6 +9,7 @@ private immutable { string authUrl = "https://login.live.com/oauth20_authorize.srf"; string redirectUrl = "https://login.live.com/oauth20_desktop.srf"; // "urn:ietf:wg:oauth:2.0:oob"; string tokenUrl = "https://login.live.com/oauth20_token.srf"; + string driveUrl = "https://api.onedrive.com/v1.0/drive"; string itemByIdUrl = "https://api.onedrive.com/v1.0/drive/items/"; string itemByPathUrl = "https://api.onedrive.com/v1.0/drive/root:/"; } @@ -61,8 +62,15 @@ final class OneDriveApi { try { refreshToken = readText(cfg.refreshTokenFilePath); + getDefaultDrive(); } catch (FileException e) { return authorize(); + } catch (OneDriveException e) { + if (e.httpStatusCode == 400 || e.httpStatusCode == 401) { + log.log("Refresh token invalid"); + return authorize(); + } + throw e; } return true; } @@ -86,6 +94,13 @@ final class OneDriveApi return true; } + // https://dev.onedrive.com/drives/default.htm + JSONValue getDefaultDrive() + { + checkAccessTokenExpired(); + return get(driveUrl); + } + // https://dev.onedrive.com/items/view_delta.htm JSONValue viewChangesById(const(char)[] id, const(char)[] statusToken) { From c9016845c339104e54df95efdca617afa9786ee7 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 25 Dec 2016 16:39:54 +0100 Subject: [PATCH 10/31] removed time comparison hack --- src/sync.d | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/sync.d b/src/sync.d index cb58566b..ac27fbf2 100644 --- a/src/sync.d +++ b/src/sync.d @@ -283,8 +283,6 @@ final class SyncEngine case ItemType.file: if (isFile(path)) { SysTime localModifiedTime = timeLastModified(path); - import core.time: Duration; - item.mtime.fracSecs = Duration.zero; // HACK if (localModifiedTime == item.mtime) { return true; } else { @@ -401,8 +399,6 @@ final class SyncEngine if (exists(path)) { if (isFile(path)) { SysTime localModifiedTime = timeLastModified(path); - import core.time: Duration; - item.mtime.fracSecs = Duration.zero; // HACK if (localModifiedTime != item.mtime) { log.vlog("The file last modified time has changed"); string id = item.id; From 6913c87d5c88164b91738b50907af8cd2cccc504 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 25 Dec 2016 19:23:33 +0100 Subject: [PATCH 11/31] schema upgrade --- src/config.d | 2 +- src/itemdb.d | 30 +++++++++++++++++------------- src/main.d | 8 ++++++++ 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/config.d b/src/config.d index 23bcdb12..66ed19a1 100644 --- a/src/config.d +++ b/src/config.d @@ -16,7 +16,7 @@ final class Config { refreshTokenFilePath = configDirName ~ "/refresh_token"; statusTokenFilePath = configDirName ~ "/status_token"; - databaseFilePath = configDirName ~ "/items.db"; + databaseFilePath = configDirName ~ "/items.sqlite3"; uploadStateFilePath = configDirName ~ "/resume_upload"; userConfigFilePath = configDirName ~ "/config"; } diff --git a/src/itemdb.d b/src/itemdb.d index 8db47ee2..9b52eee0 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -33,21 +33,25 @@ final class ItemDatabase this(const(char)[] filename) { db = Database(filename); - db.exec("CREATE TABLE IF NOT EXISTS item ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - type TEXT NOT NULL, - eTag TEXT NOT NULL, - cTag TEXT, - mtime TEXT NOT NULL, - parentId TEXT, - crc32 TEXT, - FOREIGN KEY (parentId) REFERENCES item (id) ON DELETE CASCADE - )"); - db.exec("CREATE INDEX IF NOT EXISTS name_idx ON item (name)"); + if (db.getVersion() == 0) { + db.exec("CREATE TABLE item ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + type TEXT NOT NULL, + eTag TEXT NOT NULL, + cTag TEXT, + mtime TEXT NOT NULL, + parentId TEXT, + crc32 TEXT, + FOREIGN KEY (parentId) REFERENCES item (id) ON DELETE CASCADE + )"); + db.exec("CREATE INDEX name_idx ON item (name)"); + db.setVersion(itemDatabaseVersion); + } else if (db.getVersion() != itemDatabaseVersion) { + throw new Exception("The item database is incompatible, please resync manually"); + } db.exec("PRAGMA foreign_keys = ON"); db.exec("PRAGMA recursive_triggers = ON"); - db.setVersion(itemDatabaseVersion); insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); updateItemStmt = db.prepare(" diff --git a/src/main.d b/src/main.d index 241a096f..4e657081 100644 --- a/src/main.d +++ b/src/main.d @@ -46,6 +46,14 @@ int main(string[] args) if (!exists(configDirName)) mkdir(configDirName); auto cfg = new config.Config(configDirName); cfg.init(); + + // upgrades + if (exists(configDirName ~ "/items.db")) { + remove(configDirName ~ "/items.db"); + log.log("Database schema changed, resync needed"); + resync = true; + } + if (resync || logout) { log.log("Deleting the saved status ..."); safeRemove(cfg.databaseFilePath); From bbbfeb68f3a1b2e841636c6dd933b7a89c16bff1 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 25 Dec 2016 20:17:02 +0100 Subject: [PATCH 12/31] fix condition in applyDifferences loop --- src/sync.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sync.d b/src/sync.d index ac27fbf2..f07554dd 100644 --- a/src/sync.d +++ b/src/sync.d @@ -109,7 +109,7 @@ final class SyncEngine } statusToken = changes["@delta.token"].str; std.file.write(cfg.statusTokenFilePath, statusToken); - } while ((changes.type != JSON_TYPE.OBJECT) && (("@odata.nextLink" in changes) !is null)); + } while (!((changes.type == JSON_TYPE.OBJECT) && (("@odata.nextLink" in changes) is null))); } catch (ErrnoException e) { throw new SyncException(e.msg, e); } catch (FileException e) { From 91c5ac6f3127dcc74f560eef880fd1188fb5e6d5 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 25 Dec 2016 20:53:46 +0100 Subject: [PATCH 13/31] add git clone in install instructions --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 31e228b7..d208fc15 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,12 @@ sudo apt-get update && sudo apt-get install dmd-bin ``` ### Installation -1. `make` -2. `sudo make install` +``` +git clone git@github.com:skilion/onedrive.git +cd onedrive +make +sudo make install +``` ### Configuration: You should copy the default config file into your home directory before making changes: From ff58a86b7fc3d4970bf9bf5b6926a411e22f5caa Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 25 Dec 2016 22:28:00 +0100 Subject: [PATCH 14/31] use libcurl to check connection --- src/util.d | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/util.d b/src/util.d index 6db7b0b7..b9515905 100644 --- a/src/util.d +++ b/src/util.d @@ -1,6 +1,7 @@ import std.conv; import std.digest.crc; import std.file; +import std.net.curl; import std.path; import std.regex; import std.socket; @@ -81,13 +82,9 @@ Regex!char wild2regex(const(char)[] pattern) // return true if the network connection is available bool testNetwork() { - try { - auto addr = new InternetAddress("login.live.com", 443); - auto socket = new TcpSocket(addr); - return socket.isAlive(); - } catch (SocketException) { - return false; - } + HTTP http = HTTP("https://login.live.com"); + http.method = HTTP.Method.head; + return http.perform(ThrowOnError.no) == 0; } // call globMatch for each string in pattern separated by '|' From 3a10f0f4b587af1de29c3d3124788caca842fde7 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 25 Dec 2016 23:25:24 +0100 Subject: [PATCH 15/31] fixed bug that allowed files marked for deletion to be keeped --- src/sync.d | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/sync.d b/src/sync.d index f07554dd..6dd0e284 100644 --- a/src/sync.d +++ b/src/sync.d @@ -56,7 +56,7 @@ final class SyncEngine // list of items to skip while applying the changes private string[] skippedItems; // list of items to delete after the changes has been downloaded - private string[] pathsToDelete; + private string[] idsToDelete; this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb) { @@ -117,8 +117,8 @@ final class SyncEngine } catch (OneDriveException e) { throw new SyncException(e.msg, e); } - // delete items in pathsToDelete - if (pathsToDelete.length > 0) deleteItems(); + // delete items in idsToDelete + if (idsToDelete.length > 0) deleteItems(); // empty the skipped items skippedItems.length = 0; assumeSafeAppend(skippedItems); @@ -166,10 +166,7 @@ final class SyncEngine ItemType type; if (isItemDeleted(item)) { log.vlog("The item is marked for deletion"); - if (cached) { - itemdb.deleteById(id); - pathsToDelete ~= oldPath; - } + if (cached) idsToDelete ~= id; return; } else if (isItemFile(item)) { type = ItemType.file; @@ -311,7 +308,9 @@ final class SyncEngine private void deleteItems() { log.vlog("Deleting files ..."); - foreach_reverse (path; pathsToDelete) { + foreach_reverse (id; idsToDelete) { + string path = itemdb.computePath(id); + itemdb.deleteById(id); if (exists(path)) { if (isFile(path)) { remove(path); @@ -326,8 +325,8 @@ final class SyncEngine } } } - pathsToDelete.length = 0; - assumeSafeAppend(pathsToDelete); + idsToDelete.length = 0; + assumeSafeAppend(idsToDelete); } // scan the given directory for differences From 6a7c30e34f582bb538253bcff570efc2fc904012 Mon Sep 17 00:00:00 2001 From: skilion Date: Wed, 28 Dec 2016 16:29:55 +0100 Subject: [PATCH 16/31] added missing checkAccessTokenExpired() in createByPath --- src/onedrive.d | 1 + 1 file changed, 1 insertion(+) diff --git a/src/onedrive.d b/src/onedrive.d index 9846bee3..56b6377e 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -166,6 +166,7 @@ final class OneDriveApi // https://dev.onedrive.com/items/create.htm JSONValue createByPath(const(char)[] parentPath, JSONValue item) { + checkAccessTokenExpired(); string url = itemByPathUrl ~ encodeComponent(parentPath) ~ ":/children"; http.addRequestHeader("Content-Type", "application/json"); return post(url, item.toString()); From 0afb6fa3923066f46c531f33e175011cd69e06a8 Mon Sep 17 00:00:00 2001 From: skilion Date: Wed, 28 Dec 2016 23:47:29 +0100 Subject: [PATCH 17/31] workaround for error 412 (Precondition Failed) on item delete --- src/onedrive.d | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/onedrive.d b/src/onedrive.d index 56b6377e..d75c1c2f 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -159,7 +159,8 @@ final class OneDriveApi { checkAccessTokenExpired(); char[] url = itemByIdUrl ~ id; - if (eTag) http.addRequestHeader("If-Match", eTag); + //TODO: investigate why this always fail with 412 (Precondition Failed) + //if (eTag) http.addRequestHeader("If-Match", eTag); del(url); } From 203062fc6b1416de4d9230b9401556641e2b0a24 Mon Sep 17 00:00:00 2001 From: skilion Date: Thu, 29 Dec 2016 00:36:42 +0100 Subject: [PATCH 18/31] added missing item fields in saveItem() --- src/itemdb.d | 4 ++-- src/sync.d | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/itemdb.d b/src/itemdb.d index 9b52eee0..4f5cd7f5 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -22,7 +22,7 @@ struct Item final class ItemDatabase { // increment this for every change in the db schema - immutable int itemDatabaseVersion = 1; + immutable int itemDatabaseVersion = 2; Database db; Statement insertItemStmt; @@ -35,7 +35,7 @@ final class ItemDatabase db = Database(filename); if (db.getVersion() == 0) { db.exec("CREATE TABLE item ( - id TEXT PRIMARY KEY, + id TEXT NOT NULL PRIMARY KEY, name TEXT NOT NULL, type TEXT NOT NULL, eTag TEXT NOT NULL, diff --git a/src/sync.d b/src/sync.d index 6dd0e284..b5f73b07 100644 --- a/src/sync.d +++ b/src/sync.d @@ -522,7 +522,9 @@ final class SyncEngine assert(0); } Item item = { + id: id, name: jsonItem["name"].str, + type: type, eTag: jsonItem["eTag"].str, cTag: "cTag" in jsonItem ? jsonItem["cTag"].str : null, mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str), From 1beadf25779a4395e993636486a01793d1afcac3 Mon Sep 17 00:00:00 2001 From: skilion Date: Sat, 11 Mar 2017 11:40:19 +0100 Subject: [PATCH 19/31] switch to Microsoft Graph API --- onedrive.conf | 2 +- src/onedrive.d | 79 +++++++++++++++++++++++++++++++++++++++----------- src/sync.d | 23 +++++++++------ 3 files changed, 77 insertions(+), 27 deletions(-) diff --git a/onedrive.conf b/onedrive.conf index ae7f2282..8b73f21b 100644 --- a/onedrive.conf +++ b/onedrive.conf @@ -1,4 +1,4 @@ -client_id = "000000004C15842F" +client_id = "22c49a0d-d21c-4792-aed1-8f163c982546" sync_dir = "~/OneDrive" skip_file = ".*|~*" skip_dir = ".*" diff --git a/src/onedrive.d b/src/onedrive.d index d75c1c2f..9daf2dff 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -6,12 +6,12 @@ static import log; private immutable { - string authUrl = "https://login.live.com/oauth20_authorize.srf"; - string redirectUrl = "https://login.live.com/oauth20_desktop.srf"; // "urn:ietf:wg:oauth:2.0:oob"; - string tokenUrl = "https://login.live.com/oauth20_token.srf"; - string driveUrl = "https://api.onedrive.com/v1.0/drive"; - string itemByIdUrl = "https://api.onedrive.com/v1.0/drive/items/"; - string itemByPathUrl = "https://api.onedrive.com/v1.0/drive/root:/"; + string authUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize"; + string redirectUrl = "https://login.microsoftonline.com/common/oauth2/nativeclient"; + string tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token"; + string driveUrl = "https://graph.microsoft.com/v1.0/me/drive"; + string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/"; + string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; } class OneDriveException: Exception @@ -79,7 +79,7 @@ final class OneDriveApi { import std.stdio, std.regex; char[] response; - string url = authUrl ~ "?client_id=" ~ clientId ~ "&scope=onedrive.readwrite%20offline_access&response_type=code&redirect_uri=" ~ redirectUrl; + string url = authUrl ~ "?client_id=" ~ clientId ~ "&scope=files.readwrite.all%20offline_access&response_type=code&redirect_uri=" ~ redirectUrl; log.log("Authorize this app visiting:\n"); write(url, "\n\n", "Enter the response uri: "); readln(response); @@ -105,9 +105,9 @@ final class OneDriveApi JSONValue viewChangesById(const(char)[] id, const(char)[] statusToken) { checkAccessTokenExpired(); - const(char)[] url = itemByIdUrl ~ id ~ "/view.delta"; + const(char)[] url = itemByIdUrl ~ id ~ "/delta"; url ~= "?select=id,name,eTag,cTag,deleted,file,folder,fileSystemInfo,remoteItem,parentReference"; - if (statusToken) url ~= "?token=" ~ statusToken; + if (statusToken) url ~= "&token=" ~ statusToken; return get(url); } @@ -115,7 +115,9 @@ final class OneDriveApi JSONValue viewChangesByPath(const(char)[] path, const(char)[] statusToken) { checkAccessTokenExpired(); - string url = itemByPathUrl ~ encodeComponent(path) ~ ":/view.delta"; + //string url = itemByPathUrl ~ encodeComponent(path) ~ ":/delta"; + // HACK: item by path seems to no be working + string url = "https://graph.microsoft.com/v1.0/me/drive/root/delta"; url ~= "?select=id,name,eTag,cTag,deleted,file,folder,fileSystemInfo,remoteItem,parentReference"; if (statusToken) url ~= "&token=" ~ statusToken; return get(url); @@ -138,7 +140,6 @@ final class OneDriveApi { checkAccessTokenExpired(); string url = itemByPathUrl ~ encodeComponent(remotePath) ~ ":/content"; - http.addRequestHeader("Content-Type", "application/octet-stream"); if (eTag) http.addRequestHeader("If-Match", eTag); else url ~= "?@name.conflictBehavior=fail"; return upload(localPath, url); @@ -177,7 +178,7 @@ final class OneDriveApi JSONValue createUploadSession(const(char)[] path, const(char)[] eTag = null) { checkAccessTokenExpired(); - string url = itemByPathUrl ~ encodeComponent(path) ~ ":/upload.createSession"; + string url = itemByPathUrl ~ encodeComponent(path) ~ ":/createUploadSession"; if (eTag) http.addRequestHeader("If-Match", eTag); return post(url, null); } @@ -192,7 +193,8 @@ final class OneDriveApi } http.method = HTTP.Method.put; http.url = uploadUrl; - addAccessTokenHeader(); + // when using microsoft graph the auth code is different + //addAccessTokenHeader(); import std.conv; string contentRange = "bytes " ~ to!string(offset) ~ "-" ~ to!string(offset + offsetSize - 1) ~ "/" ~ to!string(fileSize); http.addRequestHeader("Content-Range", contentRange); @@ -210,7 +212,8 @@ final class OneDriveApi JSONValue requestUploadStatus(const(char)[] uploadUrl) { checkAccessTokenExpired(); - return get(uploadUrl); + // when using microsoft graph the auth code is different + return get(uploadUrl, true); } private void redeemToken(const(char)[] authCode) @@ -254,12 +257,12 @@ final class OneDriveApi http.addRequestHeader("Authorization", accessToken); } - private JSONValue get(const(char)[] url) + private JSONValue get(const(char)[] url, bool skipToken = false) { scope(exit) http.clearRequestHeaders(); http.method = HTTP.Method.get; http.url = url; - addAccessTokenHeader(); + if (!skipToken) addAccessTokenHeader(); // HACK: requestUploadStatus auto response = perform(); checkHttpCode(response); return response; @@ -366,7 +369,15 @@ final class OneDriveApi } catch (CurlException e) { throw new OneDriveException(e.msg, e); } - return content.parseJSON(); + JSONValue json; + try { + json = content.parseJSON(); + } catch (JSONException e) { + e.msg ~= "\n"; + e.msg ~= content; + throw e; + } + return json; } private void checkHttpCode() @@ -383,3 +394,37 @@ final class OneDriveApi } } } + +unittest +{ + string configDirName = expandTilde("~/.config/onedrive"); + auto cfg = new config.Config(configDirName); + cfg.init(); + OneDriveApi onedrive = new OneDriveApi(cfg); + onedrive.init(); + std.file.write("/tmp/test", "test"); + + // simpleUpload + auto item = onedrive.simpleUpload("/tmp/test", "/test"); + try { + item = onedrive.simpleUpload("/tmp/test", "/test"); + } catch (OneDriveException e) { + assert(e.httpStatusCode == 409); + } + try { + item = onedrive.simpleUpload("/tmp/test", "/test", "123"); + } catch (OneDriveException e) { + assert(e.httpStatusCode == 412); + } + item = onedrive.simpleUpload("/tmp/test", "/test", item["eTag"].str); + + // deleteById + try { + onedrive.deleteById(item["id"].str, "123"); + } catch (OneDriveException e) { + assert(e.httpStatusCode == 412); + } + onedrive.deleteById(item["id"].str, item["eTag"].str); + + onedrive.http.shutdown(); +} diff --git a/src/sync.d b/src/sync.d index b5f73b07..816d68a1 100644 --- a/src/sync.d +++ b/src/sync.d @@ -5,21 +5,22 @@ import config, itemdb, onedrive, upload, util; static import log; // threshold after which files will be uploaded using an upload session -private long thresholdFileSize = 10 * 2^^20; // 10 MiB +private long thresholdFileSize = 4 * 2^^20; // 4 MiB private bool isItemFolder(const ref JSONValue item) { - return (("folder" in item.object) !is null); + return ("folder" in item) != null; } private bool isItemFile(const ref JSONValue item) { - return (("file" in item.object) !is null); + return ("file" in item) != null; } private bool isItemDeleted(const ref JSONValue item) { - return (("deleted" in item.object) !is null); + // HACK: fix for https://github.com/skilion/onedrive/issues/157 + return ("deleted" in item) || ("fileSystemInfo" !in item); } private bool testCrc32(string path, const(char)[] crc32) @@ -107,9 +108,15 @@ final class SyncEngine foreach (item; changes["value"].array) { applyDifference(item); } - statusToken = changes["@delta.token"].str; + // hack to reuse old code + string url; + if ("@odata.nextLink" in changes) url = changes["@odata.nextLink"].str; + if ("@odata.deltaLink" in changes) url = changes["@odata.deltaLink"].str; + auto c = matchFirst(url, r"(?:token=)([\w\d]+)"); + c.popFront(); // skip the whole match + statusToken = c.front; std.file.write(cfg.statusTokenFilePath, statusToken); - } while (!((changes.type == JSON_TYPE.OBJECT) && (("@odata.nextLink" in changes) is null))); + } while ("@odata.nextLink" in changes); } catch (ErrnoException e) { throw new SyncException(e.msg, e); } catch (FileException e) { @@ -477,7 +484,6 @@ final class SyncEngine } else { response = session.upload(path, path); } - saveItem(response); string id = response["id"].str; string cTag = response["cTag"].str; SysTime mtime = timeLastModified(path).toUTC(); @@ -512,7 +518,6 @@ final class SyncEngine private void saveItem(JSONValue jsonItem) { - string id = jsonItem["id"].str; ItemType type; if (isItemFile(jsonItem)) { type = ItemType.file; @@ -522,7 +527,7 @@ final class SyncEngine assert(0); } Item item = { - id: id, + id: jsonItem["id"].str, name: jsonItem["name"].str, type: type, eTag: jsonItem["eTag"].str, From a3b25c45f52da8a9a7806a3279b726ca49a98577 Mon Sep 17 00:00:00 2001 From: skilion Date: Sat, 11 Mar 2017 12:07:21 +0100 Subject: [PATCH 20/31] catch CurlTimeoutException --- src/sync.d | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sync.d b/src/sync.d index 816d68a1..4a01b7a8 100644 --- a/src/sync.d +++ b/src/sync.d @@ -1,3 +1,4 @@ +import std.net.curl: CurlTimeoutException; import std.exception: ErrnoException; import std.algorithm, std.datetime, std.file, std.json, std.path, std.regex; import std.stdio, std.string; @@ -121,6 +122,8 @@ final class SyncEngine throw new SyncException(e.msg, e); } catch (FileException e) { throw new SyncException(e.msg, e); + } catch (CurlTimeoutException e) { + throw new SyncException(e.msg, e); } catch (OneDriveException e) { throw new SyncException(e.msg, e); } From 1260fcfcc1856e5cc5272679559992764bb1f9b3 Mon Sep 17 00:00:00 2001 From: skilion Date: Sat, 11 Mar 2017 13:34:07 +0100 Subject: [PATCH 21/31] fix addressing root by path --- src/onedrive.d | 8 +++++--- src/sync.d | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/onedrive.d b/src/onedrive.d index 9daf2dff..5dadcab8 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -115,9 +115,9 @@ final class OneDriveApi JSONValue viewChangesByPath(const(char)[] path, const(char)[] statusToken) { checkAccessTokenExpired(); - //string url = itemByPathUrl ~ encodeComponent(path) ~ ":/delta"; - // HACK: item by path seems to no be working - string url = "https://graph.microsoft.com/v1.0/me/drive/root/delta"; + string url = itemByPathUrl ~ encodeComponent(path) ~ ":/delta"; + // HACK + if (path == ".") url = driveUrl ~ "/root/delta"; url ~= "?select=id,name,eTag,cTag,deleted,file,folder,fileSystemInfo,remoteItem,parentReference"; if (statusToken) url ~= "&token=" ~ statusToken; return get(url); @@ -170,6 +170,8 @@ final class OneDriveApi { checkAccessTokenExpired(); string url = itemByPathUrl ~ encodeComponent(parentPath) ~ ":/children"; + // HACK + if (parentPath == ".") url = driveUrl ~ "/root/children"; http.addRequestHeader("Content-Type", "application/json"); return post(url, item.toString()); } diff --git a/src/sync.d b/src/sync.d index 4a01b7a8..82e93ffd 100644 --- a/src/sync.d +++ b/src/sync.d @@ -95,7 +95,7 @@ final class SyncEngine do { // get changes from the server try { - changes = onedrive.viewChangesByPath("/", statusToken); + changes = onedrive.viewChangesByPath(".", statusToken); } catch (OneDriveException e) { if (e.httpStatusCode == 410) { log.log("Status token expired, resyncing"); @@ -474,7 +474,7 @@ final class SyncEngine log.log("Creating remote directory: ", path); JSONValue item = ["name": baseName(path).idup]; item["folder"] = parseJSON("{}"); - auto res = onedrive.createByPath(path.dirName ~ "/", item); + auto res = onedrive.createByPath(path.dirName, item); saveItem(res); } From 957b034805985d40a25d8521996791c74c055a64 Mon Sep 17 00:00:00 2001 From: skilion Date: Sat, 11 Mar 2017 14:21:00 +0100 Subject: [PATCH 22/31] exit early if the item is to be deleted --- src/sync.d | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/sync.d b/src/sync.d index 82e93ffd..5bfac586 100644 --- a/src/sync.d +++ b/src/sync.d @@ -167,6 +167,13 @@ final class SyncEngine } } + // check if the item is to be deleted + if (isItemDeleted(item)) { + log.vlog("The item is marked for deletion"); + if (cached) idsToDelete ~= id; + return; + } + // compute the path of the item string path = "."; if (parentId) { @@ -174,11 +181,7 @@ final class SyncEngine } ItemType type; - if (isItemDeleted(item)) { - log.vlog("The item is marked for deletion"); - if (cached) idsToDelete ~= id; - return; - } else if (isItemFile(item)) { + if (isItemFile(item)) { type = ItemType.file; if (!path.matchFirst(skipFile).empty) { log.vlog("Filtered out"); From ac555fd57496d921480c273bd3078cb1c8ebe59c Mon Sep 17 00:00:00 2001 From: skilion Date: Sat, 11 Mar 2017 14:52:33 +0100 Subject: [PATCH 23/31] added missing scope --- src/onedrive.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/onedrive.d b/src/onedrive.d index 5dadcab8..f42562ab 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -79,7 +79,7 @@ final class OneDriveApi { import std.stdio, std.regex; char[] response; - string url = authUrl ~ "?client_id=" ~ clientId ~ "&scope=files.readwrite.all%20offline_access&response_type=code&redirect_uri=" ~ redirectUrl; + string url = authUrl ~ "?client_id=" ~ clientId ~ "&scope=files.readwrite%20files.readwrite.all%20offline_access&response_type=code&redirect_uri=" ~ redirectUrl; log.log("Authorize this app visiting:\n"); write(url, "\n\n", "Enter the response uri: "); readln(response); From 8b204d18b47da1df3018f5d821c1ecda15cfbe0b Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 12 Mar 2017 10:30:31 +0100 Subject: [PATCH 24/31] hardcode the client id --- src/onedrive.d | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/onedrive.d b/src/onedrive.d index f42562ab..7778b83a 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -6,6 +6,7 @@ static import log; private immutable { + string clientId = "22c49a0d-d21c-4792-aed1-8f163c982546"; string authUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize"; string redirectUrl = "https://login.microsoftonline.com/common/oauth2/nativeclient"; string tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token"; @@ -45,7 +46,6 @@ class OneDriveException: Exception final class OneDriveApi { private Config cfg; - private string clientId; private string refreshToken, accessToken; private SysTime accessTokenExpiration; /* private */ HTTP http; @@ -53,7 +53,6 @@ final class OneDriveApi this(Config cfg) { this.cfg = cfg; - this.clientId = cfg.getValue("client_id"); http = HTTP(); //http.verbose = true; } From 9dd944286f381b30b0fe656e66d69900782732a2 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 12 Mar 2017 16:07:45 +0100 Subject: [PATCH 25/31] new feature selective sync removed system wide config file hardcoded sane default config updated README --- Makefile | 5 +-- README.md | 34 +++++++++++------- config | 4 +++ onedrive.conf | 4 --- src/config.d | 25 +++++++++----- src/itemdb.d | 6 ++-- src/sync.d | 95 ++++++++++++++++++++++++++++++++------------------- src/util.d | 28 +++++++++++++-- 8 files changed, 133 insertions(+), 68 deletions(-) create mode 100644 config delete mode 100644 onedrive.conf diff --git a/Makefile b/Makefile index 006d1a0e..3d8df009 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ DC = dmd DFLAGS = -ofonedrive -L-lcurl -L-lsqlite3 -L-ldl DESTDIR = /usr/local/bin -CONFDIR = /usr/local/etc SOURCES = \ src/config.d \ @@ -27,12 +26,10 @@ unittest: $(SOURCES) clean: rm -f onedrive.o onedrive -install: onedrive onedrive.conf +install: onedrive onedrive.service install onedrive $(DESTDIR)/onedrive - install -m 644 onedrive.conf $(CONFDIR)/onedrive.conf install -m 644 onedrive.service /usr/lib/systemd/user uninstall: rm -f $(DESTDIR)/onedrive - rm -f $(CONFDIR)/onedrive.conf rm -f /usr/lib/systemd/user/onedrive.service diff --git a/README.md b/README.md index d208fc15..882c0225 100644 --- a/README.md +++ b/README.md @@ -37,21 +37,30 @@ sudo make install You should copy the default config file into your home directory before making changes: ``` mkdir -p ~/.config/onedrive -cp /usr/local/etc/onedrive.conf ~/.config/onedrive/config +cp ./config ~/.config/onedrive/config ``` Available options: -* `client_id`: application identifier necessary for the [authentication][2] * `sync_dir`: directory where the files will be synced -* `skip_file`: any files that match this pattern will be skipped during sync -* `skip_dir`: any directories that match this pattern will be skipped during sync +* `skip_file`: any files or directories that match this pattern will be skipped during sync Pattern are case insensitive. -`*` and `?` [wildcards characters][3] are supported. +`*` and `?` [wildcards characters][1] are supported. Use `|` to separate multiple patterns. -[2]: https://dev.onedrive.com/auth/msa_oauth.htm -[3]: https://technet.microsoft.com/en-us/library/bb490639.aspx +[1]: https://technet.microsoft.com/en-us/library/bb490639.aspx + +### Selective sync +Selective sync allows you to sync only specific files and directories. +To enable selective sync create a file named `sync_list` in `~/.config/onedrive`. +Each line represents a path to a file or directory relative from your `sync_dir`. +``` +$ cat ~/.config/onedrive/sync_list +Backup +Documents/report.odt +Work/ProjectX +notes.txt +``` ### First run The first time you run the program you will be asked to sign in. The procedure requires a web browser. @@ -72,16 +81,17 @@ journalctl --user-unit onedrive -f ``` onedrive [OPTION]... -no option Sync and exit. -m --monitor Keep monitoring for local and remote changes. - --resync Forget the last saved state, perform a full sync. + --resync Forget the last saved state, perform a full sync. + --logout Logout the current user. + --confdir Set the directory to use to store the configuration files. -v --verbose Print more details, useful for debugging. -h --help This help information. ``` ### Notes: -* After changing the filters (`skip_file` or `skip_dir` in your configs) you must execute `onedrive --resync` -* [Windows naming conventions][4] apply +* After changing `skip_file` in your configs or the sync list, you must execute `onedrive --resync` +* [Windows naming conventions][2] apply * Use `make debug` to generate an executable for debugging -[4]: https://msdn.microsoft.com/en-us/library/aa365247 +[2]: https://msdn.microsoft.com/en-us/library/aa365247 diff --git a/config b/config new file mode 100644 index 00000000..cc4b5b5a --- /dev/null +++ b/config @@ -0,0 +1,4 @@ +# Directory where the files will be synced +sync_dir = "~/OneDrive" +# Skip files and directories that match this pattern +skip_file = ".*|~*" diff --git a/onedrive.conf b/onedrive.conf deleted file mode 100644 index 8b73f21b..00000000 --- a/onedrive.conf +++ /dev/null @@ -1,4 +0,0 @@ -client_id = "22c49a0d-d21c-4792-aed1-8f163c982546" -sync_dir = "~/OneDrive" -skip_file = ".*|~*" -skip_dir = ".*" diff --git a/src/config.d b/src/config.d index 66ed19a1..9c00638e 100644 --- a/src/config.d +++ b/src/config.d @@ -1,4 +1,4 @@ -import std.file, std.regex, std.stdio; +import std.file, std.string, std.regex, std.stdio; static import log; final class Config @@ -7,6 +7,7 @@ final class Config public string statusTokenFilePath; public string databaseFilePath; public string uploadStateFilePath; + public string syncListFilePath; private string userConfigFilePath; // hashmap for the values found in the user config file @@ -19,15 +20,16 @@ final class Config databaseFilePath = configDirName ~ "/items.sqlite3"; uploadStateFilePath = configDirName ~ "/resume_upload"; userConfigFilePath = configDirName ~ "/config"; + syncListFilePath = configDirName ~ "/sync_list"; } void init() { - bool found = false; - found |= load("/etc/onedrive.conf"); - found |= load("/usr/local/etc/onedrive.conf"); - found |= load(userConfigFilePath); - if (!found) throw new Exception("No config file found"); + setValue("sync_dir", "~/OneDrive"); + setValue("skip_file", ".*|~*"); + if (!load(userConfigFilePath)) { + log.vlog("No config file found, using defaults"); + } } string getValue(string key) @@ -50,12 +52,19 @@ final class Config } } + void setValue(string key, string value) + { + values[key] = value; + } + private bool load(string filename) { scope(failure) return false; auto file = File(filename, "r"); - auto r = regex(`^\s*(\w+)\s*=\s*"(.*)"\s*$`); + auto r = regex(`^(\w+)\s*=\s*"(.*)"\s*$`); foreach (line; file.byLine()) { + line = stripLeft(line); + if (line.length == 0 || line[0] == ';' || line[0] == '#') continue; auto c = line.matchFirst(r); if (!c.empty) { c.popFront(); // skip the whole match @@ -73,7 +82,7 @@ final class Config unittest { auto cfg = new Config(""); - cfg.load("onedrive.conf"); + cfg.load("config"); assert(cfg.getValue("sync_dir") == "~/OneDrive"); assert(cfg.getValue("empty", "default") == "default"); } diff --git a/src/itemdb.d b/src/itemdb.d index 4f5cd7f5..6b77734f 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -114,7 +114,8 @@ final class ItemDatabase bool selectByPath(const(char)[] path, out Item item) { - path = "root/" ~ path.chompPrefix("."); // HACK + // prefix with the root dir + path = "root/" ~ path.chompPrefix("."); // initialize the search string[2][] candidates; // [id, parentId] @@ -240,8 +241,7 @@ final class ItemDatabase else path = r.front[0].idup; } else { // root - if (path) path = "./" ~ path; - else path = "."; + if (!path) path = "."; break; } id = r.front[1].dup; diff --git a/src/sync.d b/src/sync.d index 5bfac586..7efb3337 100644 --- a/src/sync.d +++ b/src/sync.d @@ -51,8 +51,10 @@ final class SyncEngine private Config cfg; private OneDriveApi onedrive; private ItemDatabase itemdb; - private Regex!char skipDir, skipFile; private UploadSession session; + private Regex!char skipFile; + // list of paths to sync + private string[] selectiveSyncPaths; // token representing the last status correctly synced private string statusToken; // list of items to skip while applying the changes @@ -66,9 +68,19 @@ final class SyncEngine this.cfg = cfg; this.onedrive = onedrive; this.itemdb = itemdb; - skipDir = wild2regex(cfg.getValue("skip_dir")); skipFile = wild2regex(cfg.getValue("skip_file")); session = UploadSession(onedrive, cfg.uploadStateFilePath); + + // read the selective sync list + if (exists(cfg.syncListFilePath)) { + import std.array; + auto file = File(cfg.syncListFilePath); + selectiveSyncPaths = file + .byLine() + .map!(a => buildNormalizedPath(a)) + .filter!(a => a.length > 0) + .array; + } } void init() @@ -146,13 +158,19 @@ final class SyncEngine parentId = null; } + log.vlog(id, " ", name); + // skip unwanted items early if (skippedItems.find(parentId).length != 0) { + log.vlog("Filtered out"); + skippedItems ~= id; + return; + } + if (!name.matchFirst(skipFile).empty) { + log.vlog("Filtered out"); skippedItems ~= id; return; } - - log.vlog(id, " ", name); // rename the local item if it is unsynced and there is a new version of it Item oldItem; @@ -178,22 +196,19 @@ final class SyncEngine string path = "."; if (parentId) { path = itemdb.computePath(parentId) ~ "/" ~ name; + // selective sync + if (isPathExcluded(path, selectiveSyncPaths)) { + log.vlog("Filtered out: ", path); + skippedItems ~= id; + return; + } } ItemType type; if (isItemFile(item)) { type = ItemType.file; - if (!path.matchFirst(skipFile).empty) { - log.vlog("Filtered out"); - return; - } } else if (isItemFolder(item)) { type = ItemType.dir; - if (!path.matchFirst(skipDir).empty) { - log.vlog("Filtered out"); - skippedItems ~= id; - return; - } } else { log.vlog("The item is neither a file nor a directory, skipping"); skippedItems ~= id; @@ -365,20 +380,23 @@ final class SyncEngine private void uploadDifferences(Item item) { log.vlog(item.id, " ", item.name); + + // skip filtered items + if (!item.name.matchFirst(skipFile).empty) { + log.vlog("Filtered out"); + return; + } string path = itemdb.computePath(item.id); + if (isPathExcluded(path, selectiveSyncPaths)) { + log.vlog("Filtered out: ", path); + return; + } + final switch (item.type) { case ItemType.dir: - if (!path.matchFirst(skipDir).empty) { - log.vlog("Filtered out"); - break; - } uploadDirDifferences(item, path); break; case ItemType.file: - if (!path.matchFirst(skipFile).empty) { - log.vlog("Filtered out"); - break; - } uploadFileDifferences(item, path); break; } @@ -448,26 +466,33 @@ final class SyncEngine private void uploadNewItems(string path) { + // skip unexisting symbolic links if (isSymlink(path) && !exists(readLink(path))) { return; } + + // skip filtered items + if (!baseName(path).matchFirst(skipFile).empty) { + return; + } + if (isPathExcluded(path, selectiveSyncPaths)) { + return; + } + if (isDir(path)) { - if (path.matchFirst(skipDir).empty) { - Item item; - if (!itemdb.selectByPath(path, item)) { - uploadCreateDir(path); - } - auto entries = dirEntries(path, SpanMode.shallow, false); - foreach (DirEntry entry; entries) { - uploadNewItems(entry.name); - } + Item item; + if (!itemdb.selectByPath(path, item)) { + uploadCreateDir(path); + } + // recursively traverse children + auto entries = dirEntries(path, SpanMode.shallow, false); + foreach (DirEntry entry; entries) { + uploadNewItems(entry.name); } } else { - if (path.matchFirst(skipFile).empty) { - Item item; - if (!itemdb.selectByPath(path, item)) { - uploadNewFile(path); - } + Item item; + if (!itemdb.selectByPath(path, item)) { + uploadNewFile(path); } } } diff --git a/src/util.d b/src/util.d index b9515905..21f1edbe 100644 --- a/src/util.d +++ b/src/util.d @@ -1,3 +1,4 @@ +import std.algorithm; import std.conv; import std.digest.crc; import std.file; @@ -55,7 +56,7 @@ Regex!char wild2regex(const(char)[] pattern) { string str; str.reserve(pattern.length + 2); - str ~= "/"; + str ~= "^"; foreach (c; pattern) { switch (c) { case '*': @@ -68,7 +69,7 @@ Regex!char wild2regex(const(char)[] pattern) str ~= "[^/]"; break; case '|': - str ~= "$|/"; + str ~= "$|^"; break; default: str ~= c; @@ -98,9 +99,32 @@ bool multiGlobMatch(const(char)[] path, const(char)[] pattern) return false; } +// test if the given path is not included in the allowed paths +// if there are no allowed paths always return false +bool isPathExcluded(string path, string[] allowedPaths) +{ + // always allow the root + if (path == ".") return false; + // if there are no allowed paths always return false + if (allowedPaths.empty) return false; + + path = buildNormalizedPath(path); + foreach (allowed; allowedPaths) { + auto comm = commonPrefix(path, allowed); + if (comm.length == path.length || comm.length == allowed.length) { + return false; + } + } + return true; +} + unittest { assert(multiGlobMatch(".hidden", ".*")); assert(multiGlobMatch(".hidden", "file|.*")); assert(!multiGlobMatch("foo.bar", "foo|bar")); + assert(isPathExcluded("Documents2", ["Documents"])); + assert(isPathExcluded("Hello/World", ["Hello/John"])); + assert(!isPathExcluded("Documents", ["Documents"])); + assert(!isPathExcluded("Documents/a.txt", ["Documents"])); } From 2c1c168fabba7324ad738dd1c3a034aae164c147 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 12 Mar 2017 16:35:47 +0100 Subject: [PATCH 26/31] fix root dir skipped when searching for new files --- src/sync.d | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/sync.d b/src/sync.d index 7efb3337..4158d3a1 100644 --- a/src/sync.d +++ b/src/sync.d @@ -472,11 +472,13 @@ final class SyncEngine } // skip filtered items - if (!baseName(path).matchFirst(skipFile).empty) { - return; - } - if (isPathExcluded(path, selectiveSyncPaths)) { - return; + if (path != ".") { + if (!baseName(path).matchFirst(skipFile).empty) { + return; + } + if (isPathExcluded(path, selectiveSyncPaths)) { + return; + } } if (isDir(path)) { From 79ed0a7883678f094faf199f20be0661db0ee11f Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 12 Mar 2017 17:17:38 +0100 Subject: [PATCH 27/31] fix different time resolution when comparing --- src/sync.d | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sync.d b/src/sync.d index 4158d3a1..046f1009 100644 --- a/src/sync.d +++ b/src/sync.d @@ -429,6 +429,9 @@ final class SyncEngine if (exists(path)) { if (isFile(path)) { SysTime localModifiedTime = timeLastModified(path); + // HACK: reduce time resolution to seconds before comparing + item.mtime.fracSecs = Duration.zero; + localModifiedTime.fracSecs = Duration.zero; if (localModifiedTime != item.mtime) { log.vlog("The file last modified time has changed"); string id = item.id; From 88d2a940782813253ea6a0581267d836a8f90a4e Mon Sep 17 00:00:00 2001 From: skilion Date: Tue, 14 Mar 2017 17:20:18 +0100 Subject: [PATCH 28/31] added selective sync to monitor.d --- src/monitor.d | 55 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/src/monitor.d b/src/monitor.d index cc4dd684..2419025b 100644 --- a/src/monitor.d +++ b/src/monitor.d @@ -1,7 +1,7 @@ import core.sys.linux.sys.inotify; import core.stdc.errno; import core.sys.posix.poll, core.sys.posix.unistd; -import std.exception, std.file, std.path, std.regex, std.stdio, std.string; +import std.algorithm, std.exception, std.file, std.path, std.regex, std.stdio, std.string; import config, util; static import log; @@ -20,8 +20,10 @@ class MonitorException: ErrnoException struct Monitor { bool verbose; - // regex that match files/dirs to skip - private Regex!char skipDir, skipFile; + // regex that match files to skip + private Regex!char skipFile; + // list of paths to sync + private string[] selectiveSyncPaths; // inotify file descriptor private int fd; // map every inotify watch descriptor to its directory @@ -41,8 +43,18 @@ struct Monitor void init(Config cfg, bool verbose) { this.verbose = verbose; - skipDir = wild2regex(cfg.getValue("skip_dir")); skipFile = wild2regex(cfg.getValue("skip_file")); + // read the selective sync list + if (exists(cfg.syncListFilePath)) { + import std.array; + auto file = File(cfg.syncListFilePath); + selectiveSyncPaths = file + .byLine() + .map!(a => buildNormalizedPath(a)) + .filter!(a => a.length > 0) + .array; + } + fd = inotify_init(); if (fd == -1) throw new MonitorException("inotify_init failed"); if (!buffer) buffer = new void[4096]; @@ -57,12 +69,20 @@ struct Monitor private void addRecursive(string dirname) { - if (matchFirst(dirname, skipDir).empty) { - add(dirname); - foreach(DirEntry entry; dirEntries(dirname, SpanMode.shallow, false)) { - if (entry.isDir) { - addRecursive(entry.name); - } + // skip filtered items + if (dirname != ".") { + if (!baseName(dirname).matchFirst(skipFile).empty) { + return; + } + if (isPathExcluded(buildNormalizedPath(dirname), selectiveSyncPaths)) { + return; + } + } + + add(dirname); + foreach(DirEntry entry; dirEntries(dirname, SpanMode.shallow, false)) { + if (entry.isDir) { + addRecursive(entry.name); } } } @@ -85,7 +105,7 @@ struct Monitor } throw new MonitorException("inotify_add_watch failed"); } - wdToDirName[wd] = dirname ~ "/"; + wdToDirName[wd] = buildNormalizedPath(dirname) ~ "/"; log.vlog("Monitor directory: ", dirname); } @@ -152,14 +172,11 @@ struct Monitor // skip filtered items path = getPath(event); - if (event.mask & IN_ISDIR) { - if (!matchFirst(path, skipDir).empty) { - goto skip; - } - } else { - if (!matchFirst(path, skipFile).empty) { - goto skip; - } + if (!baseName(path).matchFirst(skipFile).empty) { + goto skip; + } + if (isPathExcluded(path, selectiveSyncPaths)) { + goto skip; } if (event.mask & IN_MOVED_FROM) { From 1c7b726994a65ce2e6b305257d89040de03050eb Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 12 Mar 2017 17:25:51 +0100 Subject: [PATCH 29/31] fix different time resolution when comparing --- src/sync.d | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sync.d b/src/sync.d index 046f1009..885f0c0a 100644 --- a/src/sync.d +++ b/src/sync.d @@ -308,6 +308,9 @@ final class SyncEngine case ItemType.file: if (isFile(path)) { SysTime localModifiedTime = timeLastModified(path); + // HACK: reduce time resolution to seconds before comparing + item.mtime.fracSecs = Duration.zero; + localModifiedTime.fracSecs = Duration.zero; if (localModifiedTime == item.mtime) { return true; } else { From 0d69ed805d45623a40510d069a104f1c9e4b17b7 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 12 Mar 2017 19:40:38 +0100 Subject: [PATCH 30/31] wip for OneDrive Biz --- src/itemdb.d | 4 ++-- src/onedrive.d | 6 +++--- src/sync.d | 19 ++++++++++++------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/itemdb.d b/src/itemdb.d index 6b77734f..556aa0de 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -22,7 +22,7 @@ struct Item final class ItemDatabase { // increment this for every change in the db schema - immutable int itemDatabaseVersion = 2; + immutable int itemDatabaseVersion = 3; Database db; Statement insertItemStmt; @@ -38,7 +38,7 @@ final class ItemDatabase id TEXT NOT NULL PRIMARY KEY, name TEXT NOT NULL, type TEXT NOT NULL, - eTag TEXT NOT NULL, + eTag TEXT, cTag TEXT, mtime TEXT NOT NULL, parentId TEXT, diff --git a/src/onedrive.d b/src/onedrive.d index 7778b83a..b400f2b9 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -83,7 +83,7 @@ final class OneDriveApi write(url, "\n\n", "Enter the response uri: "); readln(response); // match the authorization code - auto c = matchFirst(response, r"(?:code=)(([\w\d]+-){4}[\w\d]+)"); + auto c = matchFirst(response, r"(?:code=)([\w\d-]+)"); if (c.empty) { log.log("Invalid uri"); return false; @@ -105,7 +105,7 @@ final class OneDriveApi { checkAccessTokenExpired(); const(char)[] url = itemByIdUrl ~ id ~ "/delta"; - url ~= "?select=id,name,eTag,cTag,deleted,file,folder,fileSystemInfo,remoteItem,parentReference"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference"; if (statusToken) url ~= "&token=" ~ statusToken; return get(url); } @@ -117,7 +117,7 @@ final class OneDriveApi string url = itemByPathUrl ~ encodeComponent(path) ~ ":/delta"; // HACK if (path == ".") url = driveUrl ~ "/root/delta"; - url ~= "?select=id,name,eTag,cTag,deleted,file,folder,fileSystemInfo,remoteItem,parentReference"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference"; if (statusToken) url ~= "&token=" ~ statusToken; return get(url); } diff --git a/src/sync.d b/src/sync.d index 885f0c0a..d214e415 100644 --- a/src/sync.d +++ b/src/sync.d @@ -24,6 +24,11 @@ private bool isItemDeleted(const ref JSONValue item) return ("deleted" in item) || ("fileSystemInfo" !in item); } +private bool isItemRoot(const ref JSONValue item) +{ + return ("root" in item) != null; +} + private bool testCrc32(string path, const(char)[] crc32) { if (crc32) { @@ -150,16 +155,16 @@ final class SyncEngine { string id = item["id"].str; string name = item["name"].str; - string eTag = item["eTag"].str; - string parentId = item["parentReference"]["id"].str; - - // HACK: recognize the root directory - if (name == "root" && parentId[$ - 1] == '0' && parentId[$ - 2] == '!') { - parentId = null; - } log.vlog(id, " ", name); + // eTag and parentId do not exists for the root in OneDrive Biz + string eTag, parentId; + if (!isItemRoot(item)) { + eTag = item["eTag"].str; + parentId = item["parentReference"]["id"].str; + } + // skip unwanted items early if (skippedItems.find(parentId).length != 0) { log.vlog("Filtered out"); From 97a9d5391416f1bec51a80c7e154f11d0b2e203d Mon Sep 17 00:00:00 2001 From: skilion Date: Fri, 24 Mar 2017 22:30:03 +0100 Subject: [PATCH 31/31] refactored selective sync code fixed bug in selective sync --- Makefile | 1 + src/config.d | 1 + src/main.d | 9 ++++--- src/monitor.d | 39 +++++++++++---------------- src/selective.d | 72 +++++++++++++++++++++++++++++++++++++++++++++++++ src/sync.d | 39 ++++++++++----------------- src/util.d | 26 +----------------- 7 files changed, 111 insertions(+), 76 deletions(-) create mode 100644 src/selective.d diff --git a/Makefile b/Makefile index 3d8df009..363ff7fe 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ SOURCES = \ src/main.d \ src/monitor.d \ src/onedrive.d \ + src/selective.d \ src/sqlite.d \ src/sync.d \ src/upload.d \ diff --git a/src/config.d b/src/config.d index 9c00638e..653982d9 100644 --- a/src/config.d +++ b/src/config.d @@ -1,4 +1,5 @@ import std.file, std.string, std.regex, std.stdio; +import selective; static import log; final class Config diff --git a/src/main.d b/src/main.d index 4e657081..0de12d25 100644 --- a/src/main.d +++ b/src/main.d @@ -1,7 +1,7 @@ import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE; import core.memory, core.time, core.thread; import std.getopt, std.file, std.path, std.process; -import config, itemdb, monitor, onedrive, sync, util; +import config, itemdb, monitor, onedrive, selective, sync, util; static import log; int main(string[] args) @@ -87,13 +87,16 @@ int main(string[] args) chdir(syncDir); log.vlog("Initializing the Synchronization Engine ..."); - auto sync = new SyncEngine(cfg, onedrive, itemdb); + auto selectiveSync = new SelectiveSync(); + selectiveSync.load(cfg.syncListFilePath); + selectiveSync.setMask(cfg.getValue("skip_file")); + auto sync = new SyncEngine(cfg, onedrive, itemdb, selectiveSync); sync.init(); if (online) performSync(sync); if (monitor) { log.vlog("Initializing monitor ..."); - Monitor m; + Monitor m = new Monitor(selectiveSync); m.onDirCreated = delegate(string path) { log.vlog("[M] Directory created: ", path); try { diff --git a/src/monitor.d b/src/monitor.d index 2419025b..5dec10d4 100644 --- a/src/monitor.d +++ b/src/monitor.d @@ -1,8 +1,10 @@ import core.sys.linux.sys.inotify; import core.stdc.errno; import core.sys.posix.poll, core.sys.posix.unistd; -import std.algorithm, std.exception, std.file, std.path, std.regex, std.stdio, std.string; -import config, util; +import std.exception, std.file, std.path, std.regex, std.stdio, std.string; +import config; +import selective; +import util; static import log; // relevant inotify events @@ -17,13 +19,9 @@ class MonitorException: ErrnoException } } -struct Monitor +final class Monitor { bool verbose; - // regex that match files to skip - private Regex!char skipFile; - // list of paths to sync - private string[] selectiveSyncPaths; // inotify file descriptor private int fd; // map every inotify watch descriptor to its directory @@ -33,27 +31,22 @@ struct Monitor // buffer to receive the inotify events private void[] buffer; + private SelectiveSync selectiveSync; + void delegate(string path) onDirCreated; void delegate(string path) onFileChanged; void delegate(string path) onDelete; void delegate(string from, string to) onMove; - @disable this(this); + this(SelectiveSync selectiveSync) + { + assert(selectiveSync); + this.selectiveSync = selectiveSync; + } void init(Config cfg, bool verbose) { this.verbose = verbose; - skipFile = wild2regex(cfg.getValue("skip_file")); - // read the selective sync list - if (exists(cfg.syncListFilePath)) { - import std.array; - auto file = File(cfg.syncListFilePath); - selectiveSyncPaths = file - .byLine() - .map!(a => buildNormalizedPath(a)) - .filter!(a => a.length > 0) - .array; - } fd = inotify_init(); if (fd == -1) throw new MonitorException("inotify_init failed"); @@ -71,10 +64,10 @@ struct Monitor { // skip filtered items if (dirname != ".") { - if (!baseName(dirname).matchFirst(skipFile).empty) { + if (selectiveSync.isNameExcluded(baseName(dirname))) { return; } - if (isPathExcluded(buildNormalizedPath(dirname), selectiveSyncPaths)) { + if (selectiveSync.isPathExcluded(buildNormalizedPath(dirname))) { return; } } @@ -172,10 +165,10 @@ struct Monitor // skip filtered items path = getPath(event); - if (!baseName(path).matchFirst(skipFile).empty) { + if (selectiveSync.isNameExcluded(baseName(path))) { goto skip; } - if (isPathExcluded(path, selectiveSyncPaths)) { + if (selectiveSync.isPathExcluded(path)) { goto skip; } diff --git a/src/selective.d b/src/selective.d new file mode 100644 index 00000000..733a6aa6 --- /dev/null +++ b/src/selective.d @@ -0,0 +1,72 @@ +import std.algorithm; +import std.array; +import std.file; +import std.path; +import std.regex; +import std.stdio; +import util; + +final class SelectiveSync +{ + private string[] paths; + private Regex!char mask; + + void load(string filepath) + { + if (exists(filepath)) { + paths = File(filepath) + .byLine() + .map!(a => buildNormalizedPath(a)) + .filter!(a => a.length > 0) + .array; + } + } + + void setMask(const(char)[] mask) + { + this.mask = wild2regex(mask); + } + + bool isNameExcluded(string name) + { + return !name.matchFirst(mask).empty; + } + + bool isPathExcluded(string path) + { + return .isPathExcluded(path, paths); + } +} + +// test if the given path is not included in the allowed paths +// if there are no allowed paths always return false +private bool isPathExcluded(string path, string[] allowedPaths) +{ + // always allow the root + if (path == ".") return false; + // if there are no allowed paths always return false + if (allowedPaths.empty) return false; + + path = buildNormalizedPath(path); + foreach (allowed; allowedPaths) { + auto comm = commonPrefix(path, allowed); + if (comm.length == path.length) { + // the given path is contained in an allowed path + return false; + } + if (comm.length == allowed.length && path[comm.length] == '/') { + // the given path is a subitem of an allowed path + return false; + } + } + return true; +} + +unittest +{ + assert(isPathExcluded("Documents2", ["Documents"])); + assert(!isPathExcluded("Documents", ["Documents"])); + assert(!isPathExcluded("Documents/a.txt", ["Documents"])); + assert(isPathExcluded("Hello/World", ["Hello/John"])); + assert(!isPathExcluded(".", ["Documents"])); +} diff --git a/src/sync.d b/src/sync.d index d214e415..0554b4f6 100644 --- a/src/sync.d +++ b/src/sync.d @@ -1,8 +1,10 @@ +import std.algorithm; import std.net.curl: CurlTimeoutException; import std.exception: ErrnoException; -import std.algorithm, std.datetime, std.file, std.json, std.path, std.regex; +import std.datetime, std.file, std.json, std.path; +import std.regex; import std.stdio, std.string; -import config, itemdb, onedrive, upload, util; +import config, itemdb, onedrive, selective, upload, util; static import log; // threshold after which files will be uploaded using an upload session @@ -57,9 +59,7 @@ final class SyncEngine private OneDriveApi onedrive; private ItemDatabase itemdb; private UploadSession session; - private Regex!char skipFile; - // list of paths to sync - private string[] selectiveSyncPaths; + private SelectiveSync selectiveSync; // token representing the last status correctly synced private string statusToken; // list of items to skip while applying the changes @@ -67,25 +67,14 @@ final class SyncEngine // list of items to delete after the changes has been downloaded private string[] idsToDelete; - this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb) + this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync) { - assert(onedrive && itemdb); + assert(onedrive && itemdb && selectiveSync); this.cfg = cfg; this.onedrive = onedrive; this.itemdb = itemdb; - skipFile = wild2regex(cfg.getValue("skip_file")); + this.selectiveSync = selectiveSync; session = UploadSession(onedrive, cfg.uploadStateFilePath); - - // read the selective sync list - if (exists(cfg.syncListFilePath)) { - import std.array; - auto file = File(cfg.syncListFilePath); - selectiveSyncPaths = file - .byLine() - .map!(a => buildNormalizedPath(a)) - .filter!(a => a.length > 0) - .array; - } } void init() @@ -171,7 +160,7 @@ final class SyncEngine skippedItems ~= id; return; } - if (!name.matchFirst(skipFile).empty) { + if (selectiveSync.isNameExcluded(name)) { log.vlog("Filtered out"); skippedItems ~= id; return; @@ -202,7 +191,7 @@ final class SyncEngine if (parentId) { path = itemdb.computePath(parentId) ~ "/" ~ name; // selective sync - if (isPathExcluded(path, selectiveSyncPaths)) { + if (selectiveSync.isPathExcluded(path)) { log.vlog("Filtered out: ", path); skippedItems ~= id; return; @@ -390,12 +379,12 @@ final class SyncEngine log.vlog(item.id, " ", item.name); // skip filtered items - if (!item.name.matchFirst(skipFile).empty) { + if (selectiveSync.isNameExcluded(item.name)) { log.vlog("Filtered out"); return; } string path = itemdb.computePath(item.id); - if (isPathExcluded(path, selectiveSyncPaths)) { + if (selectiveSync.isPathExcluded(path)) { log.vlog("Filtered out: ", path); return; } @@ -484,10 +473,10 @@ final class SyncEngine // skip filtered items if (path != ".") { - if (!baseName(path).matchFirst(skipFile).empty) { + if (selectiveSync.isNameExcluded(baseName(path))) { return; } - if (isPathExcluded(path, selectiveSyncPaths)) { + if (selectiveSync.isPathExcluded(path)) { return; } } diff --git a/src/util.d b/src/util.d index 21f1edbe..f7512f22 100644 --- a/src/util.d +++ b/src/util.d @@ -1,4 +1,3 @@ -import std.algorithm; import std.conv; import std.digest.crc; import std.file; @@ -83,7 +82,7 @@ Regex!char wild2regex(const(char)[] pattern) // return true if the network connection is available bool testNetwork() { - HTTP http = HTTP("https://login.live.com"); + HTTP http = HTTP("https://login.microsoftonline.com"); http.method = HTTP.Method.head; return http.perform(ThrowOnError.no) == 0; } @@ -99,32 +98,9 @@ bool multiGlobMatch(const(char)[] path, const(char)[] pattern) return false; } -// test if the given path is not included in the allowed paths -// if there are no allowed paths always return false -bool isPathExcluded(string path, string[] allowedPaths) -{ - // always allow the root - if (path == ".") return false; - // if there are no allowed paths always return false - if (allowedPaths.empty) return false; - - path = buildNormalizedPath(path); - foreach (allowed; allowedPaths) { - auto comm = commonPrefix(path, allowed); - if (comm.length == path.length || comm.length == allowed.length) { - return false; - } - } - return true; -} - unittest { assert(multiGlobMatch(".hidden", ".*")); assert(multiGlobMatch(".hidden", "file|.*")); assert(!multiGlobMatch("foo.bar", "foo|bar")); - assert(isPathExcluded("Documents2", ["Documents"])); - assert(isPathExcluded("Hello/World", ["Hello/John"])); - assert(!isPathExcluded("Documents", ["Documents"])); - assert(!isPathExcluded("Documents/a.txt", ["Documents"])); }