From f887b29061e2ea56f1d446a9847ce78762f85a6f Mon Sep 17 00:00:00 2001 From: skilion Date: Wed, 16 Sep 2015 10:29:20 +0200 Subject: [PATCH] sync root folder --- src/cache.d | 244 ------------------------------------------------- src/itemdb.d | 2 +- src/main.d | 34 +++++-- src/onedrive.d | 11 ++- src/sync.d | 221 ++++++++++++++++++++++---------------------- src/util.d | 4 +- 6 files changed, 143 insertions(+), 373 deletions(-) delete mode 100644 src/cache.d diff --git a/src/cache.d b/src/cache.d deleted file mode 100644 index 862ed0b7..00000000 --- a/src/cache.d +++ /dev/null @@ -1,244 +0,0 @@ -module cache; - -import std.datetime, std.path; -import sqlite; - -enum ItemType -{ - file, - dir -} - -struct Item -{ - string id; - string path; - string name; - ItemType type; - string eTag; - string cTag; - SysTime mtime; - string parentId; - string crc32; -} - -struct ItemCache -{ - Database db; - Statement insertItemStmt; - Statement selectItemByIdStmt; - Statement selectItemByParentIdStmt; - - void init() - { - db = Database("cache.db"); - db.exec("CREATE TABLE IF NOT EXISTS item ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - type TEXT NOT NULL, - eTag TEXT NOT NULL, - cTag TEXT NOT NULL, - mtime TEXT NOT NULL, - parentId TEXT NOT NULL, - crc32 TEXT - )"); - db.exec("CREATE INDEX IF NOT EXISTS name_idx ON item (name)"); - insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); - selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32 FROM item WHERE id = ?"); - selectItemByParentIdStmt = db.prepare("SELECT id FROM item WHERE parentId = ?"); - } - - void insert(const(char)[] id, const(char)[] name, ItemType type, const(char)[] eTag, const(char)[] cTag, const(char)[] mtime, const(char)[] parentId, const(char)[] crc32) - { - with (insertItemStmt) { - bind(1, id); - bind(2, name); - string typeStr = void; - final switch (type) { - case ItemType.file: typeStr = "file"; break; - case ItemType.dir: typeStr = "dir"; break; - } - bind(3, typeStr); - bind(4, eTag); - bind(5, cTag); - bind(6, mtime); - bind(7, parentId); - bind(8, crc32); - exec(); - } - } - - // returns a range that go trough all items, depth first - auto selectAll() - { - static struct ItemRange - { - ItemCache* itemCache; - string[] stack1, stack2; - - private this(ItemCache* itemCache, string rootId) - { - this.itemCache = itemCache; - stack1.reserve(8); - stack2.reserve(8); - stack1 ~= rootId; - getChildren(); - } - - @property bool empty() - { - return stack2.length == 0; - } - - @property Item front() - { - Item item; - bool res = itemCache.selectById(stack2[$ - 1], item); - assert(res); - return item; - } - - void popFront() - { - stack2 = stack2[0 .. $ - 1]; - assumeSafeAppend(stack2); - if (stack1.length > 0) getChildren(); - } - - private void getChildren() - { - while (true) { - itemCache.selectItemByParentIdStmt.bind(1, stack1[$ - 1]); - stack2 ~= stack1[$ - 1]; - stack1 = stack1[0 .. $ - 1]; - assumeSafeAppend(stack1); - auto res = itemCache.selectItemByParentIdStmt.exec(); - if (res.empty) break; - else foreach (row; res) stack1 ~= row[0].dup; - } - } - } - - auto s = db.prepare("SELECT a.id FROM item AS a LEFT JOIN item AS b ON a.parentId = b.id WHERE b.id IS NULL"); - auto r = s.exec(); - assert(!r.empty()); - return ItemRange(&this, r.front[0].dup); - } - - bool selectById(const(char)[] id, out Item item) - { - selectItemByIdStmt.bind(1, id); - auto r = selectItemByIdStmt.exec(); - if (!r.empty) { - item = buildItem(r); - return true; - } - return false; - } - - bool selectByPath(const(char)[] path, out Item item) - { - string[2][] candidates; // [id, parentId] - auto s = db.prepare("SELECT id, parentId FROM item WHERE name = ?"); - s.bind(1, baseName(path)); - auto r = s.exec(); - foreach (row; r) candidates ~= [row[0].dup, row[1].dup]; - if (candidates.length > 1) { - s = db.prepare("SELECT parentId FROM item WHERE id = ? AND name = ?"); - do { - string[2][] newCandidates; - newCandidates.reserve(candidates.length); - path = dirName(path); - foreach (candidate; candidates) { - s.bind(1, candidate[1]); - s.bind(2, baseName(path)); - r = s.exec(); - if (!r.empty) { - string[2] c = [candidate[0], r.front[0].idup]; - newCandidates ~= c; - } - } - candidates = newCandidates; - } while (candidates.length > 1); - } - if (candidates.length == 1) return selectById(candidates[0][0], item); - return false; - } - - void deleteById(const(char)[] id) - { - auto s = db.prepare("DELETE FROM item WHERE id = ?"); - s.bind(1, id); - s.exec(); - } - - // returns true if the item has the specified parent - bool hasParent(T)(const(char)[] itemId, T parentId) - if (is(T : const(char)[]) || is(T : const(char[])[])) - { - auto s = db.prepare("SELECT parentId FROM item WHERE id = ?"); - while (true) { - s.bind(1, itemId); - auto r = s.exec(); - if (r.empty) break; - auto currParentId = r.front[0]; - static if (is(T : const(char)[])) { - if (currParentId == parentId) return true; - } else { - foreach (id; parentId) if (currParentId == id) return true; - } - itemId = currParentId.dup; - } - return false; - } - - private Item buildItem(Statement.Result result) - { - assert(!result.empty && result.front.length == 8); - Item item = { - id: result.front[0].dup, - path: computePath(result.front[0]), - name: result.front[1].dup, - eTag: result.front[3].dup, - cTag: result.front[4].dup, - mtime: SysTime.fromISOExtString(result.front[5]), - parentId: result.front[6].dup, - crc32: result.front[7].dup - }; - switch (result.front[2]) { - case "file": item.type = ItemType.file; break; - case "dir": item.type = ItemType.dir; break; - default: assert(0); - } - return item; - } - - private string computePath(const(char)[] id) - { - auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?"); - string path; - while (true) { - s.bind(1, id); - auto r = s.exec(); - if (r.empty) break; - if (path) path = r.front[0].idup ~ "/" ~ path; - else path = r.front[0].dup; - id = r.front[1].dup; - } - return path; - } - - private string computePath(const(char)[] name, const(char)[] parentId) - { - auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?"); - string path = name.dup; - while (true) { - s.bind(1, parentId); - auto r = s.exec(); - if (r.empty) break; - path = r.front[0].idup ~ "/" ~ path; - parentId = r.front[1].dup; - } - return path; - } -} diff --git a/src/itemdb.d b/src/itemdb.d index 095ff59e..b45753e1 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -224,7 +224,7 @@ final class ItemDatabase id = r.front[1].dup; } // HACK: skip "root/" - if (path.length < 5) return null; + if (path.length < 5) return "."; return path[5 .. $]; } diff --git a/src/main.d b/src/main.d index f9586294..c61b4b3b 100644 --- a/src/main.d +++ b/src/main.d @@ -1,11 +1,11 @@ import std.getopt, std.file, std.process, std.stdio; -import config, monitor, onedrive, sync; +import config, itemdb, monitor, onedrive, sync; string ver = "1.0"; void main(string[] args) { - bool monitor, resync, resetLocal, resetRemote, verbose; + bool monitor, resync, verbose; try { writeln("OneDrive Client for Linux v", ver); auto opt = getopt( @@ -29,10 +29,10 @@ void main(string[] args) string configFilePath = configDirName ~ "/config"; string refreshTokenFilePath = configDirName ~ "/refresh_token"; string statusTokenFilePath = configDirName ~ "/status_token"; - string databaseFilePath = configDirName ~ "/database"; + string databaseFilePath = configDirName ~ "/items.db"; - if (resync || resetLocal || resetRemote) { - if (verbose) writeln("Deleting the current status ..."); + if (resync) { + if (verbose) writeln("Deleting the saved status ..."); if (exists(databaseFilePath)) remove(databaseFilePath); if (exists(statusTokenFilePath)) remove(statusTokenFilePath); } @@ -54,17 +54,33 @@ void main(string[] args) } // TODO check if the token is valid + if (verbose) writeln("Opening the item database ..."); + auto itemdb = new ItemDatabase(databaseFilePath); + if (verbose) writeln("Initializing the Synchronization Engine ..."); - auto sync = new SyncEngine(cfg, onedrive); + auto sync = new SyncEngine(cfg, onedrive, itemdb, verbose); + sync.onStatusToken = (string statusToken) { + std.file.write(statusTokenFilePath, statusToken); + }; + try { + string statusToken = readText(statusTokenFilePath); + sync.setStatusToken(statusToken); + } catch (FileException e) { + // swallow exception + } + + string syncDir = cfg.get("sync_dir"); + chdir(syncDir); sync.applyDifferences(); sync.uploadDifferences(); + return; if (monitor) { if (verbose) writeln("Monitoring for changes ..."); Monitor m; m.onDirCreated = delegate(string path) { if (verbose) writeln("[M] Directory created: ", path); - sync.createFolderItem(path); + sync.uploadCreateDir(path); sync.uploadDifferences(path); }; m.onFileChanged = delegate(string path) { @@ -80,10 +96,10 @@ void main(string[] args) sync.moveItem(from, to); }; m.init(); - string syncDir = cfg.get("sync_dir"); - chdir(syncDir); m.addRecursive("test"); while (true) m.update(); // TODO download changes } + + destroy(sync); } diff --git a/src/onedrive.d b/src/onedrive.d index 152624e9..1d77d394 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -105,7 +105,7 @@ final class OneDriveApi JSONValue viewChangesByPath(const(char)[] path, const(char)[] statusToken) { checkAccessTokenExpired(); - char[] url = itemByPathUrl ~ encodeComponent(path).dup ~ ":/view.changes"; + string url = itemByPathUrl ~ encodeComponent(path) ~ ":/view.changes"; url ~= "?select=id,name,eTag,cTag,deleted,file,folder,fileSystemInfo,parentReference"; if (statusToken) url ~= "&token=" ~ statusToken; return get(url); @@ -129,7 +129,7 @@ final class OneDriveApi JSONValue simpleUpload(string localPath, const(char)[] remotePath, const(char)[] eTag = null) { checkAccessTokenExpired(); - char[] url = itemByPathUrl ~ remotePath ~ ":/content"; + string url = itemByPathUrl ~ encodeComponent(remotePath) ~ ":/content"; if (!eTag) url ~= "?@name.conflictBehavior=fail"; ubyte[] content; http.onReceive = (ubyte[] data) { @@ -137,9 +137,10 @@ final class OneDriveApi return data.length; }; if (eTag) http.addRequestHeader("If-Match", eTag); + http.addRequestHeader("Content-Type", "application/octet-stream"); upload(localPath, url, http); - // remove the if-match header - if (eTag) setAccessToken(accessToken); + // remove the headers + setAccessToken(accessToken); checkHttpCode(); return parseJSON(content); } @@ -171,7 +172,7 @@ final class OneDriveApi //https://dev.onedrive.com/items/create.htm JSONValue createByPath(const(char)[] parentPath, JSONValue item) { - char[] url = itemByPathUrl ~ parentPath ~ ":/children"; + string url = itemByPathUrl ~ encodeComponent(parentPath) ~ ":/children"; http.addRequestHeader("Content-Type", "application/json"); auto result = post(url, item.toString()); // remove the if-match header diff --git a/src/sync.d b/src/sync.d index 897ab6fd..cb683d27 100644 --- a/src/sync.d +++ b/src/sync.d @@ -1,5 +1,5 @@ import core.exception: RangeError; -import std.datetime, std.file, std.json, std.path, std.stdio; +import std.algorithm, std.datetime, std.file, std.json, std.path, std.stdio; import config, itemdb, onedrive, util; private bool isItemFolder(const ref JSONValue item) @@ -51,6 +51,7 @@ final class SyncEngine private ItemDatabase itemdb; private bool verbose; private string statusToken; + private string[] skippedItems; private string[] itemsToDelete; void delegate(string) onStatusToken; @@ -83,6 +84,9 @@ final class SyncEngine } while (changes["@changes.hasMoreChanges"].type == JSON_TYPE.TRUE); // delete items in itemsToDelete deleteItems(); + // empty the skipped items + skippedItems.length = 0; + assumeSafeAppend(skippedItems); } private void applyDifference(JSONValue item) @@ -102,9 +106,6 @@ final class SyncEngine cached = false; } - // skip items already downloaded - //if (cached && cachedItem.eTag == eTag) return; - ItemType type; if (isItemDeleted(item)) { if (verbose) writeln("The item is marked for deletion"); @@ -117,14 +118,20 @@ final class SyncEngine if (verbose) writeln("The item is a directory"); type = ItemType.dir; } else { - writeln("The item is neither a file nor a directory"); - //skippedFolders ~= id; + if (verbose) writeln("The item is neither a file nor a directory"); + skippedItems ~= id; + return; + } + + string parentId = item["parentReference"].object["id"].str; + if (skippedItems.find(parentId).length != 0) { + if (verbose) writeln("The item is a children of a skipped item"); + skippedItems ~= id; return; } string cTag = item["cTag"].str; string mtime = item["fileSystemInfo"].object["lastModifiedDateTime"].str; - string parentId = item["parentReference"].object["id"].str; string crc32; if (type == ItemType.file) { @@ -154,38 +161,6 @@ final class SyncEngine } } - private void cacheItem(JSONValue item) - { - string id = item["id"].str; - ItemType type; - if (isItemDeleted(item)) { - itemdb.deleteById(id); - } else if (isItemFile(item)) { - type = ItemType.file; - } else if (isItemFolder(item)) { - type = ItemType.dir; - } else { - writeln("The item is neither a file nor a directory, skipping"); - return; - } - string name = item["name"].str; - string eTag = item["eTag"].str; - string cTag = item["cTag"].str; - string mtime = item["fileSystemInfo"].object["lastModifiedDateTime"].str; - string parentId = item["parentReference"].object["id"].str; - string crc32; - if (type == ItemType.file) { - try { - crc32 = item["file"].object["hashes"].object["crc32Hash"].str; - } catch (JSONException e) { - writeln("The hash is not available"); - } catch (RangeError e) { - writeln("The crc32 hash is not available"); - } - } - itemdb.insert(id, name, type, eTag, cTag, mtime, parentId, crc32); - } - private void applyDeleteItem(Item item) { itemsToDelete ~= item.path; @@ -311,17 +286,22 @@ final class SyncEngine // scan the directory for unsynced files and upload them public void uploadDifferences() { - writeln("Uploading differences ..."); - string currDir = getcwd(); - string syncDir = cfg.get("sync_dir"); - chdir(syncDir); + if (verbose) writeln("Uploading differences ..."); foreach (Item item; itemdb.selectAll()) { uploadDifference(item); } - foreach (DirEntry entry; dirEntries("test", SpanMode.breadth, false)) { - uploadDifference(entry.name/*[2 .. $]*/); + // check for new files or directories + foreach (DirEntry entry; dirEntries(".", SpanMode.breadth, false)) { + string path = entry.name[2 .. $]; // HACK: skip "./" + Item item; + if (!itemdb.selectByPath(path, item)) { + if (entry.isDir) { + uploadCreateDir(path); + } else { + uploadNewFile(path); + } + } } - chdir(currDir); } public void uploadDifferences(string path) @@ -332,123 +312,140 @@ final class SyncEngine if (itemdb.selectByPath(entry.name, item)) { uploadDifference(item); } else { - uploadNewItem(entry.name); + uploadNewFile(entry.name); } } } private void uploadDifference(Item item) { - writeln(item.path); + if (verbose) writeln(item.id, " ", item.name); if (exists(item.path)) { final switch (item.type) { case ItemType.file: if (isFile(item.path)) { - updateItem(item); + uploadItemDifferences(item); } else { - deleteItem(item); - createFolderItem(item.path); + if (verbose) writeln("The item was a file but now is a directory"); + uploadDeleteItem(item); + uploadCreateDir(item.path); } break; case ItemType.dir: - if (isDir(item.path)) { - updateItem(item); + if (!isDir(item.path)) { + if (verbose) writeln("The item was a directory but now is a file"); + uploadDeleteItem(item); + uploadNewFile(item.path); } else { - deleteItem(item); - writeln("Uploading ..."); - auto res = onedrive.simpleUpload(item.path, item.path); - cacheItem(res); + if (verbose) writeln("The item has not changed"); } break; } } else { - deleteItem(item); + if (verbose) writeln("The item has been deleted"); + uploadDeleteItem(item); } } - private void uploadDifference(const(char)[] path) - { - Item item; - if (!itemdb.selectByPath(path, item)) { - writeln("New item ", path); - uploadNewItem(path); - } - } - - // HACK - void uploadDifference2(const(char)[] path) - { - assert(isFile(path)); - Item item; - if (itemdb.selectByPath(path, item)) { - uploadDifference(item); - } else { - uploadNewItem(path); - } - } - - private void deleteItem(Item item) - { - writeln("Deleting ..."); - onedrive.deleteById(item.id, item.eTag); - itemdb.deleteById(item.id); - } - - private void updateItem(Item item) + // check if the item is changed and upload the differences + private void uploadItemDifferences(Item item) { SysTime localModifiedTime = timeLastModified(item.path); import core.time: Duration; item.mtime.fracSecs = Duration.zero; // HACK if (localModifiedTime != item.mtime) { + if (verbose) writeln("The item last modified time has changed"); string id = item.id; string eTag = item.eTag; if (item.type == ItemType.file && !testCrc32(item.path, item.crc32)) { - assert(isFile(item.path)); - writeln("Uploading ..."); - JSONValue res = onedrive.simpleUpload(item.path, item.path, item.eTag); - cacheItem(res); + if (verbose) writeln("The item content has changed"); + writeln("Uploading: ", item.path); + auto res = onedrive.simpleUpload(item.path, item.path, item.eTag); + saveItem(res); id = res["id"].str; eTag = res["eTag"].str; } - updateItemLastModifiedTime(id, eTag, localModifiedTime.toUTC()); + uploadLastModifiedTime(id, eTag, localModifiedTime.toUTC()); } else { - writeln("The item is not changed"); + if (verbose) writeln("The item has not changed"); } } - void createFolderItem(const(char)[] path) + void uploadCreateDir(const(char)[] path) { - writeln("Creating folder ..."); - JSONValue item = ["name": baseName(path).dup]; + writeln("Creating remote directory: ", path); + JSONValue item = ["name": baseName(path).idup]; item["folder"] = parseJSON("{}"); auto res = onedrive.createByPath(dirName(path), item); - cacheItem(res); + saveItem(res); } - private void updateItemLastModifiedTime(const(char)[] id, const(char)[] eTag, SysTime mtime) + private void uploadNewFile(string path) + { + writeln("Uploading: ", path); + auto res = onedrive.simpleUpload(path, path); + saveItem(res); + string id = res["id"].str; + string eTag = res["eTag"].str; + uploadLastModifiedTime(id, eTag, timeLastModified(path).toUTC()); + } + + private void uploadDeleteItem(Item item) + { + writeln("Deleting remote: ", item.path); + onedrive.deleteById(item.id, item.eTag); + itemdb.deleteById(item.id); + } + + private void uploadLastModifiedTime(const(char)[] id, const(char)[] eTag, SysTime mtime) { - writeln("Updating last modified time ..."); JSONValue mtimeJson = [ "fileSystemInfo": JSONValue([ "lastModifiedDateTime": mtime.toISOExtString() ]) ]; auto res = onedrive.updateById(id, mtimeJson, eTag); - cacheItem(res); + saveItem(res); } - private void uploadNewItem(const(char)[] path) + private void saveItem(JSONValue item) { - assert(exists(path)); - if (isFile(path)) { - writeln("Uploading file ..."); - JSONValue res = onedrive.simpleUpload(path.dup, path); - cacheItem(res); - string id = res["id"].str; - string eTag = res["eTag"].str; - updateItemLastModifiedTime(id, eTag, timeLastModified(path).toUTC()); + string id = item["id"].str; + ItemType type; + if (isItemFile(item)) { + type = ItemType.file; + } else if (isItemFolder(item)) { + type = ItemType.dir; } else { - createFolderItem(path); + assert(0); + } + string name = item["name"].str; + string eTag = item["eTag"].str; + string cTag = item["cTag"].str; + string mtime = item["fileSystemInfo"].object["lastModifiedDateTime"].str; + string parentId = item["parentReference"].object["id"].str; + string crc32; + if (type == ItemType.file) { + try { + crc32 = item["file"].object["hashes"].object["crc32Hash"].str; + } catch (JSONException e) { + // swallow exception + } catch (RangeError e) { + // swallow exception + } + } + itemdb.insert(id, name, type, eTag, cTag, mtime, parentId, crc32); + } + + // HACK + void uploadDifference2(string path) + { + assert(isFile(path)); + Item item; + if (itemdb.selectByPath(path, item)) { + uploadDifference(item); + } else { + uploadNewFile(path); } } @@ -465,7 +462,7 @@ final class SyncEngine ]); writeln(diff.toPrettyString()); auto res = onedrive.updateById(item.id, diff, item.eTag); - cacheItem(res); + saveItem(res); } void deleteByPath(const(char)[] path) @@ -475,6 +472,6 @@ final class SyncEngine if (!itemdb.selectByPath(path, item)) { throw new SyncException("Can't delete a non synced item"); } - deleteItem(item); + uploadDeleteItem(item); } } diff --git a/src/util.d b/src/util.d index 19bb2f24..458959c7 100644 --- a/src/util.d +++ b/src/util.d @@ -1,10 +1,10 @@ import std.conv: to; import std.digest.crc; import std.digest.digest; -import std.stdio; -import std.string: chomp; import std.file: exists, rename; import std.path: extension; +import std.stdio; +import std.string: chomp; private string deviceName;