diff --git a/src/itemdb.d b/src/itemdb.d index 6410fbb4..e78eee7b 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -121,61 +121,18 @@ final class ItemDatabase } } - // returns a range that go trough all items, depth first - auto selectAll() + Item[] selectChildren(const(char)[] id) { - static struct ItemRange - { - ItemDatabase itemdb; - string[] stack1, stack2; - - private this(ItemDatabase itemdb, string rootId) - { - this.itemdb = itemdb; - stack1.reserve(8); - stack2.reserve(8); - stack1 ~= rootId; - getChildren(); - } - - @property bool empty() - { - return stack2.length == 0; - } - - @property Item front() - { - Item item; - bool res = itemdb.selectById(stack2[$ - 1], item); - assert(res); - return item; - } - - void popFront() - { - stack2 = stack2[0 .. $ - 1]; - assumeSafeAppend(stack2); - if (stack1.length > 0) getChildren(); - } - - private void getChildren() - { - while (true) { - itemdb.selectItemByParentIdStmt.bind(1, stack1[$ - 1]); - stack2 ~= stack1[$ - 1]; - stack1 = stack1[0 .. $ - 1]; - assumeSafeAppend(stack1); - auto res = itemdb.selectItemByParentIdStmt.exec(); - if (res.empty) break; - else foreach (row; res) stack1 ~= row[0].dup; - } - } + selectItemByParentIdStmt.bind(1, id); + auto res = selectItemByParentIdStmt.exec(); + Item[] items; + foreach (row; res) { + Item item; + bool found = selectById(row[0], item); + assert(found); + items ~= item; } - - auto s = db.prepare("SELECT id FROM item WHERE parentId IS NULL"); - auto r = s.exec(); - assert(!r.empty()); - return ItemRange(this, r.front[0].dup); + return items; } bool selectById(const(char)[] id, out Item item) @@ -191,6 +148,7 @@ final class ItemDatabase bool selectByPath(const(char)[] path, out Item item) { + if (path == ".") path = "root"; // HACK string[2][] candidates; // [id, parentId] auto s = db.prepare("SELECT id, parentId FROM item WHERE name = ?"); s.bind(1, baseName(path)); @@ -202,14 +160,24 @@ final class ItemDatabase string[2][] newCandidates; newCandidates.reserve(candidates.length); path = dirName(path); - foreach (candidate; candidates) { - s.bind(1, candidate[1]); + if (path.length != 0) { s.bind(2, baseName(path)); - r = s.exec(); - if (!r.empty) { - string[2] c = [candidate[0], r.front[0].idup]; - newCandidates ~= c; + foreach (candidate; candidates) { + s.bind(1, candidate[1]); + r = s.exec(); + if (!r.empty) { + string[2] c = [candidate[0], r.front[0].idup]; + newCandidates ~= c; + } } + } else { + // reached the root + foreach (candidate; candidates) { + if (!candidate[1]) { + newCandidates ~= candidate; + } + } + assert(newCandidates.length <= 1); } candidates = newCandidates; } while (candidates.length > 1); diff --git a/src/main.d b/src/main.d index 63defcc9..1080cf79 100644 --- a/src/main.d +++ b/src/main.d @@ -73,21 +73,20 @@ void main(string[] args) string syncDir = cfg.get("sync_dir"); chdir(syncDir); sync.applyDifferences(); - sync.uploadDifferences(); + sync.scanForDifferences("."); + return; if (monitor) { if (verbose) writeln("Initializing monitor ..."); Monitor m; m.onDirCreated = delegate(string path) { if (verbose) writeln("[M] Directory created: ", path); - sync.uploadCreateDir(path); - // the directory could be the result of a move operation - sync.uploadDifferences(path); + sync.scanForDifferences(path); }; m.onFileChanged = delegate(string path) { if (verbose) writeln("[M] File changed: ", path); try { - sync.uploadDifference(path); + sync.scanForDifferences(path); } catch(SyncException e) { writeln(e.msg); } @@ -111,7 +110,7 @@ void main(string[] args) lastCheckTime = currTime; m.shutdown(); sync.applyDifferences(); - sync.uploadDifferences(); + sync.scanForDifferences("."); m.init(cfg, verbose); } Thread.sleep(dur!"msecs"(100)); diff --git a/src/sync.d b/src/sync.d index 9f89d91a..f51ba31f 100644 --- a/src/sync.d +++ b/src/sync.d @@ -51,8 +51,11 @@ final class SyncEngine private ItemDatabase itemdb; private bool verbose; private Regex!char skipDir, skipFile; + // token representing the last status correctly synced private string statusToken; + // list of items to skip while applying the changes downloaded private string[] skippedItems; + // list of items to delete after the changes has been downloaded private string[] itemsToDelete; void delegate(string) onStatusToken; @@ -238,7 +241,7 @@ final class SyncEngine } setTimes(newItem.path, newItem.mtime, newItem.mtime); } else { - if (verbose) writeln("The item is not changed"); + if (verbose) writeln("The item has not changed"); } } @@ -302,110 +305,115 @@ final class SyncEngine assumeSafeAppend(itemsToDelete); } - // scan the root directory for unsynced files and upload them - public void uploadDifferences() + // scan the given directory for differences + public void scanForDifferences(string path) { if (verbose) writeln("Uploading differences ..."); - // check for changed files or deleted items - foreach (Item item; itemdb.selectAll()) { - uploadDifference(item); + Item item; + if (itemdb.selectByPath(path, item)) { + uploadDifferences(item); } if (verbose) writeln("Uploading new items ..."); - // check for new files or directories - foreach (DirEntry entry; dirEntries(".", SpanMode.breadth, false)) { - string path = entry.name[2 .. $]; // HACK: skip "./" - Item item; - if (!itemdb.selectByPath(path, item)) { - if (entry.isDir) { - uploadCreateDir(path); - } else { - uploadNewFile(path); - } - } - } + uploadNewItems(path); } - /* scan the specified directory for unsynced files and upload them - NOTE: this function does not check for deleted files. */ - public void uploadDifferences(string dirname) - { - foreach (DirEntry entry; dirEntries(dirname, SpanMode.breadth, false)) { - uploadDifference(entry.name); - } - } - - private void uploadDifference(Item item) + public void uploadDifferences(Item item) { if (verbose) writeln(item.id, " ", item.name); - if (!matchFirst(name, skipFile).empty) { - if (verbose) writeln("Filtered out"); - skippedItems ~= id; - return; - } + final switch (item.type) { + case ItemType.dir: + if (!matchFirst(item.name, skipDir).empty) break; + uploadDirDifferences(item); + break; + case ItemType.file: + if (!matchFirst(item.name, skipFile).empty) break; + uploadFileDifferences(item); + break; + } + } + + private void uploadDirDifferences(Item item) + { + assert(item.type == ItemType.dir); if (exists(item.path)) { - final switch (item.type) { - case ItemType.file: - if (isFile(item.path)) { - SysTime localModifiedTime = timeLastModified(item.path); - import core.time: Duration; - item.mtime.fracSecs = Duration.zero; // HACK - if (localModifiedTime != item.mtime) { - if (verbose) writeln("The item last modified time has changed"); - string id = item.id; - string eTag = item.eTag; - if (!testCrc32(item.path, item.crc32)) { - if (verbose) writeln("The item content has changed"); - writeln("Uploading: ", item.path); - auto res = onedrive.simpleUpload(item.path, item.path, item.eTag); - saveItem(res); - id = res["id"].str; - eTag = res["eTag"].str; - } - uploadLastModifiedTime(id, eTag, localModifiedTime.toUTC()); - } else { - if (verbose) writeln("The item has not changed"); - } - } else { - if (verbose) writeln("The item was a file but now is a directory"); - uploadDeleteItem(item); - uploadCreateDir(item.path); + if (!isDir(item.path)) { + if (verbose) writeln("The item was a directory but now is a file"); + uploadDeleteItem(item); + uploadNewFile(item.path); + } else { + if (verbose) writeln("The directory has not changed"); + // loop trough the children + foreach (Item child; itemdb.selectChildren(item.id)) { + uploadDifferences(child); } - break; - case ItemType.dir: - if (!isDir(item.path)) { - if (verbose) writeln("The item was a directory but now is a file"); - uploadDeleteItem(item); - uploadNewFile(item.path); - } else { - if (verbose) writeln("The item has not changed"); - } - break; } } else { - if (verbose) writeln("The item has been deleted"); + if (verbose) writeln("The directory has been deleted"); uploadDeleteItem(item); } } - void uploadDifference(string path) + private void uploadFileDifferences(Item item) { - try { - Item item; - if (itemdb.selectByPath(path, item)) { - uploadDifference(item); - } else { - if (isDir(path)) { - uploadCreateDir(path); + assert(item.type == ItemType.file); + if (exists(item.path)) { + if (isFile(item.path)) { + SysTime localModifiedTime = timeLastModified(item.path); + import core.time: Duration; + item.mtime.fracSecs = Duration.zero; // HACK + if (localModifiedTime != item.mtime) { + if (verbose) writeln("The file last modified time has changed"); + string id = item.id; + string eTag = item.eTag; + if (!testCrc32(item.path, item.crc32)) { + if (verbose) writeln("The file content has changed"); + writeln("Uploading: ", item.path); + auto res = onedrive.simpleUpload(item.path, item.path, item.eTag); + saveItem(res); + id = res["id"].str; + eTag = res["eTag"].str; + } + uploadLastModifiedTime(id, eTag, localModifiedTime.toUTC()); } else { - uploadNewFile(path); - } + if (verbose) writeln("The file has not changed"); + } + } else { + if (verbose) writeln("The item was a file but now is a directory"); + uploadDeleteItem(item); + uploadCreateDir(item.path); } - } catch (FileException e) { - throw new SyncException(e.msg, e); + } else { + if (verbose) writeln("The file has been deleted"); + uploadDeleteItem(item); } } - void uploadCreateDir(const(char)[] path) + private void uploadNewItems(string path) + { + if (isDir(path)) { + if (matchFirst(baseName(path), skipDir).empty) { + import std.string: chompPrefix; + path = chompPrefix(path, "./"); + Item item; + if (!itemdb.selectByPath(path, item)) { + uploadCreateDir(path); + } + auto entries = dirEntries(path, SpanMode.shallow, false); + foreach (DirEntry entry; entries) { + uploadNewItems(entry.name); + } + } + } else { + if (matchFirst(baseName(path), skipFile).empty) { + Item item; + if (!itemdb.selectByPath(path, item)) { + uploadNewFile(path); + } + } + } + } + + private void uploadCreateDir(const(char)[] path) { writeln("Creating remote directory: ", path); JSONValue item = ["name": baseName(path).idup];