implemented filter in sync

src/itemdb.d

@@ -121,61 +121,18 @@ final class ItemDatabase
 		}
 	}
 
-	// returns a range that go trough all items, depth first
+	Item[] selectChildren(const(char)[] id)
-	auto selectAll()
 	{
-		static struct ItemRange
+		selectItemByParentIdStmt.bind(1, id);
-		{
+		auto res = selectItemByParentIdStmt.exec();
-			ItemDatabase itemdb;
+		Item[] items;
-			string[] stack1, stack2;
+		foreach (row; res) {
-
+			Item item;
-			private this(ItemDatabase itemdb, string rootId)
+			bool found = selectById(row[0], item);
-			{
+			assert(found);
-				this.itemdb = itemdb;
+			items ~= item;
-				stack1.reserve(8);
-				stack2.reserve(8);
-				stack1 ~= rootId;
-				getChildren();
-			}
-
-			@property bool empty()
-			{
-				return stack2.length == 0;
-			}
-
-			@property Item front()
-			{
-				Item item;
-				bool res = itemdb.selectById(stack2[$ - 1], item);
-				assert(res);
-				return item;
-			}
-
-			void popFront()
-			{
-				stack2 = stack2[0 .. $ - 1];
-				assumeSafeAppend(stack2);
-				if (stack1.length > 0) getChildren();
-			}
-
-			private void getChildren()
-			{
-				while (true) {
-					itemdb.selectItemByParentIdStmt.bind(1, stack1[$ - 1]);
-					stack2 ~= stack1[$ - 1];
-					stack1 = stack1[0 .. $ - 1];
-					assumeSafeAppend(stack1);
-					auto res = itemdb.selectItemByParentIdStmt.exec();
-					if (res.empty) break;
-					else foreach (row; res) stack1 ~= row[0].dup;
-				}
-			}
 		}
-
+		return items;
-		auto s = db.prepare("SELECT id FROM item WHERE parentId IS NULL");
-		auto r = s.exec();
-		assert(!r.empty());
-		return ItemRange(this, r.front[0].dup);
 	}
 
 	bool selectById(const(char)[] id, out Item item)
@@ -191,6 +148,7 @@ final class ItemDatabase
 
 	bool selectByPath(const(char)[] path, out Item item)
 	{
+		if (path == ".") path = "root"; // HACK
 		string[2][] candidates; // [id, parentId]
 		auto s = db.prepare("SELECT id, parentId FROM item WHERE name = ?");
 		s.bind(1, baseName(path));
@@ -202,14 +160,24 @@ final class ItemDatabase
 				string[2][] newCandidates;
 				newCandidates.reserve(candidates.length);
 				path = dirName(path);
-				foreach (candidate; candidates) {
+				if (path.length != 0) {
-					s.bind(1, candidate[1]);
 					s.bind(2, baseName(path));
-					r = s.exec();
+					foreach (candidate; candidates) {
-					if (!r.empty) {
+						s.bind(1, candidate[1]);
-						string[2] c = [candidate[0], r.front[0].idup];
+						r = s.exec();
-						newCandidates ~= c;
+						if (!r.empty) {
+							string[2] c = [candidate[0], r.front[0].idup];
+							newCandidates ~= c;
+						}
 					}
+				} else {
+					// reached the root
+					foreach (candidate; candidates) {
+						if (!candidate[1]) {
+							newCandidates ~= candidate;
+						}
+					}
+					assert(newCandidates.length <= 1);
 				}
 				candidates = newCandidates;
 			} while (candidates.length > 1);

src/main.d

@@ -73,21 +73,20 @@ void main(string[] args)
 	string syncDir = cfg.get("sync_dir");
 	chdir(syncDir);
 	sync.applyDifferences();
-	sync.uploadDifferences();
+	sync.scanForDifferences(".");
+	return;
 
 	if (monitor) {
 		if (verbose) writeln("Initializing monitor ...");
 		Monitor m;
 		m.onDirCreated = delegate(string path) {
 			if (verbose) writeln("[M] Directory created: ", path);
-			sync.uploadCreateDir(path);
+			sync.scanForDifferences(path);
-			// the directory could be the result of a move operation
-			sync.uploadDifferences(path);
 		};
 		m.onFileChanged = delegate(string path) {
 			if (verbose) writeln("[M] File changed: ", path);
 			try {
-				sync.uploadDifference(path);
+				sync.scanForDifferences(path);
 			} catch(SyncException e) {
 				writeln(e.msg);
 			}
@@ -111,7 +110,7 @@ void main(string[] args)
 				lastCheckTime = currTime;
 				m.shutdown();
 				sync.applyDifferences();
-				sync.uploadDifferences();
+				sync.scanForDifferences(".");
 				m.init(cfg, verbose);
 			}
 			Thread.sleep(dur!"msecs"(100));

src/sync.d

@@ -51,8 +51,11 @@ final class SyncEngine
 	private ItemDatabase itemdb;
 	private bool verbose;
 	private Regex!char skipDir, skipFile;
+	// token representing the last status correctly synced
 	private string statusToken;
+	// list of items to skip while applying the changes downloaded
 	private string[] skippedItems;
+	// list of items to delete after the changes has been downloaded
 	private string[] itemsToDelete;
 
 	void delegate(string) onStatusToken;
@@ -238,7 +241,7 @@ final class SyncEngine
 			}
 			setTimes(newItem.path, newItem.mtime, newItem.mtime);
 		} else {
-			if (verbose) writeln("The item is not changed");
+			if (verbose) writeln("The item has not changed");
 		}
 	}
 
@@ -302,110 +305,115 @@ final class SyncEngine
 		assumeSafeAppend(itemsToDelete);
 	}
 
-	// scan the root directory for unsynced files and upload them
+	// scan the given directory for differences
-	public void uploadDifferences()
+	public void scanForDifferences(string path)
 	{
 		if (verbose) writeln("Uploading differences ...");
-		// check for changed files or deleted items
+		Item item;
-		foreach (Item item; itemdb.selectAll()) {
+		if (itemdb.selectByPath(path, item)) {
-			uploadDifference(item);
+			uploadDifferences(item);
 		}
 		if (verbose) writeln("Uploading new items ...");
-		// check for new files or directories
+		uploadNewItems(path);
-		foreach (DirEntry entry; dirEntries(".", SpanMode.breadth, false)) {
-			string path = entry.name[2 .. $]; // HACK: skip "./"
-			Item item;
-			if (!itemdb.selectByPath(path, item)) {
-				if (entry.isDir) {
-					uploadCreateDir(path);
-				} else {
-					uploadNewFile(path);
-				 }
-			}
-		}
 	}
 
-	/* scan the specified directory for unsynced files and upload them
+	public void uploadDifferences(Item item)
-	   NOTE: this function does not check for deleted files. */
-	public void uploadDifferences(string dirname)
-	{
-		foreach (DirEntry entry; dirEntries(dirname, SpanMode.breadth, false)) {
-			uploadDifference(entry.name);
-		}
-	}
-
-	private void uploadDifference(Item item)
 	{
 		if (verbose) writeln(item.id, " ", item.name);
-			if (!matchFirst(name, skipFile).empty) {
+		final switch (item.type) {
-				if (verbose) writeln("Filtered out");
+		case ItemType.dir:
-				skippedItems ~= id;
+			if (!matchFirst(item.name, skipDir).empty) break;
-				return;
+			uploadDirDifferences(item);
-			}
+			break;
+		case ItemType.file:
+			if (!matchFirst(item.name, skipFile).empty) break;
+			uploadFileDifferences(item);
+			break;
+		}
+	}
+
+	private void uploadDirDifferences(Item item)
+	{
+		assert(item.type == ItemType.dir);
 		if (exists(item.path)) {
-			final switch (item.type) {
+			if (!isDir(item.path)) {
-			case ItemType.file:
+				if (verbose) writeln("The item was a directory but now is a file");
-				if (isFile(item.path)) {
+				uploadDeleteItem(item);
-					SysTime localModifiedTime = timeLastModified(item.path);
+				uploadNewFile(item.path);
-					import core.time: Duration;
+			} else {
-					item.mtime.fracSecs = Duration.zero; // HACK
+				if (verbose) writeln("The directory has not changed");
-					if (localModifiedTime != item.mtime) {
+				// loop trough the children
-						if (verbose) writeln("The item last modified time has changed");
+				foreach (Item child; itemdb.selectChildren(item.id)) {
-						string id = item.id;
+					uploadDifferences(child);
-						string eTag = item.eTag;
-						if (!testCrc32(item.path, item.crc32)) {
-							if (verbose) writeln("The item content has changed");
-							writeln("Uploading: ", item.path);
-							auto res = onedrive.simpleUpload(item.path, item.path, item.eTag);
-							saveItem(res);
-							id = res["id"].str;
-							eTag = res["eTag"].str;
-						}
-						uploadLastModifiedTime(id, eTag, localModifiedTime.toUTC());
-					} else {
-						if (verbose) writeln("The item has not changed");
-					}
-				} else {
-					if (verbose) writeln("The item was a file but now is a directory");
-					uploadDeleteItem(item);
-					uploadCreateDir(item.path);
 				}
-				break;
-			case ItemType.dir:
-				if (!isDir(item.path)) {
-					if (verbose) writeln("The item was a directory but now is a file");
-					uploadDeleteItem(item);
-					uploadNewFile(item.path);
-				} else {
-					if (verbose) writeln("The item has not changed");
-				}
-				break;
 			}
 		} else {
-			if (verbose) writeln("The item has been deleted");
+			if (verbose) writeln("The directory has been deleted");
 			uploadDeleteItem(item);
 		}
 	}
 
-	void uploadDifference(string path)
+	private void uploadFileDifferences(Item item)
 	{
-		try {
+		assert(item.type == ItemType.file);
-			Item item;
+		if (exists(item.path)) {
-			if (itemdb.selectByPath(path, item)) {
+			if (isFile(item.path)) {
-				uploadDifference(item);
+				SysTime localModifiedTime = timeLastModified(item.path);
-			} else {
+				import core.time: Duration;
-				if (isDir(path)) {
+				item.mtime.fracSecs = Duration.zero; // HACK
-					uploadCreateDir(path);
+				if (localModifiedTime != item.mtime) {
+					if (verbose) writeln("The file last modified time has changed");
+					string id = item.id;
+					string eTag = item.eTag;
+					if (!testCrc32(item.path, item.crc32)) {
+						if (verbose) writeln("The file content has changed");
+						writeln("Uploading: ", item.path);
+						auto res = onedrive.simpleUpload(item.path, item.path, item.eTag);
+						saveItem(res);
+						id = res["id"].str;
+						eTag = res["eTag"].str;
+					}
+					uploadLastModifiedTime(id, eTag, localModifiedTime.toUTC());
 				} else {
-					uploadNewFile(path);
+					if (verbose) writeln("The file has not changed");
-				 }
+				}
+			} else {
+				if (verbose) writeln("The item was a file but now is a directory");
+				uploadDeleteItem(item);
+				uploadCreateDir(item.path);
 			}
-		} catch (FileException e) {
+		} else {
-			throw new SyncException(e.msg, e);
+			if (verbose) writeln("The file has been deleted");
+			uploadDeleteItem(item);
 		}
 	}
 
-	void uploadCreateDir(const(char)[] path)
+	private void uploadNewItems(string path)
+	{
+		if (isDir(path)) {
+			if (matchFirst(baseName(path), skipDir).empty) {
+				import std.string: chompPrefix;
+				path = chompPrefix(path, "./");
+				Item item;
+				if (!itemdb.selectByPath(path, item)) {
+					uploadCreateDir(path);
+				}
+				auto entries = dirEntries(path, SpanMode.shallow, false);
+				foreach (DirEntry entry; entries) {
+					uploadNewItems(entry.name);
+				}
+			}
+		} else {
+			if (matchFirst(baseName(path), skipFile).empty) {
+				Item item;
+				if (!itemdb.selectByPath(path, item)) {
+					uploadNewFile(path);
+				}
+			}
+		}
+	}
+
+	private void uploadCreateDir(const(char)[] path)
 	{
 		writeln("Creating remote directory: ", path);
 		JSONValue item = ["name": baseName(path).idup];