support for SHA1 and QuickXor hash algorithms

src/itemdb.d

@@ -16,13 +16,15 @@ struct Item
 	string   cTag;
 	SysTime  mtime;
 	string   parentId;
-	string   crc32;
+	string   crc32Hash;
+	string   sha1Hash;
+	string   quickXorHash;
 }
 
 final class ItemDatabase
 {
 	// increment this for every change in the db schema
-	immutable int itemDatabaseVersion = 3;
+	immutable int itemDatabaseVersion = 4;
 
 	Database db;
 	Statement insertItemStmt;
@@ -35,14 +37,16 @@ final class ItemDatabase
 		db = Database(filename);
 		if (db.getVersion() == 0) {
 			db.exec("CREATE TABLE item (
-				id       TEXT NOT NULL PRIMARY KEY,
+				id              TEXT NOT NULL PRIMARY KEY,
-				name     TEXT NOT NULL,
+				name            TEXT NOT NULL,
-				type     TEXT NOT NULL,
+				type            TEXT NOT NULL,
-				eTag     TEXT,
+				eTag            TEXT,
-				cTag     TEXT,
+				cTag            TEXT,
-				mtime    TEXT NOT NULL,
+				mtime           TEXT NOT NULL,
-				parentId TEXT,
+				parentId        TEXT,
-				crc32    TEXT,
+				crc32Hash       TEXT,
+				sha1Hash        TEXT,
+				quickXorHash    TEXT,
 				FOREIGN KEY (parentId) REFERENCES item (id) ON DELETE CASCADE
 			)");
 			db.exec("CREATE INDEX name_idx ON item (name)");
@@ -52,14 +56,13 @@ final class ItemDatabase
 		}
 		db.exec("PRAGMA foreign_keys = ON");
 		db.exec("PRAGMA recursive_triggers = ON");
-
+		insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
-		insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32) VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
 		updateItemStmt = db.prepare("
 			UPDATE item
-			SET name = ?2, type = ?3, eTag = ?4, cTag = ?5, mtime = ?6, parentId = ?7, crc32 = ?8
+			SET name = ?2, type = ?3, eTag = ?4, cTag = ?5, mtime = ?6, parentId = ?7, crc32Hash = ?8, sha1Hash = ?9, quickXorHash = ?10
 			WHERE id = ?1
 		");
-		selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32 FROM item WHERE id = ?");
+		selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash FROM item WHERE id = ?");
 		selectItemByParentIdStmt = db.prepare("SELECT id FROM item WHERE parentId = ?");
 	}
 
@@ -201,13 +204,15 @@ final class ItemDatabase
 			bind(5, cTag);
 			bind(6, mtime.toISOExtString());
 			bind(7, parentId);
-			bind(8, crc32);
+			bind(8, crc32Hash);
+			bind(9, sha1Hash);
+			bind(10, quickXorHash);
 		}
 	}
 
 	private Item buildItem(Statement.Result result)
 	{
-		assert(!result.empty && result.front.length == 8);
+		assert(!result.empty && result.front.length == 10);
 		Item item = {
 			id: result.front[0].dup,
 			name: result.front[1].dup,
@@ -215,7 +220,9 @@ final class ItemDatabase
 			cTag: result.front[4].dup,
 			mtime: SysTime.fromISOExtString(result.front[5]),
 			parentId: result.front[6].dup,
-			crc32: result.front[7].dup
+			crc32Hash: result.front[7].dup,
+			sha1Hash: result.front[8].dup,
+			quickXorHash: result.front[9].dup
 		};
 		switch (result.front[2]) {
 			case "file": item.type = ItemType.file; break;

src/sync.d

@@ -31,11 +31,55 @@ private bool isItemRoot(const ref JSONValue item)
 	return ("root" in item) != null;
 }
 
-private bool testCrc32(string path, const(char)[] crc32)
+private Item makeItem(const ref JSONValue jsonItem)
 {
-	if (crc32) {
+	ItemType type;
-		string localCrc32 = computeCrc32(path);
+	if (isItemFile(jsonItem)) {
-		if (crc32 == localCrc32) return true;
+		type = ItemType.file;
+	} else if (isItemFolder(jsonItem)) {
+		type = ItemType.dir;
+	} else {
+		assert(0);
+	}
+
+	Item item = {
+		id: jsonItem["id"].str,
+		name: jsonItem["name"].str,
+		type: type,
+		eTag: isItemRoot(jsonItem) ? null : jsonItem["eTag"].str, // eTag is not returned if for the root in OneDrive Biz
+		cTag: isItemFolder(jsonItem) ? null : jsonItem["cTag"].str,
+		mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str),
+		parentId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["id"].str
+	};
+
+	// extract the file hash
+	if (type == ItemType.file) {
+		if ("hashes" in jsonItem["file"]) {
+			if ("crc32Hash" in jsonItem["file"]["hashes"]) {
+				item.crc32Hash = jsonItem["file"]["hashes"]["crc32Hash"].str;
+			} else if ("sha1Hash" in jsonItem["file"]["hashes"]) {
+				item.sha1Hash = jsonItem["file"]["hashes"]["sha1Hash"].str;
+			} else if ("quickXorHash" in jsonItem["file"]["hashes"]) {
+				item.quickXorHash = jsonItem["file"]["hashes"]["quickXorHash"].str;
+			} else {
+				log.vlog("The file does not have any hash");
+			}
+		} else {
+			log.vlog("No hashes in the file facet");
+		}
+	}
+
+	return item;
+}
+
+private bool testFileHash(string path, const ref Item item)
+{
+	if (item.crc32Hash) {
+		if (item.crc32Hash == computeCrc32(path)) return true;
+	} else if (item.sha1Hash) {
+		if (item.sha1Hash == computeSha1Hash(path)) return true;
+	} else if (item.quickXorHash) {
+		if (item.quickXorHash == computeQuickXorHash(path)) return true;
 	}
 	return false;
 }
@@ -193,36 +237,13 @@ final class SyncEngine
 			}
 		}
 
-		ItemType type;
+		if (!isItemFile(item) && !isItemFolder(item)) {
-		if (isItemFile(item)) {
-			type = ItemType.file;
-		} else if (isItemFolder(item)) {
-			type = ItemType.dir;
-		} else {
 			log.vlog("The item is neither a file nor a directory, skipping");
 			skippedItems ~= id;
 			return;
 		}
 
-		string crc32;
+		Item newItem = makeItem(item);
-		if (type == ItemType.file) {
-			try {
-				crc32 = item["file"]["hashes"]["crc32Hash"].str;
-			} catch (JSONException e) {
-				log.vlog("The hash is not available");
-			}
-		}
-
-		Item newItem = {
-			id: id,
-			name: name,
-			type: type,
-			eTag: eTag,
-			cTag: "cTag" in item ? item["cTag"].str : null,
-			mtime: SysTime.fromISOExtString(item["fileSystemInfo"]["lastModifiedDateTime"].str),
-			parentId: parentId,
-			crc32: crc32
-		};
 
 		if (!cached) {
 			applyNewItem(newItem, path);
@@ -305,7 +326,7 @@ final class SyncEngine
 				} else {
 					log.vlog("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime);
 				}
-				if (testCrc32(path, item.crc32)) {
+				if (testFileHash(path, item)) {
 					return true;
 				} else {
 					log.vlog("The local item has a different hash");
@@ -428,7 +449,7 @@ final class SyncEngine
 					log.vlog("The file last modified time has changed");
 					string id = item.id;
 					string eTag = item.eTag;
-					if (!testCrc32(path, item.crc32)) {
+					if (!testFileHash(path, item)) {
 						log.vlog("The file content has changed");
 						log.log("Uploading: ", path);
 						JSONValue response;
@@ -546,30 +567,7 @@ final class SyncEngine
 
 	private void saveItem(JSONValue jsonItem)
 	{
-		ItemType type;
+		Item item = makeItem(jsonItem);
-		if (isItemFile(jsonItem)) {
-			type = ItemType.file;
-		} else if (isItemFolder(jsonItem)) {
-			type = ItemType.dir;
-		} else {
-			assert(0);
-		}
-		Item item = {
-			id: jsonItem["id"].str,
-			name: jsonItem["name"].str,
-			type: type,
-			eTag: jsonItem["eTag"].str,
-			cTag: "cTag" in jsonItem ? jsonItem["cTag"].str : null,
-			mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str),
-			parentId: jsonItem["parentReference"]["id"].str
-		};
-		if (type == ItemType.file) {
-			try {
-				item.crc32 = jsonItem["file"]["hashes"]["crc32Hash"].str;
-			} catch (JSONException e) {
-				log.vlog("The hash is not available");
-			}
-		}
 		itemdb.upsert(item);
 	}
 

src/util.d

@@ -1,6 +1,6 @@
 import std.base64;
 import std.conv;
-import std.digest.crc;
+import std.digest.crc, std.digest.sha;
 import std.file;
 import std.net.curl;
 import std.path;
@@ -52,6 +52,17 @@ string computeCrc32(string path)
 	return crc.finish().toHexString().dup;
 }
 
+// returns the sha1 hash hex string of a file
+string computeSha1Hash(string path)
+{
+	SHA1 sha;
+	auto file = File(path, "rb");
+	foreach (ubyte[] data; chunks(file, 4096)) {
+		sha.put(data);
+	}
+	return sha.finish().toHexString().dup;
+}
+
 // returns the quickXorHash base64 string of a file
 string computeQuickXorHash(string path)
 {