diff --git a/src/itemdb.d b/src/itemdb.d index 556aa0de..2a290adf 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -16,13 +16,15 @@ struct Item string cTag; SysTime mtime; string parentId; - string crc32; + string crc32Hash; + string sha1Hash; + string quickXorHash; } final class ItemDatabase { // increment this for every change in the db schema - immutable int itemDatabaseVersion = 3; + immutable int itemDatabaseVersion = 4; Database db; Statement insertItemStmt; @@ -35,14 +37,16 @@ final class ItemDatabase db = Database(filename); if (db.getVersion() == 0) { db.exec("CREATE TABLE item ( - id TEXT NOT NULL PRIMARY KEY, - name TEXT NOT NULL, - type TEXT NOT NULL, - eTag TEXT, - cTag TEXT, - mtime TEXT NOT NULL, - parentId TEXT, - crc32 TEXT, + id TEXT NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + type TEXT NOT NULL, + eTag TEXT, + cTag TEXT, + mtime TEXT NOT NULL, + parentId TEXT, + crc32Hash TEXT, + sha1Hash TEXT, + quickXorHash TEXT, FOREIGN KEY (parentId) REFERENCES item (id) ON DELETE CASCADE )"); db.exec("CREATE INDEX name_idx ON item (name)"); @@ -52,14 +56,13 @@ final class ItemDatabase } db.exec("PRAGMA foreign_keys = ON"); db.exec("PRAGMA recursive_triggers = ON"); - - insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); + insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); updateItemStmt = db.prepare(" UPDATE item - SET name = ?2, type = ?3, eTag = ?4, cTag = ?5, mtime = ?6, parentId = ?7, crc32 = ?8 + SET name = ?2, type = ?3, eTag = ?4, cTag = ?5, mtime = ?6, parentId = ?7, crc32Hash = ?8, sha1Hash = ?9, quickXorHash = ?10 WHERE id = ?1 "); - selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32 FROM item WHERE id = ?"); + selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash FROM item WHERE id = ?"); selectItemByParentIdStmt = db.prepare("SELECT id FROM item WHERE parentId = ?"); } @@ -201,13 +204,15 @@ final class ItemDatabase bind(5, cTag); bind(6, mtime.toISOExtString()); bind(7, parentId); - bind(8, crc32); + bind(8, crc32Hash); + bind(9, sha1Hash); + bind(10, quickXorHash); } } private Item buildItem(Statement.Result result) { - assert(!result.empty && result.front.length == 8); + assert(!result.empty && result.front.length == 10); Item item = { id: result.front[0].dup, name: result.front[1].dup, @@ -215,7 +220,9 @@ final class ItemDatabase cTag: result.front[4].dup, mtime: SysTime.fromISOExtString(result.front[5]), parentId: result.front[6].dup, - crc32: result.front[7].dup + crc32Hash: result.front[7].dup, + sha1Hash: result.front[8].dup, + quickXorHash: result.front[9].dup }; switch (result.front[2]) { case "file": item.type = ItemType.file; break; diff --git a/src/sync.d b/src/sync.d index 8916309f..0cdd3a5c 100644 --- a/src/sync.d +++ b/src/sync.d @@ -31,11 +31,55 @@ private bool isItemRoot(const ref JSONValue item) return ("root" in item) != null; } -private bool testCrc32(string path, const(char)[] crc32) +private Item makeItem(const ref JSONValue jsonItem) { - if (crc32) { - string localCrc32 = computeCrc32(path); - if (crc32 == localCrc32) return true; + ItemType type; + if (isItemFile(jsonItem)) { + type = ItemType.file; + } else if (isItemFolder(jsonItem)) { + type = ItemType.dir; + } else { + assert(0); + } + + Item item = { + id: jsonItem["id"].str, + name: jsonItem["name"].str, + type: type, + eTag: isItemRoot(jsonItem) ? null : jsonItem["eTag"].str, // eTag is not returned if for the root in OneDrive Biz + cTag: isItemFolder(jsonItem) ? null : jsonItem["cTag"].str, + mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str), + parentId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["id"].str + }; + + // extract the file hash + if (type == ItemType.file) { + if ("hashes" in jsonItem["file"]) { + if ("crc32Hash" in jsonItem["file"]["hashes"]) { + item.crc32Hash = jsonItem["file"]["hashes"]["crc32Hash"].str; + } else if ("sha1Hash" in jsonItem["file"]["hashes"]) { + item.sha1Hash = jsonItem["file"]["hashes"]["sha1Hash"].str; + } else if ("quickXorHash" in jsonItem["file"]["hashes"]) { + item.quickXorHash = jsonItem["file"]["hashes"]["quickXorHash"].str; + } else { + log.vlog("The file does not have any hash"); + } + } else { + log.vlog("No hashes in the file facet"); + } + } + + return item; +} + +private bool testFileHash(string path, const ref Item item) +{ + if (item.crc32Hash) { + if (item.crc32Hash == computeCrc32(path)) return true; + } else if (item.sha1Hash) { + if (item.sha1Hash == computeSha1Hash(path)) return true; + } else if (item.quickXorHash) { + if (item.quickXorHash == computeQuickXorHash(path)) return true; } return false; } @@ -193,36 +237,13 @@ final class SyncEngine } } - ItemType type; - if (isItemFile(item)) { - type = ItemType.file; - } else if (isItemFolder(item)) { - type = ItemType.dir; - } else { + if (!isItemFile(item) && !isItemFolder(item)) { log.vlog("The item is neither a file nor a directory, skipping"); skippedItems ~= id; return; } - string crc32; - if (type == ItemType.file) { - try { - crc32 = item["file"]["hashes"]["crc32Hash"].str; - } catch (JSONException e) { - log.vlog("The hash is not available"); - } - } - - Item newItem = { - id: id, - name: name, - type: type, - eTag: eTag, - cTag: "cTag" in item ? item["cTag"].str : null, - mtime: SysTime.fromISOExtString(item["fileSystemInfo"]["lastModifiedDateTime"].str), - parentId: parentId, - crc32: crc32 - }; + Item newItem = makeItem(item); if (!cached) { applyNewItem(newItem, path); @@ -305,7 +326,7 @@ final class SyncEngine } else { log.vlog("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime); } - if (testCrc32(path, item.crc32)) { + if (testFileHash(path, item)) { return true; } else { log.vlog("The local item has a different hash"); @@ -428,7 +449,7 @@ final class SyncEngine log.vlog("The file last modified time has changed"); string id = item.id; string eTag = item.eTag; - if (!testCrc32(path, item.crc32)) { + if (!testFileHash(path, item)) { log.vlog("The file content has changed"); log.log("Uploading: ", path); JSONValue response; @@ -546,30 +567,7 @@ final class SyncEngine private void saveItem(JSONValue jsonItem) { - ItemType type; - if (isItemFile(jsonItem)) { - type = ItemType.file; - } else if (isItemFolder(jsonItem)) { - type = ItemType.dir; - } else { - assert(0); - } - Item item = { - id: jsonItem["id"].str, - name: jsonItem["name"].str, - type: type, - eTag: jsonItem["eTag"].str, - cTag: "cTag" in jsonItem ? jsonItem["cTag"].str : null, - mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str), - parentId: jsonItem["parentReference"]["id"].str - }; - if (type == ItemType.file) { - try { - item.crc32 = jsonItem["file"]["hashes"]["crc32Hash"].str; - } catch (JSONException e) { - log.vlog("The hash is not available"); - } - } + Item item = makeItem(jsonItem); itemdb.upsert(item); } diff --git a/src/util.d b/src/util.d index 30721563..49150dc7 100644 --- a/src/util.d +++ b/src/util.d @@ -1,6 +1,6 @@ import std.base64; import std.conv; -import std.digest.crc; +import std.digest.crc, std.digest.sha; import std.file; import std.net.curl; import std.path; @@ -52,6 +52,17 @@ string computeCrc32(string path) return crc.finish().toHexString().dup; } +// returns the sha1 hash hex string of a file +string computeSha1Hash(string path) +{ + SHA1 sha; + auto file = File(path, "rb"); + foreach (ubyte[] data; chunks(file, 4096)) { + sha.put(data); + } + return sha.finish().toHexString().dup; +} + // returns the quickXorHash base64 string of a file string computeQuickXorHash(string path) {