diff --git a/src/itemdb.d b/src/itemdb.d index 28a6464a..28fc4712 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -23,9 +23,8 @@ struct Item { string cTag; SysTime mtime; string parentId; - string crc32Hash; - string sha1Hash; string quickXorHash; + string sha256Hash; string remoteDriveId; string remoteId; string syncStatus; @@ -34,7 +33,7 @@ struct Item { final class ItemDatabase { // increment this for every change in the db schema - immutable int itemDatabaseVersion = 10; + immutable int itemDatabaseVersion = 11; Database db; string insertItemStmt; @@ -100,12 +99,12 @@ final class ItemDatabase db.exec("PRAGMA locking_mode = EXCLUSIVE"); insertItemStmt = " - INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId, syncStatus) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14) + INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, quickXorHash, sha256Hash, remoteDriveId, remoteId, syncStatus) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13) "; updateItemStmt = " UPDATE item - SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, crc32Hash = ?9, sha1Hash = ?10, quickXorHash = ?11, remoteDriveId = ?12, remoteId = ?13, syncStatus = ?14 + SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, quickXorHash = ?9, sha256Hash = ?10, remoteDriveId = ?11, remoteId = ?12, syncStatus = ?13 WHERE driveId = ?1 AND id = ?2 "; selectItemByIdStmt = " @@ -136,9 +135,8 @@ final class ItemDatabase cTag TEXT, mtime TEXT NOT NULL, parentId TEXT, - crc32Hash TEXT, - sha1Hash TEXT, quickXorHash TEXT, + sha256Hash TEXT, remoteDriveId TEXT, remoteId TEXT, deltaLink TEXT, @@ -321,19 +319,18 @@ final class ItemDatabase bind(6, cTag); bind(7, mtime.toISOExtString()); bind(8, parentId); - bind(9, crc32Hash); - bind(10, sha1Hash); - bind(11, quickXorHash); - bind(12, remoteDriveId); - bind(13, remoteId); - bind(14, syncStatus); + bind(9, quickXorHash); + bind(10, sha256Hash); + bind(11, remoteDriveId); + bind(12, remoteId); + bind(13, syncStatus); } } private Item buildItem(Statement.Result result) { assert(!result.empty, "The result must not be empty"); - assert(result.front.length == 15, "The result must have 15 columns"); + assert(result.front.length == 14, "The result must have 14 columns"); Item item = { driveId: result.front[0].dup, id: result.front[1].dup, @@ -342,12 +339,11 @@ final class ItemDatabase cTag: result.front[5].dup, mtime: SysTime.fromISOExtString(result.front[6]), parentId: result.front[7].dup, - crc32Hash: result.front[8].dup, - sha1Hash: result.front[9].dup, - quickXorHash: result.front[10].dup, - remoteDriveId: result.front[11].dup, - remoteId: result.front[12].dup, - syncStatus: result.front[14].dup + quickXorHash: result.front[8].dup, + sha256Hash: result.front[9].dup, + remoteDriveId: result.front[10].dup, + remoteId: result.front[11].dup, + syncStatus: result.front[12].dup }; switch (result.front[3]) { case "file": item.type = ItemType.file; break; diff --git a/src/sync.d b/src/sync.d index 60e731d4..663a0de3 100644 --- a/src/sync.d +++ b/src/sync.d @@ -98,9 +98,9 @@ private bool hasQuickXorHash(const ref JSONValue item) return ("quickXorHash" in item["file"]["hashes"]) != null; } -private bool hasSha1Hash(const ref JSONValue item) +private bool hasSHA256Hash(const ref JSONValue item) { - return ("sha1Hash" in item["file"]["hashes"]) != null; + return ("sha256Hash" in item["file"]["hashes"]) != null; } private bool isDotFile(const(string) path) @@ -173,16 +173,24 @@ private Item makeItem(const ref JSONValue driveItem) // extract the file hash if (isItemFile(driveItem) && ("hashes" in driveItem["file"])) { - if ("crc32Hash" in driveItem["file"]["hashes"]) { - item.crc32Hash = driveItem["file"]["hashes"]["crc32Hash"].str; - } else if ("sha1Hash" in driveItem["file"]["hashes"]) { - item.sha1Hash = driveItem["file"]["hashes"]["sha1Hash"].str; - } else if ("quickXorHash" in driveItem["file"]["hashes"]) { + // Get quickXorHash + if ("quickXorHash" in driveItem["file"]["hashes"]) { item.quickXorHash = driveItem["file"]["hashes"]["quickXorHash"].str; } else { - log.vlog("The file does not have any hash"); + log.vdebug("quickXorHash is missing"); } - } + // sha256Hash + if ("sha256Hash" in driveItem["file"]["hashes"]) { + item.sha256Hash = driveItem["file"]["hashes"]["sha256Hash"].str; + } else { + log.vdebug("sha256Hash is missing"); + } + // No hashes .. + if ((item.quickXorHash.empty) && (item.sha256Hash.empty) ) { + // Odd .. no hash ...... + log.error("ERROR: OneDrive API inconsistency - the file does not have any hash"); + } + } if (isItemRemote(driveItem)) { item.remoteDriveId = driveItem["remoteItem"]["parentReference"]["driveId"].str; @@ -201,13 +209,11 @@ private Item makeItem(const ref JSONValue driveItem) private bool testFileHash(const(string) path, const ref Item item) { - // Try and compute the file hash - if (item.crc32Hash) { - if (item.crc32Hash == computeCrc32(path)) return true; - } else if (item.sha1Hash) { - if (item.sha1Hash == computeSha1Hash(path)) return true; - } else if (item.quickXorHash) { + // Generate QuickXORHash first before others + if (item.quickXorHash) { if (item.quickXorHash == computeQuickXorHash(path)) return true; + } else if (item.sha256Hash) { + if (item.sha256Hash == computeSHA256Hash(path)) return true; } return false; } @@ -3018,11 +3024,11 @@ final class SyncEngine OneDriveFileHash = fileDetails["file"]["hashes"]["quickXorHash"].str; } } - // Check for Sha1Hash - if (hasSha1Hash(fileDetails)) { - // Use the configured sha1Hash as reported by OneDrive - if (fileDetails["file"]["hashes"]["sha1Hash"].str != "") { - OneDriveFileHash = fileDetails["file"]["hashes"]["sha1Hash"].str; + // Check for sha256Hash + if (hasSHA256Hash(fileDetails)) { + // Use the configured sha256Hash as reported by OneDrive + if (fileDetails["file"]["hashes"]["sha256Hash"].str != "") { + OneDriveFileHash = fileDetails["file"]["hashes"]["sha256Hash"].str; } } } else { @@ -3152,9 +3158,8 @@ final class SyncEngine // A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on // does the file hash OneDrive reports match what we have locally? string quickXorHash = computeQuickXorHash(path); - string sha1Hash = computeSha1Hash(path); - if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash) || (OneDriveFileHash == sha1Hash)) { + if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash)) { // downloaded matches either size or hash log.vdebug("Downloaded file matches reported size and or reported file hash"); try { @@ -3173,7 +3178,7 @@ final class SyncEngine log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why."); } // hash error? - if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) { + if (OneDriveFileHash != quickXorHash) { // downloaded file hash does not match log.vdebug("Actual file hash: ", OneDriveFileHash); log.vdebug("OneDrive API reported hash: ", quickXorHash); @@ -6769,16 +6774,16 @@ final class SyncEngine // real id / eTag / cTag are different format for personal / business account auto sha1 = new SHA1Digest(); - ubyte[] hash1 = sha1.digest(path); + ubyte[] fakedOneDriveItemValues = sha1.digest(path); JSONValue fakeResponse; if (isDir(path)) { // path is a directory fakeResponse = [ - "id": JSONValue(toHexString(hash1)), - "cTag": JSONValue(toHexString(hash1)), - "eTag": JSONValue(toHexString(hash1)), + "id": JSONValue(toHexString(fakedOneDriveItemValues)), + "cTag": JSONValue(toHexString(fakedOneDriveItemValues)), + "eTag": JSONValue(toHexString(fakedOneDriveItemValues)), "fileSystemInfo": JSONValue([ "createdDateTime": mtime.toISOExtString(), "lastModifiedDateTime": mtime.toISOExtString() @@ -6797,9 +6802,9 @@ final class SyncEngine string quickXorHash = computeQuickXorHash(path); fakeResponse = [ - "id": JSONValue(toHexString(hash1)), - "cTag": JSONValue(toHexString(hash1)), - "eTag": JSONValue(toHexString(hash1)), + "id": JSONValue(toHexString(fakedOneDriveItemValues)), + "cTag": JSONValue(toHexString(fakedOneDriveItemValues)), + "eTag": JSONValue(toHexString(fakedOneDriveItemValues)), "fileSystemInfo": JSONValue([ "createdDateTime": mtime.toISOExtString(), "lastModifiedDateTime": mtime.toISOExtString() diff --git a/src/util.d b/src/util.d index 49fa2ab8..a46f63bb 100644 --- a/src/util.d +++ b/src/util.d @@ -48,28 +48,6 @@ void safeRemove(const(char)[] path) if (exists(path)) remove(path); } -// returns the crc32 hex string of a file -string computeCrc32(string path) -{ - CRC32 crc; - auto file = File(path, "rb"); - foreach (ubyte[] data; chunks(file, 4096)) { - crc.put(data); - } - return crc.finish().toHexString().dup; -} - -// returns the sha1 hash hex string of a file -string computeSha1Hash(string path) -{ - SHA1 sha; - auto file = File(path, "rb"); - foreach (ubyte[] data; chunks(file, 4096)) { - sha.put(data); - } - return sha.finish().toHexString().dup; -} - // returns the quickXorHash base64 string of a file string computeQuickXorHash(string path) { @@ -81,6 +59,16 @@ string computeQuickXorHash(string path) return Base64.encode(qxor.finish()); } +// returns the SHA256 hex string of a file +string computeSHA256Hash(string path) { + SHA256 sha256; + auto file = File(path, "rb"); + foreach (ubyte[] data; chunks(file, 4096)) { + sha256.put(data); + } + return sha256.finish().toHexString().dup; +} + // converts wildcards (*, ?) to regex Regex!char wild2regex(const(char)[] pattern) {