deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-19 14:16:36 +02:00
Code Issues Projects Releases Wiki Activity

Remove sha1 use and cleanup defunct remaining crc32 use (#2424)

Browse source 
* Remove sha1 from being used by the client as this is being depreciated by Microsoft in July 2023 - https://devblogs.microsoft.com/microsoft365dev/deprecation-of-sha1hash-on-onedrive-personal/
* Complete the removal of crc32 as this is also no longer present for a long time, but some code elements still existed
* Only compute quickXorHash, not quickXorHash and sha256Hash as computing sha256Hash is CPU expensive
* Update cache database stored items to only store quickXorHash and sha256Hash values (remove crc32 and sha1)
This commit is contained in:
abraunegg 2023-06-20 06:55:00 +10:00 committed by GitHub
parent c9fe8ad051
commit 06420c9a0a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 62 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
src/itemdb.d
View file

@ -23,9 +23,8 @@ struct Item {
string cTag; string cTag;
SysTime mtime; SysTime mtime;
string parentId; string parentId;
string crc32Hash;
string sha1Hash;
string quickXorHash; string quickXorHash;
string sha256Hash;
string remoteDriveId; string remoteDriveId;
string remoteId; string remoteId;
string syncStatus; string syncStatus;
@ -34,7 +33,7 @@ struct Item {
final class ItemDatabase final class ItemDatabase
{ {
// increment this for every change in the db schema // increment this for every change in the db schema
immutable int itemDatabaseVersion = 10; immutable int itemDatabaseVersion = 11;
Database db; Database db;
string insertItemStmt; string insertItemStmt;
@ -100,12 +99,12 @@ final class ItemDatabase
db.exec("PRAGMA locking_mode = EXCLUSIVE"); db.exec("PRAGMA locking_mode = EXCLUSIVE");
insertItemStmt = " insertItemStmt = "
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId, syncStatus) INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, quickXorHash, sha256Hash, remoteDriveId, remoteId, syncStatus)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)
"; ";
updateItemStmt = " updateItemStmt = "
UPDATE item UPDATE item
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, crc32Hash = ?9, sha1Hash = ?10, quickXorHash = ?11, remoteDriveId = ?12, remoteId = ?13, syncStatus = ?14 SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, quickXorHash = ?9, sha256Hash = ?10, remoteDriveId = ?11, remoteId = ?12, syncStatus = ?13
WHERE driveId = ?1 AND id = ?2 WHERE driveId = ?1 AND id = ?2
"; ";
selectItemByIdStmt = " selectItemByIdStmt = "
@ -136,9 +135,8 @@ final class ItemDatabase
cTag TEXT, cTag TEXT,
mtime TEXT NOT NULL, mtime TEXT NOT NULL,
parentId TEXT, parentId TEXT,
crc32Hash TEXT,
sha1Hash TEXT,
quickXorHash TEXT, quickXorHash TEXT,
sha256Hash TEXT,
remoteDriveId TEXT, remoteDriveId TEXT,
remoteId TEXT, remoteId TEXT,
deltaLink TEXT, deltaLink TEXT,
@ -321,19 +319,18 @@ final class ItemDatabase
bind(6, cTag); bind(6, cTag);
bind(7, mtime.toISOExtString()); bind(7, mtime.toISOExtString());
bind(8, parentId); bind(8, parentId);
bind(9, crc32Hash); bind(9, quickXorHash);
bind(10, sha1Hash); bind(10, sha256Hash);
bind(11, quickXorHash); bind(11, remoteDriveId);
bind(12, remoteDriveId); bind(12, remoteId);
bind(13, remoteId); bind(13, syncStatus);
bind(14, syncStatus);
} }
} }
private Item buildItem(Statement.Result result) private Item buildItem(Statement.Result result)
{ {
assert(!result.empty, "The result must not be empty"); assert(!result.empty, "The result must not be empty");
assert(result.front.length == 15, "The result must have 15 columns"); assert(result.front.length == 14, "The result must have 14 columns");
Item item = { Item item = {
driveId: result.front[0].dup, driveId: result.front[0].dup,
id: result.front[1].dup, id: result.front[1].dup,
@ -342,12 +339,11 @@ final class ItemDatabase
cTag: result.front[5].dup, cTag: result.front[5].dup,
mtime: SysTime.fromISOExtString(result.front[6]), mtime: SysTime.fromISOExtString(result.front[6]),
parentId: result.front[7].dup, parentId: result.front[7].dup,
crc32Hash: result.front[8].dup, quickXorHash: result.front[8].dup,
sha1Hash: result.front[9].dup, sha256Hash: result.front[9].dup,
quickXorHash: result.front[10].dup, remoteDriveId: result.front[10].dup,
remoteDriveId: result.front[11].dup, remoteId: result.front[11].dup,
remoteId: result.front[12].dup, syncStatus: result.front[12].dup
syncStatus: result.front[14].dup
}; };
switch (result.front[3]) { switch (result.front[3]) {
case "file": item.type = ItemType.file; break; case "file": item.type = ItemType.file; break;

65
src/sync.d
View file

@ -98,9 +98,9 @@ private bool hasQuickXorHash(const ref JSONValue item)
return ("quickXorHash" in item["file"]["hashes"]) != null; return ("quickXorHash" in item["file"]["hashes"]) != null;
} }
private bool hasSha1Hash(const ref JSONValue item) private bool hasSHA256Hash(const ref JSONValue item)
{ {
return ("sha1Hash" in item["file"]["hashes"]) != null; return ("sha256Hash" in item["file"]["hashes"]) != null;
} }
private bool isDotFile(const(string) path) private bool isDotFile(const(string) path)
@ -173,16 +173,24 @@ private Item makeItem(const ref JSONValue driveItem)
// extract the file hash // extract the file hash
if (isItemFile(driveItem) && ("hashes" in driveItem["file"])) { if (isItemFile(driveItem) && ("hashes" in driveItem["file"])) {
if ("crc32Hash" in driveItem["file"]["hashes"]) { // Get quickXorHash
item.crc32Hash = driveItem["file"]["hashes"]["crc32Hash"].str; if ("quickXorHash" in driveItem["file"]["hashes"]) {
} else if ("sha1Hash" in driveItem["file"]["hashes"]) {
item.sha1Hash = driveItem["file"]["hashes"]["sha1Hash"].str;
} else if ("quickXorHash" in driveItem["file"]["hashes"]) {
item.quickXorHash = driveItem["file"]["hashes"]["quickXorHash"].str; item.quickXorHash = driveItem["file"]["hashes"]["quickXorHash"].str;
} else { } else {
log.vlog("The file does not have any hash"); log.vdebug("quickXorHash is missing");
} }
} // sha256Hash
if ("sha256Hash" in driveItem["file"]["hashes"]) {
item.sha256Hash = driveItem["file"]["hashes"]["sha256Hash"].str;
} else {
log.vdebug("sha256Hash is missing");
}
// No hashes ..
if ((item.quickXorHash.empty) && (item.sha256Hash.empty) ) {
// Odd .. no hash ......
log.error("ERROR: OneDrive API inconsistency - the file does not have any hash");
}
}
if (isItemRemote(driveItem)) { if (isItemRemote(driveItem)) {
item.remoteDriveId = driveItem["remoteItem"]["parentReference"]["driveId"].str; item.remoteDriveId = driveItem["remoteItem"]["parentReference"]["driveId"].str;
@ -201,13 +209,11 @@ private Item makeItem(const ref JSONValue driveItem)
private bool testFileHash(const(string) path, const ref Item item) private bool testFileHash(const(string) path, const ref Item item)
{ {
// Try and compute the file hash // Generate QuickXORHash first before others
if (item.crc32Hash) { if (item.quickXorHash) {
if (item.crc32Hash == computeCrc32(path)) return true;
} else if (item.sha1Hash) {
if (item.sha1Hash == computeSha1Hash(path)) return true;
} else if (item.quickXorHash) {
if (item.quickXorHash == computeQuickXorHash(path)) return true; if (item.quickXorHash == computeQuickXorHash(path)) return true;
} else if (item.sha256Hash) {
if (item.sha256Hash == computeSHA256Hash(path)) return true;
} }
return false; return false;
} }
@ -3018,11 +3024,11 @@ final class SyncEngine
OneDriveFileHash = fileDetails["file"]["hashes"]["quickXorHash"].str; OneDriveFileHash = fileDetails["file"]["hashes"]["quickXorHash"].str;
} }
} }
// Check for Sha1Hash // Check for sha256Hash
if (hasSha1Hash(fileDetails)) { if (hasSHA256Hash(fileDetails)) {
// Use the configured sha1Hash as reported by OneDrive // Use the configured sha256Hash as reported by OneDrive
if (fileDetails["file"]["hashes"]["sha1Hash"].str != "") { if (fileDetails["file"]["hashes"]["sha256Hash"].str != "") {
OneDriveFileHash = fileDetails["file"]["hashes"]["sha1Hash"].str; OneDriveFileHash = fileDetails["file"]["hashes"]["sha256Hash"].str;
} }
} }
} else { } else {
@ -3152,9 +3158,8 @@ final class SyncEngine
// A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on // A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on
// does the file hash OneDrive reports match what we have locally? // does the file hash OneDrive reports match what we have locally?
string quickXorHash = computeQuickXorHash(path); string quickXorHash = computeQuickXorHash(path);
string sha1Hash = computeSha1Hash(path);
if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash) || (OneDriveFileHash == sha1Hash)) { if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash)) {
// downloaded matches either size or hash // downloaded matches either size or hash
log.vdebug("Downloaded file matches reported size and or reported file hash"); log.vdebug("Downloaded file matches reported size and or reported file hash");
try { try {
@ -3173,7 +3178,7 @@ final class SyncEngine
log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why."); log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why.");
} }
// hash error? // hash error?
if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) { if (OneDriveFileHash != quickXorHash) {
// downloaded file hash does not match // downloaded file hash does not match
log.vdebug("Actual file hash: ", OneDriveFileHash); log.vdebug("Actual file hash: ", OneDriveFileHash);
log.vdebug("OneDrive API reported hash: ", quickXorHash); log.vdebug("OneDrive API reported hash: ", quickXorHash);
@ -6769,16 +6774,16 @@ final class SyncEngine
// real id / eTag / cTag are different format for personal / business account // real id / eTag / cTag are different format for personal / business account
auto sha1 = new SHA1Digest(); auto sha1 = new SHA1Digest();
ubyte[] hash1 = sha1.digest(path); ubyte[] fakedOneDriveItemValues = sha1.digest(path);
JSONValue fakeResponse; JSONValue fakeResponse;
if (isDir(path)) { if (isDir(path)) {
// path is a directory // path is a directory
fakeResponse = [ fakeResponse = [
"id": JSONValue(toHexString(hash1)), "id": JSONValue(toHexString(fakedOneDriveItemValues)),
"cTag": JSONValue(toHexString(hash1)), "cTag": JSONValue(toHexString(fakedOneDriveItemValues)),
"eTag": JSONValue(toHexString(hash1)), "eTag": JSONValue(toHexString(fakedOneDriveItemValues)),
"fileSystemInfo": JSONValue([ "fileSystemInfo": JSONValue([
"createdDateTime": mtime.toISOExtString(), "createdDateTime": mtime.toISOExtString(),
"lastModifiedDateTime": mtime.toISOExtString() "lastModifiedDateTime": mtime.toISOExtString()
@ -6797,9 +6802,9 @@ final class SyncEngine
string quickXorHash = computeQuickXorHash(path); string quickXorHash = computeQuickXorHash(path);
fakeResponse = [ fakeResponse = [
"id": JSONValue(toHexString(hash1)), "id": JSONValue(toHexString(fakedOneDriveItemValues)),
"cTag": JSONValue(toHexString(hash1)), "cTag": JSONValue(toHexString(fakedOneDriveItemValues)),
"eTag": JSONValue(toHexString(hash1)), "eTag": JSONValue(toHexString(fakedOneDriveItemValues)),
"fileSystemInfo": JSONValue([ "fileSystemInfo": JSONValue([
"createdDateTime": mtime.toISOExtString(), "createdDateTime": mtime.toISOExtString(),
"lastModifiedDateTime": mtime.toISOExtString() "lastModifiedDateTime": mtime.toISOExtString()

32
src/util.d
View file

@ -48,28 +48,6 @@ void safeRemove(const(char)[] path)
if (exists(path)) remove(path); if (exists(path)) remove(path);
} }
// returns the crc32 hex string of a file
string computeCrc32(string path)
{
CRC32 crc;
auto file = File(path, "rb");
foreach (ubyte[] data; chunks(file, 4096)) {
crc.put(data);
}
return crc.finish().toHexString().dup;
}
// returns the sha1 hash hex string of a file
string computeSha1Hash(string path)
{
SHA1 sha;
auto file = File(path, "rb");
foreach (ubyte[] data; chunks(file, 4096)) {
sha.put(data);
}
return sha.finish().toHexString().dup;
}
// returns the quickXorHash base64 string of a file // returns the quickXorHash base64 string of a file
string computeQuickXorHash(string path) string computeQuickXorHash(string path)
{ {
@ -81,6 +59,16 @@ string computeQuickXorHash(string path)
return Base64.encode(qxor.finish()); return Base64.encode(qxor.finish());
} }
// returns the SHA256 hex string of a file
string computeSHA256Hash(string path) {
SHA256 sha256;
auto file = File(path, "rb");
foreach (ubyte[] data; chunks(file, 4096)) {
sha256.put(data);
}
return sha256.finish().toHexString().dup;
}
// converts wildcards (*, ?) to regex // converts wildcards (*, ?) to regex
Regex!char wild2regex(const(char)[] pattern) Regex!char wild2regex(const(char)[] pattern)
{ {