deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-02 14:12:52 +02:00
Code Issues Projects Releases Wiki Activity

Remove sha1 use and cleanup defunct remaining crc32 use (#2424)

Browse source 
* Remove sha1 from being used by the client as this is being depreciated by Microsoft in July 2023 - https://devblogs.microsoft.com/microsoft365dev/deprecation-of-sha1hash-on-onedrive-personal/
* Complete the removal of crc32 as this is also no longer present for a long time, but some code elements still existed
* Only compute quickXorHash, not quickXorHash and sha256Hash as computing sha256Hash is CPU expensive
* Update cache database stored items to only store quickXorHash and sha256Hash values (remove crc32 and sha1)
This commit is contained in:
abraunegg 2023-06-20 06:55:00 +10:00 committed by GitHub
parent c9fe8ad051
commit 06420c9a0a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 62 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
src/itemdb.d
View file

@ -23,9 +23,8 @@ struct Item {
string cTag;
SysTime mtime;
string parentId;
string crc32Hash;
string sha1Hash;
string quickXorHash;
string sha256Hash;
string remoteDriveId;
string remoteId;
string syncStatus;
@ -34,7 +33,7 @@ struct Item {
final class ItemDatabase
{
// increment this for every change in the db schema
immutable int itemDatabaseVersion = 10;
immutable int itemDatabaseVersion = 11;
Database db;
string insertItemStmt;
@ -100,12 +99,12 @@ final class ItemDatabase
db.exec("PRAGMA locking_mode = EXCLUSIVE");
insertItemStmt = "
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId, syncStatus)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, quickXorHash, sha256Hash, remoteDriveId, remoteId, syncStatus)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)
";
updateItemStmt = "
UPDATE item
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, crc32Hash = ?9, sha1Hash = ?10, quickXorHash = ?11, remoteDriveId = ?12, remoteId = ?13, syncStatus = ?14
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, quickXorHash = ?9, sha256Hash = ?10, remoteDriveId = ?11, remoteId = ?12, syncStatus = ?13
WHERE driveId = ?1 AND id = ?2
";
selectItemByIdStmt = "
@ -136,9 +135,8 @@ final class ItemDatabase
cTag TEXT,
mtime TEXT NOT NULL,
parentId TEXT,
crc32Hash TEXT,
sha1Hash TEXT,
quickXorHash TEXT,
sha256Hash TEXT,
remoteDriveId TEXT,
remoteId TEXT,
deltaLink TEXT,
@ -321,19 +319,18 @@ final class ItemDatabase
bind(6, cTag);
bind(7, mtime.toISOExtString());
bind(8, parentId);
bind(9, crc32Hash);
bind(10, sha1Hash);
bind(11, quickXorHash);
bind(12, remoteDriveId);
bind(13, remoteId);
bind(14, syncStatus);
bind(9, quickXorHash);
bind(10, sha256Hash);
bind(11, remoteDriveId);
bind(12, remoteId);
bind(13, syncStatus);
}
}
private Item buildItem(Statement.Result result)
{
assert(!result.empty, "The result must not be empty");
assert(result.front.length == 15, "The result must have 15 columns");
assert(result.front.length == 14, "The result must have 14 columns");
Item item = {
driveId: result.front[0].dup,
id: result.front[1].dup,
@ -342,12 +339,11 @@ final class ItemDatabase
cTag: result.front[5].dup,
mtime: SysTime.fromISOExtString(result.front[6]),
parentId: result.front[7].dup,
crc32Hash: result.front[8].dup,
sha1Hash: result.front[9].dup,
quickXorHash: result.front[10].dup,
remoteDriveId: result.front[11].dup,
remoteId: result.front[12].dup,
syncStatus: result.front[14].dup
quickXorHash: result.front[8].dup,
sha256Hash: result.front[9].dup,
remoteDriveId: result.front[10].dup,
remoteId: result.front[11].dup,
syncStatus: result.front[12].dup
};
switch (result.front[3]) {
case "file": item.type = ItemType.file; break;

65
src/sync.d
View file

@ -98,9 +98,9 @@ private bool hasQuickXorHash(const ref JSONValue item)
return ("quickXorHash" in item["file"]["hashes"]) != null;
}
private bool hasSha1Hash(const ref JSONValue item)
private bool hasSHA256Hash(const ref JSONValue item)
{
return ("sha1Hash" in item["file"]["hashes"]) != null;
return ("sha256Hash" in item["file"]["hashes"]) != null;
}
private bool isDotFile(const(string) path)
@ -173,16 +173,24 @@ private Item makeItem(const ref JSONValue driveItem)
// extract the file hash
if (isItemFile(driveItem) && ("hashes" in driveItem["file"])) {
if ("crc32Hash" in driveItem["file"]["hashes"]) {
item.crc32Hash = driveItem["file"]["hashes"]["crc32Hash"].str;
} else if ("sha1Hash" in driveItem["file"]["hashes"]) {
item.sha1Hash = driveItem["file"]["hashes"]["sha1Hash"].str;
} else if ("quickXorHash" in driveItem["file"]["hashes"]) {
// Get quickXorHash
if ("quickXorHash" in driveItem["file"]["hashes"]) {
item.quickXorHash = driveItem["file"]["hashes"]["quickXorHash"].str;
} else {
log.vlog("The file does not have any hash");
log.vdebug("quickXorHash is missing");
}
}
// sha256Hash
if ("sha256Hash" in driveItem["file"]["hashes"]) {
item.sha256Hash = driveItem["file"]["hashes"]["sha256Hash"].str;
} else {
log.vdebug("sha256Hash is missing");
}
// No hashes ..
if ((item.quickXorHash.empty) && (item.sha256Hash.empty) ) {
// Odd .. no hash ......
log.error("ERROR: OneDrive API inconsistency - the file does not have any hash");
}
}
if (isItemRemote(driveItem)) {
item.remoteDriveId = driveItem["remoteItem"]["parentReference"]["driveId"].str;
@ -201,13 +209,11 @@ private Item makeItem(const ref JSONValue driveItem)
private bool testFileHash(const(string) path, const ref Item item)
{
// Try and compute the file hash
if (item.crc32Hash) {
if (item.crc32Hash == computeCrc32(path)) return true;
} else if (item.sha1Hash) {
if (item.sha1Hash == computeSha1Hash(path)) return true;
} else if (item.quickXorHash) {
// Generate QuickXORHash first before others
if (item.quickXorHash) {
if (item.quickXorHash == computeQuickXorHash(path)) return true;
} else if (item.sha256Hash) {
if (item.sha256Hash == computeSHA256Hash(path)) return true;
}
return false;
}
@ -3018,11 +3024,11 @@ final class SyncEngine
OneDriveFileHash = fileDetails["file"]["hashes"]["quickXorHash"].str;
}
}
// Check for Sha1Hash
if (hasSha1Hash(fileDetails)) {
// Use the configured sha1Hash as reported by OneDrive
if (fileDetails["file"]["hashes"]["sha1Hash"].str != "") {
OneDriveFileHash = fileDetails["file"]["hashes"]["sha1Hash"].str;
// Check for sha256Hash
if (hasSHA256Hash(fileDetails)) {
// Use the configured sha256Hash as reported by OneDrive
if (fileDetails["file"]["hashes"]["sha256Hash"].str != "") {
OneDriveFileHash = fileDetails["file"]["hashes"]["sha256Hash"].str;
}
}
} else {
@ -3152,9 +3158,8 @@ final class SyncEngine
// A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on
// does the file hash OneDrive reports match what we have locally?
string quickXorHash = computeQuickXorHash(path);
string sha1Hash = computeSha1Hash(path);
if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash) || (OneDriveFileHash == sha1Hash)) {
if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash)) {
// downloaded matches either size or hash
log.vdebug("Downloaded file matches reported size and or reported file hash");
try {
@ -3173,7 +3178,7 @@ final class SyncEngine
log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why.");
}
// hash error?
if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) {
if (OneDriveFileHash != quickXorHash) {
// downloaded file hash does not match
log.vdebug("Actual file hash: ", OneDriveFileHash);
log.vdebug("OneDrive API reported hash: ", quickXorHash);
@ -6769,16 +6774,16 @@ final class SyncEngine
// real id / eTag / cTag are different format for personal / business account
auto sha1 = new SHA1Digest();
ubyte[] hash1 = sha1.digest(path);
ubyte[] fakedOneDriveItemValues = sha1.digest(path);
JSONValue fakeResponse;
if (isDir(path)) {
// path is a directory
fakeResponse = [
"id": JSONValue(toHexString(hash1)),
"cTag": JSONValue(toHexString(hash1)),
"eTag": JSONValue(toHexString(hash1)),
"id": JSONValue(toHexString(fakedOneDriveItemValues)),
"cTag": JSONValue(toHexString(fakedOneDriveItemValues)),
"eTag": JSONValue(toHexString(fakedOneDriveItemValues)),
"fileSystemInfo": JSONValue([
"createdDateTime": mtime.toISOExtString(),
"lastModifiedDateTime": mtime.toISOExtString()
@ -6797,9 +6802,9 @@ final class SyncEngine
string quickXorHash = computeQuickXorHash(path);
fakeResponse = [
"id": JSONValue(toHexString(hash1)),
"cTag": JSONValue(toHexString(hash1)),
"eTag": JSONValue(toHexString(hash1)),
"id": JSONValue(toHexString(fakedOneDriveItemValues)),
"cTag": JSONValue(toHexString(fakedOneDriveItemValues)),
"eTag": JSONValue(toHexString(fakedOneDriveItemValues)),
"fileSystemInfo": JSONValue([
"createdDateTime": mtime.toISOExtString(),
"lastModifiedDateTime": mtime.toISOExtString()

32
src/util.d
View file

@ -48,28 +48,6 @@ void safeRemove(const(char)[] path)
if (exists(path)) remove(path);
}
// returns the crc32 hex string of a file
string computeCrc32(string path)
{
CRC32 crc;
auto file = File(path, "rb");
foreach (ubyte[] data; chunks(file, 4096)) {
crc.put(data);
}
return crc.finish().toHexString().dup;
}
// returns the sha1 hash hex string of a file
string computeSha1Hash(string path)
{
SHA1 sha;
auto file = File(path, "rb");
foreach (ubyte[] data; chunks(file, 4096)) {
sha.put(data);
}
return sha.finish().toHexString().dup;
}
// returns the quickXorHash base64 string of a file
string computeQuickXorHash(string path)
{
@ -81,6 +59,16 @@ string computeQuickXorHash(string path)
return Base64.encode(qxor.finish());
}
// returns the SHA256 hex string of a file
string computeSHA256Hash(string path) {
SHA256 sha256;
auto file = File(path, "rb");
foreach (ubyte[] data; chunks(file, 4096)) {
sha256.put(data);
}
return sha256.finish().toHexString().dup;
}
// converts wildcards (*, ?) to regex
Regex!char wild2regex(const(char)[] pattern)
{