mirror of
https://github.com/abraunegg/onedrive
synced 2024-05-15 20:26:53 +02:00
support for SHA1 and QuickXor hash algorithms
This commit is contained in:
skilion
parent
691862b18f
commit
44fc36fd8d
27
src/itemdb.d
27
src/itemdb.d
|
|
@ -16,13 +16,15 @@ struct Item
|
|||
string cTag;
|
||||
SysTime mtime;
|
||||
string parentId;
|
||||
string crc32;
|
||||
string crc32Hash;
|
||||
string sha1Hash;
|
||||
string quickXorHash;
|
||||
}
|
||||
|
||||
final class ItemDatabase
|
||||
{
|
||||
// increment this for every change in the db schema
|
||||
immutable int itemDatabaseVersion = 3;
|
||||
immutable int itemDatabaseVersion = 4;
|
||||
|
||||
Database db;
|
||||
Statement insertItemStmt;
|
||||
|
|
@ -42,7 +44,9 @@ final class ItemDatabase
|
|||
cTag TEXT,
|
||||
mtime TEXT NOT NULL,
|
||||
parentId TEXT,
|
||||
crc32 TEXT,
|
||||
crc32Hash TEXT,
|
||||
sha1Hash TEXT,
|
||||
quickXorHash TEXT,
|
||||
FOREIGN KEY (parentId) REFERENCES item (id) ON DELETE CASCADE
|
||||
)");
|
||||
db.exec("CREATE INDEX name_idx ON item (name)");
|
||||
|
|
@ -52,14 +56,13 @@ final class ItemDatabase
|
|||
}
|
||||
db.exec("PRAGMA foreign_keys = ON");
|
||||
db.exec("PRAGMA recursive_triggers = ON");
|
||||
|
||||
insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32) VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
|
||||
insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
|
||||
updateItemStmt = db.prepare("
|
||||
UPDATE item
|
||||
SET name = ?2, type = ?3, eTag = ?4, cTag = ?5, mtime = ?6, parentId = ?7, crc32 = ?8
|
||||
SET name = ?2, type = ?3, eTag = ?4, cTag = ?5, mtime = ?6, parentId = ?7, crc32Hash = ?8, sha1Hash = ?9, quickXorHash = ?10
|
||||
WHERE id = ?1
|
||||
");
|
||||
selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32 FROM item WHERE id = ?");
|
||||
selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash FROM item WHERE id = ?");
|
||||
selectItemByParentIdStmt = db.prepare("SELECT id FROM item WHERE parentId = ?");
|
||||
}
|
||||
|
||||
|
|
@ -201,13 +204,15 @@ final class ItemDatabase
|
|||
bind(5, cTag);
|
||||
bind(6, mtime.toISOExtString());
|
||||
bind(7, parentId);
|
||||
bind(8, crc32);
|
||||
bind(8, crc32Hash);
|
||||
bind(9, sha1Hash);
|
||||
bind(10, quickXorHash);
|
||||
}
|
||||
}
|
||||
|
||||
private Item buildItem(Statement.Result result)
|
||||
{
|
||||
assert(!result.empty && result.front.length == 8);
|
||||
assert(!result.empty && result.front.length == 10);
|
||||
Item item = {
|
||||
id: result.front[0].dup,
|
||||
name: result.front[1].dup,
|
||||
|
|
@ -215,7 +220,9 @@ final class ItemDatabase
|
|||
cTag: result.front[4].dup,
|
||||
mtime: SysTime.fromISOExtString(result.front[5]),
|
||||
parentId: result.front[6].dup,
|
||||
crc32: result.front[7].dup
|
||||
crc32Hash: result.front[7].dup,
|
||||
sha1Hash: result.front[8].dup,
|
||||
quickXorHash: result.front[9].dup
|
||||
};
|
||||
switch (result.front[2]) {
|
||||
case "file": item.type = ItemType.file; break;
|
||||
|
|
|
|||
108
src/sync.d
108
src/sync.d
|
|
@ -31,11 +31,55 @@ private bool isItemRoot(const ref JSONValue item)
|
|||
return ("root" in item) != null;
|
||||
}
|
||||
|
||||
private bool testCrc32(string path, const(char)[] crc32)
|
||||
private Item makeItem(const ref JSONValue jsonItem)
|
||||
{
|
||||
if (crc32) {
|
||||
string localCrc32 = computeCrc32(path);
|
||||
if (crc32 == localCrc32) return true;
|
||||
ItemType type;
|
||||
if (isItemFile(jsonItem)) {
|
||||
type = ItemType.file;
|
||||
} else if (isItemFolder(jsonItem)) {
|
||||
type = ItemType.dir;
|
||||
} else {
|
||||
assert(0);
|
||||
}
|
||||
|
||||
Item item = {
|
||||
id: jsonItem["id"].str,
|
||||
name: jsonItem["name"].str,
|
||||
type: type,
|
||||
eTag: isItemRoot(jsonItem) ? null : jsonItem["eTag"].str, // eTag is not returned if for the root in OneDrive Biz
|
||||
cTag: isItemFolder(jsonItem) ? null : jsonItem["cTag"].str,
|
||||
mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str),
|
||||
parentId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["id"].str
|
||||
};
|
||||
|
||||
// extract the file hash
|
||||
if (type == ItemType.file) {
|
||||
if ("hashes" in jsonItem["file"]) {
|
||||
if ("crc32Hash" in jsonItem["file"]["hashes"]) {
|
||||
item.crc32Hash = jsonItem["file"]["hashes"]["crc32Hash"].str;
|
||||
} else if ("sha1Hash" in jsonItem["file"]["hashes"]) {
|
||||
item.sha1Hash = jsonItem["file"]["hashes"]["sha1Hash"].str;
|
||||
} else if ("quickXorHash" in jsonItem["file"]["hashes"]) {
|
||||
item.quickXorHash = jsonItem["file"]["hashes"]["quickXorHash"].str;
|
||||
} else {
|
||||
log.vlog("The file does not have any hash");
|
||||
}
|
||||
} else {
|
||||
log.vlog("No hashes in the file facet");
|
||||
}
|
||||
}
|
||||
|
||||
return item;
|
||||
}
|
||||
|
||||
private bool testFileHash(string path, const ref Item item)
|
||||
{
|
||||
if (item.crc32Hash) {
|
||||
if (item.crc32Hash == computeCrc32(path)) return true;
|
||||
} else if (item.sha1Hash) {
|
||||
if (item.sha1Hash == computeSha1Hash(path)) return true;
|
||||
} else if (item.quickXorHash) {
|
||||
if (item.quickXorHash == computeQuickXorHash(path)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
@ -193,36 +237,13 @@ final class SyncEngine
|
|||
}
|
||||
}
|
||||
|
||||
ItemType type;
|
||||
if (isItemFile(item)) {
|
||||
type = ItemType.file;
|
||||
} else if (isItemFolder(item)) {
|
||||
type = ItemType.dir;
|
||||
} else {
|
||||
if (!isItemFile(item) && !isItemFolder(item)) {
|
||||
log.vlog("The item is neither a file nor a directory, skipping");
|
||||
skippedItems ~= id;
|
||||
return;
|
||||
}
|
||||
|
||||
string crc32;
|
||||
if (type == ItemType.file) {
|
||||
try {
|
||||
crc32 = item["file"]["hashes"]["crc32Hash"].str;
|
||||
} catch (JSONException e) {
|
||||
log.vlog("The hash is not available");
|
||||
}
|
||||
}
|
||||
|
||||
Item newItem = {
|
||||
id: id,
|
||||
name: name,
|
||||
type: type,
|
||||
eTag: eTag,
|
||||
cTag: "cTag" in item ? item["cTag"].str : null,
|
||||
mtime: SysTime.fromISOExtString(item["fileSystemInfo"]["lastModifiedDateTime"].str),
|
||||
parentId: parentId,
|
||||
crc32: crc32
|
||||
};
|
||||
Item newItem = makeItem(item);
|
||||
|
||||
if (!cached) {
|
||||
applyNewItem(newItem, path);
|
||||
|
|
@ -305,7 +326,7 @@ final class SyncEngine
|
|||
} else {
|
||||
log.vlog("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime);
|
||||
}
|
||||
if (testCrc32(path, item.crc32)) {
|
||||
if (testFileHash(path, item)) {
|
||||
return true;
|
||||
} else {
|
||||
log.vlog("The local item has a different hash");
|
||||
|
|
@ -428,7 +449,7 @@ final class SyncEngine
|
|||
log.vlog("The file last modified time has changed");
|
||||
string id = item.id;
|
||||
string eTag = item.eTag;
|
||||
if (!testCrc32(path, item.crc32)) {
|
||||
if (!testFileHash(path, item)) {
|
||||
log.vlog("The file content has changed");
|
||||
log.log("Uploading: ", path);
|
||||
JSONValue response;
|
||||
|
|
@ -546,30 +567,7 @@ final class SyncEngine
|
|||
|
||||
private void saveItem(JSONValue jsonItem)
|
||||
{
|
||||
ItemType type;
|
||||
if (isItemFile(jsonItem)) {
|
||||
type = ItemType.file;
|
||||
} else if (isItemFolder(jsonItem)) {
|
||||
type = ItemType.dir;
|
||||
} else {
|
||||
assert(0);
|
||||
}
|
||||
Item item = {
|
||||
id: jsonItem["id"].str,
|
||||
name: jsonItem["name"].str,
|
||||
type: type,
|
||||
eTag: jsonItem["eTag"].str,
|
||||
cTag: "cTag" in jsonItem ? jsonItem["cTag"].str : null,
|
||||
mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str),
|
||||
parentId: jsonItem["parentReference"]["id"].str
|
||||
};
|
||||
if (type == ItemType.file) {
|
||||
try {
|
||||
item.crc32 = jsonItem["file"]["hashes"]["crc32Hash"].str;
|
||||
} catch (JSONException e) {
|
||||
log.vlog("The hash is not available");
|
||||
}
|
||||
}
|
||||
Item item = makeItem(jsonItem);
|
||||
itemdb.upsert(item);
|
||||
}
|
||||
|
||||
|
|
|
|||
13
src/util.d
13
src/util.d
|
|
@ -1,6 +1,6 @@
|
|||
import std.base64;
|
||||
import std.conv;
|
||||
import std.digest.crc;
|
||||
import std.digest.crc, std.digest.sha;
|
||||
import std.file;
|
||||
import std.net.curl;
|
||||
import std.path;
|
||||
|
|
@ -52,6 +52,17 @@ string computeCrc32(string path)
|
|||
return crc.finish().toHexString().dup;
|
||||
}
|
||||
|
||||
// returns the sha1 hash hex string of a file
|
||||
string computeSha1Hash(string path)
|
||||
{
|
||||
SHA1 sha;
|
||||
auto file = File(path, "rb");
|
||||
foreach (ubyte[] data; chunks(file, 4096)) {
|
||||
sha.put(data);
|
||||
}
|
||||
return sha.finish().toHexString().dup;
|
||||
}
|
||||
|
||||
// returns the quickXorHash base64 string of a file
|
||||
string computeQuickXorHash(string path)
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue