deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-01 21:52:50 +02:00
Code Issues Projects Releases Wiki Activity

support for SHA1 and QuickXor hash algorithms

Browse source
This commit is contained in:
skilion 2017-05-28 22:13:19 +02:00
parent 691862b18f
commit 44fc36fd8d
3 changed files with 89 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
src/itemdb.d
View file

@ -16,13 +16,15 @@ struct Item
string cTag;
SysTime mtime;
string parentId;
string crc32;
string crc32Hash;
string sha1Hash;
string quickXorHash;
}
final class ItemDatabase
{
// increment this for every change in the db schema
immutable int itemDatabaseVersion = 3;
immutable int itemDatabaseVersion = 4;
Database db;
Statement insertItemStmt;
@ -35,14 +37,16 @@ final class ItemDatabase
db = Database(filename);
if (db.getVersion() == 0) {
db.exec("CREATE TABLE item (
id TEXT NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
type TEXT NOT NULL,
eTag TEXT,
cTag TEXT,
mtime TEXT NOT NULL,
parentId TEXT,
crc32 TEXT,
id TEXT NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
type TEXT NOT NULL,
eTag TEXT,
cTag TEXT,
mtime TEXT NOT NULL,
parentId TEXT,
crc32Hash TEXT,
sha1Hash TEXT,
quickXorHash TEXT,
FOREIGN KEY (parentId) REFERENCES item (id) ON DELETE CASCADE
)");
db.exec("CREATE INDEX name_idx ON item (name)");
@ -52,14 +56,13 @@ final class ItemDatabase
}
db.exec("PRAGMA foreign_keys = ON");
db.exec("PRAGMA recursive_triggers = ON");
insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32) VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
updateItemStmt = db.prepare("
UPDATE item
SET name = ?2, type = ?3, eTag = ?4, cTag = ?5, mtime = ?6, parentId = ?7, crc32 = ?8
SET name = ?2, type = ?3, eTag = ?4, cTag = ?5, mtime = ?6, parentId = ?7, crc32Hash = ?8, sha1Hash = ?9, quickXorHash = ?10
WHERE id = ?1
");
selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32 FROM item WHERE id = ?");
selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash FROM item WHERE id = ?");
selectItemByParentIdStmt = db.prepare("SELECT id FROM item WHERE parentId = ?");
}
@ -201,13 +204,15 @@ final class ItemDatabase
bind(5, cTag);
bind(6, mtime.toISOExtString());
bind(7, parentId);
bind(8, crc32);
bind(8, crc32Hash);
bind(9, sha1Hash);
bind(10, quickXorHash);
}
}
private Item buildItem(Statement.Result result)
{
assert(!result.empty && result.front.length == 8);
assert(!result.empty && result.front.length == 10);
Item item = {
id: result.front[0].dup,
name: result.front[1].dup,
@ -215,7 +220,9 @@ final class ItemDatabase
cTag: result.front[4].dup,
mtime: SysTime.fromISOExtString(result.front[5]),
parentId: result.front[6].dup,
crc32: result.front[7].dup
crc32Hash: result.front[7].dup,
sha1Hash: result.front[8].dup,
quickXorHash: result.front[9].dup
};
switch (result.front[2]) {
case "file": item.type = ItemType.file; break;

108
src/sync.d
View file

@ -31,11 +31,55 @@ private bool isItemRoot(const ref JSONValue item)
return ("root" in item) != null;
}
private bool testCrc32(string path, const(char)[] crc32)
private Item makeItem(const ref JSONValue jsonItem)
{
if (crc32) {
string localCrc32 = computeCrc32(path);
if (crc32 == localCrc32) return true;
ItemType type;
if (isItemFile(jsonItem)) {
type = ItemType.file;
} else if (isItemFolder(jsonItem)) {
type = ItemType.dir;
} else {
assert(0);
}
Item item = {
id: jsonItem["id"].str,
name: jsonItem["name"].str,
type: type,
eTag: isItemRoot(jsonItem) ? null : jsonItem["eTag"].str, // eTag is not returned if for the root in OneDrive Biz
cTag: isItemFolder(jsonItem) ? null : jsonItem["cTag"].str,
mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str),
parentId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["id"].str
};
// extract the file hash
if (type == ItemType.file) {
if ("hashes" in jsonItem["file"]) {
if ("crc32Hash" in jsonItem["file"]["hashes"]) {
item.crc32Hash = jsonItem["file"]["hashes"]["crc32Hash"].str;
} else if ("sha1Hash" in jsonItem["file"]["hashes"]) {
item.sha1Hash = jsonItem["file"]["hashes"]["sha1Hash"].str;
} else if ("quickXorHash" in jsonItem["file"]["hashes"]) {
item.quickXorHash = jsonItem["file"]["hashes"]["quickXorHash"].str;
} else {
log.vlog("The file does not have any hash");
}
} else {
log.vlog("No hashes in the file facet");
}
}
return item;
}
private bool testFileHash(string path, const ref Item item)
{
if (item.crc32Hash) {
if (item.crc32Hash == computeCrc32(path)) return true;
} else if (item.sha1Hash) {
if (item.sha1Hash == computeSha1Hash(path)) return true;
} else if (item.quickXorHash) {
if (item.quickXorHash == computeQuickXorHash(path)) return true;
}
return false;
}
@ -193,36 +237,13 @@ final class SyncEngine
}
}
ItemType type;
if (isItemFile(item)) {
type = ItemType.file;
} else if (isItemFolder(item)) {
type = ItemType.dir;
} else {
if (!isItemFile(item) && !isItemFolder(item)) {
log.vlog("The item is neither a file nor a directory, skipping");
skippedItems ~= id;
return;
}
string crc32;
if (type == ItemType.file) {
try {
crc32 = item["file"]["hashes"]["crc32Hash"].str;
} catch (JSONException e) {
log.vlog("The hash is not available");
}
}
Item newItem = {
id: id,
name: name,
type: type,
eTag: eTag,
cTag: "cTag" in item ? item["cTag"].str : null,
mtime: SysTime.fromISOExtString(item["fileSystemInfo"]["lastModifiedDateTime"].str),
parentId: parentId,
crc32: crc32
};
Item newItem = makeItem(item);
if (!cached) {
applyNewItem(newItem, path);
@ -305,7 +326,7 @@ final class SyncEngine
} else {
log.vlog("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime);
}
if (testCrc32(path, item.crc32)) {
if (testFileHash(path, item)) {
return true;
} else {
log.vlog("The local item has a different hash");
@ -428,7 +449,7 @@ final class SyncEngine
log.vlog("The file last modified time has changed");
string id = item.id;
string eTag = item.eTag;
if (!testCrc32(path, item.crc32)) {
if (!testFileHash(path, item)) {
log.vlog("The file content has changed");
log.log("Uploading: ", path);
JSONValue response;
@ -546,30 +567,7 @@ final class SyncEngine
private void saveItem(JSONValue jsonItem)
{
ItemType type;
if (isItemFile(jsonItem)) {
type = ItemType.file;
} else if (isItemFolder(jsonItem)) {
type = ItemType.dir;
} else {
assert(0);
}
Item item = {
id: jsonItem["id"].str,
name: jsonItem["name"].str,
type: type,
eTag: jsonItem["eTag"].str,
cTag: "cTag" in jsonItem ? jsonItem["cTag"].str : null,
mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str),
parentId: jsonItem["parentReference"]["id"].str
};
if (type == ItemType.file) {
try {
item.crc32 = jsonItem["file"]["hashes"]["crc32Hash"].str;
} catch (JSONException e) {
log.vlog("The hash is not available");
}
}
Item item = makeItem(jsonItem);
itemdb.upsert(item);
}

13
src/util.d
View file

@ -1,6 +1,6 @@
import std.base64;
import std.conv;
import std.digest.crc;
import std.digest.crc, std.digest.sha;
import std.file;
import std.net.curl;
import std.path;
@ -52,6 +52,17 @@ string computeCrc32(string path)
return crc.finish().toHexString().dup;
}
// returns the sha1 hash hex string of a file
string computeSha1Hash(string path)
{
SHA1 sha;
auto file = File(path, "rb");
foreach (ubyte[] data; chunks(file, 4096)) {
sha.put(data);
}
return sha.finish().toHexString().dup;
}
// returns the quickXorHash base64 string of a file
string computeQuickXorHash(string path)
{