deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-06-03 14:32:24 +02:00
Code Issues Projects Releases Wiki Activity

itemdb support for remote items

Browse source
This commit is contained in:
skilion 2017-06-14 15:50:02 +02:00
parent 74c931cdba
commit 35f81b2115
2 changed files with 139 additions and 141 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

212
src/itemdb.d
View file

@ -4,17 +4,20 @@ import sqlite;
enum ItemType enum ItemType
{ {
file, file,
dir dir,
remote
} }
struct Item struct Item
{ {
string driveId;
string id; string id;
string name; string name;
ItemType type; ItemType type;
string eTag; string eTag;
string cTag; string cTag;
SysTime mtime; SysTime mtime;
string parentDriveId;
string parentId; string parentId;
string crc32Hash; string crc32Hash;
string sha1Hash; string sha1Hash;
@ -24,30 +27,37 @@ struct Item
final class ItemDatabase final class ItemDatabase
{ {
// increment this for every change in the db schema // increment this for every change in the db schema
immutable int itemDatabaseVersion = 4; immutable int itemDatabaseVersion = 5;
Database db; Database db;
Statement insertItemStmt; Statement insertItemStmt;
Statement updateItemStmt; Statement updateItemStmt;
Statement selectItemByIdStmt; Statement selectItemByIdStmt;
Statement selectItemByParentIdStmt; Statement selectItemByParentIdStmt;
Statement deleteItemByIdStmt;
this(const(char)[] filename) this(const(char)[] filename)
{ {
db = Database(filename); db = Database(filename);
if (db.getVersion() == 0) { if (db.getVersion() == 0) {
db.exec("CREATE TABLE item ( db.exec("CREATE TABLE item (
id TEXT NOT NULL PRIMARY KEY, driveId TEXT NOT NULL,
id TEXT NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
type TEXT NOT NULL, type TEXT NOT NULL,
eTag TEXT, eTag TEXT,
cTag TEXT, cTag TEXT,
mtime TEXT NOT NULL, mtime TEXT NOT NULL,
parentDriveId TEXT,
parentId TEXT, parentId TEXT,
crc32Hash TEXT, crc32Hash TEXT,
sha1Hash TEXT, sha1Hash TEXT,
quickXorHash TEXT, quickXorHash TEXT,
FOREIGN KEY (parentId) REFERENCES item (id) ON DELETE CASCADE PRIMARY KEY (driveId, id),
FOREIGN KEY (parentDriveId, parentId)
REFERENCES item (driveId, id)
ON DELETE CASCADE
ON UPDATE RESTRICT
)"); )");
db.exec("CREATE INDEX name_idx ON item (name)"); db.exec("CREATE INDEX name_idx ON item (name)");
db.setVersion(itemDatabaseVersion); db.setVersion(itemDatabaseVersion);
@ -56,14 +66,22 @@ final class ItemDatabase
} }
db.exec("PRAGMA foreign_keys = ON"); db.exec("PRAGMA foreign_keys = ON");
db.exec("PRAGMA recursive_triggers = ON"); db.exec("PRAGMA recursive_triggers = ON");
insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); insertItemStmt = db.prepare("
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentDriveId, parentId, crc32Hash, sha1Hash, quickXorHash)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
");
updateItemStmt = db.prepare(" updateItemStmt = db.prepare("
UPDATE item UPDATE item
SET name = ?2, type = ?3, eTag = ?4, cTag = ?5, mtime = ?6, parentId = ?7, crc32Hash = ?8, sha1Hash = ?9, quickXorHash = ?10 SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentDriveId = ?8, parentId = ?9, crc32Hash = ?10, sha1Hash = ?11, quickXorHash = ?12
WHERE id = ?1 WHERE driveId = ?1 AND id = ?2
"); ");
selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash FROM item WHERE id = ?"); selectItemByIdStmt = db.prepare("
selectItemByParentIdStmt = db.prepare("SELECT id FROM item WHERE parentId = ?"); SELECT *
FROM item
WHERE driveId = ?1 AND id = ?2
");
selectItemByParentIdStmt = db.prepare("SELECT driveId, id FROM item WHERE parentId = ? AND id = ?");
deleteItemByIdStmt = db.prepare("DELETE FROM item WHERE driveId = ? AND id = ?");
} }
void insert(const ref Item item) void insert(const ref Item item)
@ -80,8 +98,9 @@ final class ItemDatabase
void upsert(const ref Item item) void upsert(const ref Item item)
{ {
auto s = db.prepare("SELECT COUNT(*) FROM item WHERE id = ?"); auto s = db.prepare("SELECT COUNT(*) FROM item WHERE driveId = ? AND id = ?");
s.bind(1, item.id); s.bind(1, item.driveId);
s.bind(2, item.id);
auto r = s.exec(); auto r = s.exec();
Statement* stmt; Statement* stmt;
if (r.front[0] == "0") stmt = &insertItemStmt; if (r.front[0] == "0") stmt = &insertItemStmt;
@ -90,23 +109,25 @@ final class ItemDatabase
stmt.exec(); stmt.exec();
} }
Item[] selectChildren(const(char)[] id) Item[] selectChildren(const(char)[] driveId, const(char)[] id)
{ {
selectItemByParentIdStmt.bind(1, id); selectItemByParentIdStmt.bind(1, driveId);
selectItemByParentIdStmt.bind(2, id);
auto res = selectItemByParentIdStmt.exec(); auto res = selectItemByParentIdStmt.exec();
Item[] items; Item[] items;
foreach (row; res) { foreach (row; res) {
Item item; Item item;
bool found = selectById(row[0], item); bool found = selectById(row[0], row[1], item);
assert(found); assert(found, "Could not select the child of the item");
items ~= item; items ~= item;
} }
return items; return items;
} }
bool selectById(const(char)[] id, out Item item) bool selectById(const(char)[] driveId, const(char)[] id, out Item item)
{ {
selectItemByIdStmt.bind(1, id); selectItemByIdStmt.bind(1, driveId);
selectItemByIdStmt.bind(2, id);
auto r = selectItemByIdStmt.exec(); auto r = selectItemByIdStmt.exec();
if (!r.empty) { if (!r.empty) {
item = buildItem(r); item = buildItem(r);
@ -115,143 +136,110 @@ final class ItemDatabase
return false; return false;
} }
// returns the item with the given path
// the path is relative to the sync directory ex: "./Music/Turbo Killer.mp3"
bool selectByPath(const(char)[] path, out Item item) bool selectByPath(const(char)[] path, out Item item)
{ {
// prefix with the root dir Item currItem;
path = "root/" ~ path.chompPrefix("."); path = "root/" ~ path.chompPrefix(".");
auto s = db.prepare("SELECT * FROM item WHERE name IS ?1 AND parentDriveId IS ?2 AND parentId IS ?3");
// initialize the search foreach (name; pathSplitter(path)) {
string[2][] candidates; // [id, parentId] s.bind(1, name);
auto s = db.prepare("SELECT id, parentId FROM item WHERE name = ?"); s.bind(2, currItem.driveId);
s.bind(1, baseName(path)); s.bind(3, currItem.id);
auto r = s.exec(); auto r = s.exec();
foreach (row; r) candidates ~= [row[0].dup, row[1].dup]; if (r.empty) return false;
path = dirName(path); currItem = buildItem(r);
// if the item of type remote jump to the child
if (path != ".") { if (currItem.type == ItemType.remote) {
s = db.prepare("SELECT parentId FROM item WHERE id = ? AND name = ?"); auto children = selectChildren(currItem.driveId, currItem.id);
// discard the candidates that do not have the correct parent enforce(children.length == 1, "The remote item has more than 1 child");
do { currItem = children[0];
s.bind(2, baseName(path));
string[2][] newCandidates;
newCandidates.reserve(candidates.length);
foreach (candidate; candidates) {
s.bind(1, candidate[1]);
r = s.exec();
if (!r.empty) {
string[2] c = [candidate[0], r.front[0].idup];
newCandidates ~= c;
} }
} }
candidates = newCandidates; item = currItem;
path = dirName(path); return true;
} while (path != ".");
} }
// reached the root void deleteById(const(char)[] driveId, const(char)[] id)
string[2][] newCandidates;
foreach (candidate; candidates) {
if (!candidate[1]) {
newCandidates ~= candidate;
}
}
candidates = newCandidates;
assert(candidates.length <= 1);
if (candidates.length == 1) return selectById(candidates[0][0], item);
return false;
}
void deleteById(const(char)[] id)
{ {
auto s = db.prepare("DELETE FROM item WHERE id = ?"); deleteItemByIdStmt.bind(1, driveId);
s.bind(1, id); deleteItemByIdStmt.bind(2, id);
s.exec(); deleteItemByIdStmt.exec();
}
// returns true if the item has the specified parent
bool hasParent(T)(const(char)[] itemId, T parentId)
if (is(T : const(char)[]) || is(T : const(char[])[]))
{
auto s = db.prepare("SELECT parentId FROM item WHERE id = ?");
while (true) {
s.bind(1, itemId);
auto r = s.exec();
if (r.empty) break;
auto currParentId = r.front[0];
static if (is(T : const(char)[])) {
if (currParentId == parentId) return true;
} else {
foreach (id; parentId) if (currParentId == id) return true;
}
itemId = currParentId.dup;
}
return false;
} }
private void bindItem(const ref Item item, ref Statement stmt) private void bindItem(const ref Item item, ref Statement stmt)
{ {
with (stmt) with (item) { with (stmt) with (item) {
bind(1, id); bind(1, driveId);
bind(2, name); bind(2, id);
bind(3, name);
string typeStr = null; string typeStr = null;
final switch (type) with (ItemType) { final switch (type) with (ItemType) {
case file: typeStr = "file"; break; case file: typeStr = "file"; break;
case dir: typeStr = "dir"; break; case dir: typeStr = "dir"; break;
case remote: typeStr = "remote"; break;
} }
bind(3, typeStr); bind(4, typeStr);
bind(4, eTag); bind(5, eTag);
bind(5, cTag); bind(6, cTag);
bind(6, mtime.toISOExtString()); bind(7, mtime.toISOExtString());
bind(7, parentId); bind(8, parentDriveId);
bind(8, crc32Hash); bind(9, parentId);
bind(9, sha1Hash); bind(10, crc32Hash);
bind(10, quickXorHash); bind(11, sha1Hash);
bind(12, quickXorHash);
} }
} }
private Item buildItem(Statement.Result result) private Item buildItem(Statement.Result result)
{ {
assert(!result.empty && result.front.length == 10); assert(!result.empty, "The result must not be empty");
assert(result.front.length == 12, "The result must have 12 columns");
Item item = { Item item = {
id: result.front[0].dup, driveId: result.front[0].dup,
name: result.front[1].dup, id: result.front[1].dup,
eTag: result.front[3].dup, name: result.front[2].dup,
cTag: result.front[4].dup, eTag: result.front[4].dup,
mtime: SysTime.fromISOExtString(result.front[5]), cTag: result.front[5].dup,
parentId: result.front[6].dup, mtime: SysTime.fromISOExtString(result.front[6]),
crc32Hash: result.front[7].dup, parentDriveId: result.front[7].dup,
sha1Hash: result.front[8].dup, parentId: result.front[8].dup,
quickXorHash: result.front[9].dup crc32Hash: result.front[9].dup,
sha1Hash: result.front[10].dup,
quickXorHash: result.front[11].dup
}; };
switch (result.front[2]) { switch (result.front[3]) {
case "file": item.type = ItemType.file; break; case "file": item.type = ItemType.file; break;
case "dir": item.type = ItemType.dir; break; case "dir": item.type = ItemType.dir; break;
case "remote": item.type = ItemType.remote; break;
default: assert(0); default: assert(0);
} }
return item; return item;
} }
// computes the path of the given item id // computes the path of the given item id
// the path is relative to the sync directory ex: "./Music/Turbo Killer.mp3" // the path is relative to the sync directory ex: "Music/Turbo Killer.mp3"
// a trailing slash is never added // a trailing slash is not added if the item is a directory
string computePath(const(char)[] id) string computePath(const(char)[] driveId, const(char)[] id)
{ {
string path; string path;
auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?"); Item item;
while (true) { while (true) {
s.bind(1, id); enforce(selectById(driveId, id, item), "Unknow item id");
auto r = s.exec(); if (item.type == ItemType.remote) {
enforce(!r.empty, "Unknow item id"); // substitute the last name with the current
if (r.front[1]) { path = item.name ~ path[indexOf(path, '/') .. $];
if (path) path = r.front[0].idup ~ "/" ~ path; } else if (item.parentId) {
else path = r.front[0].idup; if (path) path = item.name ~ "/" ~ path;
else path = item.name;
} else { } else {
// root // root
if (!path) path = "."; if (!path) path = ".";
break; break;
} }
id = r.front[1].dup; driveId = item.parentDriveId;
id = item.parentId;
} }
return path; return path;
} }

34
src/sync.d
View file

@ -43,12 +43,14 @@ private Item makeItem(const ref JSONValue jsonItem)
} }
Item item = { Item item = {
driveId: jsonItem["parentReference"]["driveId"].str,
id: jsonItem["id"].str, id: jsonItem["id"].str,
name: jsonItem["name"].str, name: jsonItem["name"].str,
type: type, type: type,
eTag: isItemRoot(jsonItem) ? null : jsonItem["eTag"].str, // eTag is not returned for the root in OneDrive Biz eTag: isItemRoot(jsonItem) ? null : jsonItem["eTag"].str, // eTag is not returned for the root in OneDrive Biz
cTag: "cTag" !in jsonItem ? null : jsonItem["cTag"].str, // cTag is missing in old files (plus all folders) cTag: "cTag" !in jsonItem ? null : jsonItem["cTag"].str, // cTag is missing in old files (plus all folders)
mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str), mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str),
parentDriveId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["driveId"].str,
parentId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["id"].str parentId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["id"].str
}; };
@ -108,7 +110,7 @@ final class SyncEngine
// list of items to skip while applying the changes // list of items to skip while applying the changes
private string[] skippedItems; private string[] skippedItems;
// list of items to delete after the changes has been downloaded // list of items to delete after the changes has been downloaded
private string[] idsToDelete; private string[2][] idsToDelete;
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync) this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
{ {
@ -182,6 +184,7 @@ final class SyncEngine
private void applyDifference(JSONValue item) private void applyDifference(JSONValue item)
{ {
string driveId = item["parentReference"]["driveId"].str;
string id = item["id"].str; string id = item["id"].str;
string name = item["name"].str; string name = item["name"].str;
@ -209,9 +212,9 @@ final class SyncEngine
// rename the local item if it is unsynced and there is a new version of it // rename the local item if it is unsynced and there is a new version of it
Item oldItem; Item oldItem;
string oldPath; string oldPath;
bool cached = itemdb.selectById(id, oldItem); bool cached = itemdb.selectById(driveId, id, oldItem);
if (cached && eTag != oldItem.eTag) { if (cached && eTag != oldItem.eTag) {
oldPath = itemdb.computePath(id); oldPath = itemdb.computePath(driveId, id);
if (!isItemSynced(oldItem, oldPath)) { if (!isItemSynced(oldItem, oldPath)) {
log.vlog("The local item is unsynced, renaming"); log.vlog("The local item is unsynced, renaming");
if (exists(oldPath)) safeRename(oldPath); if (exists(oldPath)) safeRename(oldPath);
@ -222,14 +225,14 @@ final class SyncEngine
// check if the item is to be deleted // check if the item is to be deleted
if (isItemDeleted(item)) { if (isItemDeleted(item)) {
log.vlog("The item is marked for deletion"); log.vlog("The item is marked for deletion");
if (cached) idsToDelete ~= id; if (cached) idsToDelete ~= [driveId, id];
return; return;
} }
// compute the path of the item // compute the path of the item
string path = "."; string path = ".";
if (parentId) { if (parentId) {
path = itemdb.computePath(parentId) ~ "/" ~ name; path = itemdb.computePath(driveId, parentId) ~ "/" ~ name;
// selective sync // selective sync
if (selectiveSync.isPathExcluded(path)) { if (selectiveSync.isPathExcluded(path)) {
log.vlog("Filtered out: ", path); log.vlog("Filtered out: ", path);
@ -282,17 +285,20 @@ final class SyncEngine
log.log("Creating directory: ", path); log.log("Creating directory: ", path);
mkdir(path); mkdir(path);
break; break;
case ItemType.remote:
assert(0);
} }
setTimes(path, item.mtime, item.mtime); setTimes(path, item.mtime, item.mtime);
} }
private void applyChangedItem(Item oldItem, Item newItem, string newPath) private void applyChangedItem(Item oldItem, Item newItem, string newPath)
{ {
assert(oldItem.driveId == newItem.driveId);
assert(oldItem.id == newItem.id); assert(oldItem.id == newItem.id);
assert(oldItem.type == newItem.type); assert(oldItem.type == newItem.type);
if (oldItem.eTag != newItem.eTag) { if (oldItem.eTag != newItem.eTag) {
string oldPath = itemdb.computePath(oldItem.id); string oldPath = itemdb.computePath(oldItem.driveId, oldItem.id);
if (oldPath != newPath) { if (oldPath != newPath) {
log.log("Moving: ", oldPath, " -> ", newPath); log.log("Moving: ", oldPath, " -> ", newPath);
if (exists(newPath)) { if (exists(newPath)) {
@ -343,6 +349,8 @@ final class SyncEngine
log.vlog("The local item is a file but should be a directory"); log.vlog("The local item is a file but should be a directory");
} }
break; break;
case ItemType.remote:
assert(0);
} }
return false; return false;
} }
@ -350,9 +358,9 @@ final class SyncEngine
private void deleteItems() private void deleteItems()
{ {
log.vlog("Deleting files ..."); log.vlog("Deleting files ...");
foreach_reverse (id; idsToDelete) { foreach_reverse (i; idsToDelete) {
string path = itemdb.computePath(id); string path = itemdb.computePath(i[0], i[1]);
itemdb.deleteById(id); itemdb.deleteById(i[0], i[1]);
if (exists(path)) { if (exists(path)) {
if (isFile(path)) { if (isFile(path)) {
remove(path); remove(path);
@ -400,7 +408,7 @@ final class SyncEngine
log.vlog("Filtered out"); log.vlog("Filtered out");
return; return;
} }
string path = itemdb.computePath(item.id); string path = itemdb.computePath(item.driveId, item.id);
if (selectiveSync.isPathExcluded(path)) { if (selectiveSync.isPathExcluded(path)) {
log.vlog("Filtered out: ", path); log.vlog("Filtered out: ", path);
return; return;
@ -413,6 +421,8 @@ final class SyncEngine
case ItemType.file: case ItemType.file:
uploadFileDifferences(item, path); uploadFileDifferences(item, path);
break; break;
case ItemType.remote:
assert(0);
} }
} }
@ -427,7 +437,7 @@ final class SyncEngine
} else { } else {
log.vlog("The directory has not changed"); log.vlog("The directory has not changed");
// loop trough the children // loop trough the children
foreach (Item child; itemdb.selectChildren(item.id)) { foreach (Item child; itemdb.selectChildren(item.driveId, item.id)) {
uploadDifferences(child); uploadDifferences(child);
} }
} }
@ -552,7 +562,7 @@ final class SyncEngine
if (e.httpStatusCode == 404) log.log(e.msg); if (e.httpStatusCode == 404) log.log(e.msg);
else throw e; else throw e;
} }
itemdb.deleteById(item.id); itemdb.deleteById(item.driveId, item.id);
} }
private void uploadLastModifiedTime(const(char)[] id, const(char)[] eTag, SysTime mtime) private void uploadLastModifiedTime(const(char)[] id, const(char)[] eTag, SysTime mtime)