From 35f81b21158932cd3051ddcb845c61afa63a191b Mon Sep 17 00:00:00 2001 From: skilion Date: Wed, 14 Jun 2017 15:50:02 +0200 Subject: [PATCH] itemdb support for remote items --- src/itemdb.d | 244 ++++++++++++++++++++++++--------------------------- src/sync.d | 36 +++++--- 2 files changed, 139 insertions(+), 141 deletions(-) diff --git a/src/itemdb.d b/src/itemdb.d index 2a290adf..73c791dc 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -4,17 +4,20 @@ import sqlite; enum ItemType { file, - dir + dir, + remote } struct Item { + string driveId; string id; string name; ItemType type; string eTag; string cTag; SysTime mtime; + string parentDriveId; string parentId; string crc32Hash; string sha1Hash; @@ -24,30 +27,37 @@ struct Item final class ItemDatabase { // increment this for every change in the db schema - immutable int itemDatabaseVersion = 4; + immutable int itemDatabaseVersion = 5; Database db; Statement insertItemStmt; Statement updateItemStmt; Statement selectItemByIdStmt; Statement selectItemByParentIdStmt; + Statement deleteItemByIdStmt; this(const(char)[] filename) { db = Database(filename); if (db.getVersion() == 0) { db.exec("CREATE TABLE item ( - id TEXT NOT NULL PRIMARY KEY, - name TEXT NOT NULL, - type TEXT NOT NULL, - eTag TEXT, - cTag TEXT, - mtime TEXT NOT NULL, - parentId TEXT, - crc32Hash TEXT, - sha1Hash TEXT, - quickXorHash TEXT, - FOREIGN KEY (parentId) REFERENCES item (id) ON DELETE CASCADE + driveId TEXT NOT NULL, + id TEXT NOT NULL, + name TEXT NOT NULL, + type TEXT NOT NULL, + eTag TEXT, + cTag TEXT, + mtime TEXT NOT NULL, + parentDriveId TEXT, + parentId TEXT, + crc32Hash TEXT, + sha1Hash TEXT, + quickXorHash TEXT, + PRIMARY KEY (driveId, id), + FOREIGN KEY (parentDriveId, parentId) + REFERENCES item (driveId, id) + ON DELETE CASCADE + ON UPDATE RESTRICT )"); db.exec("CREATE INDEX name_idx ON item (name)"); db.setVersion(itemDatabaseVersion); @@ -56,14 +66,22 @@ final class ItemDatabase } db.exec("PRAGMA foreign_keys = ON"); db.exec("PRAGMA recursive_triggers = ON"); - insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + insertItemStmt = db.prepare(" + INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentDriveId, parentId, crc32Hash, sha1Hash, quickXorHash) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + "); updateItemStmt = db.prepare(" UPDATE item - SET name = ?2, type = ?3, eTag = ?4, cTag = ?5, mtime = ?6, parentId = ?7, crc32Hash = ?8, sha1Hash = ?9, quickXorHash = ?10 - WHERE id = ?1 + SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentDriveId = ?8, parentId = ?9, crc32Hash = ?10, sha1Hash = ?11, quickXorHash = ?12 + WHERE driveId = ?1 AND id = ?2 "); - selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash FROM item WHERE id = ?"); - selectItemByParentIdStmt = db.prepare("SELECT id FROM item WHERE parentId = ?"); + selectItemByIdStmt = db.prepare(" + SELECT * + FROM item + WHERE driveId = ?1 AND id = ?2 + "); + selectItemByParentIdStmt = db.prepare("SELECT driveId, id FROM item WHERE parentId = ? AND id = ?"); + deleteItemByIdStmt = db.prepare("DELETE FROM item WHERE driveId = ? AND id = ?"); } void insert(const ref Item item) @@ -80,8 +98,9 @@ final class ItemDatabase void upsert(const ref Item item) { - auto s = db.prepare("SELECT COUNT(*) FROM item WHERE id = ?"); - s.bind(1, item.id); + auto s = db.prepare("SELECT COUNT(*) FROM item WHERE driveId = ? AND id = ?"); + s.bind(1, item.driveId); + s.bind(2, item.id); auto r = s.exec(); Statement* stmt; if (r.front[0] == "0") stmt = &insertItemStmt; @@ -90,23 +109,25 @@ final class ItemDatabase stmt.exec(); } - Item[] selectChildren(const(char)[] id) + Item[] selectChildren(const(char)[] driveId, const(char)[] id) { - selectItemByParentIdStmt.bind(1, id); + selectItemByParentIdStmt.bind(1, driveId); + selectItemByParentIdStmt.bind(2, id); auto res = selectItemByParentIdStmt.exec(); Item[] items; foreach (row; res) { Item item; - bool found = selectById(row[0], item); - assert(found); + bool found = selectById(row[0], row[1], item); + assert(found, "Could not select the child of the item"); items ~= item; } return items; } - bool selectById(const(char)[] id, out Item item) + bool selectById(const(char)[] driveId, const(char)[] id, out Item item) { - selectItemByIdStmt.bind(1, id); + selectItemByIdStmt.bind(1, driveId); + selectItemByIdStmt.bind(2, id); auto r = selectItemByIdStmt.exec(); if (!r.empty) { item = buildItem(r); @@ -115,143 +136,110 @@ final class ItemDatabase return false; } + // returns the item with the given path + // the path is relative to the sync directory ex: "./Music/Turbo Killer.mp3" bool selectByPath(const(char)[] path, out Item item) { - // prefix with the root dir + Item currItem; path = "root/" ~ path.chompPrefix("."); - - // initialize the search - string[2][] candidates; // [id, parentId] - auto s = db.prepare("SELECT id, parentId FROM item WHERE name = ?"); - s.bind(1, baseName(path)); - auto r = s.exec(); - foreach (row; r) candidates ~= [row[0].dup, row[1].dup]; - path = dirName(path); - - if (path != ".") { - s = db.prepare("SELECT parentId FROM item WHERE id = ? AND name = ?"); - // discard the candidates that do not have the correct parent - do { - s.bind(2, baseName(path)); - string[2][] newCandidates; - newCandidates.reserve(candidates.length); - foreach (candidate; candidates) { - s.bind(1, candidate[1]); - r = s.exec(); - if (!r.empty) { - string[2] c = [candidate[0], r.front[0].idup]; - newCandidates ~= c; - } - } - candidates = newCandidates; - path = dirName(path); - } while (path != "."); - } - - // reached the root - string[2][] newCandidates; - foreach (candidate; candidates) { - if (!candidate[1]) { - newCandidates ~= candidate; - } - } - candidates = newCandidates; - assert(candidates.length <= 1); - - if (candidates.length == 1) return selectById(candidates[0][0], item); - return false; - } - - void deleteById(const(char)[] id) - { - auto s = db.prepare("DELETE FROM item WHERE id = ?"); - s.bind(1, id); - s.exec(); - } - - // returns true if the item has the specified parent - bool hasParent(T)(const(char)[] itemId, T parentId) - if (is(T : const(char)[]) || is(T : const(char[])[])) - { - auto s = db.prepare("SELECT parentId FROM item WHERE id = ?"); - while (true) { - s.bind(1, itemId); + auto s = db.prepare("SELECT * FROM item WHERE name IS ?1 AND parentDriveId IS ?2 AND parentId IS ?3"); + foreach (name; pathSplitter(path)) { + s.bind(1, name); + s.bind(2, currItem.driveId); + s.bind(3, currItem.id); auto r = s.exec(); - if (r.empty) break; - auto currParentId = r.front[0]; - static if (is(T : const(char)[])) { - if (currParentId == parentId) return true; - } else { - foreach (id; parentId) if (currParentId == id) return true; + if (r.empty) return false; + currItem = buildItem(r); + // if the item of type remote jump to the child + if (currItem.type == ItemType.remote) { + auto children = selectChildren(currItem.driveId, currItem.id); + enforce(children.length == 1, "The remote item has more than 1 child"); + currItem = children[0]; } - itemId = currParentId.dup; } - return false; + item = currItem; + return true; + } + + void deleteById(const(char)[] driveId, const(char)[] id) + { + deleteItemByIdStmt.bind(1, driveId); + deleteItemByIdStmt.bind(2, id); + deleteItemByIdStmt.exec(); } private void bindItem(const ref Item item, ref Statement stmt) { with (stmt) with (item) { - bind(1, id); - bind(2, name); + bind(1, driveId); + bind(2, id); + bind(3, name); string typeStr = null; final switch (type) with (ItemType) { - case file: typeStr = "file"; break; - case dir: typeStr = "dir"; break; + case file: typeStr = "file"; break; + case dir: typeStr = "dir"; break; + case remote: typeStr = "remote"; break; } - bind(3, typeStr); - bind(4, eTag); - bind(5, cTag); - bind(6, mtime.toISOExtString()); - bind(7, parentId); - bind(8, crc32Hash); - bind(9, sha1Hash); - bind(10, quickXorHash); + bind(4, typeStr); + bind(5, eTag); + bind(6, cTag); + bind(7, mtime.toISOExtString()); + bind(8, parentDriveId); + bind(9, parentId); + bind(10, crc32Hash); + bind(11, sha1Hash); + bind(12, quickXorHash); } } private Item buildItem(Statement.Result result) { - assert(!result.empty && result.front.length == 10); + assert(!result.empty, "The result must not be empty"); + assert(result.front.length == 12, "The result must have 12 columns"); Item item = { - id: result.front[0].dup, - name: result.front[1].dup, - eTag: result.front[3].dup, - cTag: result.front[4].dup, - mtime: SysTime.fromISOExtString(result.front[5]), - parentId: result.front[6].dup, - crc32Hash: result.front[7].dup, - sha1Hash: result.front[8].dup, - quickXorHash: result.front[9].dup + driveId: result.front[0].dup, + id: result.front[1].dup, + name: result.front[2].dup, + eTag: result.front[4].dup, + cTag: result.front[5].dup, + mtime: SysTime.fromISOExtString(result.front[6]), + parentDriveId: result.front[7].dup, + parentId: result.front[8].dup, + crc32Hash: result.front[9].dup, + sha1Hash: result.front[10].dup, + quickXorHash: result.front[11].dup }; - switch (result.front[2]) { - case "file": item.type = ItemType.file; break; - case "dir": item.type = ItemType.dir; break; + switch (result.front[3]) { + case "file": item.type = ItemType.file; break; + case "dir": item.type = ItemType.dir; break; + case "remote": item.type = ItemType.remote; break; default: assert(0); } return item; } // computes the path of the given item id - // the path is relative to the sync directory ex: "./Music/Turbo Killer.mp3" - // a trailing slash is never added - string computePath(const(char)[] id) + // the path is relative to the sync directory ex: "Music/Turbo Killer.mp3" + // a trailing slash is not added if the item is a directory + string computePath(const(char)[] driveId, const(char)[] id) { string path; - auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?"); + Item item; while (true) { - s.bind(1, id); - auto r = s.exec(); - enforce(!r.empty, "Unknow item id"); - if (r.front[1]) { - if (path) path = r.front[0].idup ~ "/" ~ path; - else path = r.front[0].idup; + enforce(selectById(driveId, id, item), "Unknow item id"); + if (item.type == ItemType.remote) { + // substitute the last name with the current + path = item.name ~ path[indexOf(path, '/') .. $]; + } else if (item.parentId) { + if (path) path = item.name ~ "/" ~ path; + else path = item.name; } else { // root if (!path) path = "."; break; } - id = r.front[1].dup; + driveId = item.parentDriveId; + id = item.parentId; } return path; } diff --git a/src/sync.d b/src/sync.d index a481ffba..fee2bd69 100644 --- a/src/sync.d +++ b/src/sync.d @@ -43,12 +43,14 @@ private Item makeItem(const ref JSONValue jsonItem) } Item item = { + driveId: jsonItem["parentReference"]["driveId"].str, id: jsonItem["id"].str, name: jsonItem["name"].str, type: type, eTag: isItemRoot(jsonItem) ? null : jsonItem["eTag"].str, // eTag is not returned for the root in OneDrive Biz cTag: "cTag" !in jsonItem ? null : jsonItem["cTag"].str, // cTag is missing in old files (plus all folders) mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str), + parentDriveId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["driveId"].str, parentId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["id"].str }; @@ -108,7 +110,7 @@ final class SyncEngine // list of items to skip while applying the changes private string[] skippedItems; // list of items to delete after the changes has been downloaded - private string[] idsToDelete; + private string[2][] idsToDelete; this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync) { @@ -182,6 +184,7 @@ final class SyncEngine private void applyDifference(JSONValue item) { + string driveId = item["parentReference"]["driveId"].str; string id = item["id"].str; string name = item["name"].str; @@ -209,9 +212,9 @@ final class SyncEngine // rename the local item if it is unsynced and there is a new version of it Item oldItem; string oldPath; - bool cached = itemdb.selectById(id, oldItem); + bool cached = itemdb.selectById(driveId, id, oldItem); if (cached && eTag != oldItem.eTag) { - oldPath = itemdb.computePath(id); + oldPath = itemdb.computePath(driveId, id); if (!isItemSynced(oldItem, oldPath)) { log.vlog("The local item is unsynced, renaming"); if (exists(oldPath)) safeRename(oldPath); @@ -222,14 +225,14 @@ final class SyncEngine // check if the item is to be deleted if (isItemDeleted(item)) { log.vlog("The item is marked for deletion"); - if (cached) idsToDelete ~= id; + if (cached) idsToDelete ~= [driveId, id]; return; } // compute the path of the item string path = "."; if (parentId) { - path = itemdb.computePath(parentId) ~ "/" ~ name; + path = itemdb.computePath(driveId, parentId) ~ "/" ~ name; // selective sync if (selectiveSync.isPathExcluded(path)) { log.vlog("Filtered out: ", path); @@ -281,18 +284,21 @@ final class SyncEngine case ItemType.dir: log.log("Creating directory: ", path); mkdir(path); - break; + break; + case ItemType.remote: + assert(0); } setTimes(path, item.mtime, item.mtime); } private void applyChangedItem(Item oldItem, Item newItem, string newPath) { + assert(oldItem.driveId == newItem.driveId); assert(oldItem.id == newItem.id); assert(oldItem.type == newItem.type); if (oldItem.eTag != newItem.eTag) { - string oldPath = itemdb.computePath(oldItem.id); + string oldPath = itemdb.computePath(oldItem.driveId, oldItem.id); if (oldPath != newPath) { log.log("Moving: ", oldPath, " -> ", newPath); if (exists(newPath)) { @@ -343,6 +349,8 @@ final class SyncEngine log.vlog("The local item is a file but should be a directory"); } break; + case ItemType.remote: + assert(0); } return false; } @@ -350,9 +358,9 @@ final class SyncEngine private void deleteItems() { log.vlog("Deleting files ..."); - foreach_reverse (id; idsToDelete) { - string path = itemdb.computePath(id); - itemdb.deleteById(id); + foreach_reverse (i; idsToDelete) { + string path = itemdb.computePath(i[0], i[1]); + itemdb.deleteById(i[0], i[1]); if (exists(path)) { if (isFile(path)) { remove(path); @@ -400,7 +408,7 @@ final class SyncEngine log.vlog("Filtered out"); return; } - string path = itemdb.computePath(item.id); + string path = itemdb.computePath(item.driveId, item.id); if (selectiveSync.isPathExcluded(path)) { log.vlog("Filtered out: ", path); return; @@ -413,6 +421,8 @@ final class SyncEngine case ItemType.file: uploadFileDifferences(item, path); break; + case ItemType.remote: + assert(0); } } @@ -427,7 +437,7 @@ final class SyncEngine } else { log.vlog("The directory has not changed"); // loop trough the children - foreach (Item child; itemdb.selectChildren(item.id)) { + foreach (Item child; itemdb.selectChildren(item.driveId, item.id)) { uploadDifferences(child); } } @@ -552,7 +562,7 @@ final class SyncEngine if (e.httpStatusCode == 404) log.log(e.msg); else throw e; } - itemdb.deleteById(item.id); + itemdb.deleteById(item.driveId, item.id); } private void uploadLastModifiedTime(const(char)[] id, const(char)[] eTag, SysTime mtime)