mirror of
https://github.com/abraunegg/onedrive
synced 2024-05-01 13:42:53 +02:00
itemdb support for remote items
This commit is contained in:
skilion
parent
74c931cdba
commit
35f81b2115
244
src/itemdb.d
244
src/itemdb.d
|
|
@ -4,17 +4,20 @@ import sqlite;
|
|||
enum ItemType
|
||||
{
|
||||
file,
|
||||
dir
|
||||
dir,
|
||||
remote
|
||||
}
|
||||
|
||||
struct Item
|
||||
{
|
||||
string driveId;
|
||||
string id;
|
||||
string name;
|
||||
ItemType type;
|
||||
string eTag;
|
||||
string cTag;
|
||||
SysTime mtime;
|
||||
string parentDriveId;
|
||||
string parentId;
|
||||
string crc32Hash;
|
||||
string sha1Hash;
|
||||
|
|
@ -24,30 +27,37 @@ struct Item
|
|||
final class ItemDatabase
|
||||
{
|
||||
// increment this for every change in the db schema
|
||||
immutable int itemDatabaseVersion = 4;
|
||||
immutable int itemDatabaseVersion = 5;
|
||||
|
||||
Database db;
|
||||
Statement insertItemStmt;
|
||||
Statement updateItemStmt;
|
||||
Statement selectItemByIdStmt;
|
||||
Statement selectItemByParentIdStmt;
|
||||
Statement deleteItemByIdStmt;
|
||||
|
||||
this(const(char)[] filename)
|
||||
{
|
||||
db = Database(filename);
|
||||
if (db.getVersion() == 0) {
|
||||
db.exec("CREATE TABLE item (
|
||||
id TEXT NOT NULL PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
eTag TEXT,
|
||||
cTag TEXT,
|
||||
mtime TEXT NOT NULL,
|
||||
parentId TEXT,
|
||||
crc32Hash TEXT,
|
||||
sha1Hash TEXT,
|
||||
quickXorHash TEXT,
|
||||
FOREIGN KEY (parentId) REFERENCES item (id) ON DELETE CASCADE
|
||||
driveId TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
eTag TEXT,
|
||||
cTag TEXT,
|
||||
mtime TEXT NOT NULL,
|
||||
parentDriveId TEXT,
|
||||
parentId TEXT,
|
||||
crc32Hash TEXT,
|
||||
sha1Hash TEXT,
|
||||
quickXorHash TEXT,
|
||||
PRIMARY KEY (driveId, id),
|
||||
FOREIGN KEY (parentDriveId, parentId)
|
||||
REFERENCES item (driveId, id)
|
||||
ON DELETE CASCADE
|
||||
ON UPDATE RESTRICT
|
||||
)");
|
||||
db.exec("CREATE INDEX name_idx ON item (name)");
|
||||
db.setVersion(itemDatabaseVersion);
|
||||
|
|
@ -56,14 +66,22 @@ final class ItemDatabase
|
|||
}
|
||||
db.exec("PRAGMA foreign_keys = ON");
|
||||
db.exec("PRAGMA recursive_triggers = ON");
|
||||
insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
|
||||
insertItemStmt = db.prepare("
|
||||
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentDriveId, parentId, crc32Hash, sha1Hash, quickXorHash)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
");
|
||||
updateItemStmt = db.prepare("
|
||||
UPDATE item
|
||||
SET name = ?2, type = ?3, eTag = ?4, cTag = ?5, mtime = ?6, parentId = ?7, crc32Hash = ?8, sha1Hash = ?9, quickXorHash = ?10
|
||||
WHERE id = ?1
|
||||
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentDriveId = ?8, parentId = ?9, crc32Hash = ?10, sha1Hash = ?11, quickXorHash = ?12
|
||||
WHERE driveId = ?1 AND id = ?2
|
||||
");
|
||||
selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash FROM item WHERE id = ?");
|
||||
selectItemByParentIdStmt = db.prepare("SELECT id FROM item WHERE parentId = ?");
|
||||
selectItemByIdStmt = db.prepare("
|
||||
SELECT *
|
||||
FROM item
|
||||
WHERE driveId = ?1 AND id = ?2
|
||||
");
|
||||
selectItemByParentIdStmt = db.prepare("SELECT driveId, id FROM item WHERE parentId = ? AND id = ?");
|
||||
deleteItemByIdStmt = db.prepare("DELETE FROM item WHERE driveId = ? AND id = ?");
|
||||
}
|
||||
|
||||
void insert(const ref Item item)
|
||||
|
|
@ -80,8 +98,9 @@ final class ItemDatabase
|
|||
|
||||
void upsert(const ref Item item)
|
||||
{
|
||||
auto s = db.prepare("SELECT COUNT(*) FROM item WHERE id = ?");
|
||||
s.bind(1, item.id);
|
||||
auto s = db.prepare("SELECT COUNT(*) FROM item WHERE driveId = ? AND id = ?");
|
||||
s.bind(1, item.driveId);
|
||||
s.bind(2, item.id);
|
||||
auto r = s.exec();
|
||||
Statement* stmt;
|
||||
if (r.front[0] == "0") stmt = &insertItemStmt;
|
||||
|
|
@ -90,23 +109,25 @@ final class ItemDatabase
|
|||
stmt.exec();
|
||||
}
|
||||
|
||||
Item[] selectChildren(const(char)[] id)
|
||||
Item[] selectChildren(const(char)[] driveId, const(char)[] id)
|
||||
{
|
||||
selectItemByParentIdStmt.bind(1, id);
|
||||
selectItemByParentIdStmt.bind(1, driveId);
|
||||
selectItemByParentIdStmt.bind(2, id);
|
||||
auto res = selectItemByParentIdStmt.exec();
|
||||
Item[] items;
|
||||
foreach (row; res) {
|
||||
Item item;
|
||||
bool found = selectById(row[0], item);
|
||||
assert(found);
|
||||
bool found = selectById(row[0], row[1], item);
|
||||
assert(found, "Could not select the child of the item");
|
||||
items ~= item;
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
bool selectById(const(char)[] id, out Item item)
|
||||
bool selectById(const(char)[] driveId, const(char)[] id, out Item item)
|
||||
{
|
||||
selectItemByIdStmt.bind(1, id);
|
||||
selectItemByIdStmt.bind(1, driveId);
|
||||
selectItemByIdStmt.bind(2, id);
|
||||
auto r = selectItemByIdStmt.exec();
|
||||
if (!r.empty) {
|
||||
item = buildItem(r);
|
||||
|
|
@ -115,143 +136,110 @@ final class ItemDatabase
|
|||
return false;
|
||||
}
|
||||
|
||||
// returns the item with the given path
|
||||
// the path is relative to the sync directory ex: "./Music/Turbo Killer.mp3"
|
||||
bool selectByPath(const(char)[] path, out Item item)
|
||||
{
|
||||
// prefix with the root dir
|
||||
Item currItem;
|
||||
path = "root/" ~ path.chompPrefix(".");
|
||||
|
||||
// initialize the search
|
||||
string[2][] candidates; // [id, parentId]
|
||||
auto s = db.prepare("SELECT id, parentId FROM item WHERE name = ?");
|
||||
s.bind(1, baseName(path));
|
||||
auto r = s.exec();
|
||||
foreach (row; r) candidates ~= [row[0].dup, row[1].dup];
|
||||
path = dirName(path);
|
||||
|
||||
if (path != ".") {
|
||||
s = db.prepare("SELECT parentId FROM item WHERE id = ? AND name = ?");
|
||||
// discard the candidates that do not have the correct parent
|
||||
do {
|
||||
s.bind(2, baseName(path));
|
||||
string[2][] newCandidates;
|
||||
newCandidates.reserve(candidates.length);
|
||||
foreach (candidate; candidates) {
|
||||
s.bind(1, candidate[1]);
|
||||
r = s.exec();
|
||||
if (!r.empty) {
|
||||
string[2] c = [candidate[0], r.front[0].idup];
|
||||
newCandidates ~= c;
|
||||
}
|
||||
}
|
||||
candidates = newCandidates;
|
||||
path = dirName(path);
|
||||
} while (path != ".");
|
||||
}
|
||||
|
||||
// reached the root
|
||||
string[2][] newCandidates;
|
||||
foreach (candidate; candidates) {
|
||||
if (!candidate[1]) {
|
||||
newCandidates ~= candidate;
|
||||
}
|
||||
}
|
||||
candidates = newCandidates;
|
||||
assert(candidates.length <= 1);
|
||||
|
||||
if (candidates.length == 1) return selectById(candidates[0][0], item);
|
||||
return false;
|
||||
}
|
||||
|
||||
void deleteById(const(char)[] id)
|
||||
{
|
||||
auto s = db.prepare("DELETE FROM item WHERE id = ?");
|
||||
s.bind(1, id);
|
||||
s.exec();
|
||||
}
|
||||
|
||||
// returns true if the item has the specified parent
|
||||
bool hasParent(T)(const(char)[] itemId, T parentId)
|
||||
if (is(T : const(char)[]) || is(T : const(char[])[]))
|
||||
{
|
||||
auto s = db.prepare("SELECT parentId FROM item WHERE id = ?");
|
||||
while (true) {
|
||||
s.bind(1, itemId);
|
||||
auto s = db.prepare("SELECT * FROM item WHERE name IS ?1 AND parentDriveId IS ?2 AND parentId IS ?3");
|
||||
foreach (name; pathSplitter(path)) {
|
||||
s.bind(1, name);
|
||||
s.bind(2, currItem.driveId);
|
||||
s.bind(3, currItem.id);
|
||||
auto r = s.exec();
|
||||
if (r.empty) break;
|
||||
auto currParentId = r.front[0];
|
||||
static if (is(T : const(char)[])) {
|
||||
if (currParentId == parentId) return true;
|
||||
} else {
|
||||
foreach (id; parentId) if (currParentId == id) return true;
|
||||
if (r.empty) return false;
|
||||
currItem = buildItem(r);
|
||||
// if the item of type remote jump to the child
|
||||
if (currItem.type == ItemType.remote) {
|
||||
auto children = selectChildren(currItem.driveId, currItem.id);
|
||||
enforce(children.length == 1, "The remote item has more than 1 child");
|
||||
currItem = children[0];
|
||||
}
|
||||
itemId = currParentId.dup;
|
||||
}
|
||||
return false;
|
||||
item = currItem;
|
||||
return true;
|
||||
}
|
||||
|
||||
void deleteById(const(char)[] driveId, const(char)[] id)
|
||||
{
|
||||
deleteItemByIdStmt.bind(1, driveId);
|
||||
deleteItemByIdStmt.bind(2, id);
|
||||
deleteItemByIdStmt.exec();
|
||||
}
|
||||
|
||||
private void bindItem(const ref Item item, ref Statement stmt)
|
||||
{
|
||||
with (stmt) with (item) {
|
||||
bind(1, id);
|
||||
bind(2, name);
|
||||
bind(1, driveId);
|
||||
bind(2, id);
|
||||
bind(3, name);
|
||||
string typeStr = null;
|
||||
final switch (type) with (ItemType) {
|
||||
case file: typeStr = "file"; break;
|
||||
case dir: typeStr = "dir"; break;
|
||||
case file: typeStr = "file"; break;
|
||||
case dir: typeStr = "dir"; break;
|
||||
case remote: typeStr = "remote"; break;
|
||||
}
|
||||
bind(3, typeStr);
|
||||
bind(4, eTag);
|
||||
bind(5, cTag);
|
||||
bind(6, mtime.toISOExtString());
|
||||
bind(7, parentId);
|
||||
bind(8, crc32Hash);
|
||||
bind(9, sha1Hash);
|
||||
bind(10, quickXorHash);
|
||||
bind(4, typeStr);
|
||||
bind(5, eTag);
|
||||
bind(6, cTag);
|
||||
bind(7, mtime.toISOExtString());
|
||||
bind(8, parentDriveId);
|
||||
bind(9, parentId);
|
||||
bind(10, crc32Hash);
|
||||
bind(11, sha1Hash);
|
||||
bind(12, quickXorHash);
|
||||
}
|
||||
}
|
||||
|
||||
private Item buildItem(Statement.Result result)
|
||||
{
|
||||
assert(!result.empty && result.front.length == 10);
|
||||
assert(!result.empty, "The result must not be empty");
|
||||
assert(result.front.length == 12, "The result must have 12 columns");
|
||||
Item item = {
|
||||
id: result.front[0].dup,
|
||||
name: result.front[1].dup,
|
||||
eTag: result.front[3].dup,
|
||||
cTag: result.front[4].dup,
|
||||
mtime: SysTime.fromISOExtString(result.front[5]),
|
||||
parentId: result.front[6].dup,
|
||||
crc32Hash: result.front[7].dup,
|
||||
sha1Hash: result.front[8].dup,
|
||||
quickXorHash: result.front[9].dup
|
||||
driveId: result.front[0].dup,
|
||||
id: result.front[1].dup,
|
||||
name: result.front[2].dup,
|
||||
eTag: result.front[4].dup,
|
||||
cTag: result.front[5].dup,
|
||||
mtime: SysTime.fromISOExtString(result.front[6]),
|
||||
parentDriveId: result.front[7].dup,
|
||||
parentId: result.front[8].dup,
|
||||
crc32Hash: result.front[9].dup,
|
||||
sha1Hash: result.front[10].dup,
|
||||
quickXorHash: result.front[11].dup
|
||||
};
|
||||
switch (result.front[2]) {
|
||||
case "file": item.type = ItemType.file; break;
|
||||
case "dir": item.type = ItemType.dir; break;
|
||||
switch (result.front[3]) {
|
||||
case "file": item.type = ItemType.file; break;
|
||||
case "dir": item.type = ItemType.dir; break;
|
||||
case "remote": item.type = ItemType.remote; break;
|
||||
default: assert(0);
|
||||
}
|
||||
return item;
|
||||
}
|
||||
|
||||
// computes the path of the given item id
|
||||
// the path is relative to the sync directory ex: "./Music/Turbo Killer.mp3"
|
||||
// a trailing slash is never added
|
||||
string computePath(const(char)[] id)
|
||||
// the path is relative to the sync directory ex: "Music/Turbo Killer.mp3"
|
||||
// a trailing slash is not added if the item is a directory
|
||||
string computePath(const(char)[] driveId, const(char)[] id)
|
||||
{
|
||||
string path;
|
||||
auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?");
|
||||
Item item;
|
||||
while (true) {
|
||||
s.bind(1, id);
|
||||
auto r = s.exec();
|
||||
enforce(!r.empty, "Unknow item id");
|
||||
if (r.front[1]) {
|
||||
if (path) path = r.front[0].idup ~ "/" ~ path;
|
||||
else path = r.front[0].idup;
|
||||
enforce(selectById(driveId, id, item), "Unknow item id");
|
||||
if (item.type == ItemType.remote) {
|
||||
// substitute the last name with the current
|
||||
path = item.name ~ path[indexOf(path, '/') .. $];
|
||||
} else if (item.parentId) {
|
||||
if (path) path = item.name ~ "/" ~ path;
|
||||
else path = item.name;
|
||||
} else {
|
||||
// root
|
||||
if (!path) path = ".";
|
||||
break;
|
||||
}
|
||||
id = r.front[1].dup;
|
||||
driveId = item.parentDriveId;
|
||||
id = item.parentId;
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
|
|
|||
36
src/sync.d
36
src/sync.d
|
|
@ -43,12 +43,14 @@ private Item makeItem(const ref JSONValue jsonItem)
|
|||
}
|
||||
|
||||
Item item = {
|
||||
driveId: jsonItem["parentReference"]["driveId"].str,
|
||||
id: jsonItem["id"].str,
|
||||
name: jsonItem["name"].str,
|
||||
type: type,
|
||||
eTag: isItemRoot(jsonItem) ? null : jsonItem["eTag"].str, // eTag is not returned for the root in OneDrive Biz
|
||||
cTag: "cTag" !in jsonItem ? null : jsonItem["cTag"].str, // cTag is missing in old files (plus all folders)
|
||||
mtime: SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str),
|
||||
parentDriveId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["driveId"].str,
|
||||
parentId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["id"].str
|
||||
};
|
||||
|
||||
|
|
@ -108,7 +110,7 @@ final class SyncEngine
|
|||
// list of items to skip while applying the changes
|
||||
private string[] skippedItems;
|
||||
// list of items to delete after the changes has been downloaded
|
||||
private string[] idsToDelete;
|
||||
private string[2][] idsToDelete;
|
||||
|
||||
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
|
||||
{
|
||||
|
|
@ -182,6 +184,7 @@ final class SyncEngine
|
|||
|
||||
private void applyDifference(JSONValue item)
|
||||
{
|
||||
string driveId = item["parentReference"]["driveId"].str;
|
||||
string id = item["id"].str;
|
||||
string name = item["name"].str;
|
||||
|
||||
|
|
@ -209,9 +212,9 @@ final class SyncEngine
|
|||
// rename the local item if it is unsynced and there is a new version of it
|
||||
Item oldItem;
|
||||
string oldPath;
|
||||
bool cached = itemdb.selectById(id, oldItem);
|
||||
bool cached = itemdb.selectById(driveId, id, oldItem);
|
||||
if (cached && eTag != oldItem.eTag) {
|
||||
oldPath = itemdb.computePath(id);
|
||||
oldPath = itemdb.computePath(driveId, id);
|
||||
if (!isItemSynced(oldItem, oldPath)) {
|
||||
log.vlog("The local item is unsynced, renaming");
|
||||
if (exists(oldPath)) safeRename(oldPath);
|
||||
|
|
@ -222,14 +225,14 @@ final class SyncEngine
|
|||
// check if the item is to be deleted
|
||||
if (isItemDeleted(item)) {
|
||||
log.vlog("The item is marked for deletion");
|
||||
if (cached) idsToDelete ~= id;
|
||||
if (cached) idsToDelete ~= [driveId, id];
|
||||
return;
|
||||
}
|
||||
|
||||
// compute the path of the item
|
||||
string path = ".";
|
||||
if (parentId) {
|
||||
path = itemdb.computePath(parentId) ~ "/" ~ name;
|
||||
path = itemdb.computePath(driveId, parentId) ~ "/" ~ name;
|
||||
// selective sync
|
||||
if (selectiveSync.isPathExcluded(path)) {
|
||||
log.vlog("Filtered out: ", path);
|
||||
|
|
@ -281,18 +284,21 @@ final class SyncEngine
|
|||
case ItemType.dir:
|
||||
log.log("Creating directory: ", path);
|
||||
mkdir(path);
|
||||
break;
|
||||
break;
|
||||
case ItemType.remote:
|
||||
assert(0);
|
||||
}
|
||||
setTimes(path, item.mtime, item.mtime);
|
||||
}
|
||||
|
||||
private void applyChangedItem(Item oldItem, Item newItem, string newPath)
|
||||
{
|
||||
assert(oldItem.driveId == newItem.driveId);
|
||||
assert(oldItem.id == newItem.id);
|
||||
assert(oldItem.type == newItem.type);
|
||||
|
||||
if (oldItem.eTag != newItem.eTag) {
|
||||
string oldPath = itemdb.computePath(oldItem.id);
|
||||
string oldPath = itemdb.computePath(oldItem.driveId, oldItem.id);
|
||||
if (oldPath != newPath) {
|
||||
log.log("Moving: ", oldPath, " -> ", newPath);
|
||||
if (exists(newPath)) {
|
||||
|
|
@ -343,6 +349,8 @@ final class SyncEngine
|
|||
log.vlog("The local item is a file but should be a directory");
|
||||
}
|
||||
break;
|
||||
case ItemType.remote:
|
||||
assert(0);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
@ -350,9 +358,9 @@ final class SyncEngine
|
|||
private void deleteItems()
|
||||
{
|
||||
log.vlog("Deleting files ...");
|
||||
foreach_reverse (id; idsToDelete) {
|
||||
string path = itemdb.computePath(id);
|
||||
itemdb.deleteById(id);
|
||||
foreach_reverse (i; idsToDelete) {
|
||||
string path = itemdb.computePath(i[0], i[1]);
|
||||
itemdb.deleteById(i[0], i[1]);
|
||||
if (exists(path)) {
|
||||
if (isFile(path)) {
|
||||
remove(path);
|
||||
|
|
@ -400,7 +408,7 @@ final class SyncEngine
|
|||
log.vlog("Filtered out");
|
||||
return;
|
||||
}
|
||||
string path = itemdb.computePath(item.id);
|
||||
string path = itemdb.computePath(item.driveId, item.id);
|
||||
if (selectiveSync.isPathExcluded(path)) {
|
||||
log.vlog("Filtered out: ", path);
|
||||
return;
|
||||
|
|
@ -413,6 +421,8 @@ final class SyncEngine
|
|||
case ItemType.file:
|
||||
uploadFileDifferences(item, path);
|
||||
break;
|
||||
case ItemType.remote:
|
||||
assert(0);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -427,7 +437,7 @@ final class SyncEngine
|
|||
} else {
|
||||
log.vlog("The directory has not changed");
|
||||
// loop trough the children
|
||||
foreach (Item child; itemdb.selectChildren(item.id)) {
|
||||
foreach (Item child; itemdb.selectChildren(item.driveId, item.id)) {
|
||||
uploadDifferences(child);
|
||||
}
|
||||
}
|
||||
|
|
@ -552,7 +562,7 @@ final class SyncEngine
|
|||
if (e.httpStatusCode == 404) log.log(e.msg);
|
||||
else throw e;
|
||||
}
|
||||
itemdb.deleteById(item.id);
|
||||
itemdb.deleteById(item.driveId, item.id);
|
||||
}
|
||||
|
||||
private void uploadLastModifiedTime(const(char)[] id, const(char)[] eTag, SysTime mtime)
|
||||
|
|
|
|||
Loading…
Reference in a new issue