deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-01 21:52:50 +02:00
Code Issues Projects Releases Wiki Activity

partial remote items support

Browse source
This commit is contained in:
skilion 2017-12-27 15:13:28 +01:00
parent d9c9915bc3
commit 789ec85e0c
2 changed files with 185 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
src/itemdb.d
View file

@ -1,6 +1,8 @@
import std.datetime, std.path, std.exception, std.string;
import sqlite;
import std.stdio;
enum ItemType
{
file,
@ -22,12 +24,15 @@ struct Item
string crc32Hash;
string sha1Hash;
string quickXorHash;
string remoteDriveId;
string remoteId;
string deltaLink;
}
final class ItemDatabase
{
// increment this for every change in the db schema
immutable int itemDatabaseVersion = 5;
immutable int itemDatabaseVersion = 6 ;
Database db;
Statement insertItemStmt;
@ -53,26 +58,33 @@ final class ItemDatabase
crc32Hash TEXT,
sha1Hash TEXT,
quickXorHash TEXT,
remoteDriveId TEXT,
remoteId TEXT,
deltaLink TEXT,
PRIMARY KEY (driveId, id),
FOREIGN KEY (parentDriveId, parentId)
REFERENCES item (driveId, id)
FOREIGN KEY (parentDriveId, parentId)
REFERENCES item (driveId, id)
ON DELETE CASCADE
ON UPDATE RESTRICT
)");
db.exec("CREATE INDEX name_idx ON item (name)");
/*db.exec("CREATE INDEX name_idx ON item (name)");
db.exec("CREATE INDEX id_idx ON item (id)");
db.exec("CREATE INDEX remoteId_idx ON item (remoteId)");*/
db.setVersion(itemDatabaseVersion);
} else if (db.getVersion() != itemDatabaseVersion) {
throw new Exception("The item database is incompatible, please resync manually");
}
db.exec("PRAGMA foreign_keys = ON");
db.exec("PRAGMA recursive_triggers = ON");
/*db.exec("PRAGMA foreign_keys = ON");
db.exec("PRAGMA recursive_triggers = ON");*/
insertItemStmt = db.prepare("
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentDriveId, parentId, crc32Hash, sha1Hash, quickXorHash)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentDriveId, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId, deltaLink)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
");
updateItemStmt = db.prepare("
UPDATE item
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentDriveId = ?8, parentId = ?9, crc32Hash = ?10, sha1Hash = ?11, quickXorHash = ?12
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentDriveId = ?8, parentId = ?9, crc32Hash = ?10, sha1Hash = ?11, quickXorHash = ?12, remoteDriveId = ?13, remoteId = ?14, deltaLink = ?15
WHERE driveId = ?1 AND id = ?2
");
selectItemByIdStmt = db.prepare("
@ -143,6 +155,7 @@ final class ItemDatabase
Item currItem;
path = "root/" ~ path.chompPrefix(".");
auto s = db.prepare("SELECT * FROM item WHERE name IS ?1 AND parentDriveId IS ?2 AND parentId IS ?3");
writeln("selectByPath " ~ path);
foreach (name; pathSplitter(path)) {
s.bind(1, name);
s.bind(2, currItem.driveId);
@ -152,12 +165,13 @@ final class ItemDatabase
currItem = buildItem(r);
// if the item is of type remote substitute it with the child
if (currItem.type == ItemType.remote) {
auto children = selectChildren(currItem.driveId, currItem.id);
enforce(children.length == 1, "The remote item does not have exactly 1 child");
// keep the name of the remote item
children[0].name = currItem.name;
currItem = children[0];
Item child;
if (selectById(currItem.remoteDriveId, currItem.remoteId, child)) {
assert(child.type != ItemType.remote, "The type of the child cannot be remote");
currItem = child;
}
}
writeln(currItem.id);
}
item = currItem;
return true;
@ -191,13 +205,16 @@ final class ItemDatabase
bind(10, crc32Hash);
bind(11, sha1Hash);
bind(12, quickXorHash);
bind(13, remoteDriveId);
bind(14, remoteId);
bind(15, deltaLink);
}
}
private Item buildItem(Statement.Result result)
{
assert(!result.empty, "The result must not be empty");
assert(result.front.length == 12, "The result must have 12 columns");
assert(result.front.length == 15, "The result must have 15 columns");
Item item = {
driveId: result.front[0].dup,
id: result.front[1].dup,
@ -209,39 +226,74 @@ final class ItemDatabase
parentId: result.front[8].dup,
crc32Hash: result.front[9].dup,
sha1Hash: result.front[10].dup,
quickXorHash: result.front[11].dup
quickXorHash: result.front[11].dup,
remoteDriveId: result.front[12].dup,
remoteId: result.front[13].dup,
deltaLink: result.front[14].dup
};
switch (result.front[3]) {
case "file": item.type = ItemType.file; break;
case "dir": item.type = ItemType.dir; break;
case "remote": item.type = ItemType.remote; break;
default: assert(0);
default: assert(0, "Invalid item type");
}
return item;
}
// computes the path of the given item id
// the path is relative to the sync directory ex: "Music/Turbo Killer.mp3"
// a trailing slash is not added if the item is a directory
// the trailing slash is not added even if the item is a directory
string computePath(const(char)[] driveId, const(char)[] id)
{
assert(driveId && id);
string path;
Item item;
auto s = db.prepare("SELECT * FROM item WHERE driveId = ?1 AND id = ?2");
auto s2 = db.prepare("SELECT driveId, id FROM item WHERE remoteDriveId = ?1 AND remoteId = ?2");
writeln("computePath " ~ id);
while (true) {
enforce(selectById(driveId, id, item), "Unknow item id");
if (item.type == ItemType.remote) {
// substitute the last name with the current
path = item.name ~ path[indexOf(path, '/') .. $];
} else if (item.parentId) {
if (path) path = item.name ~ "/" ~ path;
else path = item.name;
s.bind(1, driveId);
s.bind(2, id);
auto r = s.exec();
if (!r.empty) {
item = buildItem(r);
if (item.type == ItemType.remote) {
// substitute the last name with the current
ptrdiff_t idx = indexOf(path, '/');
path = idx >= 0 ? item.name ~ path[idx .. $] : item.name;
} else {
if (path) path = item.name ~ "/" ~ path;
else path = item.name;
}
driveId = item.parentDriveId;
id = item.parentId;
} else {
// root
if (!path) path = ".";
break;
if (id == null) {
// check for remoteItem
s2.bind(1, item.driveId);
s2.bind(2, item.id);
auto r2 = s2.exec();
if (r2.empty) {
// root reached
assert(path.length >= 4);
// remove "root"
if (path.length >= 5) path = path[5 .. $];
else path = path[4 .. $];
// special case of computing the path of the root itself
if (path.length == 0) path = ".";
writeln(path);
break;
} else {
// remote folder
driveId = r2.front[0].dup;
id = r2.front[1].dup;
}
} else {
// broken tree
assert(0);
}
}
driveId = item.parentDriveId;
id = item.parentId;
writeln(path);
}
return path;
}

178
src/sync.d
View file

@ -28,7 +28,7 @@ private bool isItemDeleted(const ref JSONValue item)
private bool isItemRoot(const ref JSONValue item)
{
return ("root" in item) != null;
return ("root" in item) != null || ("parentReference" in item) == null;
}
private bool isItemRemote(const ref JSONValue item)
@ -58,7 +58,7 @@ private Item makeItem(const ref JSONValue jsonItem)
eTag: "eTag" in jsonItem ? jsonItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Biz
cTag: "cTag" in jsonItem ? jsonItem["cTag"].str : null, // cTag is missing in old files (and all folders)
mtime: "fileSystemInfo" in jsonItem ? SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str) : SysTime(0),
parentDriveId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["driveId"].str,
parentDriveId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["driveId"].str, // root and remote items do not have parentReference
parentId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["id"].str
};
@ -77,6 +77,11 @@ private Item makeItem(const ref JSONValue jsonItem)
}
}
if (isItemRemote(jsonItem)) {
item.remoteDriveId = jsonItem["remoteItem"]["parentReference"]["driveId"].str;
item.remoteId = jsonItem["remoteItem"]["id"].str;
}
return item;
}
@ -141,46 +146,10 @@ final class SyncEngine
{
log.vlog("Applying differences ...");
// restore the last known state
string deltaLink;
try {
deltaLink = readText(cfg.deltaLinkFilePath);
} catch (FileException e) {
// swallow exception
}
try {
defaultDriveId = onedrive.getDefaultDrive()["id"].str;
JSONValue changes;
do {
// get changes from the server
try {
changes = onedrive.viewChangesByPath(".", deltaLink);
} catch (OneDriveException e) {
if (e.httpStatusCode == 410) {
log.log("Delta link expired, resyncing");
deltaLink = null;
continue;
} else {
throw e;
}
}
foreach (item; changes["value"].array) {
applyDifference(item);
}
if ("@odata.nextLink" in changes) deltaLink = changes["@odata.nextLink"].str;
if ("@odata.deltaLink" in changes) deltaLink = changes["@odata.deltaLink"].str;
std.file.write(cfg.deltaLinkFilePath, deltaLink);
} while ("@odata.nextLink" in changes);
} catch (ErrnoException e) {
throw new SyncException(e.msg, e);
} catch (FileException e) {
throw new SyncException(e.msg, e);
} catch (CurlTimeoutException e) {
throw new SyncException(e.msg, e);
} catch (OneDriveException e) {
throw new SyncException(e.msg, e);
}
string driveId = onedrive.getDefaultDrive()["id"].str;
string rootId = onedrive.getDefaultRoot["id"].str;
applyDifferences(driveId, rootId);
// delete items in idsToDelete
if (idsToDelete.length > 0) deleteItems();
// empty the skipped items
@ -188,16 +157,59 @@ final class SyncEngine
assumeSafeAppend(skippedItems);
}
void applyDifferences(const(char)[] driveId, const(char)[] id)
{
// HACK
string h = driveId.dup;
// restore the last known state
string deltaLink;
Item beginItem;
if (itemdb.selectById(driveId, id, beginItem)) {
deltaLink = beginItem.deltaLink;
}
JSONValue changes;
do {
// HACK
defaultDriveId = h;
// get changes from the server
try {
changes = onedrive.viewChangesById(driveId, id, deltaLink);
} catch (OneDriveException e) {
if (e.httpStatusCode == 410) {
log.log("Delta link expired, resyncing");
deltaLink = null;
continue;
} else {
throw e;
}
}
foreach (item; changes["value"].array) {
applyDifference(item);
}
if ("@odata.deltaLink" in changes) deltaLink = changes["@odata.deltaLink"].str;
// save the state
import std.exception;
enforce(itemdb.selectById(driveId, id, beginItem));
beginItem.deltaLink = deltaLink;
itemdb.upsert(beginItem);
if ("@odata.nextLink" in changes) deltaLink = changes["@odata.nextLink"].str;
} while ("@odata.nextLink" in changes);
}
private void applyDifference(JSONValue jsonItem)
{
log.vlog(jsonItem["id"].str, " ", "name" in jsonItem ? jsonItem["name"].str : null);
Item item = makeItem(jsonItem);
string path = ".";
bool unwanted;
unwanted |= skippedItems.find(item.parentId).length != 0;
unwanted |= selectiveSync.isNameExcluded(item.name);
string path = ".";
if (!unwanted && !isItemRoot(jsonItem)) {
// delay path computation after assuring the item parent is not excluded
path = itemdb.computePath(item.parentDriveId, item.parentId) ~ "/" ~ item.name;
@ -214,13 +226,8 @@ final class SyncEngine
// check the item type
if (isItemRemote(jsonItem)) {
// TODO
// check name change
// scan the children later
// fix child references
log.vlog("Remote items are not supported yet");
skippedItems ~= item.id;
return;
log.vlog("Remote item");
assert(isItemFolder(jsonItem["remoteItem"]), "The remote item is not a folder");
} else if (!isItemFile(jsonItem) && !isItemFolder(jsonItem) && !isItemDeleted(jsonItem)) {
log.vlog("The item is neither a file nor a directory, skipping");
skippedItems ~= item.id;
@ -255,18 +262,25 @@ final class SyncEngine
}
}
if (!cached) {
applyNewItem(item, path);
} else {
// update the item
if (cached) {
applyChangedItem(oldItem, oldPath, item, path);
} else {
applyNewItem(item, path);
}
// save the item in the db
if (oldItem.id) {
if (cached) {
itemdb.update(item);
} else {
itemdb.insert(item);
}
// sync remote folder
if (isItemRemote(jsonItem)) {
log.log("Syncing remote folder: ", path);
applyDifferences(item.remoteDriveId, item.remoteId);
}
}
private void applyNewItem(Item item, string path)
@ -274,8 +288,6 @@ final class SyncEngine
if (exists(path)) {
if (isItemSynced(item, path)) {
log.vlog("The item is already present");
// ensure the modified time is correct
setTimes(path, item.mtime, item.mtime);
return;
} else {
log.vlog("The local item is out of sync, renaming...");
@ -285,17 +297,15 @@ final class SyncEngine
final switch (item.type) {
case ItemType.file:
log.log("Downloading: ", path);
onedrive.downloadById(item.id, path);
onedrive.downloadById(item.driveId, item.id, path);
setTimes(path, item.mtime, item.mtime);
break;
case ItemType.dir:
case ItemType.remote:
log.log("Creating directory: ", path);
//Use mkdirRecuse to deal nested dir
mkdirRecurse(path);
break;
case ItemType.remote:
assert(0);
}
setTimes(path, item.mtime, item.mtime);
}
// update a local item
@ -307,6 +317,7 @@ final class SyncEngine
assert(oldItem.type == newItem.type);
if (oldItem.eTag != newItem.eTag) {
// handle changed path
if (oldPath != newPath) {
log.log("Moving: ", oldPath, " -> ", newPath);
if (exists(newPath)) {
@ -315,11 +326,27 @@ final class SyncEngine
}
rename(oldPath, newPath);
}
if (newItem.type == ItemType.file && oldItem.cTag != newItem.cTag) {
log.log("Downloading: ", newPath);
onedrive.downloadById(newItem.id, newPath);
// handle changed content
if (oldItem.cTag != newItem.cTag) {
final switch (newItem.type) {
case ItemType.file:
log.log("Downloading: ", newPath);
onedrive.downloadById(newItem.driveId, newItem.id, newPath);
break;
case ItemType.dir:
// nothing to do
break;
case ItemType.remote:
assert(oldItem.remoteDriveId == newItem.remoteDriveId);
assert(oldItem.remoteId == newItem.remoteId);
// nothing to do
break;
}
}
// handle changed time
if (newItem.type == ItemType.file) {
setTimes(newPath, newItem.mtime, newItem.mtime);
}
setTimes(newPath, newItem.mtime, newItem.mtime);
} else {
log.vlog("The item has not changed");
}
@ -351,14 +378,13 @@ final class SyncEngine
}
break;
case ItemType.dir:
case ItemType.remote:
if (isDir(path)) {
return true;
} else {
log.vlog("The local item is a file but should be a directory");
}
break;
case ItemType.remote:
assert(0);
}
return false;
}
@ -371,6 +397,7 @@ final class SyncEngine
if (!itemdb.selectById(i[0], i[1], item)) continue; // check if the item is in the db
string path = itemdb.computePath(i[0], i[1]);
itemdb.deleteById(i[0], i[1]);
// TODO CHECK REMOTE ITEM
if (exists(path)) {
if (isFile(path)) {
remove(path);
@ -468,6 +495,7 @@ final class SyncEngine
localModifiedTime.fracSecs = Duration.zero;
if (localModifiedTime != item.mtime) {
log.vlog("The file last modified time has changed");
string driveId = item.driveId;
string id = item.id;
string eTag = item.eTag;
if (!testFileHash(path, item)) {
@ -481,12 +509,13 @@ final class SyncEngine
}
saveItem(response);
id = response["id"].str;
driveId = response["parentReference"]["driveId"].str;
/* use the cTag instead of the eTag because Onedrive changes the
* metadata of some type of files (ex. images) AFTER they have been
* uploaded */
eTag = response["cTag"].str;
}
uploadLastModifiedTime(id, eTag, localModifiedTime.toUTC());
uploadLastModifiedTime(driveId, id, eTag, localModifiedTime.toUTC());
} else {
log.vlog("The file has not changed");
}
@ -503,6 +532,7 @@ final class SyncEngine
private void uploadNewItems(string path)
{
writeln("uploadNewItems " ~ path);
// skip unexisting symbolic links
if (isSymlink(path) && !exists(readLink(path))) {
return;
@ -554,13 +584,14 @@ final class SyncEngine
} else {
response = session.upload(path, path);
}
string driveId = response["parentReference"]["driveId"].str;
string id = response["id"].str;
string cTag = response["cTag"].str;
SysTime mtime = timeLastModified(path).toUTC();
/* use the cTag instead of the eTag because Onedrive changes the
* metadata of some type of files (ex. images) AFTER they have been
* uploaded */
uploadLastModifiedTime(id, cTag, mtime);
uploadLastModifiedTime(driveId, id, cTag, mtime);
}
private void uploadDeleteItem(Item item, const(char)[] path)
@ -575,14 +606,14 @@ final class SyncEngine
itemdb.deleteById(item.driveId, item.id);
}
private void uploadLastModifiedTime(const(char)[] id, const(char)[] eTag, SysTime mtime)
private void uploadLastModifiedTime(const(char)[] driveId, const(char)[] id, const(char)[] eTag, SysTime mtime)
{
JSONValue mtimeJson = [
"fileSystemInfo": JSONValue([
"lastModifiedDateTime": mtime.toISOExtString()
])
];
auto res = onedrive.updateById(id, mtimeJson, eTag);
auto res = onedrive.updateById(driveId, id, mtimeJson, eTag);
saveItem(res);
}
@ -610,11 +641,12 @@ final class SyncEngine
diff["parentReference"] = JSONValue([
"id": parentItem.id
]);
auto res = onedrive.updateById(fromItem.id, diff, fromItem.eTag);
auto res = onedrive.updateById(fromItem.driveId, fromItem.id, diff, fromItem.eTag);
saveItem(res);
string driveId = res["parentReference"]["driveId"].str;
string id = res["id"].str;
string eTag = res["eTag"].str;
uploadLastModifiedTime(id, eTag, timeLastModified(to).toUTC());
uploadLastModifiedTime(driveId, id, eTag, timeLastModified(to).toUTC());
}
void deleteByPath(const(char)[] path)