deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-10 17:56:44 +02:00
Code Issues Projects Releases Wiki Activity

WIP on remote folders

Browse source
This commit is contained in:
skilion 2017-12-31 02:30:31 +01:00
parent b250214577
commit b7adc4d0cc
3 changed files with 187 additions and 142 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
src/itemdb.d
View file

@ -26,13 +26,12 @@ struct Item
string quickXorHash;
string remoteDriveId;
string remoteId;
string deltaLink;
}
final class ItemDatabase
{
// increment this for every change in the db schema
immutable int itemDatabaseVersion = 6 ;
immutable int itemDatabaseVersion = 6;
Database db;
Statement insertItemStmt;
@ -76,12 +75,12 @@ final class ItemDatabase
db.exec("PRAGMA foreign_keys = ON");
db.exec("PRAGMA recursive_triggers = ON");
insertItemStmt = db.prepare("
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentDriveId, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId, deltaLink)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentDriveId, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
");
updateItemStmt = db.prepare("
UPDATE item
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentDriveId = ?8, parentId = ?9, crc32Hash = ?10, sha1Hash = ?11, quickXorHash = ?12, remoteDriveId = ?13, remoteId = ?14, deltaLink = ?15
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentDriveId = ?8, parentId = ?9, crc32Hash = ?10, sha1Hash = ?11, quickXorHash = ?12, remoteDriveId = ?13, remoteId = ?14
WHERE driveId = ?1 AND id = ?2
");
selectItemByIdStmt = db.prepare("
@ -202,7 +201,6 @@ final class ItemDatabase
bind(12, quickXorHash);
bind(13, remoteDriveId);
bind(14, remoteId);
bind(15, deltaLink);
}
}
@ -223,8 +221,7 @@ final class ItemDatabase
sha1Hash: result.front[10].dup,
quickXorHash: result.front[11].dup,
remoteDriveId: result.front[12].dup,
remoteId: result.front[13].dup,
deltaLink: result.front[14].dup
remoteId: result.front[13].dup
};
switch (result.front[3]) {
case "file": item.type = ItemType.file; break;
@ -292,4 +289,38 @@ final class ItemDatabase
}
return path;
}
Item[] selectRemoteItems()
{
Item[] items;
auto stmt = db.prepare("SELECT * FROM item WHERE remoteDriveId IS NOT NULL");
auto res = stmt.exec();
while (!res.empty) {
items ~= buildItem(res);
res.step();
}
return items;
}
string getDeltaLink(const(char)[] driveId, const(char)[] id)
{
assert(driveId && id);
auto stmt = db.prepare("SELECT deltaLink FROM item WHERE driveId = ?1 AND id = ?2");
stmt.bind(1, driveId);
stmt.bind(2, id);
auto res = stmt.exec();
if (res.empty) return null;
return res.front[0].dup;
}
void setDeltaLink(const(char)[] driveId, const(char)[] id, const(char)[] deltaLink)
{
assert(driveId && id);
assert(deltaLink);
auto stmt = db.prepare("UPDATE item SET deltaLink = ?3 WHERE driveId = ?1 AND id = ?2");
stmt.bind(1, driveId);
stmt.bind(2, id);
stmt.bind(3, deltaLink);
stmt.exec();
}
}

25
src/onedrive.d
View file

@ -136,13 +136,34 @@ final class OneDriveApi
download(url, saveToPath);
}
// https://dev.onedrive.com/items/upload_put.htm
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
JSONValue simpleUpload(string localPath, const(char)[] remotePath, const(char)[] eTag = null)
{
checkAccessTokenExpired();
string url = itemByPathUrl ~ encodeComponent(remotePath) ~ ":/content";
// TODO: investigate why this fails for remote folders
//if (eTag) http.addRequestHeader("If-Match", eTag);
/*else*/ http.addRequestHeader("If-None-Match", "*");
return upload(localPath, url);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
JSONValue simpleUpload(string localPath, string parentDriveId, string parentId, string filename, const(char)[] eTag = null)
{
checkAccessTokenExpired();
string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/content";
// TODO: investigate why this fails for remote folders
//if (eTag) http.addRequestHeader("If-Match", eTag);
/*else http.addRequestHeader("If-None-Match", "*");*/
return upload(localPath, url);
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
JSONValue simpleUploadById(string localPath, string driveId, string id, const(char)[] eTag = null)
{
checkAccessTokenExpired();
string url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content";
if (eTag) http.addRequestHeader("If-Match", eTag);
else url ~= "?@name.conflictBehavior=fail";
return upload(localPath, url);
}

257
src/sync.d
View file

@ -1,7 +1,5 @@
import std.algorithm;
import std.array: array;
import std.net.curl: CurlTimeoutException;
import std.exception: ErrnoException;
import std.datetime, std.file, std.json, std.path;
import std.regex;
import std.stdio, std.string;
@ -36,54 +34,50 @@ private bool isItemRemote(const ref JSONValue item)
return ("remoteItem" in item) != null;
}
// HACK: OneDrive Biz does not return parentReference for the root
string defaultDriveId;
private Item makeItem(const ref JSONValue jsonItem)
// construct an Item struct from a JSON driveItem
private Item makeItem(const ref JSONValue driveItem)
{
ItemType type;
if (isItemFile(jsonItem)) {
type = ItemType.file;
} else if (isItemFolder(jsonItem)) {
type = ItemType.dir;
} else if (isItemRemote(jsonItem)) {
type = ItemType.remote;
}
Item item = {
driveId: "parentReference" in jsonItem ? jsonItem["parentReference"]["driveId"].str : defaultDriveId, // HACK
id: jsonItem["id"].str,
name: "name" in jsonItem ? jsonItem["name"].str : null, // name may be missing for deleted files in OneDrive Biz
type: type,
eTag: "eTag" in jsonItem ? jsonItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Biz
cTag: "cTag" in jsonItem ? jsonItem["cTag"].str : null, // cTag is missing in old files (and all folders)
mtime: "fileSystemInfo" in jsonItem ? SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str) : SysTime(0),
id: driveItem["id"].str,
name: "name" in driveItem ? driveItem["name"].str : null, // name may be missing for deleted files in OneDrive Biz
eTag: "eTag" in driveItem ? driveItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Biz
cTag: "cTag" in driveItem ? driveItem["cTag"].str : null, // cTag is missing in old files (and all folders in OneDrive Biz)
mtime: "fileSystemInfo" in driveItem ? SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str) : SysTime(0),
};
if (isItemFile(driveItem)) {
item.type = ItemType.file;
} else if (isItemFolder(driveItem)) {
item.type = ItemType.dir;
} else if (isItemRemote(driveItem)) {
item.type = ItemType.remote;
} else {
assert(0);
}
// root and remote items do not have parentReference
if (!isItemRoot(jsonItem) && ("parentReference" in jsonItem) != null) {
item.parentDriveId = jsonItem["parentReference"]["driveId"].str;
item.parentId = jsonItem["parentReference"]["id"].str;
if (!isItemRoot(driveItem) && ("parentReference" in driveItem) != null) {
item.driveId = driveItem["parentReference"]["driveId"].str,
item.parentDriveId = item.driveId; // TODO: parentDriveId is redundant
item.parentId = driveItem["parentReference"]["id"].str;
}
// extract the file hash
if (isItemFile(jsonItem)) {
if ("hashes" in jsonItem["file"]) {
if ("crc32Hash" in jsonItem["file"]["hashes"]) {
item.crc32Hash = jsonItem["file"]["hashes"]["crc32Hash"].str;
} else if ("sha1Hash" in jsonItem["file"]["hashes"]) {
item.sha1Hash = jsonItem["file"]["hashes"]["sha1Hash"].str;
} else if ("quickXorHash" in jsonItem["file"]["hashes"]) {
item.quickXorHash = jsonItem["file"]["hashes"]["quickXorHash"].str;
} else {
log.vlog("The file does not have any hash");
}
if (isItemFile(driveItem) && ("hashes" in driveItem["file"])) {
if ("crc32Hash" in driveItem["file"]["hashes"]) {
item.crc32Hash = driveItem["file"]["hashes"]["crc32Hash"].str;
} else if ("sha1Hash" in driveItem["file"]["hashes"]) {
item.sha1Hash = driveItem["file"]["hashes"]["sha1Hash"].str;
} else if ("quickXorHash" in driveItem["file"]["hashes"]) {
item.quickXorHash = driveItem["file"]["hashes"]["quickXorHash"].str;
} else {
log.vlog("The file does not have any hash");
}
}
if (isItemRemote(jsonItem)) {
item.remoteDriveId = jsonItem["remoteItem"]["parentReference"]["driveId"].str;
item.remoteId = jsonItem["remoteItem"]["id"].str;
if (isItemRemote(driveItem)) {
item.remoteDriveId = driveItem["remoteItem"]["parentReference"]["driveId"].str;
item.remoteId = driveItem["remoteItem"]["id"].str;
}
return item;
@ -109,6 +103,7 @@ class SyncException: Exception
}
}
// This class handle the synchronization of one folder (ex. the root)
final class SyncEngine
{
private Config cfg;
@ -141,42 +136,33 @@ final class SyncEngine
}
}
// download all new changes from OneDrive
void applyDifferences()
{
log.vlog("Applying differences ...");
// root folder
string driveId = onedrive.getDefaultDrive()["id"].str;
string rootId = onedrive.getDefaultRoot["id"].str;
applyDifferences(driveId, rootId);
// delete items in idsToDelete
if (idsToDelete.length > 0) deleteItems();
// empty the skipped items
skippedItems.length = 0;
assumeSafeAppend(skippedItems);
// check all remote folders
// https://github.com/OneDrive/onedrive-api-docs/issues/764
Item[] items = itemdb.selectRemoteItems();
foreach (item; items) applyDifferences(item.remoteDriveId, item.remoteId);
}
void applyDifferences(const(char)[] driveId, const(char)[] id)
// download all new changes from OneDrive
void applyDifferences(string driveId, const(char)[] id)
{
// HACK
string h = driveId.dup;
// restore the last known state
string deltaLink;
Item beginItem;
if (itemdb.selectById(driveId, id, beginItem)) {
deltaLink = beginItem.deltaLink;
}
JSONValue changes;
string deltaLink = itemdb.getDeltaLink(driveId, id);
log.vlog("Downloading changes of " ~ id);
do {
// get changes from the server
try {
changes = onedrive.viewChangesById(driveId, id, deltaLink);
} catch (OneDriveException e) {
if (e.httpStatusCode == 410) {
log.log("Delta link expired, resyncing");
log.vlog("Delta link expired, resyncing...");
deltaLink = null;
continue;
} else {
@ -184,29 +170,31 @@ final class SyncEngine
}
}
foreach (item; changes["value"].array) {
// HACK
defaultDriveId = h;
applyDifference(item);
applyDifference(item, driveId);
}
// the response may contain either @odata.deltaLink or @odata.nextLink
if ("@odata.deltaLink" in changes) deltaLink = changes["@odata.deltaLink"].str;
// save the state
import std.exception;
enforce(itemdb.selectById(driveId, id, beginItem));
beginItem.deltaLink = deltaLink;
itemdb.upsert(beginItem);
if (deltaLink) itemdb.setDeltaLink(driveId, id, deltaLink);
if ("@odata.nextLink" in changes) deltaLink = changes["@odata.nextLink"].str;
} while ("@odata.nextLink" in changes);
// delete items in idsToDelete
if (idsToDelete.length > 0) deleteItems();
// empty the skipped items
skippedItems.length = 0;
assumeSafeAppend(skippedItems);
}
private void applyDifference(JSONValue jsonItem)
// process the change of a single DriveItem
private void applyDifference(JSONValue driveItem, string driveId)
{
log.vlog(jsonItem["id"].str, " ", "name" in jsonItem ? jsonItem["name"].str : null);
Item item = makeItem(jsonItem);
Item item = makeItem(driveItem);
log.vlog("Processing ", item.id, " ", item.name);
if (!isItemRoot(jsonItem) && !item.parentDriveId) {
log.vlog("Root of remote item");
// this the root of a remote item, it can be skipped safely
if (isItemRoot(driveItem) || !item.parentDriveId) {
log.vlog("Root");
item.driveId = driveId; // HACK: makeItem() cannot set the driveId propery of the root
itemdb.upsert(item);
return;
}
@ -215,14 +203,27 @@ final class SyncEngine
unwanted |= skippedItems.find(item.parentId).length != 0;
unwanted |= selectiveSync.isNameExcluded(item.name);
string path = ".";
if (!unwanted && !isItemRoot(jsonItem)) {
// delay path computation after assuring the item parent is not excluded
// check the item type
if (!unwanted) {
if (isItemFile(driveItem)) {
log.vlog("File");
} else if (isItemFolder(driveItem)) {
log.vlog("Folder");
} else if (isItemRemote(driveItem)) {
log.vlog("Remote item");
assert(isItemFolder(driveItem["remoteItem"]), "The remote item is not a folder");
} else {
log.vlog("The item type is not supported");
unwanted = true;
}
}
// check for selective sync
string path;
if (!unwanted) {
path = itemdb.computePath(item.parentDriveId, item.parentId) ~ "/" ~ item.name;
// ensure path matches the path generated by itemdb.computePath()
path = asNormalizedPath(path).array;
// selective sync
unwanted |= selectiveSync.isPathExcluded(path);
path = buildNormalizedPath(path);
unwanted = selectiveSync.isPathExcluded(path);
}
// skip unwanted items early
@ -232,22 +233,12 @@ final class SyncEngine
return;
}
// check the item type
if (isItemRemote(jsonItem)) {
log.vlog("Remote item");
assert(isItemFolder(jsonItem["remoteItem"]), "The remote item is not a folder");
} else if (!isItemFile(jsonItem) && !isItemFolder(jsonItem) && !isItemDeleted(jsonItem)) {
log.vlog("The item is neither a file nor a directory, skipping");
skippedItems ~= item.id;
return;
}
// check if the item has been seen before
Item oldItem;
bool cached = itemdb.selectById(item.driveId, item.id, oldItem);
// check if the item is going to be deleted
if (isItemDeleted(jsonItem)) {
if (isItemDeleted(driveItem)) {
log.vlog("The item is marked for deletion");
if (cached) {
// flag to delete
@ -285,12 +276,13 @@ final class SyncEngine
}
// sync remote folder
if (isItemRemote(jsonItem)) {
/*if (isItemRemote(driveItem)) {
log.log("Syncing remote folder: ", path);
applyDifferences(item.remoteDriveId, item.remoteId);
}
}*/
}
// download an item that was not synced before
private void applyNewItem(Item item, string path)
{
if (exists(path)) {
@ -298,19 +290,18 @@ final class SyncEngine
log.vlog("The item is already present");
return;
} else {
// TODO: force remote sync by deleting local item
log.vlog("The local item is out of sync, renaming...");
safeRename(path);
}
}
final switch (item.type) {
case ItemType.file:
log.log("Downloading: ", path);
onedrive.downloadById(item.driveId, item.id, path);
setTimes(path, item.mtime, item.mtime);
downloadFileItem(item, path);
break;
case ItemType.dir:
case ItemType.remote:
log.log("Creating directory: ", path);
log.log("Creating directory ", path);
mkdirRecurse(path);
break;
}
@ -323,45 +314,47 @@ final class SyncEngine
assert(oldItem.driveId == newItem.driveId);
assert(oldItem.id == newItem.id);
assert(oldItem.type == newItem.type);
assert(oldItem.remoteDriveId == newItem.remoteDriveId);
assert(oldItem.remoteId == newItem.remoteId);
if (oldItem.eTag != newItem.eTag) {
// handle changed path
// handle changed name/path
if (oldPath != newPath) {
log.log("Moving: ", oldPath, " -> ", newPath);
if (exists(newPath)) {
// TODO: force remote sync by deleting local item
log.vlog("The destination is occupied, renaming the conflicting file...");
safeRename(newPath);
}
rename(oldPath, newPath);
}
// handle changed content
if (oldItem.cTag != newItem.cTag) {
final switch (newItem.type) {
case ItemType.file:
log.log("Downloading: ", newPath);
onedrive.downloadById(newItem.driveId, newItem.id, newPath);
break;
case ItemType.dir:
// nothing to do
break;
case ItemType.remote:
assert(oldItem.remoteDriveId == newItem.remoteDriveId);
assert(oldItem.remoteId == newItem.remoteId);
// nothing to do
break;
}
// handle changed content and mtime
// HACK: use mtime instead of cTag because of https://github.com/OneDrive/onedrive-api-docs/issues/765
if (newItem.type == ItemType.file && oldItem.mtime != newItem.mtime) {
downloadFileItem(newItem, newPath);
} else {
log.vlog("The item content has not changed");
}
// handle changed time
/* redundant because of the previous HACK
if (newItem.type == ItemType.file) {
setTimes(newPath, newItem.mtime, newItem.mtime);
}
}*/
} else {
log.vlog("The item has not changed");
}
}
// downloads a File resource
private void downloadFileItem(Item item, string path)
{
assert(item.type == ItemType.file);
write("Downloading ", path, "...");
onedrive.downloadById(item.driveId, item.id, path);
setTimes(path, item.mtime, item.mtime);
writeln(" done.");
}
// returns true if the given item corresponds to the local one
private bool isItemSynced(Item item, string path)
{
@ -369,7 +362,7 @@ final class SyncEngine
final switch (item.type) {
case ItemType.file:
if (isFile(path)) {
SysTime localModifiedTime = timeLastModified(path);
SysTime localModifiedTime = timeLastModified(path).toUTC();
// HACK: reduce time resolution to seconds before comparing
item.mtime.fracSecs = Duration.zero;
localModifiedTime.fracSecs = Duration.zero;
@ -427,7 +420,7 @@ final class SyncEngine
}
// scan the given directory for differences
public void scanForDifferences(string path)
void scanForDifferences(string path = ".")
{
log.vlog("Uploading differences ...");
Item item;
@ -494,33 +487,32 @@ final class SyncEngine
assert(item.type == ItemType.file);
if (exists(path)) {
if (isFile(path)) {
SysTime localModifiedTime = timeLastModified(path);
SysTime localModifiedTime = timeLastModified(path).toUTC();
// HACK: reduce time resolution to seconds before comparing
item.mtime.fracSecs = Duration.zero;
localModifiedTime.fracSecs = Duration.zero;
if (localModifiedTime != item.mtime) {
log.vlog("The file last modified time has changed");
string driveId = item.driveId;
string id = item.id;
/*string driveId = item.driveId;
string id = item.id;*/
string eTag = item.eTag;
if (!testFileHash(path, item)) {
log.vlog("The file content has changed");
log.log("Uploading: ", path);
JSONValue response;
if (getSize(path) <= thresholdFileSize) {
response = onedrive.simpleUpload(path, path, eTag);
writeln("upload by id");
response = onedrive.simpleUploadById(path, item.driveId, item.id, item.eTag);
} else {
response = session.upload(path, path, eTag);
}
saveItem(response);
/*saveItem(response);
id = response["id"].str;
driveId = response["parentReference"]["driveId"].str;
/* use the cTag instead of the eTag because Onedrive changes the
* metadata of some type of files (ex. images) AFTER they have been
* uploaded */
driveId = response["parentReference"]["driveId"].str;*/
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
eTag = response["cTag"].str;
}
uploadLastModifiedTime(driveId, id, eTag, localModifiedTime.toUTC());
uploadLastModifiedTime(item.driveId, item.id, eTag, localModifiedTime.toUTC());
} else {
log.vlog("The file has not changed");
}
@ -613,13 +605,14 @@ final class SyncEngine
private void uploadLastModifiedTime(const(char)[] driveId, const(char)[] id, const(char)[] eTag, SysTime mtime)
{
JSONValue mtimeJson = [
JSONValue data = [
"fileSystemInfo": JSONValue([
"lastModifiedDateTime": mtime.toISOExtString()
])
];
auto res = onedrive.updateById(driveId, id, mtimeJson, eTag);
saveItem(res);
auto response = onedrive.updateById(driveId, id, data, eTag);
writeln(response);
saveItem(response);
}
private void saveItem(JSONValue jsonItem)