mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-03 06:22:18 +02:00
WIP on remote folders
This commit is contained in:
parent
b250214577
commit
b7adc4d0cc
47
src/itemdb.d
47
src/itemdb.d
|
@ -26,13 +26,12 @@ struct Item
|
||||||
string quickXorHash;
|
string quickXorHash;
|
||||||
string remoteDriveId;
|
string remoteDriveId;
|
||||||
string remoteId;
|
string remoteId;
|
||||||
string deltaLink;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
final class ItemDatabase
|
final class ItemDatabase
|
||||||
{
|
{
|
||||||
// increment this for every change in the db schema
|
// increment this for every change in the db schema
|
||||||
immutable int itemDatabaseVersion = 6 ;
|
immutable int itemDatabaseVersion = 6;
|
||||||
|
|
||||||
Database db;
|
Database db;
|
||||||
Statement insertItemStmt;
|
Statement insertItemStmt;
|
||||||
|
@ -76,12 +75,12 @@ final class ItemDatabase
|
||||||
db.exec("PRAGMA foreign_keys = ON");
|
db.exec("PRAGMA foreign_keys = ON");
|
||||||
db.exec("PRAGMA recursive_triggers = ON");
|
db.exec("PRAGMA recursive_triggers = ON");
|
||||||
insertItemStmt = db.prepare("
|
insertItemStmt = db.prepare("
|
||||||
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentDriveId, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId, deltaLink)
|
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentDriveId, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
");
|
");
|
||||||
updateItemStmt = db.prepare("
|
updateItemStmt = db.prepare("
|
||||||
UPDATE item
|
UPDATE item
|
||||||
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentDriveId = ?8, parentId = ?9, crc32Hash = ?10, sha1Hash = ?11, quickXorHash = ?12, remoteDriveId = ?13, remoteId = ?14, deltaLink = ?15
|
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentDriveId = ?8, parentId = ?9, crc32Hash = ?10, sha1Hash = ?11, quickXorHash = ?12, remoteDriveId = ?13, remoteId = ?14
|
||||||
WHERE driveId = ?1 AND id = ?2
|
WHERE driveId = ?1 AND id = ?2
|
||||||
");
|
");
|
||||||
selectItemByIdStmt = db.prepare("
|
selectItemByIdStmt = db.prepare("
|
||||||
|
@ -202,7 +201,6 @@ final class ItemDatabase
|
||||||
bind(12, quickXorHash);
|
bind(12, quickXorHash);
|
||||||
bind(13, remoteDriveId);
|
bind(13, remoteDriveId);
|
||||||
bind(14, remoteId);
|
bind(14, remoteId);
|
||||||
bind(15, deltaLink);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -223,8 +221,7 @@ final class ItemDatabase
|
||||||
sha1Hash: result.front[10].dup,
|
sha1Hash: result.front[10].dup,
|
||||||
quickXorHash: result.front[11].dup,
|
quickXorHash: result.front[11].dup,
|
||||||
remoteDriveId: result.front[12].dup,
|
remoteDriveId: result.front[12].dup,
|
||||||
remoteId: result.front[13].dup,
|
remoteId: result.front[13].dup
|
||||||
deltaLink: result.front[14].dup
|
|
||||||
};
|
};
|
||||||
switch (result.front[3]) {
|
switch (result.front[3]) {
|
||||||
case "file": item.type = ItemType.file; break;
|
case "file": item.type = ItemType.file; break;
|
||||||
|
@ -292,4 +289,38 @@ final class ItemDatabase
|
||||||
}
|
}
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Item[] selectRemoteItems()
|
||||||
|
{
|
||||||
|
Item[] items;
|
||||||
|
auto stmt = db.prepare("SELECT * FROM item WHERE remoteDriveId IS NOT NULL");
|
||||||
|
auto res = stmt.exec();
|
||||||
|
while (!res.empty) {
|
||||||
|
items ~= buildItem(res);
|
||||||
|
res.step();
|
||||||
|
}
|
||||||
|
return items;
|
||||||
|
}
|
||||||
|
|
||||||
|
string getDeltaLink(const(char)[] driveId, const(char)[] id)
|
||||||
|
{
|
||||||
|
assert(driveId && id);
|
||||||
|
auto stmt = db.prepare("SELECT deltaLink FROM item WHERE driveId = ?1 AND id = ?2");
|
||||||
|
stmt.bind(1, driveId);
|
||||||
|
stmt.bind(2, id);
|
||||||
|
auto res = stmt.exec();
|
||||||
|
if (res.empty) return null;
|
||||||
|
return res.front[0].dup;
|
||||||
|
}
|
||||||
|
|
||||||
|
void setDeltaLink(const(char)[] driveId, const(char)[] id, const(char)[] deltaLink)
|
||||||
|
{
|
||||||
|
assert(driveId && id);
|
||||||
|
assert(deltaLink);
|
||||||
|
auto stmt = db.prepare("UPDATE item SET deltaLink = ?3 WHERE driveId = ?1 AND id = ?2");
|
||||||
|
stmt.bind(1, driveId);
|
||||||
|
stmt.bind(2, id);
|
||||||
|
stmt.bind(3, deltaLink);
|
||||||
|
stmt.exec();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -136,13 +136,34 @@ final class OneDriveApi
|
||||||
download(url, saveToPath);
|
download(url, saveToPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
// https://dev.onedrive.com/items/upload_put.htm
|
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
|
||||||
JSONValue simpleUpload(string localPath, const(char)[] remotePath, const(char)[] eTag = null)
|
JSONValue simpleUpload(string localPath, const(char)[] remotePath, const(char)[] eTag = null)
|
||||||
{
|
{
|
||||||
checkAccessTokenExpired();
|
checkAccessTokenExpired();
|
||||||
string url = itemByPathUrl ~ encodeComponent(remotePath) ~ ":/content";
|
string url = itemByPathUrl ~ encodeComponent(remotePath) ~ ":/content";
|
||||||
|
// TODO: investigate why this fails for remote folders
|
||||||
|
//if (eTag) http.addRequestHeader("If-Match", eTag);
|
||||||
|
/*else*/ http.addRequestHeader("If-None-Match", "*");
|
||||||
|
return upload(localPath, url);
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
|
||||||
|
JSONValue simpleUpload(string localPath, string parentDriveId, string parentId, string filename, const(char)[] eTag = null)
|
||||||
|
{
|
||||||
|
checkAccessTokenExpired();
|
||||||
|
string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/content";
|
||||||
|
// TODO: investigate why this fails for remote folders
|
||||||
|
//if (eTag) http.addRequestHeader("If-Match", eTag);
|
||||||
|
/*else http.addRequestHeader("If-None-Match", "*");*/
|
||||||
|
return upload(localPath, url);
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
|
||||||
|
JSONValue simpleUploadById(string localPath, string driveId, string id, const(char)[] eTag = null)
|
||||||
|
{
|
||||||
|
checkAccessTokenExpired();
|
||||||
|
string url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content";
|
||||||
if (eTag) http.addRequestHeader("If-Match", eTag);
|
if (eTag) http.addRequestHeader("If-Match", eTag);
|
||||||
else url ~= "?@name.conflictBehavior=fail";
|
|
||||||
return upload(localPath, url);
|
return upload(localPath, url);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
257
src/sync.d
257
src/sync.d
|
@ -1,7 +1,5 @@
|
||||||
import std.algorithm;
|
import std.algorithm;
|
||||||
import std.array: array;
|
import std.array: array;
|
||||||
import std.net.curl: CurlTimeoutException;
|
|
||||||
import std.exception: ErrnoException;
|
|
||||||
import std.datetime, std.file, std.json, std.path;
|
import std.datetime, std.file, std.json, std.path;
|
||||||
import std.regex;
|
import std.regex;
|
||||||
import std.stdio, std.string;
|
import std.stdio, std.string;
|
||||||
|
@ -36,54 +34,50 @@ private bool isItemRemote(const ref JSONValue item)
|
||||||
return ("remoteItem" in item) != null;
|
return ("remoteItem" in item) != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// HACK: OneDrive Biz does not return parentReference for the root
|
// construct an Item struct from a JSON driveItem
|
||||||
string defaultDriveId;
|
private Item makeItem(const ref JSONValue driveItem)
|
||||||
|
|
||||||
private Item makeItem(const ref JSONValue jsonItem)
|
|
||||||
{
|
{
|
||||||
ItemType type;
|
|
||||||
if (isItemFile(jsonItem)) {
|
|
||||||
type = ItemType.file;
|
|
||||||
} else if (isItemFolder(jsonItem)) {
|
|
||||||
type = ItemType.dir;
|
|
||||||
} else if (isItemRemote(jsonItem)) {
|
|
||||||
type = ItemType.remote;
|
|
||||||
}
|
|
||||||
|
|
||||||
Item item = {
|
Item item = {
|
||||||
driveId: "parentReference" in jsonItem ? jsonItem["parentReference"]["driveId"].str : defaultDriveId, // HACK
|
id: driveItem["id"].str,
|
||||||
id: jsonItem["id"].str,
|
name: "name" in driveItem ? driveItem["name"].str : null, // name may be missing for deleted files in OneDrive Biz
|
||||||
name: "name" in jsonItem ? jsonItem["name"].str : null, // name may be missing for deleted files in OneDrive Biz
|
eTag: "eTag" in driveItem ? driveItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Biz
|
||||||
type: type,
|
cTag: "cTag" in driveItem ? driveItem["cTag"].str : null, // cTag is missing in old files (and all folders in OneDrive Biz)
|
||||||
eTag: "eTag" in jsonItem ? jsonItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Biz
|
mtime: "fileSystemInfo" in driveItem ? SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str) : SysTime(0),
|
||||||
cTag: "cTag" in jsonItem ? jsonItem["cTag"].str : null, // cTag is missing in old files (and all folders)
|
|
||||||
mtime: "fileSystemInfo" in jsonItem ? SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str) : SysTime(0),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if (isItemFile(driveItem)) {
|
||||||
|
item.type = ItemType.file;
|
||||||
|
} else if (isItemFolder(driveItem)) {
|
||||||
|
item.type = ItemType.dir;
|
||||||
|
} else if (isItemRemote(driveItem)) {
|
||||||
|
item.type = ItemType.remote;
|
||||||
|
} else {
|
||||||
|
assert(0);
|
||||||
|
}
|
||||||
|
|
||||||
// root and remote items do not have parentReference
|
// root and remote items do not have parentReference
|
||||||
if (!isItemRoot(jsonItem) && ("parentReference" in jsonItem) != null) {
|
if (!isItemRoot(driveItem) && ("parentReference" in driveItem) != null) {
|
||||||
item.parentDriveId = jsonItem["parentReference"]["driveId"].str;
|
item.driveId = driveItem["parentReference"]["driveId"].str,
|
||||||
item.parentId = jsonItem["parentReference"]["id"].str;
|
item.parentDriveId = item.driveId; // TODO: parentDriveId is redundant
|
||||||
|
item.parentId = driveItem["parentReference"]["id"].str;
|
||||||
}
|
}
|
||||||
|
|
||||||
// extract the file hash
|
// extract the file hash
|
||||||
if (isItemFile(jsonItem)) {
|
if (isItemFile(driveItem) && ("hashes" in driveItem["file"])) {
|
||||||
if ("hashes" in jsonItem["file"]) {
|
if ("crc32Hash" in driveItem["file"]["hashes"]) {
|
||||||
if ("crc32Hash" in jsonItem["file"]["hashes"]) {
|
item.crc32Hash = driveItem["file"]["hashes"]["crc32Hash"].str;
|
||||||
item.crc32Hash = jsonItem["file"]["hashes"]["crc32Hash"].str;
|
} else if ("sha1Hash" in driveItem["file"]["hashes"]) {
|
||||||
} else if ("sha1Hash" in jsonItem["file"]["hashes"]) {
|
item.sha1Hash = driveItem["file"]["hashes"]["sha1Hash"].str;
|
||||||
item.sha1Hash = jsonItem["file"]["hashes"]["sha1Hash"].str;
|
} else if ("quickXorHash" in driveItem["file"]["hashes"]) {
|
||||||
} else if ("quickXorHash" in jsonItem["file"]["hashes"]) {
|
item.quickXorHash = driveItem["file"]["hashes"]["quickXorHash"].str;
|
||||||
item.quickXorHash = jsonItem["file"]["hashes"]["quickXorHash"].str;
|
} else {
|
||||||
} else {
|
log.vlog("The file does not have any hash");
|
||||||
log.vlog("The file does not have any hash");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isItemRemote(jsonItem)) {
|
if (isItemRemote(driveItem)) {
|
||||||
item.remoteDriveId = jsonItem["remoteItem"]["parentReference"]["driveId"].str;
|
item.remoteDriveId = driveItem["remoteItem"]["parentReference"]["driveId"].str;
|
||||||
item.remoteId = jsonItem["remoteItem"]["id"].str;
|
item.remoteId = driveItem["remoteItem"]["id"].str;
|
||||||
}
|
}
|
||||||
|
|
||||||
return item;
|
return item;
|
||||||
|
@ -109,6 +103,7 @@ class SyncException: Exception
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// This class handle the synchronization of one folder (ex. the root)
|
||||||
final class SyncEngine
|
final class SyncEngine
|
||||||
{
|
{
|
||||||
private Config cfg;
|
private Config cfg;
|
||||||
|
@ -141,42 +136,33 @@ final class SyncEngine
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// download all new changes from OneDrive
|
||||||
void applyDifferences()
|
void applyDifferences()
|
||||||
{
|
{
|
||||||
log.vlog("Applying differences ...");
|
// root folder
|
||||||
|
|
||||||
string driveId = onedrive.getDefaultDrive()["id"].str;
|
string driveId = onedrive.getDefaultDrive()["id"].str;
|
||||||
string rootId = onedrive.getDefaultRoot["id"].str;
|
string rootId = onedrive.getDefaultRoot["id"].str;
|
||||||
applyDifferences(driveId, rootId);
|
applyDifferences(driveId, rootId);
|
||||||
|
|
||||||
// delete items in idsToDelete
|
// check all remote folders
|
||||||
if (idsToDelete.length > 0) deleteItems();
|
// https://github.com/OneDrive/onedrive-api-docs/issues/764
|
||||||
// empty the skipped items
|
Item[] items = itemdb.selectRemoteItems();
|
||||||
skippedItems.length = 0;
|
foreach (item; items) applyDifferences(item.remoteDriveId, item.remoteId);
|
||||||
assumeSafeAppend(skippedItems);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void applyDifferences(const(char)[] driveId, const(char)[] id)
|
|
||||||
|
// download all new changes from OneDrive
|
||||||
|
void applyDifferences(string driveId, const(char)[] id)
|
||||||
{
|
{
|
||||||
// HACK
|
|
||||||
string h = driveId.dup;
|
|
||||||
|
|
||||||
// restore the last known state
|
|
||||||
string deltaLink;
|
|
||||||
Item beginItem;
|
|
||||||
if (itemdb.selectById(driveId, id, beginItem)) {
|
|
||||||
deltaLink = beginItem.deltaLink;
|
|
||||||
}
|
|
||||||
|
|
||||||
JSONValue changes;
|
JSONValue changes;
|
||||||
|
string deltaLink = itemdb.getDeltaLink(driveId, id);
|
||||||
|
log.vlog("Downloading changes of " ~ id);
|
||||||
do {
|
do {
|
||||||
|
|
||||||
// get changes from the server
|
|
||||||
try {
|
try {
|
||||||
changes = onedrive.viewChangesById(driveId, id, deltaLink);
|
changes = onedrive.viewChangesById(driveId, id, deltaLink);
|
||||||
} catch (OneDriveException e) {
|
} catch (OneDriveException e) {
|
||||||
if (e.httpStatusCode == 410) {
|
if (e.httpStatusCode == 410) {
|
||||||
log.log("Delta link expired, resyncing");
|
log.vlog("Delta link expired, resyncing...");
|
||||||
deltaLink = null;
|
deltaLink = null;
|
||||||
continue;
|
continue;
|
||||||
} else {
|
} else {
|
||||||
|
@ -184,29 +170,31 @@ final class SyncEngine
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
foreach (item; changes["value"].array) {
|
foreach (item; changes["value"].array) {
|
||||||
// HACK
|
applyDifference(item, driveId);
|
||||||
defaultDriveId = h;
|
|
||||||
applyDifference(item);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// the response may contain either @odata.deltaLink or @odata.nextLink
|
||||||
if ("@odata.deltaLink" in changes) deltaLink = changes["@odata.deltaLink"].str;
|
if ("@odata.deltaLink" in changes) deltaLink = changes["@odata.deltaLink"].str;
|
||||||
// save the state
|
if (deltaLink) itemdb.setDeltaLink(driveId, id, deltaLink);
|
||||||
import std.exception;
|
|
||||||
enforce(itemdb.selectById(driveId, id, beginItem));
|
|
||||||
beginItem.deltaLink = deltaLink;
|
|
||||||
itemdb.upsert(beginItem);
|
|
||||||
if ("@odata.nextLink" in changes) deltaLink = changes["@odata.nextLink"].str;
|
if ("@odata.nextLink" in changes) deltaLink = changes["@odata.nextLink"].str;
|
||||||
} while ("@odata.nextLink" in changes);
|
} while ("@odata.nextLink" in changes);
|
||||||
|
|
||||||
|
// delete items in idsToDelete
|
||||||
|
if (idsToDelete.length > 0) deleteItems();
|
||||||
|
// empty the skipped items
|
||||||
|
skippedItems.length = 0;
|
||||||
|
assumeSafeAppend(skippedItems);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void applyDifference(JSONValue jsonItem)
|
// process the change of a single DriveItem
|
||||||
|
private void applyDifference(JSONValue driveItem, string driveId)
|
||||||
{
|
{
|
||||||
log.vlog(jsonItem["id"].str, " ", "name" in jsonItem ? jsonItem["name"].str : null);
|
Item item = makeItem(driveItem);
|
||||||
Item item = makeItem(jsonItem);
|
log.vlog("Processing ", item.id, " ", item.name);
|
||||||
|
|
||||||
if (!isItemRoot(jsonItem) && !item.parentDriveId) {
|
if (isItemRoot(driveItem) || !item.parentDriveId) {
|
||||||
log.vlog("Root of remote item");
|
log.vlog("Root");
|
||||||
// this the root of a remote item, it can be skipped safely
|
item.driveId = driveId; // HACK: makeItem() cannot set the driveId propery of the root
|
||||||
itemdb.upsert(item);
|
itemdb.upsert(item);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -215,14 +203,27 @@ final class SyncEngine
|
||||||
unwanted |= skippedItems.find(item.parentId).length != 0;
|
unwanted |= skippedItems.find(item.parentId).length != 0;
|
||||||
unwanted |= selectiveSync.isNameExcluded(item.name);
|
unwanted |= selectiveSync.isNameExcluded(item.name);
|
||||||
|
|
||||||
string path = ".";
|
// check the item type
|
||||||
if (!unwanted && !isItemRoot(jsonItem)) {
|
if (!unwanted) {
|
||||||
// delay path computation after assuring the item parent is not excluded
|
if (isItemFile(driveItem)) {
|
||||||
|
log.vlog("File");
|
||||||
|
} else if (isItemFolder(driveItem)) {
|
||||||
|
log.vlog("Folder");
|
||||||
|
} else if (isItemRemote(driveItem)) {
|
||||||
|
log.vlog("Remote item");
|
||||||
|
assert(isItemFolder(driveItem["remoteItem"]), "The remote item is not a folder");
|
||||||
|
} else {
|
||||||
|
log.vlog("The item type is not supported");
|
||||||
|
unwanted = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// check for selective sync
|
||||||
|
string path;
|
||||||
|
if (!unwanted) {
|
||||||
path = itemdb.computePath(item.parentDriveId, item.parentId) ~ "/" ~ item.name;
|
path = itemdb.computePath(item.parentDriveId, item.parentId) ~ "/" ~ item.name;
|
||||||
// ensure path matches the path generated by itemdb.computePath()
|
path = buildNormalizedPath(path);
|
||||||
path = asNormalizedPath(path).array;
|
unwanted = selectiveSync.isPathExcluded(path);
|
||||||
// selective sync
|
|
||||||
unwanted |= selectiveSync.isPathExcluded(path);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// skip unwanted items early
|
// skip unwanted items early
|
||||||
|
@ -232,22 +233,12 @@ final class SyncEngine
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// check the item type
|
|
||||||
if (isItemRemote(jsonItem)) {
|
|
||||||
log.vlog("Remote item");
|
|
||||||
assert(isItemFolder(jsonItem["remoteItem"]), "The remote item is not a folder");
|
|
||||||
} else if (!isItemFile(jsonItem) && !isItemFolder(jsonItem) && !isItemDeleted(jsonItem)) {
|
|
||||||
log.vlog("The item is neither a file nor a directory, skipping");
|
|
||||||
skippedItems ~= item.id;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// check if the item has been seen before
|
// check if the item has been seen before
|
||||||
Item oldItem;
|
Item oldItem;
|
||||||
bool cached = itemdb.selectById(item.driveId, item.id, oldItem);
|
bool cached = itemdb.selectById(item.driveId, item.id, oldItem);
|
||||||
|
|
||||||
// check if the item is going to be deleted
|
// check if the item is going to be deleted
|
||||||
if (isItemDeleted(jsonItem)) {
|
if (isItemDeleted(driveItem)) {
|
||||||
log.vlog("The item is marked for deletion");
|
log.vlog("The item is marked for deletion");
|
||||||
if (cached) {
|
if (cached) {
|
||||||
// flag to delete
|
// flag to delete
|
||||||
|
@ -285,12 +276,13 @@ final class SyncEngine
|
||||||
}
|
}
|
||||||
|
|
||||||
// sync remote folder
|
// sync remote folder
|
||||||
if (isItemRemote(jsonItem)) {
|
/*if (isItemRemote(driveItem)) {
|
||||||
log.log("Syncing remote folder: ", path);
|
log.log("Syncing remote folder: ", path);
|
||||||
applyDifferences(item.remoteDriveId, item.remoteId);
|
applyDifferences(item.remoteDriveId, item.remoteId);
|
||||||
}
|
}*/
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// download an item that was not synced before
|
||||||
private void applyNewItem(Item item, string path)
|
private void applyNewItem(Item item, string path)
|
||||||
{
|
{
|
||||||
if (exists(path)) {
|
if (exists(path)) {
|
||||||
|
@ -298,19 +290,18 @@ final class SyncEngine
|
||||||
log.vlog("The item is already present");
|
log.vlog("The item is already present");
|
||||||
return;
|
return;
|
||||||
} else {
|
} else {
|
||||||
|
// TODO: force remote sync by deleting local item
|
||||||
log.vlog("The local item is out of sync, renaming...");
|
log.vlog("The local item is out of sync, renaming...");
|
||||||
safeRename(path);
|
safeRename(path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
final switch (item.type) {
|
final switch (item.type) {
|
||||||
case ItemType.file:
|
case ItemType.file:
|
||||||
log.log("Downloading: ", path);
|
downloadFileItem(item, path);
|
||||||
onedrive.downloadById(item.driveId, item.id, path);
|
|
||||||
setTimes(path, item.mtime, item.mtime);
|
|
||||||
break;
|
break;
|
||||||
case ItemType.dir:
|
case ItemType.dir:
|
||||||
case ItemType.remote:
|
case ItemType.remote:
|
||||||
log.log("Creating directory: ", path);
|
log.log("Creating directory ", path);
|
||||||
mkdirRecurse(path);
|
mkdirRecurse(path);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
@ -323,45 +314,47 @@ final class SyncEngine
|
||||||
assert(oldItem.driveId == newItem.driveId);
|
assert(oldItem.driveId == newItem.driveId);
|
||||||
assert(oldItem.id == newItem.id);
|
assert(oldItem.id == newItem.id);
|
||||||
assert(oldItem.type == newItem.type);
|
assert(oldItem.type == newItem.type);
|
||||||
|
assert(oldItem.remoteDriveId == newItem.remoteDriveId);
|
||||||
|
assert(oldItem.remoteId == newItem.remoteId);
|
||||||
|
|
||||||
if (oldItem.eTag != newItem.eTag) {
|
if (oldItem.eTag != newItem.eTag) {
|
||||||
// handle changed path
|
// handle changed name/path
|
||||||
if (oldPath != newPath) {
|
if (oldPath != newPath) {
|
||||||
log.log("Moving: ", oldPath, " -> ", newPath);
|
log.log("Moving: ", oldPath, " -> ", newPath);
|
||||||
if (exists(newPath)) {
|
if (exists(newPath)) {
|
||||||
|
// TODO: force remote sync by deleting local item
|
||||||
log.vlog("The destination is occupied, renaming the conflicting file...");
|
log.vlog("The destination is occupied, renaming the conflicting file...");
|
||||||
safeRename(newPath);
|
safeRename(newPath);
|
||||||
}
|
}
|
||||||
rename(oldPath, newPath);
|
rename(oldPath, newPath);
|
||||||
}
|
}
|
||||||
// handle changed content
|
// handle changed content and mtime
|
||||||
if (oldItem.cTag != newItem.cTag) {
|
// HACK: use mtime instead of cTag because of https://github.com/OneDrive/onedrive-api-docs/issues/765
|
||||||
final switch (newItem.type) {
|
if (newItem.type == ItemType.file && oldItem.mtime != newItem.mtime) {
|
||||||
case ItemType.file:
|
downloadFileItem(newItem, newPath);
|
||||||
log.log("Downloading: ", newPath);
|
|
||||||
onedrive.downloadById(newItem.driveId, newItem.id, newPath);
|
|
||||||
break;
|
|
||||||
case ItemType.dir:
|
|
||||||
// nothing to do
|
|
||||||
break;
|
|
||||||
case ItemType.remote:
|
|
||||||
assert(oldItem.remoteDriveId == newItem.remoteDriveId);
|
|
||||||
assert(oldItem.remoteId == newItem.remoteId);
|
|
||||||
// nothing to do
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
log.vlog("The item content has not changed");
|
log.vlog("The item content has not changed");
|
||||||
}
|
}
|
||||||
// handle changed time
|
// handle changed time
|
||||||
|
/* redundant because of the previous HACK
|
||||||
if (newItem.type == ItemType.file) {
|
if (newItem.type == ItemType.file) {
|
||||||
setTimes(newPath, newItem.mtime, newItem.mtime);
|
setTimes(newPath, newItem.mtime, newItem.mtime);
|
||||||
}
|
}*/
|
||||||
} else {
|
} else {
|
||||||
log.vlog("The item has not changed");
|
log.vlog("The item has not changed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// downloads a File resource
|
||||||
|
private void downloadFileItem(Item item, string path)
|
||||||
|
{
|
||||||
|
assert(item.type == ItemType.file);
|
||||||
|
write("Downloading ", path, "...");
|
||||||
|
onedrive.downloadById(item.driveId, item.id, path);
|
||||||
|
setTimes(path, item.mtime, item.mtime);
|
||||||
|
writeln(" done.");
|
||||||
|
}
|
||||||
|
|
||||||
// returns true if the given item corresponds to the local one
|
// returns true if the given item corresponds to the local one
|
||||||
private bool isItemSynced(Item item, string path)
|
private bool isItemSynced(Item item, string path)
|
||||||
{
|
{
|
||||||
|
@ -369,7 +362,7 @@ final class SyncEngine
|
||||||
final switch (item.type) {
|
final switch (item.type) {
|
||||||
case ItemType.file:
|
case ItemType.file:
|
||||||
if (isFile(path)) {
|
if (isFile(path)) {
|
||||||
SysTime localModifiedTime = timeLastModified(path);
|
SysTime localModifiedTime = timeLastModified(path).toUTC();
|
||||||
// HACK: reduce time resolution to seconds before comparing
|
// HACK: reduce time resolution to seconds before comparing
|
||||||
item.mtime.fracSecs = Duration.zero;
|
item.mtime.fracSecs = Duration.zero;
|
||||||
localModifiedTime.fracSecs = Duration.zero;
|
localModifiedTime.fracSecs = Duration.zero;
|
||||||
|
@ -427,7 +420,7 @@ final class SyncEngine
|
||||||
}
|
}
|
||||||
|
|
||||||
// scan the given directory for differences
|
// scan the given directory for differences
|
||||||
public void scanForDifferences(string path)
|
void scanForDifferences(string path = ".")
|
||||||
{
|
{
|
||||||
log.vlog("Uploading differences ...");
|
log.vlog("Uploading differences ...");
|
||||||
Item item;
|
Item item;
|
||||||
|
@ -494,33 +487,32 @@ final class SyncEngine
|
||||||
assert(item.type == ItemType.file);
|
assert(item.type == ItemType.file);
|
||||||
if (exists(path)) {
|
if (exists(path)) {
|
||||||
if (isFile(path)) {
|
if (isFile(path)) {
|
||||||
SysTime localModifiedTime = timeLastModified(path);
|
SysTime localModifiedTime = timeLastModified(path).toUTC();
|
||||||
// HACK: reduce time resolution to seconds before comparing
|
// HACK: reduce time resolution to seconds before comparing
|
||||||
item.mtime.fracSecs = Duration.zero;
|
item.mtime.fracSecs = Duration.zero;
|
||||||
localModifiedTime.fracSecs = Duration.zero;
|
localModifiedTime.fracSecs = Duration.zero;
|
||||||
if (localModifiedTime != item.mtime) {
|
if (localModifiedTime != item.mtime) {
|
||||||
log.vlog("The file last modified time has changed");
|
log.vlog("The file last modified time has changed");
|
||||||
string driveId = item.driveId;
|
/*string driveId = item.driveId;
|
||||||
string id = item.id;
|
string id = item.id;*/
|
||||||
string eTag = item.eTag;
|
string eTag = item.eTag;
|
||||||
if (!testFileHash(path, item)) {
|
if (!testFileHash(path, item)) {
|
||||||
log.vlog("The file content has changed");
|
log.vlog("The file content has changed");
|
||||||
log.log("Uploading: ", path);
|
log.log("Uploading: ", path);
|
||||||
JSONValue response;
|
JSONValue response;
|
||||||
if (getSize(path) <= thresholdFileSize) {
|
if (getSize(path) <= thresholdFileSize) {
|
||||||
response = onedrive.simpleUpload(path, path, eTag);
|
writeln("upload by id");
|
||||||
|
response = onedrive.simpleUploadById(path, item.driveId, item.id, item.eTag);
|
||||||
} else {
|
} else {
|
||||||
response = session.upload(path, path, eTag);
|
response = session.upload(path, path, eTag);
|
||||||
}
|
}
|
||||||
saveItem(response);
|
/*saveItem(response);
|
||||||
id = response["id"].str;
|
id = response["id"].str;
|
||||||
driveId = response["parentReference"]["driveId"].str;
|
driveId = response["parentReference"]["driveId"].str;*/
|
||||||
/* use the cTag instead of the eTag because Onedrive changes the
|
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
|
||||||
* metadata of some type of files (ex. images) AFTER they have been
|
|
||||||
* uploaded */
|
|
||||||
eTag = response["cTag"].str;
|
eTag = response["cTag"].str;
|
||||||
}
|
}
|
||||||
uploadLastModifiedTime(driveId, id, eTag, localModifiedTime.toUTC());
|
uploadLastModifiedTime(item.driveId, item.id, eTag, localModifiedTime.toUTC());
|
||||||
} else {
|
} else {
|
||||||
log.vlog("The file has not changed");
|
log.vlog("The file has not changed");
|
||||||
}
|
}
|
||||||
|
@ -613,13 +605,14 @@ final class SyncEngine
|
||||||
|
|
||||||
private void uploadLastModifiedTime(const(char)[] driveId, const(char)[] id, const(char)[] eTag, SysTime mtime)
|
private void uploadLastModifiedTime(const(char)[] driveId, const(char)[] id, const(char)[] eTag, SysTime mtime)
|
||||||
{
|
{
|
||||||
JSONValue mtimeJson = [
|
JSONValue data = [
|
||||||
"fileSystemInfo": JSONValue([
|
"fileSystemInfo": JSONValue([
|
||||||
"lastModifiedDateTime": mtime.toISOExtString()
|
"lastModifiedDateTime": mtime.toISOExtString()
|
||||||
])
|
])
|
||||||
];
|
];
|
||||||
auto res = onedrive.updateById(driveId, id, mtimeJson, eTag);
|
auto response = onedrive.updateById(driveId, id, data, eTag);
|
||||||
saveItem(res);
|
writeln(response);
|
||||||
|
saveItem(response);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void saveItem(JSONValue jsonItem)
|
private void saveItem(JSONValue jsonItem)
|
||||||
|
|
Loading…
Reference in a new issue