2017-03-24 22:30:03 +01:00
|
|
|
import std.algorithm;
|
2017-03-11 12:07:21 +01:00
|
|
|
import std.net.curl: CurlTimeoutException;
|
2015-09-22 11:52:28 +02:00
|
|
|
import std.exception: ErrnoException;
|
2017-03-24 22:30:03 +01:00
|
|
|
import std.datetime, std.file, std.json, std.path;
|
|
|
|
import std.regex;
|
2015-09-20 19:07:16 +02:00
|
|
|
import std.stdio, std.string;
|
2017-03-24 22:30:03 +01:00
|
|
|
import config, itemdb, onedrive, selective, upload, util;
|
2016-08-04 23:35:58 +02:00
|
|
|
static import log;
|
2015-09-27 18:47:41 +02:00
|
|
|
|
|
|
|
// threshold after which files will be uploaded using an upload session
|
2017-03-11 11:40:19 +01:00
|
|
|
private long thresholdFileSize = 4 * 2^^20; // 4 MiB
|
2015-09-01 20:45:34 +02:00
|
|
|
|
|
|
|
private bool isItemFolder(const ref JSONValue item)
|
|
|
|
{
|
2017-03-11 11:40:19 +01:00
|
|
|
return ("folder" in item) != null;
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
private bool isItemFile(const ref JSONValue item)
|
|
|
|
{
|
2017-03-11 11:40:19 +01:00
|
|
|
return ("file" in item) != null;
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
private bool isItemDeleted(const ref JSONValue item)
|
|
|
|
{
|
2017-03-11 11:40:19 +01:00
|
|
|
// HACK: fix for https://github.com/skilion/onedrive/issues/157
|
2017-06-12 16:53:15 +02:00
|
|
|
return ("deleted" in item) || ("fileSystemInfo" !in item && "remoteItem" !in item);
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
|
2017-03-12 19:40:38 +01:00
|
|
|
private bool isItemRoot(const ref JSONValue item)
|
|
|
|
{
|
|
|
|
return ("root" in item) != null;
|
|
|
|
}
|
|
|
|
|
2017-06-14 22:39:36 +02:00
|
|
|
private bool isItemRemote(const ref JSONValue item)
|
|
|
|
{
|
|
|
|
return ("remoteItem" in item) != null;
|
|
|
|
}
|
|
|
|
|
2017-05-28 22:13:19 +02:00
|
|
|
private Item makeItem(const ref JSONValue jsonItem)
|
2015-09-08 18:25:41 +02:00
|
|
|
{
|
2017-05-28 22:13:19 +02:00
|
|
|
ItemType type;
|
|
|
|
if (isItemFile(jsonItem)) {
|
|
|
|
type = ItemType.file;
|
|
|
|
} else if (isItemFolder(jsonItem)) {
|
|
|
|
type = ItemType.dir;
|
2017-06-14 22:39:36 +02:00
|
|
|
} else if (isItemRemote(jsonItem)) {
|
|
|
|
type = ItemType.remote;
|
2017-05-28 22:13:19 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
Item item = {
|
2017-06-14 15:50:02 +02:00
|
|
|
driveId: jsonItem["parentReference"]["driveId"].str,
|
2017-05-28 22:13:19 +02:00
|
|
|
id: jsonItem["id"].str,
|
|
|
|
name: jsonItem["name"].str,
|
|
|
|
type: type,
|
2017-05-28 23:15:03 +02:00
|
|
|
eTag: isItemRoot(jsonItem) ? null : jsonItem["eTag"].str, // eTag is not returned for the root in OneDrive Biz
|
|
|
|
cTag: "cTag" !in jsonItem ? null : jsonItem["cTag"].str, // cTag is missing in old files (plus all folders)
|
2017-06-14 22:39:36 +02:00
|
|
|
mtime: isItemRemote(jsonItem) ? SysTime(0) : SysTime.fromISOExtString(jsonItem["fileSystemInfo"]["lastModifiedDateTime"].str),
|
2017-06-14 15:50:02 +02:00
|
|
|
parentDriveId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["driveId"].str,
|
2017-05-28 22:13:19 +02:00
|
|
|
parentId: isItemRoot(jsonItem) ? null : jsonItem["parentReference"]["id"].str
|
|
|
|
};
|
|
|
|
|
|
|
|
// extract the file hash
|
|
|
|
if (type == ItemType.file) {
|
|
|
|
if ("hashes" in jsonItem["file"]) {
|
|
|
|
if ("crc32Hash" in jsonItem["file"]["hashes"]) {
|
|
|
|
item.crc32Hash = jsonItem["file"]["hashes"]["crc32Hash"].str;
|
|
|
|
} else if ("sha1Hash" in jsonItem["file"]["hashes"]) {
|
|
|
|
item.sha1Hash = jsonItem["file"]["hashes"]["sha1Hash"].str;
|
|
|
|
} else if ("quickXorHash" in jsonItem["file"]["hashes"]) {
|
|
|
|
item.quickXorHash = jsonItem["file"]["hashes"]["quickXorHash"].str;
|
|
|
|
} else {
|
|
|
|
log.vlog("The file does not have any hash");
|
|
|
|
}
|
|
|
|
} else {
|
2017-05-28 22:45:09 +02:00
|
|
|
// 'hashes' is missing in old files
|
|
|
|
log.vlog("No hashes facet");
|
2017-05-28 22:13:19 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return item;
|
|
|
|
}
|
|
|
|
|
|
|
|
private bool testFileHash(string path, const ref Item item)
|
|
|
|
{
|
|
|
|
if (item.crc32Hash) {
|
|
|
|
if (item.crc32Hash == computeCrc32(path)) return true;
|
|
|
|
} else if (item.sha1Hash) {
|
|
|
|
if (item.sha1Hash == computeSha1Hash(path)) return true;
|
|
|
|
} else if (item.quickXorHash) {
|
|
|
|
if (item.quickXorHash == computeQuickXorHash(path)) return true;
|
2015-09-08 18:25:41 +02:00
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-09-01 20:45:34 +02:00
|
|
|
class SyncException: Exception
|
|
|
|
{
|
|
|
|
@nogc @safe pure nothrow this(string msg, string file = __FILE__, size_t line = __LINE__, Throwable next = null)
|
|
|
|
{
|
|
|
|
super(msg, file, line, next);
|
|
|
|
}
|
|
|
|
|
|
|
|
@nogc @safe pure nothrow this(string msg, Throwable next, string file = __FILE__, size_t line = __LINE__)
|
|
|
|
{
|
|
|
|
super(msg, file, line, next);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
final class SyncEngine
|
|
|
|
{
|
2015-09-14 23:56:14 +02:00
|
|
|
private Config cfg;
|
|
|
|
private OneDriveApi onedrive;
|
|
|
|
private ItemDatabase itemdb;
|
2015-09-27 18:47:41 +02:00
|
|
|
private UploadSession session;
|
2017-03-24 22:30:03 +01:00
|
|
|
private SelectiveSync selectiveSync;
|
2015-12-29 19:38:15 +01:00
|
|
|
// list of items to skip while applying the changes
|
2015-09-16 10:29:20 +02:00
|
|
|
private string[] skippedItems;
|
2015-09-18 21:42:27 +02:00
|
|
|
// list of items to delete after the changes has been downloaded
|
2017-06-14 15:50:02 +02:00
|
|
|
private string[2][] idsToDelete;
|
2015-09-14 23:56:14 +02:00
|
|
|
|
2017-03-24 22:30:03 +01:00
|
|
|
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
|
2015-09-01 20:45:34 +02:00
|
|
|
{
|
2017-03-24 22:30:03 +01:00
|
|
|
assert(onedrive && itemdb && selectiveSync);
|
2015-09-01 20:45:34 +02:00
|
|
|
this.cfg = cfg;
|
|
|
|
this.onedrive = onedrive;
|
2015-09-14 23:56:14 +02:00
|
|
|
this.itemdb = itemdb;
|
2017-03-24 22:30:03 +01:00
|
|
|
this.selectiveSync = selectiveSync;
|
2016-08-04 23:35:58 +02:00
|
|
|
session = UploadSession(onedrive, cfg.uploadStateFilePath);
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
|
2016-08-04 23:35:58 +02:00
|
|
|
void init()
|
2015-09-01 20:45:34 +02:00
|
|
|
{
|
2015-09-27 18:47:41 +02:00
|
|
|
// check if there is an interrupted upload session
|
|
|
|
if (session.restore()) {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.log("Continuing the upload session ...");
|
2015-09-27 18:47:41 +02:00
|
|
|
auto item = session.upload();
|
|
|
|
saveItem(item);
|
|
|
|
}
|
2015-09-14 23:56:14 +02:00
|
|
|
}
|
2015-09-01 20:45:34 +02:00
|
|
|
|
2015-09-14 23:56:14 +02:00
|
|
|
void applyDifferences()
|
|
|
|
{
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("Applying differences ...");
|
2017-05-28 20:54:57 +02:00
|
|
|
|
|
|
|
// restore the last known state
|
|
|
|
string deltaLink;
|
|
|
|
try {
|
|
|
|
deltaLink = readText(cfg.deltaLinkFilePath);
|
|
|
|
} catch (FileException e) {
|
|
|
|
// swallow exception
|
|
|
|
}
|
|
|
|
|
2015-09-22 11:52:28 +02:00
|
|
|
try {
|
|
|
|
JSONValue changes;
|
|
|
|
do {
|
2016-12-14 15:17:20 +01:00
|
|
|
// get changes from the server
|
|
|
|
try {
|
2017-05-28 20:54:57 +02:00
|
|
|
changes = onedrive.viewChangesByPath(".", deltaLink);
|
2016-12-14 15:17:20 +01:00
|
|
|
} catch (OneDriveException e) {
|
|
|
|
if (e.httpStatusCode == 410) {
|
2017-05-28 20:54:57 +02:00
|
|
|
log.log("Delta link expired, resyncing");
|
|
|
|
deltaLink = null;
|
2016-12-14 15:17:20 +01:00
|
|
|
continue;
|
2017-05-28 20:54:57 +02:00
|
|
|
} else {
|
2016-12-14 15:17:20 +01:00
|
|
|
throw e;
|
|
|
|
}
|
|
|
|
}
|
2015-09-22 11:52:28 +02:00
|
|
|
foreach (item; changes["value"].array) {
|
|
|
|
applyDifference(item);
|
|
|
|
}
|
2017-05-28 20:54:57 +02:00
|
|
|
if ("@odata.nextLink" in changes) deltaLink = changes["@odata.nextLink"].str;
|
|
|
|
if ("@odata.deltaLink" in changes) deltaLink = changes["@odata.deltaLink"].str;
|
|
|
|
std.file.write(cfg.deltaLinkFilePath, deltaLink);
|
2017-03-11 11:40:19 +01:00
|
|
|
} while ("@odata.nextLink" in changes);
|
2015-09-22 11:52:28 +02:00
|
|
|
} catch (ErrnoException e) {
|
|
|
|
throw new SyncException(e.msg, e);
|
|
|
|
} catch (FileException e) {
|
|
|
|
throw new SyncException(e.msg, e);
|
2017-03-11 12:07:21 +01:00
|
|
|
} catch (CurlTimeoutException e) {
|
|
|
|
throw new SyncException(e.msg, e);
|
2015-09-22 11:52:28 +02:00
|
|
|
} catch (OneDriveException e) {
|
|
|
|
throw new SyncException(e.msg, e);
|
|
|
|
}
|
2016-12-25 23:25:24 +01:00
|
|
|
// delete items in idsToDelete
|
|
|
|
if (idsToDelete.length > 0) deleteItems();
|
2015-09-16 10:29:20 +02:00
|
|
|
// empty the skipped items
|
|
|
|
skippedItems.length = 0;
|
|
|
|
assumeSafeAppend(skippedItems);
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
|
2017-06-14 22:39:36 +02:00
|
|
|
private void applyDifference(JSONValue jsonItem)
|
2015-09-01 20:45:34 +02:00
|
|
|
{
|
2017-06-14 22:39:36 +02:00
|
|
|
Item item = makeItem(jsonItem);
|
|
|
|
log.vlog(item.id, " ", item.name);
|
2015-09-19 15:38:43 +02:00
|
|
|
|
2017-06-14 22:39:36 +02:00
|
|
|
// skip unwanted items early
|
|
|
|
bool unwanted;
|
|
|
|
unwanted |= skippedItems.find(item.parentId).length != 0;
|
|
|
|
unwanted |= selectiveSync.isNameExcluded(item.name);
|
|
|
|
unwanted |= selectiveSync.isPathExcluded(path);
|
|
|
|
if (unwanted) {
|
|
|
|
log.vlog("Filtered out");
|
|
|
|
skippedItems ~= item.id;
|
|
|
|
return;
|
|
|
|
}
|
2017-03-12 16:07:45 +01:00
|
|
|
|
2017-06-14 22:39:36 +02:00
|
|
|
// compute the path of the item
|
|
|
|
string path = ".";
|
|
|
|
if (!isItemRoot(jsonItem)) {
|
|
|
|
path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name;
|
2017-03-12 19:40:38 +01:00
|
|
|
}
|
|
|
|
|
2017-06-14 22:39:36 +02:00
|
|
|
// check if the item is to be deleted
|
|
|
|
if (isItemDeleted(jsonItem)) {
|
|
|
|
log.vlog("The item is marked for deletion");
|
|
|
|
idsToDelete ~= [item.driveId, item.id];
|
2017-03-12 16:07:45 +01:00
|
|
|
return;
|
|
|
|
}
|
2017-06-14 22:39:36 +02:00
|
|
|
|
|
|
|
// check the item type
|
|
|
|
if (isItemRemote(jsonItem)) {
|
|
|
|
// TODO
|
|
|
|
// check name change
|
|
|
|
// scan the children later
|
|
|
|
// fix child references
|
|
|
|
log.vlog("Remote items are not supported yet");
|
|
|
|
skippedItems ~= item.id;
|
|
|
|
return;
|
|
|
|
} else if (!isItemFile(jsonItem) && !isItemFolder(jsonItem)) {
|
|
|
|
log.vlog("The item is neither a file nor a directory, skipping");
|
|
|
|
skippedItems ~= item.id;
|
2015-09-19 15:38:43 +02:00
|
|
|
return;
|
|
|
|
}
|
2015-09-14 23:56:14 +02:00
|
|
|
|
2015-09-30 15:53:49 +02:00
|
|
|
// rename the local item if it is unsynced and there is a new version of it
|
2015-09-19 15:38:43 +02:00
|
|
|
Item oldItem;
|
|
|
|
string oldPath;
|
2017-06-14 22:39:36 +02:00
|
|
|
bool cached = itemdb.selectById(item.driveId, item.id, oldItem);
|
|
|
|
if (cached && item.eTag != oldItem.eTag) {
|
|
|
|
oldPath = itemdb.computePath(item.driveId, item.id);
|
2015-09-19 15:38:43 +02:00
|
|
|
if (!isItemSynced(oldItem, oldPath)) {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The local item is unsynced, renaming");
|
2015-09-19 15:38:43 +02:00
|
|
|
if (exists(oldPath)) safeRename(oldPath);
|
|
|
|
cached = false;
|
|
|
|
}
|
|
|
|
}
|
2015-09-14 23:56:14 +02:00
|
|
|
|
2015-09-19 15:38:43 +02:00
|
|
|
if (!cached) {
|
2017-06-14 22:39:36 +02:00
|
|
|
applyNewItem(item, path);
|
2015-09-17 16:28:24 +02:00
|
|
|
} else {
|
2017-06-14 22:39:36 +02:00
|
|
|
applyChangedItem(oldItem, oldPath, item, path);
|
2015-09-17 16:28:24 +02:00
|
|
|
}
|
2015-09-01 20:45:34 +02:00
|
|
|
|
2015-09-19 15:38:43 +02:00
|
|
|
// save the item in the db
|
|
|
|
if (oldItem.id) {
|
2017-06-14 22:39:36 +02:00
|
|
|
itemdb.update(item);
|
2015-09-19 15:38:43 +02:00
|
|
|
} else {
|
2017-06-14 22:39:36 +02:00
|
|
|
itemdb.insert(item);
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-09-19 15:38:43 +02:00
|
|
|
private void applyNewItem(Item item, string path)
|
2015-09-01 20:45:34 +02:00
|
|
|
{
|
2015-09-19 15:38:43 +02:00
|
|
|
if (exists(path)) {
|
|
|
|
if (isItemSynced(item, path)) {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The item is already present");
|
2015-09-14 23:56:14 +02:00
|
|
|
// ensure the modified time is correct
|
2015-09-19 15:38:43 +02:00
|
|
|
setTimes(path, item.mtime, item.mtime);
|
2015-09-01 20:45:34 +02:00
|
|
|
return;
|
|
|
|
} else {
|
2017-06-14 22:39:36 +02:00
|
|
|
log.vlog("The local item is out of sync, renaming...");
|
2015-09-19 15:38:43 +02:00
|
|
|
safeRename(path);
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
final switch (item.type) {
|
|
|
|
case ItemType.file:
|
2016-08-04 23:35:58 +02:00
|
|
|
log.log("Downloading: ", path);
|
2015-09-22 11:52:28 +02:00
|
|
|
onedrive.downloadById(item.id, path);
|
2015-09-01 20:45:34 +02:00
|
|
|
break;
|
|
|
|
case ItemType.dir:
|
2016-08-04 23:35:58 +02:00
|
|
|
log.log("Creating directory: ", path);
|
2015-09-19 15:38:43 +02:00
|
|
|
mkdir(path);
|
2017-06-14 15:50:02 +02:00
|
|
|
break;
|
2017-06-14 22:39:36 +02:00
|
|
|
case ItemType.remote:
|
|
|
|
assert(0);
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
2015-09-19 15:38:43 +02:00
|
|
|
setTimes(path, item.mtime, item.mtime);
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
|
2017-06-14 22:39:36 +02:00
|
|
|
// update a local item
|
|
|
|
// the local item is assumed to be in sync with the local db
|
|
|
|
private void applyChangedItem(Item oldItem, string oldPath, Item newItem, string newPath)
|
2015-09-01 20:45:34 +02:00
|
|
|
{
|
2017-06-14 15:50:02 +02:00
|
|
|
assert(oldItem.driveId == newItem.driveId);
|
2015-09-01 20:45:34 +02:00
|
|
|
assert(oldItem.id == newItem.id);
|
2015-09-14 23:56:14 +02:00
|
|
|
assert(oldItem.type == newItem.type);
|
|
|
|
|
|
|
|
if (oldItem.eTag != newItem.eTag) {
|
2015-09-19 15:38:43 +02:00
|
|
|
if (oldPath != newPath) {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.log("Moving: ", oldPath, " -> ", newPath);
|
2015-09-19 15:38:43 +02:00
|
|
|
if (exists(newPath)) {
|
2017-06-14 22:39:36 +02:00
|
|
|
log.vlog("The destination is occupied, renaming the conflicting file...");
|
2015-09-19 15:38:43 +02:00
|
|
|
safeRename(newPath);
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
2015-09-19 15:38:43 +02:00
|
|
|
rename(oldPath, newPath);
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
2015-09-14 23:56:14 +02:00
|
|
|
if (newItem.type == ItemType.file && oldItem.cTag != newItem.cTag) {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.log("Downloading: ", newPath);
|
2015-09-19 15:38:43 +02:00
|
|
|
onedrive.downloadById(newItem.id, newPath);
|
2015-09-14 23:56:14 +02:00
|
|
|
}
|
2015-09-19 15:38:43 +02:00
|
|
|
setTimes(newPath, newItem.mtime, newItem.mtime);
|
2015-09-01 20:45:34 +02:00
|
|
|
} else {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The item has not changed");
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// returns true if the given item corresponds to the local one
|
2015-09-19 15:38:43 +02:00
|
|
|
private bool isItemSynced(Item item, string path)
|
2015-09-01 20:45:34 +02:00
|
|
|
{
|
2015-09-19 15:38:43 +02:00
|
|
|
if (!exists(path)) return false;
|
2015-09-01 20:45:34 +02:00
|
|
|
final switch (item.type) {
|
|
|
|
case ItemType.file:
|
2015-09-19 15:38:43 +02:00
|
|
|
if (isFile(path)) {
|
|
|
|
SysTime localModifiedTime = timeLastModified(path);
|
2017-03-12 17:25:51 +01:00
|
|
|
// HACK: reduce time resolution to seconds before comparing
|
|
|
|
item.mtime.fracSecs = Duration.zero;
|
|
|
|
localModifiedTime.fracSecs = Duration.zero;
|
2015-09-14 23:56:14 +02:00
|
|
|
if (localModifiedTime == item.mtime) {
|
|
|
|
return true;
|
|
|
|
} else {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime);
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
2017-05-28 22:13:19 +02:00
|
|
|
if (testFileHash(path, item)) {
|
2015-09-19 15:38:43 +02:00
|
|
|
return true;
|
|
|
|
} else {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The local item has a different hash");
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
} else {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The local item is a directory but should be a file");
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
case ItemType.dir:
|
2015-09-19 15:38:43 +02:00
|
|
|
if (isDir(path)) {
|
2015-09-14 23:56:14 +02:00
|
|
|
return true;
|
|
|
|
} else {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The local item is a file but should be a directory");
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
break;
|
2017-06-14 15:50:02 +02:00
|
|
|
case ItemType.remote:
|
|
|
|
assert(0);
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-09-14 23:56:14 +02:00
|
|
|
private void deleteItems()
|
2015-09-01 20:45:34 +02:00
|
|
|
{
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("Deleting files ...");
|
2017-06-14 15:50:02 +02:00
|
|
|
foreach_reverse (i; idsToDelete) {
|
|
|
|
string path = itemdb.computePath(i[0], i[1]);
|
|
|
|
itemdb.deleteById(i[0], i[1]);
|
2015-09-14 23:56:14 +02:00
|
|
|
if (exists(path)) {
|
|
|
|
if (isFile(path)) {
|
|
|
|
remove(path);
|
2016-08-04 23:35:58 +02:00
|
|
|
log.log("Deleted file: ", path);
|
2015-09-28 13:42:58 +02:00
|
|
|
} else {
|
|
|
|
try {
|
|
|
|
rmdir(path);
|
2016-08-04 23:35:58 +02:00
|
|
|
log.log("Deleted directory: ", path);
|
2015-09-28 13:42:58 +02:00
|
|
|
} catch (FileException e) {
|
|
|
|
// directory not empty
|
|
|
|
}
|
2015-09-06 22:42:44 +02:00
|
|
|
}
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
|
|
|
}
|
2016-12-25 23:25:24 +01:00
|
|
|
idsToDelete.length = 0;
|
|
|
|
assumeSafeAppend(idsToDelete);
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|
2015-09-04 21:00:22 +02:00
|
|
|
|
2015-09-18 21:42:27 +02:00
|
|
|
// scan the given directory for differences
|
|
|
|
public void scanForDifferences(string path)
|
2015-09-04 21:00:22 +02:00
|
|
|
{
|
2015-09-20 21:21:51 +02:00
|
|
|
try {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("Uploading differences ...");
|
2015-09-20 21:21:51 +02:00
|
|
|
Item item;
|
|
|
|
if (itemdb.selectByPath(path, item)) {
|
|
|
|
uploadDifferences(item);
|
|
|
|
}
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("Uploading new items ...");
|
2015-09-20 21:21:51 +02:00
|
|
|
uploadNewItems(path);
|
2015-09-22 11:52:28 +02:00
|
|
|
} catch (ErrnoException e) {
|
|
|
|
throw new SyncException(e.msg, e);
|
2015-09-20 21:21:51 +02:00
|
|
|
} catch (FileException e) {
|
|
|
|
throw new SyncException(e.msg, e);
|
|
|
|
} catch (OneDriveException e) {
|
|
|
|
throw new SyncException(e.msg, e);
|
2015-09-08 18:25:41 +02:00
|
|
|
}
|
2015-09-04 21:00:22 +02:00
|
|
|
}
|
|
|
|
|
2015-09-20 21:21:51 +02:00
|
|
|
private void uploadDifferences(Item item)
|
2015-09-11 18:33:22 +02:00
|
|
|
{
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog(item.id, " ", item.name);
|
2017-03-12 16:07:45 +01:00
|
|
|
|
|
|
|
// skip filtered items
|
2017-03-24 22:30:03 +01:00
|
|
|
if (selectiveSync.isNameExcluded(item.name)) {
|
2017-03-12 16:07:45 +01:00
|
|
|
log.vlog("Filtered out");
|
|
|
|
return;
|
|
|
|
}
|
2017-06-14 15:50:02 +02:00
|
|
|
string path = itemdb.computePath(item.driveId, item.id);
|
2017-03-24 22:30:03 +01:00
|
|
|
if (selectiveSync.isPathExcluded(path)) {
|
2017-03-12 16:07:45 +01:00
|
|
|
log.vlog("Filtered out: ", path);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2015-09-18 21:42:27 +02:00
|
|
|
final switch (item.type) {
|
|
|
|
case ItemType.dir:
|
2015-09-19 15:38:43 +02:00
|
|
|
uploadDirDifferences(item, path);
|
2015-09-18 21:42:27 +02:00
|
|
|
break;
|
|
|
|
case ItemType.file:
|
2015-09-19 15:38:43 +02:00
|
|
|
uploadFileDifferences(item, path);
|
2015-09-18 21:42:27 +02:00
|
|
|
break;
|
2017-06-14 15:50:02 +02:00
|
|
|
case ItemType.remote:
|
|
|
|
assert(0);
|
2015-09-11 18:33:22 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-09-19 15:38:43 +02:00
|
|
|
private void uploadDirDifferences(Item item, string path)
|
2015-09-04 21:00:22 +02:00
|
|
|
{
|
2015-09-18 21:42:27 +02:00
|
|
|
assert(item.type == ItemType.dir);
|
2015-09-19 15:38:43 +02:00
|
|
|
if (exists(path)) {
|
|
|
|
if (!isDir(path)) {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The item was a directory but now is a file");
|
2015-09-19 15:38:43 +02:00
|
|
|
uploadDeleteItem(item, path);
|
|
|
|
uploadNewFile(path);
|
2015-09-18 21:42:27 +02:00
|
|
|
} else {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The directory has not changed");
|
2015-09-18 21:42:27 +02:00
|
|
|
// loop trough the children
|
2017-06-14 15:50:02 +02:00
|
|
|
foreach (Item child; itemdb.selectChildren(item.driveId, item.id)) {
|
2015-09-18 21:42:27 +02:00
|
|
|
uploadDifferences(child);
|
|
|
|
}
|
2015-09-17 17:34:58 +02:00
|
|
|
}
|
2015-09-18 21:42:27 +02:00
|
|
|
} else {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The directory has been deleted");
|
2015-09-19 15:38:43 +02:00
|
|
|
uploadDeleteItem(item, path);
|
2015-09-18 21:42:27 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-09-19 15:38:43 +02:00
|
|
|
private void uploadFileDifferences(Item item, string path)
|
2015-09-18 21:42:27 +02:00
|
|
|
{
|
|
|
|
assert(item.type == ItemType.file);
|
2015-09-19 15:38:43 +02:00
|
|
|
if (exists(path)) {
|
|
|
|
if (isFile(path)) {
|
|
|
|
SysTime localModifiedTime = timeLastModified(path);
|
2017-03-12 17:17:38 +01:00
|
|
|
// HACK: reduce time resolution to seconds before comparing
|
|
|
|
item.mtime.fracSecs = Duration.zero;
|
|
|
|
localModifiedTime.fracSecs = Duration.zero;
|
2015-09-18 21:42:27 +02:00
|
|
|
if (localModifiedTime != item.mtime) {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The file last modified time has changed");
|
2015-09-18 21:42:27 +02:00
|
|
|
string id = item.id;
|
|
|
|
string eTag = item.eTag;
|
2017-05-28 22:13:19 +02:00
|
|
|
if (!testFileHash(path, item)) {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The file content has changed");
|
|
|
|
log.log("Uploading: ", path);
|
2015-09-27 18:47:41 +02:00
|
|
|
JSONValue response;
|
|
|
|
if (getSize(path) <= thresholdFileSize) {
|
2015-09-30 15:14:39 +02:00
|
|
|
response = onedrive.simpleUpload(path, path, eTag);
|
2015-09-27 18:47:41 +02:00
|
|
|
} else {
|
2015-09-30 15:14:39 +02:00
|
|
|
response = session.upload(path, path, eTag);
|
2015-09-27 18:47:41 +02:00
|
|
|
}
|
|
|
|
saveItem(response);
|
|
|
|
id = response["id"].str;
|
2015-09-21 17:23:12 +02:00
|
|
|
/* use the cTag instead of the eTag because Onedrive changes the
|
|
|
|
* metadata of some type of files (ex. images) AFTER they have been
|
|
|
|
* uploaded */
|
2015-09-27 18:47:41 +02:00
|
|
|
eTag = response["cTag"].str;
|
2015-09-17 16:28:24 +02:00
|
|
|
}
|
2015-09-18 21:42:27 +02:00
|
|
|
uploadLastModifiedTime(id, eTag, localModifiedTime.toUTC());
|
2015-09-06 22:42:44 +02:00
|
|
|
} else {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The file has not changed");
|
2015-09-06 22:42:44 +02:00
|
|
|
}
|
2015-09-18 21:42:27 +02:00
|
|
|
} else {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The item was a file but now is a directory");
|
2015-09-19 15:38:43 +02:00
|
|
|
uploadDeleteItem(item, path);
|
|
|
|
uploadCreateDir(path);
|
2015-09-04 21:00:22 +02:00
|
|
|
}
|
|
|
|
} else {
|
2016-08-04 23:35:58 +02:00
|
|
|
log.vlog("The file has been deleted");
|
2015-09-19 15:38:43 +02:00
|
|
|
uploadDeleteItem(item, path);
|
2015-09-11 18:33:22 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-09-18 21:42:27 +02:00
|
|
|
private void uploadNewItems(string path)
|
2015-09-08 18:25:41 +02:00
|
|
|
{
|
2017-03-12 16:07:45 +01:00
|
|
|
// skip unexisting symbolic links
|
2016-08-22 04:45:24 +02:00
|
|
|
if (isSymlink(path) && !exists(readLink(path))) {
|
|
|
|
return;
|
|
|
|
}
|
2017-03-12 16:07:45 +01:00
|
|
|
|
|
|
|
// skip filtered items
|
2017-03-12 16:35:47 +01:00
|
|
|
if (path != ".") {
|
2017-03-24 22:30:03 +01:00
|
|
|
if (selectiveSync.isNameExcluded(baseName(path))) {
|
2017-03-12 16:35:47 +01:00
|
|
|
return;
|
|
|
|
}
|
2017-03-24 22:30:03 +01:00
|
|
|
if (selectiveSync.isPathExcluded(path)) {
|
2017-03-12 16:35:47 +01:00
|
|
|
return;
|
|
|
|
}
|
2017-03-12 16:07:45 +01:00
|
|
|
}
|
|
|
|
|
2015-09-18 21:42:27 +02:00
|
|
|
if (isDir(path)) {
|
2017-03-12 16:07:45 +01:00
|
|
|
Item item;
|
|
|
|
if (!itemdb.selectByPath(path, item)) {
|
|
|
|
uploadCreateDir(path);
|
|
|
|
}
|
|
|
|
// recursively traverse children
|
|
|
|
auto entries = dirEntries(path, SpanMode.shallow, false);
|
|
|
|
foreach (DirEntry entry; entries) {
|
|
|
|
uploadNewItems(entry.name);
|
2015-09-18 21:42:27 +02:00
|
|
|
}
|
|
|
|
} else {
|
2017-03-12 16:07:45 +01:00
|
|
|
Item item;
|
|
|
|
if (!itemdb.selectByPath(path, item)) {
|
|
|
|
uploadNewFile(path);
|
2015-09-08 18:25:41 +02:00
|
|
|
}
|
2015-09-06 22:42:44 +02:00
|
|
|
}
|
2015-09-08 18:25:41 +02:00
|
|
|
}
|
|
|
|
|
2015-09-18 21:42:27 +02:00
|
|
|
private void uploadCreateDir(const(char)[] path)
|
2015-09-08 18:25:41 +02:00
|
|
|
{
|
2016-08-04 23:35:58 +02:00
|
|
|
log.log("Creating remote directory: ", path);
|
2015-09-16 10:29:20 +02:00
|
|
|
JSONValue item = ["name": baseName(path).idup];
|
2015-09-14 12:57:47 +02:00
|
|
|
item["folder"] = parseJSON("{}");
|
2017-03-11 13:34:07 +01:00
|
|
|
auto res = onedrive.createByPath(path.dirName, item);
|
2015-09-16 10:29:20 +02:00
|
|
|
saveItem(res);
|
|
|
|
}
|
|
|
|
|
|
|
|
private void uploadNewFile(string path)
|
|
|
|
{
|
2016-08-04 23:35:58 +02:00
|
|
|
log.log("Uploading: ", path);
|
2015-09-27 18:47:41 +02:00
|
|
|
JSONValue response;
|
|
|
|
if (getSize(path) <= thresholdFileSize) {
|
|
|
|
response = onedrive.simpleUpload(path, path);
|
|
|
|
} else {
|
|
|
|
response = session.upload(path, path);
|
|
|
|
}
|
|
|
|
string id = response["id"].str;
|
|
|
|
string cTag = response["cTag"].str;
|
2015-09-20 21:21:51 +02:00
|
|
|
SysTime mtime = timeLastModified(path).toUTC();
|
2015-09-21 17:23:12 +02:00
|
|
|
/* use the cTag instead of the eTag because Onedrive changes the
|
|
|
|
* metadata of some type of files (ex. images) AFTER they have been
|
|
|
|
* uploaded */
|
|
|
|
uploadLastModifiedTime(id, cTag, mtime);
|
2015-09-16 10:29:20 +02:00
|
|
|
}
|
|
|
|
|
2015-09-19 15:38:43 +02:00
|
|
|
private void uploadDeleteItem(Item item, const(char)[] path)
|
2015-09-16 10:29:20 +02:00
|
|
|
{
|
2016-08-04 23:35:58 +02:00
|
|
|
log.log("Deleting remote item: ", path);
|
2015-10-04 16:25:31 +02:00
|
|
|
try {
|
|
|
|
onedrive.deleteById(item.id, item.eTag);
|
|
|
|
} catch (OneDriveException e) {
|
2016-12-14 15:17:20 +01:00
|
|
|
if (e.httpStatusCode == 404) log.log(e.msg);
|
2015-10-04 16:25:31 +02:00
|
|
|
else throw e;
|
|
|
|
}
|
2017-06-14 15:50:02 +02:00
|
|
|
itemdb.deleteById(item.driveId, item.id);
|
2015-09-08 18:25:41 +02:00
|
|
|
}
|
|
|
|
|
2015-09-16 10:29:20 +02:00
|
|
|
private void uploadLastModifiedTime(const(char)[] id, const(char)[] eTag, SysTime mtime)
|
2015-09-08 18:25:41 +02:00
|
|
|
{
|
|
|
|
JSONValue mtimeJson = [
|
|
|
|
"fileSystemInfo": JSONValue([
|
|
|
|
"lastModifiedDateTime": mtime.toISOExtString()
|
|
|
|
])
|
|
|
|
];
|
2015-09-11 18:33:22 +02:00
|
|
|
auto res = onedrive.updateById(id, mtimeJson, eTag);
|
2015-09-16 10:29:20 +02:00
|
|
|
saveItem(res);
|
2015-09-08 18:25:41 +02:00
|
|
|
}
|
|
|
|
|
2016-12-24 14:12:20 +01:00
|
|
|
private void saveItem(JSONValue jsonItem)
|
2015-09-08 18:25:41 +02:00
|
|
|
{
|
2017-05-28 22:13:19 +02:00
|
|
|
Item item = makeItem(jsonItem);
|
2016-12-24 14:12:20 +01:00
|
|
|
itemdb.upsert(item);
|
2015-09-16 10:29:20 +02:00
|
|
|
}
|
|
|
|
|
2015-09-19 15:38:43 +02:00
|
|
|
void uploadMoveItem(string from, string to)
|
2015-09-16 10:29:20 +02:00
|
|
|
{
|
2016-08-04 23:35:58 +02:00
|
|
|
log.log("Moving remote item: ", from, " -> ", to);
|
2015-09-20 19:07:16 +02:00
|
|
|
Item fromItem, toItem, parentItem;
|
|
|
|
if (!itemdb.selectByPath(from, fromItem)) {
|
2015-09-20 21:21:51 +02:00
|
|
|
throw new SyncException("Can't move an unsynced item");
|
2015-09-11 18:33:22 +02:00
|
|
|
}
|
2015-09-20 19:07:16 +02:00
|
|
|
if (itemdb.selectByPath(to, toItem)) {
|
|
|
|
// the destination has been overridden
|
|
|
|
uploadDeleteItem(toItem, to);
|
|
|
|
}
|
|
|
|
if (!itemdb.selectByPath(to.dirName, parentItem)) {
|
2015-09-20 21:21:51 +02:00
|
|
|
throw new SyncException("Can't move an item to an unsynced directory");
|
2015-09-17 16:28:24 +02:00
|
|
|
}
|
2015-09-11 18:33:22 +02:00
|
|
|
JSONValue diff = ["name": baseName(to)];
|
|
|
|
diff["parentReference"] = JSONValue([
|
2015-09-20 19:07:16 +02:00
|
|
|
"id": parentItem.id
|
2015-09-11 18:33:22 +02:00
|
|
|
]);
|
2015-09-20 19:07:16 +02:00
|
|
|
auto res = onedrive.updateById(fromItem.id, diff, fromItem.eTag);
|
2015-09-16 10:29:20 +02:00
|
|
|
saveItem(res);
|
2015-09-17 16:28:24 +02:00
|
|
|
string id = res["id"].str;
|
|
|
|
string eTag = res["eTag"].str;
|
|
|
|
uploadLastModifiedTime(id, eTag, timeLastModified(to).toUTC());
|
2015-09-11 18:33:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void deleteByPath(const(char)[] path)
|
|
|
|
{
|
|
|
|
Item item;
|
2015-09-14 23:56:14 +02:00
|
|
|
if (!itemdb.selectByPath(path, item)) {
|
2015-09-17 00:16:23 +02:00
|
|
|
throw new SyncException("Can't delete an unsynced item");
|
2015-09-11 18:33:22 +02:00
|
|
|
}
|
2015-10-04 16:25:31 +02:00
|
|
|
try {
|
|
|
|
uploadDeleteItem(item, path);
|
|
|
|
} catch (OneDriveException e) {
|
2016-12-14 15:17:20 +01:00
|
|
|
if (e.httpStatusCode == 404) log.log(e.msg);
|
2015-10-04 16:25:31 +02:00
|
|
|
else throw e;
|
|
|
|
}
|
2015-09-11 18:33:22 +02:00
|
|
|
}
|
2015-09-01 20:45:34 +02:00
|
|
|
}
|