mirror of
https://github.com/abraunegg/onedrive
synced 2024-05-15 20:26:53 +02:00
ccd4dfbaf7
* Resolve where '--single-directory' is used to sync a single directory, but a file / folder is 'moved' on OneDrive to outside the scope of the focus of the '--single-directory' path. Before this change, the file would remain in the local path (original location) whilst it would reside in the new OneDrive path. This patch looks for this mis match and deletes the local file / folder to reflect it is no longer in the same OneDrive path.
1128 lines
38 KiB
D
1128 lines
38 KiB
D
import std.algorithm;
|
|
import std.array: array;
|
|
import std.datetime;
|
|
import std.exception: enforce;
|
|
import std.file, std.json, std.path;
|
|
import std.regex;
|
|
import std.stdio, std.string, std.uni, std.uri;
|
|
import config, itemdb, onedrive, selective, upload, util;
|
|
static import log;
|
|
|
|
// threshold after which files will be uploaded using an upload session
|
|
private long thresholdFileSize = 4 * 2^^20; // 4 MiB
|
|
|
|
private bool isItemFolder(const ref JSONValue item)
|
|
{
|
|
return ("folder" in item) != null;
|
|
}
|
|
|
|
private bool isItemFile(const ref JSONValue item)
|
|
{
|
|
return ("file" in item) != null;
|
|
}
|
|
|
|
private bool isItemDeleted(const ref JSONValue item)
|
|
{
|
|
return ("deleted" in item) != null;
|
|
}
|
|
|
|
private bool isItemRoot(const ref JSONValue item)
|
|
{
|
|
return ("root" in item) != null;
|
|
}
|
|
|
|
private bool isItemRemote(const ref JSONValue item)
|
|
{
|
|
return ("remoteItem" in item) != null;
|
|
}
|
|
|
|
private bool changeHasParentReferenceId(const ref JSONValue item)
|
|
{
|
|
return ("id" in item["parentReference"]) != null;
|
|
}
|
|
|
|
// construct an Item struct from a JSON driveItem
|
|
private Item makeItem(const ref JSONValue driveItem)
|
|
{
|
|
Item item = {
|
|
id: driveItem["id"].str,
|
|
name: "name" in driveItem ? driveItem["name"].str : null, // name may be missing for deleted files in OneDrive Biz
|
|
eTag: "eTag" in driveItem ? driveItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Biz
|
|
cTag: "cTag" in driveItem ? driveItem["cTag"].str : null, // cTag is missing in old files (and all folders in OneDrive Biz)
|
|
//mtime: "fileSystemInfo" in driveItem ? SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str) : SysTime(0),
|
|
};
|
|
|
|
// OneDrive API Change: https://github.com/OneDrive/onedrive-api-docs/issues/834
|
|
// OneDrive no longer returns lastModifiedDateTime if the item is deleted by OneDrive
|
|
if(isItemDeleted(driveItem)){
|
|
// Set mtime to SysTime(0)
|
|
item.mtime = SysTime(0);
|
|
} else {
|
|
// Item is not in a deleted state
|
|
item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str);
|
|
}
|
|
|
|
if (isItemFile(driveItem)) {
|
|
item.type = ItemType.file;
|
|
} else if (isItemFolder(driveItem)) {
|
|
item.type = ItemType.dir;
|
|
} else if (isItemRemote(driveItem)) {
|
|
item.type = ItemType.remote;
|
|
} else {
|
|
// do not throw exception, item will be removed in applyDifferences()
|
|
}
|
|
|
|
// root and remote items do not have parentReference
|
|
if (!isItemRoot(driveItem) && ("parentReference" in driveItem) != null) {
|
|
item.driveId = driveItem["parentReference"]["driveId"].str,
|
|
item.parentId = driveItem["parentReference"]["id"].str;
|
|
}
|
|
|
|
// extract the file hash
|
|
if (isItemFile(driveItem) && ("hashes" in driveItem["file"])) {
|
|
if ("crc32Hash" in driveItem["file"]["hashes"]) {
|
|
item.crc32Hash = driveItem["file"]["hashes"]["crc32Hash"].str;
|
|
} else if ("sha1Hash" in driveItem["file"]["hashes"]) {
|
|
item.sha1Hash = driveItem["file"]["hashes"]["sha1Hash"].str;
|
|
} else if ("quickXorHash" in driveItem["file"]["hashes"]) {
|
|
item.quickXorHash = driveItem["file"]["hashes"]["quickXorHash"].str;
|
|
} else {
|
|
log.vlog("The file does not have any hash");
|
|
}
|
|
}
|
|
|
|
if (isItemRemote(driveItem)) {
|
|
item.remoteDriveId = driveItem["remoteItem"]["parentReference"]["driveId"].str;
|
|
item.remoteId = driveItem["remoteItem"]["id"].str;
|
|
}
|
|
|
|
return item;
|
|
}
|
|
|
|
private bool testFileHash(string path, const ref Item item)
|
|
{
|
|
if (item.crc32Hash) {
|
|
if (item.crc32Hash == computeCrc32(path)) return true;
|
|
} else if (item.sha1Hash) {
|
|
if (item.sha1Hash == computeSha1Hash(path)) return true;
|
|
} else if (item.quickXorHash) {
|
|
if (item.quickXorHash == computeQuickXorHash(path)) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
class SyncException: Exception
|
|
{
|
|
@nogc @safe pure nothrow this(string msg, string file = __FILE__, size_t line = __LINE__)
|
|
{
|
|
super(msg, file, line);
|
|
}
|
|
}
|
|
|
|
final class SyncEngine
|
|
{
|
|
private Config cfg;
|
|
private OneDriveApi onedrive;
|
|
private ItemDatabase itemdb;
|
|
private UploadSession session;
|
|
private SelectiveSync selectiveSync;
|
|
// list of items to skip while applying the changes
|
|
private string[] skippedItems;
|
|
// list of items to delete after the changes has been downloaded
|
|
private string[2][] idsToDelete;
|
|
// default drive id
|
|
private string defaultDriveId;
|
|
// default root id
|
|
private string defaultRootId;
|
|
// type of OneDrive account
|
|
private string accountType;
|
|
// free space remaining at init()
|
|
private long remainingFreeSpace;
|
|
|
|
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
|
|
{
|
|
assert(onedrive && itemdb && selectiveSync);
|
|
this.cfg = cfg;
|
|
this.onedrive = onedrive;
|
|
this.itemdb = itemdb;
|
|
this.selectiveSync = selectiveSync;
|
|
session = UploadSession(onedrive, cfg.uploadStateFilePath);
|
|
}
|
|
|
|
void init()
|
|
{
|
|
// Set accountType, defaultDriveId, defaultRootId & remainingFreeSpace once and reuse where possible
|
|
auto oneDriveDetails = onedrive.getDefaultDrive();
|
|
accountType = oneDriveDetails["driveType"].str;
|
|
defaultDriveId = oneDriveDetails["id"].str;
|
|
defaultRootId = onedrive.getDefaultRoot["id"].str;
|
|
remainingFreeSpace = oneDriveDetails["quota"]["remaining"].integer;
|
|
|
|
// Display accountType, defaultDriveId, defaultRootId & remainingFreeSpace for verbose logging purposes
|
|
log.vlog("Account Type: ", accountType);
|
|
log.vlog("Default Drive ID: ", defaultDriveId);
|
|
log.vlog("Default Root ID: ", defaultRootId);
|
|
log.vlog("Remaining Free Space: ", remainingFreeSpace);
|
|
|
|
// Check the local database to ensure the OneDrive Root details are in the database
|
|
checkDatabaseForOneDriveRoot();
|
|
|
|
// check if there is an interrupted upload session
|
|
if (session.restore()) {
|
|
log.log("Continuing the upload session ...");
|
|
auto item = session.upload();
|
|
saveItem(item);
|
|
}
|
|
}
|
|
|
|
// download all new changes from OneDrive
|
|
void applyDifferences()
|
|
{
|
|
// Set defaults for the root folder
|
|
// Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls
|
|
string driveId = defaultDriveId;
|
|
string rootId = defaultRootId;
|
|
applyDifferences(driveId, rootId);
|
|
|
|
// check all remote folders
|
|
// https://github.com/OneDrive/onedrive-api-docs/issues/764
|
|
Item[] items = itemdb.selectRemoteItems();
|
|
foreach (item; items) applyDifferences(item.remoteDriveId, item.remoteId);
|
|
}
|
|
|
|
// download all new changes from a specified folder on OneDrive
|
|
void applyDifferencesSingleDirectory(string path)
|
|
{
|
|
// test if the path we are going to sync from actually exists on OneDrive
|
|
try {
|
|
onedrive.getPathDetails(path);
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 404) {
|
|
// The directory was not found
|
|
log.vlog("ERROR: The requested single directory to sync was not found on OneDrive");
|
|
return;
|
|
}
|
|
}
|
|
// OK - the path on OneDrive should exist, get the driveId and rootId for this folder
|
|
log.vlog("Getting path details from OneDrive ...");
|
|
JSONValue onedrivePathDetails = onedrive.getPathDetails(path); // Returns a JSON String for the OneDrive Path
|
|
|
|
// Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls
|
|
string driveId = defaultDriveId;
|
|
string folderId = onedrivePathDetails["id"].str; // Should give something like 12345ABCDE1234A1!101
|
|
|
|
// Apply any differences found on OneDrive for this path (download data)
|
|
applyDifferences(driveId, folderId);
|
|
}
|
|
|
|
// make sure the OneDrive root is in our database
|
|
auto checkDatabaseForOneDriveRoot()
|
|
{
|
|
log.vlog("Fetching details for OneDrive Root");
|
|
JSONValue rootPathDetails = onedrive.getDefaultRoot(); // Returns a JSON Value
|
|
Item rootPathItem = makeItem(rootPathDetails);
|
|
|
|
// configure driveId and rootId for the OneDrive Root
|
|
|
|
// Set defaults for the root folder
|
|
string driveId = rootPathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1
|
|
string rootId = rootPathDetails["id"].str; // Should give something like 12345ABCDE1234A1!101
|
|
|
|
// Query the database
|
|
if (!itemdb.selectById(driveId, rootId, rootPathItem)) {
|
|
log.vlog("OneDrive Root does not exist in the database. We need to add it.");
|
|
applyDifference(rootPathDetails, driveId, true);
|
|
log.vlog("Added OneDrive Root to the local database");
|
|
} else {
|
|
log.vlog("OneDrive Root exists in the database");
|
|
}
|
|
}
|
|
|
|
// create a directory on OneDrive without syncing
|
|
auto createDirectoryNoSync(string path)
|
|
{
|
|
// Attempt to create the requested path within OneDrive without performing a sync
|
|
log.vlog("Attempting to create the requested path within OneDrive");
|
|
|
|
// Handle the remote folder creation and updating of the local database without performing a sync
|
|
uploadCreateDir(path);
|
|
}
|
|
|
|
// delete a directory on OneDrive without syncing
|
|
auto deleteDirectoryNoSync(string path)
|
|
{
|
|
// Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls
|
|
string rootId = defaultRootId;
|
|
|
|
// Attempt to delete the requested path within OneDrive without performing a sync
|
|
log.vlog("Attempting to delete the requested path within OneDrive");
|
|
|
|
// test if the path we are going to exists on OneDrive
|
|
try {
|
|
onedrive.getPathDetails(path);
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 404) {
|
|
// The directory was not found on OneDrive - no need to delete it
|
|
log.vlog("The requested directory to create was not found on OneDrive - skipping removing the remote directory as it doesnt exist");
|
|
return;
|
|
}
|
|
}
|
|
|
|
Item item;
|
|
if (!itemdb.selectByPath(path, defaultDriveId, item)) {
|
|
// this is odd .. this directory is not in the local database - just go delete it
|
|
log.vlog("The requested directory to delete was not found in the local database - pushing delete request direct to OneDrive");
|
|
uploadDeleteItem(item, path);
|
|
} else {
|
|
// the folder was in the local database
|
|
// Handle the deletion and saving any update to the local database
|
|
log.vlog("The requested directory to delete was found in the local database. Processing the delection normally");
|
|
deleteByPath(path);
|
|
}
|
|
}
|
|
|
|
// rename a directory on OneDrive without syncing
|
|
auto renameDirectoryNoSync(string source, string destination)
|
|
{
|
|
try {
|
|
// test if the local path exists on OneDrive
|
|
onedrive.getPathDetails(source);
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 404) {
|
|
// The directory was not found
|
|
log.vlog("The requested directory to rename was not found on OneDrive");
|
|
return;
|
|
}
|
|
}
|
|
// The OneDrive API returned a 200 OK status, so the folder exists
|
|
// Rename the requested directory on OneDrive without performing a sync
|
|
moveByPath(source, destination);
|
|
}
|
|
|
|
// download the new changes of a specific item
|
|
// id is the root of the drive or a shared folder
|
|
private void applyDifferences(string driveId, const(char)[] id)
|
|
{
|
|
log.vlog("Applying changes of Path ID: " ~ id);
|
|
JSONValue changes;
|
|
string syncFolderName;
|
|
string deltaLink = itemdb.getDeltaLink(driveId, id);
|
|
JSONValue idDetails = onedrive.getPathDetailsById(id);
|
|
|
|
// Set the name of this folder
|
|
if ((idDetails["id"].str == id) && (isItemFolder(idDetails))){
|
|
syncFolderName = idDetails["name"].str;
|
|
}
|
|
|
|
for (;;) {
|
|
try {
|
|
// Due to differences in OneDrive API's between personal and business we need to get changes only from defaultRootId
|
|
// If we used the 'id' passed in & when using --single-directory with a business account we get:
|
|
// 'HTTP request returned status code 501 (Not Implemented): view.delta can only be called on the root.'
|
|
// To view changes correctly, we need to use 'defaultRootId'
|
|
changes = onedrive.viewChangesById(driveId, defaultRootId, deltaLink);
|
|
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 410) {
|
|
log.vlog("Delta link expired, resyncing...");
|
|
deltaLink = null;
|
|
continue;
|
|
} else {
|
|
throw e;
|
|
}
|
|
}
|
|
foreach (item; changes["value"].array) {
|
|
bool isRoot = false;
|
|
string thisItemPath;
|
|
|
|
// Deleted items returned from onedrive.viewChangesById (/delta) do not have a 'name' attribute
|
|
// Thus we cannot name check for 'root' below on deleted items
|
|
if(!isItemDeleted(item)){
|
|
// This is not a deleted item
|
|
// Test is this is the OneDrive Root?
|
|
// Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls
|
|
if ((id == defaultRootId) && (item["name"].str == "root")) { // fix for https://github.com/skilion/onedrive/issues/269
|
|
// This IS the OneDrive Root
|
|
isRoot = true;
|
|
}
|
|
}
|
|
|
|
// How do we handle this change?
|
|
if (isRoot || !changeHasParentReferenceId(item) || isItemDeleted(item)){
|
|
// Is a root item, has no id in parentReference or is a OneDrive deleted item
|
|
applyDifference(item, driveId, isRoot);
|
|
} else {
|
|
// What is this item's path?
|
|
thisItemPath = item["parentReference"]["path"].str;
|
|
// Check this item's path to see if this is a change on the path we want
|
|
if ( (item["id"].str == id) || (item["parentReference"]["id"].str == id) || (canFind(thisItemPath, syncFolderName)) ){
|
|
// This is a change we want to apply
|
|
applyDifference(item, driveId, isRoot);
|
|
} else {
|
|
// No item ID match or folder sync match
|
|
// Before discarding change - does this ID still exist on OneDrive - as in IS this
|
|
// potentially a --single-directory sync and the user 'moved' the file out of the 'sync-dir' to another OneDrive folder
|
|
// This is a corner edge case - https://github.com/skilion/onedrive/issues/341
|
|
JSONValue oneDriveMovedNotDeleted;
|
|
try {
|
|
oneDriveMovedNotDeleted = onedrive.getPathDetailsById(item["id"].str);
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 404) {
|
|
// No .. that ID is GONE
|
|
return;
|
|
}
|
|
}
|
|
// Yes .. ID is still on OneDrive but elsewhere .... #341 edge case handling
|
|
// What is the original local path for this ID in the database? Does it match 'syncFolderName'
|
|
string originalLocalPath = itemdb.computePath(driveId, item["id"].str);
|
|
|
|
if (canFind(originalLocalPath, syncFolderName)){
|
|
// This 'change' relates to an item that WAS in 'syncFolderName' but is now
|
|
// stored elsewhere on OneDrive - outside the path we are syncing from
|
|
// Remove this item locally as it's local path is now obsolete
|
|
idsToDelete ~= [driveId, item["id"].str];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// the response may contain either @odata.deltaLink or @odata.nextLink
|
|
if ("@odata.deltaLink" in changes) deltaLink = changes["@odata.deltaLink"].str;
|
|
if (deltaLink) itemdb.setDeltaLink(driveId, id, deltaLink);
|
|
if ("@odata.nextLink" in changes) deltaLink = changes["@odata.nextLink"].str;
|
|
else break;
|
|
}
|
|
|
|
// delete items in idsToDelete
|
|
if (idsToDelete.length > 0) deleteItems();
|
|
// empty the skipped items
|
|
skippedItems.length = 0;
|
|
assumeSafeAppend(skippedItems);
|
|
}
|
|
|
|
// process the change of a single DriveItem
|
|
private void applyDifference(JSONValue driveItem, string driveId, bool isRoot)
|
|
{
|
|
Item item = makeItem(driveItem);
|
|
//log.vlog("Processing item to apply differences");
|
|
|
|
if (isItemRoot(driveItem) || !item.parentId || isRoot) {
|
|
item.parentId = null; // ensures that it has no parent
|
|
item.driveId = driveId; // HACK: makeItem() cannot set the driveId propery of the root
|
|
itemdb.upsert(item);
|
|
return;
|
|
}
|
|
|
|
bool unwanted;
|
|
unwanted |= skippedItems.find(item.parentId).length != 0;
|
|
unwanted |= selectiveSync.isNameExcluded(item.name);
|
|
|
|
// check the item type
|
|
if (!unwanted) {
|
|
if (isItemFile(driveItem)) {
|
|
//log.vlog("The item we are syncing is a file");
|
|
} else if (isItemFolder(driveItem)) {
|
|
//log.vlog("The item we are syncing is a folder");
|
|
} else if (isItemRemote(driveItem)) {
|
|
//log.vlog("The item we are syncing is a remote item");
|
|
assert(isItemFolder(driveItem["remoteItem"]), "The remote item is not a folder");
|
|
} else {
|
|
log.vlog("This item type (", item.name, ") is not supported");
|
|
unwanted = true;
|
|
}
|
|
}
|
|
|
|
// check for selective sync
|
|
string path;
|
|
if (!unwanted) {
|
|
path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name;
|
|
path = buildNormalizedPath(path);
|
|
unwanted = selectiveSync.isPathExcluded(path);
|
|
}
|
|
|
|
// skip unwanted items early
|
|
if (unwanted) {
|
|
log.vlog("Filtered out");
|
|
skippedItems ~= item.id;
|
|
return;
|
|
}
|
|
|
|
// check if the item has been seen before
|
|
Item oldItem;
|
|
bool cached = itemdb.selectById(item.driveId, item.id, oldItem);
|
|
|
|
// check if the item is going to be deleted
|
|
if (isItemDeleted(driveItem)) {
|
|
log.vlog("The item is marked for deletion");
|
|
if (cached) {
|
|
// flag to delete
|
|
idsToDelete ~= [item.driveId, item.id];
|
|
} else {
|
|
// flag to ignore
|
|
skippedItems ~= item.id;
|
|
}
|
|
return;
|
|
}
|
|
|
|
// rename the local item if it is unsynced and there is a new version of it
|
|
string oldPath;
|
|
if (cached && item.eTag != oldItem.eTag) {
|
|
oldPath = itemdb.computePath(item.driveId, item.id);
|
|
if (!isItemSynced(oldItem, oldPath)) {
|
|
log.vlog("The local item is unsynced, renaming");
|
|
if (exists(oldPath)) safeRename(oldPath);
|
|
cached = false;
|
|
}
|
|
}
|
|
|
|
// update the item
|
|
if (cached) {
|
|
applyChangedItem(oldItem, oldPath, item, path);
|
|
} else {
|
|
applyNewItem(item, path);
|
|
}
|
|
|
|
// save the item in the db
|
|
if (cached) {
|
|
itemdb.update(item);
|
|
} else {
|
|
itemdb.insert(item);
|
|
}
|
|
|
|
// sync remote folder
|
|
// https://github.com/OneDrive/onedrive-api-docs/issues/764
|
|
/*if (isItemRemote(driveItem)) {
|
|
log.log("Syncing remote folder: ", path);
|
|
applyDifferences(item.remoteDriveId, item.remoteId);
|
|
}*/
|
|
}
|
|
|
|
// download an item that was not synced before
|
|
private void applyNewItem(Item item, string path)
|
|
{
|
|
if (exists(path)) {
|
|
if (isItemSynced(item, path)) {
|
|
//log.vlog("The item is already present");
|
|
return;
|
|
} else {
|
|
// TODO: force remote sync by deleting local item
|
|
log.vlog("The local item is out of sync, renaming...");
|
|
safeRename(path);
|
|
}
|
|
}
|
|
final switch (item.type) {
|
|
case ItemType.file:
|
|
downloadFileItem(item, path);
|
|
break;
|
|
case ItemType.dir:
|
|
case ItemType.remote:
|
|
log.log("Creating directory ", path);
|
|
mkdirRecurse(path);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// update a local item
|
|
// the local item is assumed to be in sync with the local db
|
|
private void applyChangedItem(Item oldItem, string oldPath, Item newItem, string newPath)
|
|
{
|
|
assert(oldItem.driveId == newItem.driveId);
|
|
assert(oldItem.id == newItem.id);
|
|
assert(oldItem.type == newItem.type);
|
|
assert(oldItem.remoteDriveId == newItem.remoteDriveId);
|
|
assert(oldItem.remoteId == newItem.remoteId);
|
|
|
|
if (oldItem.eTag != newItem.eTag) {
|
|
// handle changed name/path
|
|
if (oldPath != newPath) {
|
|
log.log("Moving ", oldPath, " to ", newPath);
|
|
if (exists(newPath)) {
|
|
// TODO: force remote sync by deleting local item
|
|
log.vlog("The destination is occupied, renaming the conflicting file...");
|
|
safeRename(newPath);
|
|
}
|
|
rename(oldPath, newPath);
|
|
}
|
|
// handle changed content and mtime
|
|
// HACK: use mtime+hash instead of cTag because of https://github.com/OneDrive/onedrive-api-docs/issues/765
|
|
if (newItem.type == ItemType.file && oldItem.mtime != newItem.mtime && !testFileHash(newPath, newItem)) {
|
|
downloadFileItem(newItem, newPath);
|
|
} else {
|
|
//log.vlog("The item content has not changed");
|
|
}
|
|
// handle changed time
|
|
if (newItem.type == ItemType.file && oldItem.mtime != newItem.mtime) {
|
|
setTimes(newPath, newItem.mtime, newItem.mtime);
|
|
}
|
|
} else {
|
|
//log.vlog("", oldItem.name, " has not changed");
|
|
}
|
|
}
|
|
|
|
// downloads a File resource
|
|
private void downloadFileItem(Item item, string path)
|
|
{
|
|
assert(item.type == ItemType.file);
|
|
write("Downloading ", path, "...");
|
|
onedrive.downloadById(item.driveId, item.id, path);
|
|
setTimes(path, item.mtime, item.mtime);
|
|
writeln(" done.");
|
|
log.log("Downloading ", path, "... done.");
|
|
}
|
|
|
|
// returns true if the given item corresponds to the local one
|
|
private bool isItemSynced(Item item, string path)
|
|
{
|
|
if (!exists(path)) return false;
|
|
final switch (item.type) {
|
|
case ItemType.file:
|
|
if (isFile(path)) {
|
|
SysTime localModifiedTime = timeLastModified(path).toUTC();
|
|
// HACK: reduce time resolution to seconds before comparing
|
|
item.mtime.fracSecs = Duration.zero;
|
|
localModifiedTime.fracSecs = Duration.zero;
|
|
if (localModifiedTime == item.mtime) {
|
|
return true;
|
|
} else {
|
|
log.vlog("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime);
|
|
}
|
|
if (testFileHash(path, item)) {
|
|
return true;
|
|
} else {
|
|
log.vlog("The local item has a different hash");
|
|
}
|
|
} else {
|
|
log.vlog("The local item is a directory but should be a file");
|
|
}
|
|
break;
|
|
case ItemType.dir:
|
|
case ItemType.remote:
|
|
if (isDir(path)) {
|
|
return true;
|
|
} else {
|
|
log.vlog("The local item is a file but should be a directory");
|
|
}
|
|
break;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private void deleteItems()
|
|
{
|
|
foreach_reverse (i; idsToDelete) {
|
|
Item item;
|
|
if (!itemdb.selectById(i[0], i[1], item)) continue; // check if the item is in the db
|
|
string path = itemdb.computePath(i[0], i[1]);
|
|
log.log("Deleting item ", path);
|
|
itemdb.deleteById(item.driveId, item.id);
|
|
if (item.remoteDriveId != null) {
|
|
// delete the linked remote folder
|
|
itemdb.deleteById(item.remoteDriveId, item.remoteId);
|
|
}
|
|
if (exists(path)) {
|
|
// path exists on the local system
|
|
if (isFile(path)) {
|
|
remove(path);
|
|
} else {
|
|
try {
|
|
// Remove any children of this path if they still exist
|
|
// Resolve 'Directory not empty' error when deleting local files
|
|
foreach (DirEntry child; dirEntries(path, SpanMode.depth, false)) {
|
|
attrIsDir(child.linkAttributes) ? rmdir(child.name) : remove(child.name);
|
|
}
|
|
// Remove the path now that it is empty of children
|
|
rmdirRecurse(path);
|
|
} catch (FileException e) {
|
|
log.log(e.msg);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
idsToDelete.length = 0;
|
|
assumeSafeAppend(idsToDelete);
|
|
}
|
|
|
|
// scan the given directory for differences and new items
|
|
void scanForDifferences(string path)
|
|
{
|
|
// scan for changes
|
|
log.vlog("Uploading differences of ", path);
|
|
Item item;
|
|
if (itemdb.selectByPath(path, defaultDriveId, item)) {
|
|
uploadDifferences(item);
|
|
}
|
|
log.vlog("Uploading new items of ", path);
|
|
uploadNewItems(path);
|
|
}
|
|
|
|
private void uploadDifferences(Item item)
|
|
{
|
|
log.vlog("Processing ", item.name);
|
|
|
|
string path;
|
|
bool unwanted = selectiveSync.isNameExcluded(item.name);
|
|
if (!unwanted) {
|
|
path = itemdb.computePath(item.driveId, item.id);
|
|
unwanted = selectiveSync.isPathExcluded(path);
|
|
}
|
|
|
|
// skip unwanted items
|
|
if (unwanted) {
|
|
log.vlog("Filtered out");
|
|
return;
|
|
}
|
|
|
|
// Restriction and limitations about windows naming files
|
|
if (!isValidName(path)) {
|
|
log.vlog("Skipping item - invalid name (Microsoft Naming Convention): ", path);
|
|
return;
|
|
}
|
|
|
|
final switch (item.type) {
|
|
case ItemType.dir:
|
|
uploadDirDifferences(item, path);
|
|
break;
|
|
case ItemType.file:
|
|
uploadFileDifferences(item, path);
|
|
break;
|
|
case ItemType.remote:
|
|
uploadRemoteDirDifferences(item, path);
|
|
break;
|
|
}
|
|
}
|
|
|
|
private void uploadDirDifferences(Item item, string path)
|
|
{
|
|
assert(item.type == ItemType.dir);
|
|
if (exists(path)) {
|
|
if (!isDir(path)) {
|
|
log.vlog("The item was a directory but now it is a file");
|
|
uploadDeleteItem(item, path);
|
|
uploadNewFile(path);
|
|
} else {
|
|
log.vlog("The directory has not changed");
|
|
// loop trough the children
|
|
foreach (Item child; itemdb.selectChildren(item.driveId, item.id)) {
|
|
uploadDifferences(child);
|
|
}
|
|
}
|
|
} else {
|
|
log.vlog("The directory has been deleted");
|
|
uploadDeleteItem(item, path);
|
|
}
|
|
}
|
|
|
|
private void uploadRemoteDirDifferences(Item item, string path)
|
|
{
|
|
assert(item.type == ItemType.remote);
|
|
if (exists(path)) {
|
|
if (!isDir(path)) {
|
|
log.vlog("The item was a directory but now it is a file");
|
|
uploadDeleteItem(item, path);
|
|
uploadNewFile(path);
|
|
} else {
|
|
log.vlog("The directory has not changed");
|
|
// continue trough the linked folder
|
|
assert(item.remoteDriveId && item.remoteId);
|
|
Item remoteItem;
|
|
bool found = itemdb.selectById(item.remoteDriveId, item.remoteId, remoteItem);
|
|
assert(found);
|
|
uploadDifferences(remoteItem);
|
|
}
|
|
} else {
|
|
log.vlog("The directory has been deleted");
|
|
uploadDeleteItem(item, path);
|
|
}
|
|
}
|
|
|
|
private void uploadFileDifferences(Item item, string path)
|
|
{
|
|
assert(item.type == ItemType.file);
|
|
if (exists(path)) {
|
|
if (isFile(path)) {
|
|
SysTime localModifiedTime = timeLastModified(path).toUTC();
|
|
// HACK: reduce time resolution to seconds before comparing
|
|
item.mtime.fracSecs = Duration.zero;
|
|
localModifiedTime.fracSecs = Duration.zero;
|
|
if (localModifiedTime != item.mtime) {
|
|
log.vlog("The file last modified time has changed");
|
|
string eTag = item.eTag;
|
|
if (!testFileHash(path, item)) {
|
|
log.vlog("The file content has changed");
|
|
write("Uploading file ", path, "...");
|
|
JSONValue response;
|
|
if (getSize(path) <= thresholdFileSize) {
|
|
response = onedrive.simpleUploadReplace(path, item.driveId, item.id, item.eTag);
|
|
writeln(" done.");
|
|
} else {
|
|
writeln("");
|
|
response = session.upload(path, item.driveId, item.parentId, baseName(path), eTag);
|
|
}
|
|
log.log("Uploading file ", path, "... done.");
|
|
// saveItem(response); redundant
|
|
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
|
|
eTag = response["cTag"].str;
|
|
}
|
|
uploadLastModifiedTime(item.driveId, item.id, eTag, localModifiedTime.toUTC());
|
|
} else {
|
|
log.vlog("The file has not changed");
|
|
}
|
|
} else {
|
|
log.vlog("The item was a file but now is a directory");
|
|
uploadDeleteItem(item, path);
|
|
uploadCreateDir(path);
|
|
}
|
|
} else {
|
|
log.vlog("The file has been deleted");
|
|
uploadDeleteItem(item, path);
|
|
}
|
|
}
|
|
|
|
private void uploadNewItems(string path)
|
|
{
|
|
// https://github.com/OneDrive/onedrive-api-docs/issues/443
|
|
// If the path is greater than 430 characters, then one drive will return a '400 - Bad Request'
|
|
// Need to ensure that the URI is encoded before the check is made
|
|
if(encodeComponent(path).length < 430){
|
|
// path is less than 430 characters
|
|
|
|
// skip unexisting symbolic links
|
|
if (isSymlink(path) && !exists(readLink(path))) {
|
|
log.vlog("Skipping item - symbolic link: ", path);
|
|
return;
|
|
}
|
|
|
|
// Restriction and limitations about windows naming files
|
|
if (!isValidName(path)) {
|
|
log.vlog("Skipping item - invalid name (Microsoft Naming Convention): ", path);
|
|
return;
|
|
}
|
|
|
|
// filter out user configured items to skip
|
|
if (path != ".") {
|
|
if (selectiveSync.isNameExcluded(baseName(path))) {
|
|
log.vlog("Skipping item - excluded by skip_file config: ", path);
|
|
return;
|
|
}
|
|
if (selectiveSync.isPathExcluded(path)) {
|
|
log.vlog("Skipping item - path excluded: ", path);
|
|
return;
|
|
}
|
|
}
|
|
|
|
// This item passed all the unwanted checks
|
|
// We want to upload this new item
|
|
if (isDir(path)) {
|
|
Item item;
|
|
if (!itemdb.selectByPath(path, defaultDriveId, item)) {
|
|
uploadCreateDir(path);
|
|
}
|
|
// recursively traverse children
|
|
auto entries = dirEntries(path, SpanMode.shallow, false);
|
|
foreach (DirEntry entry; entries) {
|
|
uploadNewItems(entry.name);
|
|
}
|
|
} else {
|
|
// This item is a file
|
|
// Can we upload this file - is there enough free space? - https://github.com/skilion/onedrive/issues/73
|
|
auto fileSize = getSize(path);
|
|
if ((remainingFreeSpace - fileSize) > 0){
|
|
Item item;
|
|
if (!itemdb.selectByPath(path, defaultDriveId, item)) {
|
|
uploadNewFile(path);
|
|
remainingFreeSpace = (remainingFreeSpace - fileSize);
|
|
log.vlog("Remaining free space: ", remainingFreeSpace);
|
|
}
|
|
} else {
|
|
// Not enough free space
|
|
log.log("Skipping item '", path, "' due to insufficient free space available on OneDrive");
|
|
}
|
|
}
|
|
} else {
|
|
// This path was skipped - why?
|
|
log.log("Skipping item '", path, "' due to the full path exceeding 430 characters (Microsoft OneDrive limitation)");
|
|
}
|
|
}
|
|
|
|
private void uploadCreateDir(const(string) path)
|
|
{
|
|
log.vlog("OneDrive Client requested to create remote path: ", path);
|
|
JSONValue onedrivePathDetails;
|
|
Item parent;
|
|
|
|
// Was the path entered the root path?
|
|
if (path != "."){
|
|
// If this is null or empty - we cant query the database properly
|
|
if ((parent.driveId == "") && (parent.id == "")){
|
|
// What path to use?
|
|
string parentPath = dirName(path); // will be either . or something else
|
|
|
|
try {
|
|
onedrivePathDetails = onedrive.getPathDetails(parentPath);
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 404) {
|
|
// Parent does not exist ... need to create parent
|
|
uploadCreateDir(parentPath);
|
|
}
|
|
}
|
|
|
|
// configure the data
|
|
parent.driveId = onedrivePathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1
|
|
parent.id = onedrivePathDetails["id"].str; // This item's ID. Should give something like 12345ABCDE1234A1!101
|
|
}
|
|
|
|
// test if the path we are going to create already exists on OneDrive
|
|
try {
|
|
onedrive.getPathDetails(path);
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 404) {
|
|
// The directory was not found
|
|
log.vlog("The requested directory to create was not found on OneDrive - creating remote directory: ", path);
|
|
|
|
// Perform the database lookup
|
|
enforce(itemdb.selectById(parent.driveId, parent.id, parent), "The parent item id is not in the database");
|
|
JSONValue driveItem = [
|
|
"name": JSONValue(baseName(path)),
|
|
"folder": parseJSON("{}")
|
|
];
|
|
|
|
// Submit the creation request
|
|
auto res = onedrive.createById(parent.driveId, parent.id, driveItem);
|
|
saveItem(res);
|
|
log.vlog("Sucessfully created the remote directory ", path, " on OneDrive");
|
|
return;
|
|
}
|
|
}
|
|
log.vlog("The requested directory to create was found on OneDrive - skipping creating the directory: ", path );
|
|
|
|
// Check that this path is in the database
|
|
if (!itemdb.selectById(parent.driveId, parent.id, parent)){
|
|
// parent for 'path' is NOT in the database
|
|
log.vlog("The parent for this path is not in the local database - need to add parent to local database");
|
|
string parentPath = dirName(path);
|
|
uploadCreateDir(parentPath);
|
|
} else {
|
|
// parent is in database
|
|
log.vlog("The parent for this path is in the local database - adding requested path (", path ,") to database");
|
|
auto res = onedrive.getPathDetails(path);
|
|
saveItem(res);
|
|
}
|
|
}
|
|
}
|
|
|
|
private void uploadNewFile(string path)
|
|
{
|
|
Item parent;
|
|
|
|
// Check the database for the parent
|
|
enforce(itemdb.selectByPath(dirName(path), defaultDriveId, parent), "The parent item is not in the local database");
|
|
|
|
// Maximum file size upload
|
|
// https://support.microsoft.com/en-au/help/3125202/restrictions-and-limitations-when-you-sync-files-and-folders
|
|
// 1. OneDrive Business say's 15GB
|
|
// 2. Another article updated April 2018 says 20GB:
|
|
// https://answers.microsoft.com/en-us/onedrive/forum/odoptions-oddesktop-sdwin10/personal-onedrive-file-upload-size-max/a3621fc9-b766-4a99-99f8-bcc01ccb025f
|
|
|
|
// Use smaller size for now
|
|
auto maxUploadFileSize = 16106127360; // 15GB
|
|
//auto maxUploadFileSize = 21474836480; // 20GB
|
|
auto thisFileSize = getSize(path);
|
|
|
|
if (thisFileSize <= maxUploadFileSize){
|
|
// Resolves: https://github.com/skilion/onedrive/issues/121, https://github.com/skilion/onedrive/issues/294, https://github.com/skilion/onedrive/issues/329
|
|
|
|
// To avoid a 409 Conflict error - does the file actually exist on OneDrive already?
|
|
JSONValue fileDetailsFromOneDrive;
|
|
|
|
// Does this 'file' already exist on OneDrive?
|
|
try {
|
|
// test if the local path exists on OneDrive
|
|
fileDetailsFromOneDrive = onedrive.getPathDetails(path);
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 404) {
|
|
// The file was not found on OneDrive, need to upload it
|
|
write("Uploading file ", path, "...");
|
|
JSONValue response;
|
|
if (getSize(path) <= thresholdFileSize) {
|
|
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
|
writeln(" done.");
|
|
} else {
|
|
writeln("");
|
|
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
|
}
|
|
log.log("Uploading file ", path, "... done.");
|
|
string id = response["id"].str;
|
|
string cTag = response["cTag"].str;
|
|
SysTime mtime = timeLastModified(path).toUTC();
|
|
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
|
|
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
|
return;
|
|
}
|
|
}
|
|
|
|
log.vlog("Requested file to upload exists on OneDrive - local database is out of sync for this file: ", path);
|
|
|
|
// Is the local file newer than the uploaded file?
|
|
SysTime localFileModifiedTime = timeLastModified(path).toUTC();
|
|
SysTime remoteFileModifiedTime = SysTime.fromISOExtString(fileDetailsFromOneDrive["fileSystemInfo"]["lastModifiedDateTime"].str);
|
|
|
|
if (localFileModifiedTime > remoteFileModifiedTime){
|
|
// local file is newer
|
|
log.vlog("Requested file to upload is newer than existing file on OneDrive");
|
|
|
|
write("Uploading file ", path, "...");
|
|
JSONValue response;
|
|
if (getSize(path) <= thresholdFileSize) {
|
|
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
|
writeln(" done.");
|
|
} else {
|
|
writeln("");
|
|
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
|
}
|
|
log.log("Uploading file ", path, "... done.");
|
|
string id = response["id"].str;
|
|
string cTag = response["cTag"].str;
|
|
SysTime mtime = timeLastModified(path).toUTC();
|
|
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
|
|
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
|
} else {
|
|
// Save the details of the file that we got from OneDrive
|
|
log.vlog("Updating the local database with details for this file: ", path);
|
|
saveItem(fileDetailsFromOneDrive);
|
|
}
|
|
} else {
|
|
// Skip file - too large
|
|
log.log("Skipping uploading this new file as it exceeds the maximum size allowed by OneDrive: ", path);
|
|
}
|
|
}
|
|
|
|
private void uploadDeleteItem(Item item, string path)
|
|
{
|
|
log.log("Deleting item from OneDrive: ", path);
|
|
|
|
if ((item.driveId == "") && (item.id == "") && (item.eTag == "")){
|
|
// These are empty ... we cannot delete if this is empty ....
|
|
JSONValue onedrivePathDetails = onedrive.getPathDetails(path); // Returns a JSON String for the OneDrive Path
|
|
item.driveId = onedrivePathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1
|
|
item.id = onedrivePathDetails["id"].str; // This item's ID. Should give something like 12345ABCDE1234A1!101
|
|
item.eTag = onedrivePathDetails["eTag"].str; // Should be something like aNjM2NjJFRUVGQjY2NjJFMSE5MzUuMA
|
|
}
|
|
|
|
try {
|
|
onedrive.deleteById(item.driveId, item.id, item.eTag);
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 404) log.vlog("OneDrive reported: The resource could not be found.");
|
|
else throw e;
|
|
}
|
|
itemdb.deleteById(item.driveId, item.id);
|
|
if (item.remoteId != null) {
|
|
itemdb.deleteById(item.remoteDriveId, item.remoteId);
|
|
}
|
|
}
|
|
|
|
private void uploadLastModifiedTime(const(char)[] driveId, const(char)[] id, const(char)[] eTag, SysTime mtime)
|
|
{
|
|
JSONValue data = [
|
|
"fileSystemInfo": JSONValue([
|
|
"lastModifiedDateTime": mtime.toISOExtString()
|
|
])
|
|
];
|
|
auto response = onedrive.updateById(driveId, id, data, eTag);
|
|
saveItem(response);
|
|
}
|
|
|
|
private void saveItem(JSONValue jsonItem)
|
|
{
|
|
// Takes a JSON input and formats to an item which can be used by the database
|
|
Item item = makeItem(jsonItem);
|
|
|
|
// Add to the local database
|
|
itemdb.upsert(item);
|
|
}
|
|
|
|
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_move
|
|
void uploadMoveItem(string from, string to)
|
|
{
|
|
log.log("Moving ", from, " to ", to);
|
|
Item fromItem, toItem, parentItem;
|
|
if (!itemdb.selectByPath(from, defaultDriveId, fromItem)) {
|
|
throw new SyncException("Can't move an unsynced item");
|
|
}
|
|
if (fromItem.parentId == null) {
|
|
// the item is a remote folder, need to do the operation on the parent
|
|
enforce(itemdb.selectByPathNoRemote(from, defaultDriveId, fromItem));
|
|
}
|
|
if (itemdb.selectByPath(to, defaultDriveId, toItem)) {
|
|
// the destination has been overwritten
|
|
uploadDeleteItem(toItem, to);
|
|
}
|
|
if (!itemdb.selectByPath(dirName(to), defaultDriveId, parentItem)) {
|
|
throw new SyncException("Can't move an item to an unsynced directory");
|
|
}
|
|
if (fromItem.driveId != parentItem.driveId) {
|
|
// items cannot be moved between drives
|
|
uploadDeleteItem(fromItem, from);
|
|
uploadNewFile(to);
|
|
} else {
|
|
SysTime mtime = timeLastModified(to).toUTC();
|
|
JSONValue diff = [
|
|
"name": JSONValue(baseName(to)),
|
|
"parentReference": JSONValue([
|
|
"id": parentItem.id
|
|
]),
|
|
"fileSystemInfo": JSONValue([
|
|
"lastModifiedDateTime": mtime.toISOExtString()
|
|
])
|
|
];
|
|
auto res = onedrive.updateById(fromItem.driveId, fromItem.id, diff, fromItem.eTag);
|
|
// update itemdb
|
|
saveItem(res);
|
|
}
|
|
}
|
|
|
|
void deleteByPath(string path)
|
|
{
|
|
Item item;
|
|
if (!itemdb.selectByPath(path, defaultDriveId, item)) {
|
|
throw new SyncException("The item to delete is not in the local database");
|
|
}
|
|
if (item.parentId == null) {
|
|
// the item is a remote folder, need to do the operation on the parent
|
|
enforce(itemdb.selectByPathNoRemote(path, defaultDriveId, item));
|
|
}
|
|
try {
|
|
uploadDeleteItem(item, path);
|
|
} catch (OneDriveException e) {
|
|
if (e.httpStatusCode == 404) log.log(e.msg);
|
|
else throw e;
|
|
}
|
|
}
|
|
|
|
// move a OneDrive folder from one name to another
|
|
void moveByPath(const(string) source, const(string) destination)
|
|
{
|
|
log.vlog("Moving remote folder: ", source, " -> ", destination);
|
|
|
|
// Source and Destination are relative to ~/OneDrive
|
|
string sourcePath = source;
|
|
string destinationBasePath = dirName(destination).idup;
|
|
|
|
// if destinationBasePath == '.' then destinationBasePath needs to be ""
|
|
if (destinationBasePath == ".") {
|
|
destinationBasePath = "";
|
|
}
|
|
|
|
string newFolderName = baseName(destination).idup;
|
|
string destinationPathString = "/drive/root:/" ~ destinationBasePath;
|
|
|
|
// Build up the JSON changes
|
|
JSONValue moveData = ["name": newFolderName];
|
|
JSONValue destinationPath = ["path": destinationPathString];
|
|
moveData["parentReference"] = destinationPath;
|
|
|
|
// Make the change on OneDrive
|
|
auto res = onedrive.moveByPath(sourcePath, moveData);
|
|
}
|
|
|
|
}
|