import core.exception: RangeError; import std.datetime, std.file, std.json, std.path, std.stdio; import cache, config, onedrive, util; private string statusTokenFile = "status_token"; private bool isItemFolder(const ref JSONValue item) { scope (failure) return false; JSONValue folder = item["folder"]; return true; } private bool isItemFile(const ref JSONValue item) { scope (failure) return false; JSONValue folder = item["file"]; return true; } private bool isItemDeleted(const ref JSONValue item) { scope (failure) return false; return !item["deleted"].isNull(); } private bool testCrc32(string path, const(char)[] crc32) { if (crc32) { string localCrc32 = computeCrc32(path); if (crc32 == localCrc32) return true; } return false; } class SyncException: Exception { @nogc @safe pure nothrow this(string msg, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { super(msg, file, line, next); } @nogc @safe pure nothrow this(string msg, Throwable next, string file = __FILE__, size_t line = __LINE__) { super(msg, file, line, next); } } final class SyncEngine { Config cfg; OneDriveApi onedrive; ItemCache itemCache; string[] itemToDelete; // array of items to be deleted JSONValue folderItem; this(Config cfg, OneDriveApi onedrive) { assert(onedrive); this.cfg = cfg; this.onedrive = onedrive; itemCache.init(); } void applyDifferences() { string statusToken; try { statusToken = readText(statusTokenFile); } catch (FileException e) { writeln("Welcome !"); } writeln("Applying differences ..."); string currDir = getcwd(); string syncDir = cfg.get("sync_dir"); JSONValue changes; do { chdir(syncDir); changes = onedrive.viewChangesByPath("test", statusToken); foreach (item; changes["value"].array) { applyDifference(item); } statusToken = changes["@changes.token"].str; chdir(currDir); std.file.write(statusTokenFile, statusToken); } while (changes["@changes.hasMoreChanges"].type == JSON_TYPE.TRUE); chdir(syncDir); deleteFiles(); chdir(currDir); } private void applyDifference(JSONValue item) { string id = item["id"].str; string name = item["name"].str; string eTag = item["eTag"].str; Item cachedItem; bool cached = itemCache.selectById(id, cachedItem); // skip items already downloaded //if (cached && cachedItem.eTag == eTag) return; writeln("Item ", id, " ", name); ItemType type; if (isItemDeleted(item)) { writeln("The item is marked for deletion"); if (cached) applyDelete(cachedItem); return; } else if (isItemFile(item)) { type = ItemType.file; writeln("The item is a file"); } else if (isItemFolder(item)) { type = ItemType.dir; writeln("The item is a directory"); } else { writeln("The item is neither a file nor a directory, skipping"); //skippedFolders ~= id; return; } string cTag = item["cTag"].str; string mtime = item["fileSystemInfo"].object["lastModifiedDateTime"].str; string parentId = item["parentReference"].object["id"].str; string crc32; if (type == ItemType.file) { try { crc32 = item["file"].object["hashes"].object["crc32Hash"].str; } catch (JSONException e) { writeln("The hash is not available"); } catch (RangeError e) { writeln("The crc32 hash is not available"); } } Item newItem; itemCache.insert(id, name, type, eTag, cTag, mtime, parentId, crc32); itemCache.selectById(id, newItem); writeln("Path: ", newItem.path); try { if (!cached) { applyNewItem(newItem); } else { applyChangedItem(cachedItem, newItem); } } catch (SyncException e) { itemCache.deleteById(id); throw e; } } private void cacheItem(JSONValue item) { string id = item["id"].str; ItemType type; if (isItemDeleted(item)) { itemCache.deleteById(id); } else if (isItemFile(item)) { type = ItemType.file; } else if (isItemFolder(item)) { type = ItemType.dir; } else { writeln("The item is neither a file nor a directory, skipping"); return; } string name = item["name"].str; string eTag = item["eTag"].str; string cTag = item["cTag"].str; string mtime = item["fileSystemInfo"].object["lastModifiedDateTime"].str; string parentId = item["parentReference"].object["id"].str; string crc32; if (type == ItemType.file) { try { crc32 = item["file"].object["hashes"].object["crc32Hash"].str; } catch (JSONException e) { writeln("The hash is not available"); } catch (RangeError e) { writeln("The crc32 hash is not available"); } } itemCache.insert(id, name, type, eTag, cTag, mtime, parentId, crc32); } private void applyDelete(Item item) { if (exists(item.path)) { if (isItemSynced(item)) { addFileToDelete(item.path); } else { writeln("The local item is not synced, renaming ..."); safeRename(item.path); } } else { writeln("The local item is already deleted"); } itemCache.deleteById(item.id); } private void applyNewItem(Item item) { assert(item.id); if (exists(item.path)) { if (isItemSynced(item)) { writeln("The item is already present"); // ensure the modified time is synced setTimes(item.path, item.mtime, item.mtime); return; } else { writeln("The item is not synced, renaming ..."); safeRename(item.path); } } final switch (item.type) { case ItemType.file: writeln("Downloading ..."); try { onedrive.downloadById(item.id, item.path); } catch (OneDriveException e) { throw new SyncException("Sync error", e); } break; case ItemType.dir: writeln("Creating local directory..."); mkdir(item.path); break; } setTimes(item.path, item.mtime, item.mtime); } private void applyChangedItem(Item oldItem, Item newItem) { assert(oldItem.id == newItem.id); if (exists(oldItem.path)) { if (isItemSynced(oldItem)) { if (oldItem.eTag != newItem.eTag) { assert(oldItem.type == newItem.type); if (oldItem.path != newItem.path) { writeln("Moved item ", oldItem.path, " to ", newItem.path); if (exists(newItem.path)) { writeln("The destination is occupied, renaming ..."); safeRename(newItem.path); } rename(oldItem.path, newItem.path); } if (oldItem.type == ItemType.file && oldItem.cTag != newItem.cTag) { writeln("Downloading ..."); onedrive.downloadById(oldItem.id, oldItem.path); } setTimes(newItem.path, newItem.mtime, newItem.mtime); writeln("Updated last modified time"); } else { writeln("The item is not changed"); } } else { writeln("The item is not synced, renaming ..."); safeRename(oldItem.path); applyNewItem(newItem); } } else { applyNewItem(newItem); } } // returns true if the given item corresponds to the local one private bool isItemSynced(Item item) { final switch (item.type) { case ItemType.file: if (isFile(item.path)) { SysTime localModifiedTime = timeLastModified(item.path); import core.time: Duration; item.mtime.fracSecs = Duration.zero; // HACK if (localModifiedTime == item.mtime) return true; else { writeln("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime); } if (item.crc32) { string localCrc32 = computeCrc32(item.path); if (localCrc32 == item.crc32) return true; else { writeln("The local item has a different hash"); } } } else { writeln("The local item is a directory but should be a file"); } break; case ItemType.dir: if (isDir(item.path)) return true; else { writeln("The local item is a file but should be a directory"); } break; } return false; } private void addFileToDelete(string path) { itemToDelete ~= path; } private void deleteFiles() { writeln("Deleting marked files ..."); foreach_reverse (ref path; itemToDelete) { if (isFile(path)) { remove(path); } else { try { rmdir(path); } catch (FileException e) { writeln("Keeping dir \"", path, "\" not empty"); } } } itemToDelete.length = 0; assumeSafeAppend(itemToDelete); } // scan the directory for unsynced files and upload them public void uploadDifferences() { writeln("Uploading differences ..."); string currDir = getcwd(); string syncDir = cfg.get("sync_dir"); chdir(syncDir); foreach (Item item; itemCache.selectAll()) { uploadDifference(item); } foreach (DirEntry entry; dirEntries("test", SpanMode.breadth, false)) { uploadDifference(entry.name/*[2 .. $]*/); } chdir(currDir); } public void uploadDifferences(string path) { assert(isDir(path)); Item item; foreach (DirEntry entry; dirEntries(path, SpanMode.breadth, false)) { if (itemCache.selectByPath(entry.name, item)) { uploadDifference(item); } else { uploadNewItem(entry.name); } } } private void uploadDifference(Item item) { writeln(item.path); if (exists(item.path)) { final switch (item.type) { case ItemType.file: if (isFile(item.path)) { updateItem(item); } else { deleteItem(item); createFolderItem(item.path); } break; case ItemType.dir: if (isDir(item.path)) { updateItem(item); } else { deleteItem(item); writeln("Uploading ..."); auto res = onedrive.simpleUpload(item.path, item.path); cacheItem(res); } break; } } else { deleteItem(item); } } private void uploadDifference(const(char)[] path) { Item item; if (!itemCache.selectByPath(path, item)) { writeln("New item ", path); uploadNewItem(path); } } // HACK void uploadDifference2(const(char)[] path) { assert(isFile(path)); Item item; if (itemCache.selectByPath(path, item)) { uploadDifference(item); } else { uploadNewItem(path); } } private void deleteItem(Item item) { writeln("Deleting ..."); onedrive.deleteById(item.id, item.eTag); itemCache.deleteById(item.id); } private void updateItem(Item item) { SysTime localModifiedTime = timeLastModified(item.path); import core.time: Duration; item.mtime.fracSecs = Duration.zero; // HACK if (localModifiedTime != item.mtime) { string id = item.id; string eTag = item.eTag; if (item.type == ItemType.file && !testCrc32(item.path, item.crc32)) { assert(isFile(item.path)); writeln("Uploading ..."); JSONValue res = onedrive.simpleUpload(item.path, item.path, item.eTag); cacheItem(res); id = res["id"].str; eTag = res["eTag"].str; } updateItemLastModifiedTime(id, eTag, localModifiedTime.toUTC()); } else { writeln("The item is not changed"); } } void createFolderItem(const(char)[] path) { writeln("Creating folder ..."); JSONValue item = ["name": baseName(path).dup]; item["folder"] = parseJSON("{}"); auto res = onedrive.createByPath(dirName(path), item); cacheItem(res); } private void updateItemLastModifiedTime(const(char)[] id, const(char)[] eTag, SysTime mtime) { writeln("Updating last modified time ..."); JSONValue mtimeJson = [ "fileSystemInfo": JSONValue([ "lastModifiedDateTime": mtime.toISOExtString() ]) ]; auto res = onedrive.updateById(id, mtimeJson, eTag); cacheItem(res); } private void uploadNewItem(const(char)[] path) { assert(exists(path)); if (isFile(path)) { writeln("Uploading file ..."); JSONValue res = onedrive.simpleUpload(path.dup, path); cacheItem(res); string id = res["id"].str; string eTag = res["eTag"].str; updateItemLastModifiedTime(id, eTag, timeLastModified(path).toUTC()); } else { createFolderItem(path); } } void moveItem(const(char)[] from, string to) { writeln("Moving ", from, " to ", to, " ..."); Item item; if (!itemCache.selectByPath(from, item)) { throw new SyncException("Can't move a non synced item"); } JSONValue diff = ["name": baseName(to)]; diff["parentReference"] = JSONValue([ "path": "/drive/root:/" ~ dirName(to) ]); writeln(diff.toPrettyString()); auto res = onedrive.updateById(item.id, diff, item.eTag); cacheItem(res); } void deleteByPath(const(char)[] path) { writeln("Deleting: ", path); Item item; if (!itemCache.selectByPath(path, item)) { throw new SyncException("Can't delete a non synced item"); } deleteItem(item); } }