refactoring

This commit is contained in:
skilion 2015-09-14 23:56:14 +02:00
parent ff07f13cd6
commit 2d4cf32be5
4 changed files with 347 additions and 120 deletions

View file

@ -5,8 +5,8 @@ SOURCES = \
/usr/include/dlang/dmd/core/sys/posix/poll.d \
/usr/include/dlang/dmd/etc/c/curl.d \
/usr/include/dlang/dmd/std/net/curl.d \
src/cache.d \
src/config.d \
src/itemdb.d \
src/main.d \
src/monitor.d \
src/onedrive.d \

242
src/itemdb.d Normal file
View file

@ -0,0 +1,242 @@
import std.datetime, std.path;
import sqlite;
enum ItemType
{
file,
dir
}
struct Item
{
string id;
string path;
string name;
ItemType type;
string eTag;
string cTag;
SysTime mtime;
string parentId;
string crc32;
}
final class ItemDatabase
{
Database db;
Statement insertItemStmt;
Statement selectItemByIdStmt;
Statement selectItemByParentIdStmt;
this(const(char)[] filename)
{
db = Database(filename);
db.exec("CREATE TABLE IF NOT EXISTS item (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
type TEXT NOT NULL,
eTag TEXT NOT NULL,
cTag TEXT NOT NULL,
mtime TEXT NOT NULL,
parentId TEXT NOT NULL,
crc32 TEXT
)");
db.exec("CREATE INDEX IF NOT EXISTS name_idx ON item (name)");
insertItemStmt = db.prepare("INSERT OR REPLACE INTO item (id, name, type, eTag, cTag, mtime, parentId, crc32) VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
selectItemByIdStmt = db.prepare("SELECT id, name, type, eTag, cTag, mtime, parentId, crc32 FROM item WHERE id = ?");
selectItemByParentIdStmt = db.prepare("SELECT id FROM item WHERE parentId = ?");
}
void insert(const(char)[] id, const(char)[] name, ItemType type, const(char)[] eTag, const(char)[] cTag, const(char)[] mtime, const(char)[] parentId, const(char)[] crc32)
{
with (insertItemStmt) {
bind(1, id);
bind(2, name);
string typeStr = void;
final switch (type) {
case ItemType.file: typeStr = "file"; break;
case ItemType.dir: typeStr = "dir"; break;
}
bind(3, typeStr);
bind(4, eTag);
bind(5, cTag);
bind(6, mtime);
bind(7, parentId);
bind(8, crc32);
exec();
}
}
// returns a range that go trough all items, depth first
auto selectAll()
{
static struct ItemRange
{
ItemDatabase itemdb;
string[] stack1, stack2;
private this(ItemDatabase itemdb, string rootId)
{
this.itemdb = itemdb;
stack1.reserve(8);
stack2.reserve(8);
stack1 ~= rootId;
getChildren();
}
@property bool empty()
{
return stack2.length == 0;
}
@property Item front()
{
Item item;
bool res = itemdb.selectById(stack2[$ - 1], item);
assert(res);
return item;
}
void popFront()
{
stack2 = stack2[0 .. $ - 1];
assumeSafeAppend(stack2);
if (stack1.length > 0) getChildren();
}
private void getChildren()
{
while (true) {
itemdb.selectItemByParentIdStmt.bind(1, stack1[$ - 1]);
stack2 ~= stack1[$ - 1];
stack1 = stack1[0 .. $ - 1];
assumeSafeAppend(stack1);
auto res = itemdb.selectItemByParentIdStmt.exec();
if (res.empty) break;
else foreach (row; res) stack1 ~= row[0].dup;
}
}
}
auto s = db.prepare("SELECT a.id FROM item AS a LEFT JOIN item AS b ON a.parentId = b.id WHERE b.id IS NULL");
auto r = s.exec();
assert(!r.empty());
return ItemRange(this, r.front[0].dup);
}
bool selectById(const(char)[] id, out Item item)
{
selectItemByIdStmt.bind(1, id);
auto r = selectItemByIdStmt.exec();
if (!r.empty) {
item = buildItem(r);
return true;
}
return false;
}
bool selectByPath(const(char)[] path, out Item item)
{
string[2][] candidates; // [id, parentId]
auto s = db.prepare("SELECT id, parentId FROM item WHERE name = ?");
s.bind(1, baseName(path));
auto r = s.exec();
foreach (row; r) candidates ~= [row[0].dup, row[1].dup];
if (candidates.length > 1) {
s = db.prepare("SELECT parentId FROM item WHERE id = ? AND name = ?");
do {
string[2][] newCandidates;
newCandidates.reserve(candidates.length);
path = dirName(path);
foreach (candidate; candidates) {
s.bind(1, candidate[1]);
s.bind(2, baseName(path));
r = s.exec();
if (!r.empty) {
string[2] c = [candidate[0], r.front[0].idup];
newCandidates ~= c;
}
}
candidates = newCandidates;
} while (candidates.length > 1);
}
if (candidates.length == 1) return selectById(candidates[0][0], item);
return false;
}
void deleteById(const(char)[] id)
{
auto s = db.prepare("DELETE FROM item WHERE id = ?");
s.bind(1, id);
s.exec();
}
// returns true if the item has the specified parent
bool hasParent(T)(const(char)[] itemId, T parentId)
if (is(T : const(char)[]) || is(T : const(char[])[]))
{
auto s = db.prepare("SELECT parentId FROM item WHERE id = ?");
while (true) {
s.bind(1, itemId);
auto r = s.exec();
if (r.empty) break;
auto currParentId = r.front[0];
static if (is(T : const(char)[])) {
if (currParentId == parentId) return true;
} else {
foreach (id; parentId) if (currParentId == id) return true;
}
itemId = currParentId.dup;
}
return false;
}
private Item buildItem(Statement.Result result)
{
assert(!result.empty && result.front.length == 8);
Item item = {
id: result.front[0].dup,
path: computePath(result.front[0]),
name: result.front[1].dup,
eTag: result.front[3].dup,
cTag: result.front[4].dup,
mtime: SysTime.fromISOExtString(result.front[5]),
parentId: result.front[6].dup,
crc32: result.front[7].dup
};
switch (result.front[2]) {
case "file": item.type = ItemType.file; break;
case "dir": item.type = ItemType.dir; break;
default: assert(0);
}
return item;
}
private string computePath(const(char)[] id)
{
auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?");
string path;
while (true) {
s.bind(1, id);
auto r = s.exec();
if (r.empty) break;
if (path) path = r.front[0].idup ~ "/" ~ path;
else path = r.front[0].dup;
id = r.front[1].dup;
}
return path[5 .. $]; // HACK: skip "root/"
}
/*private string computePath(const(char)[] name, const(char)[] parentId)
{
auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?");
string path = name.dup;
while (true) {
s.bind(1, parentId);
auto r = s.exec();
if (r.empty) break;
path = r.front[0].idup ~ "/" ~ path;
parentId = r.front[1].dup;
}
return path;
}*/
}

View file

@ -36,7 +36,7 @@ final class OneDriveApi
this.clientId = cfg.get("client_id");
this.clientSecret = cfg.get("client_secret");
http = HTTP();
http.verbose = verbose;
//http.verbose = verbose;
}
~this()

View file

@ -1,8 +1,6 @@
import core.exception: RangeError;
import std.datetime, std.file, std.json, std.path, std.stdio;
import cache, config, onedrive, util;
private string statusTokenFile = "status_token";
import config, itemdb, onedrive, util;
private bool isItemFolder(const ref JSONValue item)
{
@ -48,47 +46,43 @@ class SyncException: Exception
final class SyncEngine
{
Config cfg;
OneDriveApi onedrive;
ItemCache itemCache;
string[] itemToDelete; // array of items to be deleted
JSONValue folderItem;
private Config cfg;
private OneDriveApi onedrive;
private ItemDatabase itemdb;
private bool verbose;
private string statusToken;
private string[] itemsToDelete;
this(Config cfg, OneDriveApi onedrive)
void delegate(string) onStatusToken;
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, bool verbose)
{
assert(onedrive);
assert(onedrive && itemdb);
this.cfg = cfg;
this.onedrive = onedrive;
itemCache.init();
this.itemdb = itemdb;
this.verbose = verbose;
}
void setStatusToken(string statusToken)
{
this.statusToken = statusToken;
}
void applyDifferences()
{
string statusToken;
try {
statusToken = readText(statusTokenFile);
} catch (FileException e) {
writeln("Welcome !");
}
writeln("Applying differences ...");
string currDir = getcwd();
string syncDir = cfg.get("sync_dir");
if (verbose) writeln("Applying differences ...");
JSONValue changes;
do {
chdir(syncDir);
changes = onedrive.viewChangesByPath("test", statusToken);
changes = onedrive.viewChangesByPath("/", statusToken);
foreach (item; changes["value"].array) {
applyDifference(item);
}
statusToken = changes["@changes.token"].str;
chdir(currDir);
std.file.write(statusTokenFile, statusToken);
onStatusToken(statusToken);
} while (changes["@changes.hasMoreChanges"].type == JSON_TYPE.TRUE);
chdir(syncDir);
deleteFiles();
chdir(currDir);
// delete items in itemsToDelete
deleteItems();
}
private void applyDifference(JSONValue item)
@ -97,27 +91,33 @@ final class SyncEngine
string name = item["name"].str;
string eTag = item["eTag"].str;
if (verbose) writeln(id, " ", name);
Item cachedItem;
bool cached = itemCache.selectById(id, cachedItem);
bool cached = itemdb.selectById(id, cachedItem);
if (cached && !isItemSynced(cachedItem)) {
if (verbose) writeln("The local item is out of sync, renaming ...");
safeRename(cachedItem.path);
cached = false;
}
// skip items already downloaded
//if (cached && cachedItem.eTag == eTag) return;
writeln("Item ", id, " ", name);
ItemType type;
if (isItemDeleted(item)) {
writeln("The item is marked for deletion");
if (cached) applyDelete(cachedItem);
if (verbose) writeln("The item is marked for deletion");
if (cached) applyDeleteItem(cachedItem);
return;
} else if (isItemFile(item)) {
if (verbose) writeln("The item is a file");
type = ItemType.file;
writeln("The item is a file");
} else if (isItemFolder(item)) {
if (verbose) writeln("The item is a directory");
type = ItemType.dir;
writeln("The item is a directory");
} else {
writeln("The item is neither a file nor a directory, skipping");
writeln("The item is neither a file nor a directory");
//skippedFolders ~= id;
return;
}
@ -131,18 +131,17 @@ final class SyncEngine
try {
crc32 = item["file"].object["hashes"].object["crc32Hash"].str;
} catch (JSONException e) {
writeln("The hash is not available");
if (verbose) writeln("The hash is not available");
} catch (RangeError e) {
writeln("The crc32 hash is not available");
if (verbose) writeln("The crc32 hash is not available");
}
}
Item newItem;
itemCache.insert(id, name, type, eTag, cTag, mtime, parentId, crc32);
itemCache.selectById(id, newItem);
writeln("Path: ", newItem.path);
itemdb.insert(id, name, type, eTag, cTag, mtime, parentId, crc32);
itemdb.selectById(id, newItem);
// TODO add item in the db anly if correctly downloaded
try {
if (!cached) {
applyNewItem(newItem);
@ -150,7 +149,7 @@ final class SyncEngine
applyChangedItem(cachedItem, newItem);
}
} catch (SyncException e) {
itemCache.deleteById(id);
itemdb.deleteById(id);
throw e;
}
}
@ -160,7 +159,7 @@ final class SyncEngine
string id = item["id"].str;
ItemType type;
if (isItemDeleted(item)) {
itemCache.deleteById(id);
itemdb.deleteById(id);
} else if (isItemFile(item)) {
type = ItemType.file;
} else if (isItemFolder(item)) {
@ -184,40 +183,32 @@ final class SyncEngine
writeln("The crc32 hash is not available");
}
}
itemCache.insert(id, name, type, eTag, cTag, mtime, parentId, crc32);
itemdb.insert(id, name, type, eTag, cTag, mtime, parentId, crc32);
}
private void applyDelete(Item item)
private void applyDeleteItem(Item item)
{
if (exists(item.path)) {
if (isItemSynced(item)) {
addFileToDelete(item.path);
} else {
writeln("The local item is not synced, renaming ...");
safeRename(item.path);
}
} else {
writeln("The local item is already deleted");
}
itemCache.deleteById(item.id);
itemsToDelete ~= item.path;
itemdb.deleteById(item.id);
}
private void applyNewItem(Item item)
{
assert(item.id);
if (exists(item.path)) {
if (isItemSynced(item)) {
writeln("The item is already present");
// ensure the modified time is synced
if (verbose) writeln("The item is already present");
// ensure the modified time is correct
setTimes(item.path, item.mtime, item.mtime);
return;
} else {
writeln("The item is not synced, renaming ...");
if (verbose) writeln("The local item is out of sync, renaming ...");
safeRename(item.path);
}
}
final switch (item.type) {
case ItemType.file:
writeln("Downloading ...");
writeln("Downloading: ", item.path);
try {
onedrive.downloadById(item.id, item.path);
} catch (OneDriveException e) {
@ -225,7 +216,7 @@ final class SyncEngine
}
break;
case ItemType.dir:
writeln("Creating local directory...");
writeln("Creating directory: ", item.path);
mkdir(item.path);
break;
}
@ -235,92 +226,86 @@ final class SyncEngine
private void applyChangedItem(Item oldItem, Item newItem)
{
assert(oldItem.id == newItem.id);
if (exists(oldItem.path)) {
if (isItemSynced(oldItem)) {
if (oldItem.eTag != newItem.eTag) {
assert(oldItem.type == newItem.type);
if (oldItem.path != newItem.path) {
writeln("Moved item ", oldItem.path, " to ", newItem.path);
if (exists(newItem.path)) {
writeln("The destination is occupied, renaming ...");
safeRename(newItem.path);
}
rename(oldItem.path, newItem.path);
}
if (oldItem.type == ItemType.file && oldItem.cTag != newItem.cTag) {
writeln("Downloading ...");
onedrive.downloadById(oldItem.id, oldItem.path);
}
setTimes(newItem.path, newItem.mtime, newItem.mtime);
writeln("Updated last modified time");
} else {
writeln("The item is not changed");
assert(oldItem.type == newItem.type);
assert(exists(oldItem.path));
if (oldItem.eTag != newItem.eTag) {
if (oldItem.path != newItem.path) {
writeln("Moving: ", oldItem.path, " -> ", newItem.path);
if (exists(newItem.path)) {
if (verbose) writeln("The destination is occupied, renaming ...");
safeRename(newItem.path);
}
} else {
writeln("The item is not synced, renaming ...");
safeRename(oldItem.path);
applyNewItem(newItem);
rename(oldItem.path, newItem.path);
}
if (newItem.type == ItemType.file && oldItem.cTag != newItem.cTag) {
writeln("Downloading: ", newItem.path);
onedrive.downloadById(newItem.id, newItem.path);
}
setTimes(newItem.path, newItem.mtime, newItem.mtime);
} else {
applyNewItem(newItem);
if (verbose) writeln("The item is not changed");
}
}
// returns true if the given item corresponds to the local one
private bool isItemSynced(Item item)
{
if (!exists(item.path)) return false;
final switch (item.type) {
case ItemType.file:
if (isFile(item.path)) {
SysTime localModifiedTime = timeLastModified(item.path);
import core.time: Duration;
item.mtime.fracSecs = Duration.zero; // HACK
if (localModifiedTime == item.mtime) return true;
else {
writeln("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime);
if (localModifiedTime == item.mtime) {
return true;
} else {
if (verbose) writeln("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime);
}
if (item.crc32) {
string localCrc32 = computeCrc32(item.path);
if (localCrc32 == item.crc32) return true;
else {
writeln("The local item has a different hash");
if (localCrc32 == item.crc32) {
return true;
} else {
if (verbose) writeln("The local item has a different hash");
}
}
} else {
writeln("The local item is a directory but should be a file");
if (verbose) writeln("The local item is a directory but should be a file");
}
break;
case ItemType.dir:
if (isDir(item.path)) return true;
else {
writeln("The local item is a file but should be a directory");
if (isDir(item.path)) {
return true;
} else {
if (verbose) writeln("The local item is a file but should be a directory");
}
break;
}
return false;
}
private void addFileToDelete(string path)
private void deleteItems()
{
itemToDelete ~= path;
}
private void deleteFiles()
{
writeln("Deleting marked files ...");
foreach_reverse (ref path; itemToDelete) {
if (isFile(path)) {
remove(path);
if (verbose) writeln("Deleting files ...");
foreach_reverse (path; itemsToDelete) {
if (exists(path)) {
if (isFile(path)) {
remove(path);
writeln("Deleted file: ", path);
}
} else {
try {
rmdir(path);
writeln("Deleted dir: ", path);
} catch (FileException e) {
writeln("Keeping dir \"", path, "\" not empty");
writeln("Keeping dir: ", path);
}
}
}
itemToDelete.length = 0;
assumeSafeAppend(itemToDelete);
itemsToDelete.length = 0;
assumeSafeAppend(itemsToDelete);
}
// scan the directory for unsynced files and upload them
@ -330,7 +315,7 @@ final class SyncEngine
string currDir = getcwd();
string syncDir = cfg.get("sync_dir");
chdir(syncDir);
foreach (Item item; itemCache.selectAll()) {
foreach (Item item; itemdb.selectAll()) {
uploadDifference(item);
}
foreach (DirEntry entry; dirEntries("test", SpanMode.breadth, false)) {
@ -344,7 +329,7 @@ final class SyncEngine
assert(isDir(path));
Item item;
foreach (DirEntry entry; dirEntries(path, SpanMode.breadth, false)) {
if (itemCache.selectByPath(entry.name, item)) {
if (itemdb.selectByPath(entry.name, item)) {
uploadDifference(item);
} else {
uploadNewItem(entry.name);
@ -384,7 +369,7 @@ final class SyncEngine
private void uploadDifference(const(char)[] path)
{
Item item;
if (!itemCache.selectByPath(path, item)) {
if (!itemdb.selectByPath(path, item)) {
writeln("New item ", path);
uploadNewItem(path);
}
@ -395,7 +380,7 @@ final class SyncEngine
{
assert(isFile(path));
Item item;
if (itemCache.selectByPath(path, item)) {
if (itemdb.selectByPath(path, item)) {
uploadDifference(item);
} else {
uploadNewItem(path);
@ -406,7 +391,7 @@ final class SyncEngine
{
writeln("Deleting ...");
onedrive.deleteById(item.id, item.eTag);
itemCache.deleteById(item.id);
itemdb.deleteById(item.id);
}
private void updateItem(Item item)
@ -471,7 +456,7 @@ final class SyncEngine
{
writeln("Moving ", from, " to ", to, " ...");
Item item;
if (!itemCache.selectByPath(from, item)) {
if (!itemdb.selectByPath(from, item)) {
throw new SyncException("Can't move a non synced item");
}
JSONValue diff = ["name": baseName(to)];
@ -487,7 +472,7 @@ final class SyncEngine
{
writeln("Deleting: ", path);
Item item;
if (!itemCache.selectByPath(path, item)) {
if (!itemdb.selectByPath(path, item)) {
throw new SyncException("Can't delete a non synced item");
}
deleteItem(item);