deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-06-19 06:05:18 +02:00
Code Issues Projects Releases Wiki Activity

removed path dependency in Item

Browse source
This commit is contained in:
skilion 2015-09-19 15:38:43 +02:00
parent 8d00ad5672
commit 2d50e43674
3 changed files with 125 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
src/itemdb.d
View file

@ -10,7 +10,6 @@ enum ItemType
struct Item struct Item
{ {
string id; string id;
string path;
string name; string name;
ItemType type; ItemType type;
string eTag; string eTag;
@ -218,7 +217,6 @@ final class ItemDatabase
assert(!result.empty && result.front.length == 8); assert(!result.empty && result.front.length == 8);
Item item = { Item item = {
id: result.front[0].dup, id: result.front[0].dup,
path: computePath(result.front[0]),
name: result.front[1].dup, name: result.front[1].dup,
eTag: result.front[3].dup, eTag: result.front[3].dup,
cTag: result.front[4].dup, cTag: result.front[4].dup,
@ -234,10 +232,11 @@ final class ItemDatabase
return item; return item;
} }
private string computePath(const(char)[] id) string computePath(const(char)[] id)
{ {
auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?"); if (!id) return null;
string path; string path;
auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?");
while (true) { while (true) {
s.bind(1, id); s.bind(1, id);
auto r = s.exec(); auto r = s.exec();
@ -250,18 +249,4 @@ final class ItemDatabase
if (path.length < 5) return "."; if (path.length < 5) return ".";
return path[5 .. $]; return path[5 .. $];
} }
/*private string computePath(const(char)[] name, const(char)[] parentId)
{
auto s = db.prepare("SELECT name, parentId FROM item WHERE id = ?");
string path = name.dup;
while (true) {
s.bind(1, parentId);
auto r = s.exec();
if (r.empty) break;
path = r.front[0].idup ~ "/" ~ path;
parentId = r.front[1].dup;
}
return path;
}*/
} }

5
src/main.d
View file

@ -8,15 +8,14 @@ void main(string[] args)
{ {
bool monitor, resync, verbose; bool monitor, resync, verbose;
try { try {
writeln("OneDrive Client for Linux v", ver);
auto opt = getopt( auto opt = getopt(
args, args,
"monitor|m", "Keep monitoring for local and remote changes.", &monitor, "monitor|m", "Keep monitoring for local and remote changes.", &monitor,
"resync", "Perform a full synchronization.", &resync, "resync", "Forget the local state and perform a full synchronization.", &resync,
"verbose|v", "Print more details, useful for debugging.", &verbose "verbose|v", "Print more details, useful for debugging.", &verbose
); );
if (opt.helpWanted) { if (opt.helpWanted) {
defaultGetoptPrinter("Available options:", opt.options); defaultGetoptPrinter("OneDrive Free Client for Linux v" ~ ver ~ "\nAvailable options:", opt.options);
return; return;
} }
} catch (GetOptException e) { } catch (GetOptException e) {

231
src/sync.d
View file

@ -56,7 +56,7 @@ final class SyncEngine
// list of items to skip while applying the changes downloaded // list of items to skip while applying the changes downloaded
private string[] skippedItems; private string[] skippedItems;
// list of items to delete after the changes has been downloaded // list of items to delete after the changes has been downloaded
private string[] itemsToDelete; private string[] pathsToDelete;
void delegate(string) onStatusToken; void delegate(string) onStatusToken;
@ -88,8 +88,8 @@ final class SyncEngine
statusToken = changes["@changes.token"].str; statusToken = changes["@changes.token"].str;
onStatusToken(statusToken); onStatusToken(statusToken);
} while (changes["@changes.hasMoreChanges"].type == JSON_TYPE.TRUE); } while (changes["@changes.hasMoreChanges"].type == JSON_TYPE.TRUE);
// delete items in itemsToDelete // delete items in pathsToDelete
if (itemsToDelete.length > 0) deleteItems(); if (pathsToDelete.length > 0) deleteItems();
// empty the skipped items // empty the skipped items
skippedItems.length = 0; skippedItems.length = 0;
assumeSafeAppend(skippedItems); assumeSafeAppend(skippedItems);
@ -99,34 +99,59 @@ final class SyncEngine
{ {
string id = item["id"].str; string id = item["id"].str;
string name = item["name"].str; string name = item["name"].str;
string eTag = item["eTag"].str; string parentId = item["parentReference"].object["id"].str;
// HACK: recognize the root directory
if (name == "root" && parentId[$ - 1] == '0' && parentId[$ - 2] == '!') {
parentId = null;
}
// skip unwanted items early
if (skippedItems.find(parentId).length != 0) {
skippedItems ~= id;
return;
}
if (verbose) writeln(id, " ", name); if (verbose) writeln(id, " ", name);
Item cachedItem; // check if the cached item is still synced
bool cached = itemdb.selectById(id, cachedItem); Item oldItem;
string oldPath;
bool cached = itemdb.selectById(id, oldItem);
if (cached) {
oldPath = itemdb.computePath(id);
if (!isItemSynced(oldItem, oldPath)) {
if (verbose) writeln("The local item is out of sync, renaming");
if (exists(oldPath)) safeRename(oldPath);
cached = false;
}
}
if (cached && !isItemSynced(cachedItem)) { // compute the path of the item
if (verbose) writeln("The local item is out of sync, renaming: ", cachedItem.path); string path;
if (exists(cachedItem.path)) safeRename(cachedItem.path); if (parentId) {
cached = false; path = itemdb.computePath(parentId) ~ "/" ~ name;
} else {
path = name;
} }
ItemType type; ItemType type;
if (isItemDeleted(item)) { if (isItemDeleted(item)) {
if (verbose) writeln("The item is marked for deletion"); if (verbose) writeln("The item is marked for deletion");
if (cached) applyDeleteItem(cachedItem); if (cached) {
itemdb.deleteById(id);
pathsToDelete ~= oldPath;
}
return; return;
} else if (isItemFile(item)) { } else if (isItemFile(item)) {
type = ItemType.file; type = ItemType.file;
if (!matchFirst(name, skipFile).empty) { if (!path.matchFirst(skipFile).empty) {
if (verbose) writeln("Filtered out"); if (verbose) writeln("Filtered out");
skippedItems ~= id;
return; return;
} }
} else if (isItemFolder(item)) { } else if (isItemFolder(item)) {
type = ItemType.dir; type = ItemType.dir;
if (!matchFirst(name, skipDir).empty) { if (!path.matchFirst(skipDir).empty) {
if (verbose) writeln("Filtered out"); if (verbose) writeln("Filtered out");
skippedItems ~= id; skippedItems ~= id;
return; return;
@ -137,17 +162,7 @@ final class SyncEngine
return; return;
} }
string parentId = item["parentReference"].object["id"].str; string eTag = item["eTag"].str;
if (name == "root" && parentId[$ - 1] == '0' && parentId[$ - 2] == '!') {
// HACK: recognize the root directory
parentId = null;
}
if (skippedItems.find(parentId).length != 0) {
if (verbose) writeln("The item is a children of a skipped item");
skippedItems ~= id;
return;
}
string cTag = item["cTag"].str; string cTag = item["cTag"].str;
string mtime = item["fileSystemInfo"].object["lastModifiedDateTime"].str; string mtime = item["fileSystemInfo"].object["lastModifiedDateTime"].str;
@ -162,98 +177,94 @@ final class SyncEngine
} }
} }
if (cached) { Item newItem = {
id: id,
name: name,
type: type,
eTag: eTag,
cTag: cTag,
mtime: SysTime.fromISOExtString(mtime),
parentId: parentId,
crc32: crc32
};
if (!cached) {
applyNewItem(newItem, path);
} else {
applyChangedItem(oldItem, newItem, path);
}
// save the item in the db
if (oldItem.id) {
itemdb.update(id, name, type, eTag, cTag, mtime, parentId, crc32); itemdb.update(id, name, type, eTag, cTag, mtime, parentId, crc32);
} else { } else {
itemdb.insert(id, name, type, eTag, cTag, mtime, parentId, crc32); itemdb.insert(id, name, type, eTag, cTag, mtime, parentId, crc32);
} }
Item newItem;
bool found = itemdb.selectById(id, newItem);
assert(found);
// TODO add item in the db only if correctly downloaded
try {
if (!cached) {
applyNewItem(newItem);
} else {
applyChangedItem(cachedItem, newItem);
}
} catch (SyncException e) {
itemdb.deleteById(id);
throw e;
}
} }
private void applyDeleteItem(Item item) private void applyNewItem(Item item, string path)
{ {
itemsToDelete ~= item.path; if (exists(path)) {
itemdb.deleteById(item.id); if (isItemSynced(item, path)) {
}
private void applyNewItem(Item item)
{
assert(item.id);
if (exists(item.path)) {
if (isItemSynced(item)) {
if (verbose) writeln("The item is already present"); if (verbose) writeln("The item is already present");
// ensure the modified time is correct // ensure the modified time is correct
setTimes(item.path, item.mtime, item.mtime); setTimes(path, item.mtime, item.mtime);
return; return;
} else { } else {
if (verbose) writeln("The local item is out of sync, renaming ..."); if (verbose) writeln("The local item is out of sync, renaming ...");
safeRename(item.path); safeRename(path);
} }
} }
final switch (item.type) { final switch (item.type) {
case ItemType.file: case ItemType.file:
writeln("Downloading: ", item.path); writeln("Downloading: ", path);
try { try {
onedrive.downloadById(item.id, item.path); onedrive.downloadById(item.id, path);
} catch (OneDriveException e) { } catch (OneDriveException e) {
throw new SyncException("Sync error", e); throw new SyncException("Sync error", e);
} }
break; break;
case ItemType.dir: case ItemType.dir:
writeln("Creating directory: ", item.path); writeln("Creating directory: ", path);
mkdir(item.path); mkdir(path);
break; break;
} }
setTimes(item.path, item.mtime, item.mtime); setTimes(path, item.mtime, item.mtime);
} }
private void applyChangedItem(Item oldItem, Item newItem) private void applyChangedItem(Item oldItem, Item newItem, string newPath)
{ {
assert(oldItem.id == newItem.id); assert(oldItem.id == newItem.id);
assert(oldItem.type == newItem.type); assert(oldItem.type == newItem.type);
assert(exists(oldItem.path));
if (oldItem.eTag != newItem.eTag) { if (oldItem.eTag != newItem.eTag) {
if (oldItem.path != newItem.path) { string oldPath = itemdb.computePath(oldItem.id);
writeln("Moving: ", oldItem.path, " -> ", newItem.path); if (oldPath != newPath) {
if (exists(newItem.path)) { writeln("Moving: ", oldPath, " -> ", newPath);
if (exists(newPath)) {
if (verbose) writeln("The destination is occupied, renaming ..."); if (verbose) writeln("The destination is occupied, renaming ...");
safeRename(newItem.path); safeRename(newPath);
} }
rename(oldItem.path, newItem.path); rename(oldPath, newPath);
} }
if (newItem.type == ItemType.file && oldItem.cTag != newItem.cTag) { if (newItem.type == ItemType.file && oldItem.cTag != newItem.cTag) {
writeln("Downloading: ", newItem.path); writeln("Downloading: ", newPath);
onedrive.downloadById(newItem.id, newItem.path); onedrive.downloadById(newItem.id, newPath);
} }
setTimes(newItem.path, newItem.mtime, newItem.mtime); setTimes(newPath, newItem.mtime, newItem.mtime);
} else { } else {
if (verbose) writeln("The item has not changed"); if (verbose) writeln("The item has not changed");
} }
} }
// returns true if the given item corresponds to the local one // returns true if the given item corresponds to the local one
private bool isItemSynced(Item item) private bool isItemSynced(Item item, string path)
{ {
if (!exists(item.path)) return false; if (!exists(path)) return false;
final switch (item.type) { final switch (item.type) {
case ItemType.file: case ItemType.file:
if (isFile(item.path)) { if (isFile(path)) {
SysTime localModifiedTime = timeLastModified(item.path); SysTime localModifiedTime = timeLastModified(path);
import core.time: Duration; import core.time: Duration;
item.mtime.fracSecs = Duration.zero; // HACK item.mtime.fracSecs = Duration.zero; // HACK
if (localModifiedTime == item.mtime) { if (localModifiedTime == item.mtime) {
@ -261,20 +272,17 @@ final class SyncEngine
} else { } else {
if (verbose) writeln("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime); if (verbose) writeln("The local item has a different modified time ", localModifiedTime, " remote is ", item.mtime);
} }
if (item.crc32) { if (testCrc32(path, item.crc32)) {
string localCrc32 = computeCrc32(item.path); return true;
if (localCrc32 == item.crc32) { } else {
return true; if (verbose) writeln("The local item has a different hash");
} else {
if (verbose) writeln("The local item has a different hash");
}
} }
} else { } else {
if (verbose) writeln("The local item is a directory but should be a file"); if (verbose) writeln("The local item is a directory but should be a file");
} }
break; break;
case ItemType.dir: case ItemType.dir:
if (isDir(item.path)) { if (isDir(path)) {
return true; return true;
} else { } else {
if (verbose) writeln("The local item is a file but should be a directory"); if (verbose) writeln("The local item is a file but should be a directory");
@ -287,7 +295,7 @@ final class SyncEngine
private void deleteItems() private void deleteItems()
{ {
if (verbose) writeln("Deleting files ..."); if (verbose) writeln("Deleting files ...");
foreach_reverse (path; itemsToDelete) { foreach_reverse (path; pathsToDelete) {
if (exists(path)) { if (exists(path)) {
if (isFile(path)) { if (isFile(path)) {
remove(path); remove(path);
@ -302,8 +310,8 @@ final class SyncEngine
} }
} }
} }
itemsToDelete.length = 0; pathsToDelete.length = 0;
assumeSafeAppend(itemsToDelete); assumeSafeAppend(pathsToDelete);
} }
// scan the given directory for differences // scan the given directory for differences
@ -321,32 +329,33 @@ final class SyncEngine
public void uploadDifferences(Item item) public void uploadDifferences(Item item)
{ {
if (verbose) writeln(item.id, " ", item.name); if (verbose) writeln(item.id, " ", item.name);
string path = itemdb.computePath(item.id);
final switch (item.type) { final switch (item.type) {
case ItemType.dir: case ItemType.dir:
if (!matchFirst(item.name, skipDir).empty) { if (!path.matchFirst(skipDir).empty) {
if (verbose) writeln("Filtered out"); if (verbose) writeln("Filtered out");
break; break;
} }
uploadDirDifferences(item); uploadDirDifferences(item, path);
break; break;
case ItemType.file: case ItemType.file:
if (!matchFirst(item.name, skipFile).empty) { if (!path.matchFirst(skipFile).empty) {
if (verbose) writeln("Filtered out"); if (verbose) writeln("Filtered out");
break; break;
} }
uploadFileDifferences(item); uploadFileDifferences(item, path);
break; break;
} }
} }
private void uploadDirDifferences(Item item) private void uploadDirDifferences(Item item, string path)
{ {
assert(item.type == ItemType.dir); assert(item.type == ItemType.dir);
if (exists(item.path)) { if (exists(path)) {
if (!isDir(item.path)) { if (!isDir(path)) {
if (verbose) writeln("The item was a directory but now is a file"); if (verbose) writeln("The item was a directory but now is a file");
uploadDeleteItem(item); uploadDeleteItem(item, path);
uploadNewFile(item.path); uploadNewFile(path);
} else { } else {
if (verbose) writeln("The directory has not changed"); if (verbose) writeln("The directory has not changed");
// loop trough the children // loop trough the children
@ -356,26 +365,26 @@ final class SyncEngine
} }
} else { } else {
if (verbose) writeln("The directory has been deleted"); if (verbose) writeln("The directory has been deleted");
uploadDeleteItem(item); uploadDeleteItem(item, path);
} }
} }
private void uploadFileDifferences(Item item) private void uploadFileDifferences(Item item, string path)
{ {
assert(item.type == ItemType.file); assert(item.type == ItemType.file);
if (exists(item.path)) { if (exists(path)) {
if (isFile(item.path)) { if (isFile(path)) {
SysTime localModifiedTime = timeLastModified(item.path); SysTime localModifiedTime = timeLastModified(path);
import core.time: Duration; import core.time: Duration;
item.mtime.fracSecs = Duration.zero; // HACK item.mtime.fracSecs = Duration.zero; // HACK
if (localModifiedTime != item.mtime) { if (localModifiedTime != item.mtime) {
if (verbose) writeln("The file last modified time has changed"); if (verbose) writeln("The file last modified time has changed");
string id = item.id; string id = item.id;
string eTag = item.eTag; string eTag = item.eTag;
if (!testCrc32(item.path, item.crc32)) { if (!testCrc32(path, item.crc32)) {
if (verbose) writeln("The file content has changed"); if (verbose) writeln("The file content has changed");
writeln("Uploading: ", item.path); writeln("Uploading: ", path);
auto res = onedrive.simpleUpload(item.path, item.path, item.eTag); auto res = onedrive.simpleUpload(path, path, item.eTag);
saveItem(res); saveItem(res);
id = res["id"].str; id = res["id"].str;
eTag = res["eTag"].str; eTag = res["eTag"].str;
@ -386,19 +395,19 @@ final class SyncEngine
} }
} else { } else {
if (verbose) writeln("The item was a file but now is a directory"); if (verbose) writeln("The item was a file but now is a directory");
uploadDeleteItem(item); uploadDeleteItem(item, path);
uploadCreateDir(item.path); uploadCreateDir(path);
} }
} else { } else {
if (verbose) writeln("The file has been deleted"); if (verbose) writeln("The file has been deleted");
uploadDeleteItem(item); uploadDeleteItem(item, path);
} }
} }
private void uploadNewItems(string path) private void uploadNewItems(string path)
{ {
if (isDir(path)) { if (isDir(path)) {
if (matchFirst(baseName(path), skipDir).empty) { if (path.matchFirst(skipDir).empty) {
import std.string: chompPrefix; import std.string: chompPrefix;
path = chompPrefix(path, "./"); path = chompPrefix(path, "./");
Item item; Item item;
@ -411,7 +420,7 @@ final class SyncEngine
} }
} }
} else { } else {
if (matchFirst(baseName(path), skipFile).empty) { if (path.matchFirst(skipFile).empty) {
Item item; Item item;
if (!itemdb.selectByPath(path, item)) { if (!itemdb.selectByPath(path, item)) {
uploadNewFile(path); uploadNewFile(path);
@ -452,9 +461,9 @@ final class SyncEngine
uploadLastModifiedTime(id, eTag, mtime); uploadLastModifiedTime(id, eTag, mtime);
} }
private void uploadDeleteItem(Item item) private void uploadDeleteItem(Item item, const(char)[] path)
{ {
writeln("Deleting remote item: ", item.path); writeln("Deleting remote item: ", path);
onedrive.deleteById(item.id, item.eTag); onedrive.deleteById(item.id, item.eTag);
itemdb.deleteById(item.id); itemdb.deleteById(item.id);
} }
@ -499,16 +508,16 @@ final class SyncEngine
itemdb.upsert(id, name, type, eTag, cTag, mtime, parentId, crc32); itemdb.upsert(id, name, type, eTag, cTag, mtime, parentId, crc32);
} }
void uploadMoveItem(const(char)[] from, string to) void uploadMoveItem(string from, string to)
{ {
writeln("Moving remote item: ", from, " -> ", to); writeln("Moving remote item: ", from, " -> ", to);
Item item; Item item;
if (!itemdb.selectByPath(from, item) || !isItemSynced(item)) { if (!itemdb.selectByPath(from, item) || !isItemSynced(item, from)) {
writeln("Can't move an unsynced item"); writeln("Can't move an unsynced item");
return; return;
} }
if (itemdb.selectByPath(to, item)) { if (itemdb.selectByPath(to, item)) {
uploadDeleteItem(item); uploadDeleteItem(item, to);
} }
JSONValue diff = ["name": baseName(to)]; JSONValue diff = ["name": baseName(to)];
diff["parentReference"] = JSONValue([ diff["parentReference"] = JSONValue([
@ -527,6 +536,6 @@ final class SyncEngine
if (!itemdb.selectByPath(path, item)) { if (!itemdb.selectByPath(path, item)) {
throw new SyncException("Can't delete an unsynced item"); throw new SyncException("Can't delete an unsynced item");
} }
uploadDeleteItem(item); uploadDeleteItem(item, path);
} }
} }