deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-15 20:26:53 +02:00
Code Issues Projects Releases Wiki Activity

implemented filter in sync

Browse source
This commit is contained in:
skilion 2015-09-18 21:42:27 +02:00
parent 9aa1f221c7
commit 7126c4fefc
3 changed files with 123 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
src/itemdb.d
View file

@ -121,61 +121,18 @@ final class ItemDatabase
}
}
// returns a range that go trough all items, depth first
auto selectAll()
Item[] selectChildren(const(char)[] id)
{
static struct ItemRange
{
ItemDatabase itemdb;
string[] stack1, stack2;
private this(ItemDatabase itemdb, string rootId)
{
this.itemdb = itemdb;
stack1.reserve(8);
stack2.reserve(8);
stack1 ~= rootId;
getChildren();
}
@property bool empty()
{
return stack2.length == 0;
}
@property Item front()
{
Item item;
bool res = itemdb.selectById(stack2[$ - 1], item);
assert(res);
return item;
}
void popFront()
{
stack2 = stack2[0 .. $ - 1];
assumeSafeAppend(stack2);
if (stack1.length > 0) getChildren();
}
private void getChildren()
{
while (true) {
itemdb.selectItemByParentIdStmt.bind(1, stack1[$ - 1]);
stack2 ~= stack1[$ - 1];
stack1 = stack1[0 .. $ - 1];
assumeSafeAppend(stack1);
auto res = itemdb.selectItemByParentIdStmt.exec();
if (res.empty) break;
else foreach (row; res) stack1 ~= row[0].dup;
}
}
selectItemByParentIdStmt.bind(1, id);
auto res = selectItemByParentIdStmt.exec();
Item[] items;
foreach (row; res) {
Item item;
bool found = selectById(row[0], item);
assert(found);
items ~= item;
}
auto s = db.prepare("SELECT id FROM item WHERE parentId IS NULL");
auto r = s.exec();
assert(!r.empty());
return ItemRange(this, r.front[0].dup);
return items;
}
bool selectById(const(char)[] id, out Item item)
@ -191,6 +148,7 @@ final class ItemDatabase
bool selectByPath(const(char)[] path, out Item item)
{
if (path == ".") path = "root"; // HACK
string[2][] candidates; // [id, parentId]
auto s = db.prepare("SELECT id, parentId FROM item WHERE name = ?");
s.bind(1, baseName(path));
@ -202,14 +160,24 @@ final class ItemDatabase
string[2][] newCandidates;
newCandidates.reserve(candidates.length);
path = dirName(path);
foreach (candidate; candidates) {
s.bind(1, candidate[1]);
if (path.length != 0) {
s.bind(2, baseName(path));
r = s.exec();
if (!r.empty) {
string[2] c = [candidate[0], r.front[0].idup];
newCandidates ~= c;
foreach (candidate; candidates) {
s.bind(1, candidate[1]);
r = s.exec();
if (!r.empty) {
string[2] c = [candidate[0], r.front[0].idup];
newCandidates ~= c;
}
}
} else {
// reached the root
foreach (candidate; candidates) {
if (!candidate[1]) {
newCandidates ~= candidate;
}
}
assert(newCandidates.length <= 1);
}
candidates = newCandidates;
} while (candidates.length > 1);

11
src/main.d
View file

@ -73,21 +73,20 @@ void main(string[] args)
string syncDir = cfg.get("sync_dir");
chdir(syncDir);
sync.applyDifferences();
sync.uploadDifferences();
sync.scanForDifferences(".");
return;
if (monitor) {
if (verbose) writeln("Initializing monitor ...");
Monitor m;
m.onDirCreated = delegate(string path) {
if (verbose) writeln("[M] Directory created: ", path);
sync.uploadCreateDir(path);
// the directory could be the result of a move operation
sync.uploadDifferences(path);
sync.scanForDifferences(path);
};
m.onFileChanged = delegate(string path) {
if (verbose) writeln("[M] File changed: ", path);
try {
sync.uploadDifference(path);
sync.scanForDifferences(path);
} catch(SyncException e) {
writeln(e.msg);
}
@ -111,7 +110,7 @@ void main(string[] args)
lastCheckTime = currTime;
m.shutdown();
sync.applyDifferences();
sync.uploadDifferences();
sync.scanForDifferences(".");
m.init(cfg, verbose);
}
Thread.sleep(dur!"msecs"(100));

174
src/sync.d
View file

@ -51,8 +51,11 @@ final class SyncEngine
private ItemDatabase itemdb;
private bool verbose;
private Regex!char skipDir, skipFile;
// token representing the last status correctly synced
private string statusToken;
// list of items to skip while applying the changes downloaded
private string[] skippedItems;
// list of items to delete after the changes has been downloaded
private string[] itemsToDelete;
void delegate(string) onStatusToken;
@ -238,7 +241,7 @@ final class SyncEngine
}
setTimes(newItem.path, newItem.mtime, newItem.mtime);
} else {
if (verbose) writeln("The item is not changed");
if (verbose) writeln("The item has not changed");
}
}
@ -302,110 +305,115 @@ final class SyncEngine
assumeSafeAppend(itemsToDelete);
}
// scan the root directory for unsynced files and upload them
public void uploadDifferences()
// scan the given directory for differences
public void scanForDifferences(string path)
{
if (verbose) writeln("Uploading differences ...");
// check for changed files or deleted items
foreach (Item item; itemdb.selectAll()) {
uploadDifference(item);
Item item;
if (itemdb.selectByPath(path, item)) {
uploadDifferences(item);
}
if (verbose) writeln("Uploading new items ...");
// check for new files or directories
foreach (DirEntry entry; dirEntries(".", SpanMode.breadth, false)) {
string path = entry.name[2 .. $]; // HACK: skip "./"
Item item;
if (!itemdb.selectByPath(path, item)) {
if (entry.isDir) {
uploadCreateDir(path);
} else {
uploadNewFile(path);
}
}
}
uploadNewItems(path);
}
/* scan the specified directory for unsynced files and upload them
NOTE: this function does not check for deleted files. */
public void uploadDifferences(string dirname)
{
foreach (DirEntry entry; dirEntries(dirname, SpanMode.breadth, false)) {
uploadDifference(entry.name);
}
}
private void uploadDifference(Item item)
public void uploadDifferences(Item item)
{
if (verbose) writeln(item.id, " ", item.name);
if (!matchFirst(name, skipFile).empty) {
if (verbose) writeln("Filtered out");
skippedItems ~= id;
return;
}
final switch (item.type) {
case ItemType.dir:
if (!matchFirst(item.name, skipDir).empty) break;
uploadDirDifferences(item);
break;
case ItemType.file:
if (!matchFirst(item.name, skipFile).empty) break;
uploadFileDifferences(item);
break;
}
}
private void uploadDirDifferences(Item item)
{
assert(item.type == ItemType.dir);
if (exists(item.path)) {
final switch (item.type) {
case ItemType.file:
if (isFile(item.path)) {
SysTime localModifiedTime = timeLastModified(item.path);
import core.time: Duration;
item.mtime.fracSecs = Duration.zero; // HACK
if (localModifiedTime != item.mtime) {
if (verbose) writeln("The item last modified time has changed");
string id = item.id;
string eTag = item.eTag;
if (!testCrc32(item.path, item.crc32)) {
if (verbose) writeln("The item content has changed");
writeln("Uploading: ", item.path);
auto res = onedrive.simpleUpload(item.path, item.path, item.eTag);
saveItem(res);
id = res["id"].str;
eTag = res["eTag"].str;
}
uploadLastModifiedTime(id, eTag, localModifiedTime.toUTC());
} else {
if (verbose) writeln("The item has not changed");
}
} else {
if (verbose) writeln("The item was a file but now is a directory");
uploadDeleteItem(item);
uploadCreateDir(item.path);
if (!isDir(item.path)) {
if (verbose) writeln("The item was a directory but now is a file");
uploadDeleteItem(item);
uploadNewFile(item.path);
} else {
if (verbose) writeln("The directory has not changed");
// loop trough the children
foreach (Item child; itemdb.selectChildren(item.id)) {
uploadDifferences(child);
}
break;
case ItemType.dir:
if (!isDir(item.path)) {
if (verbose) writeln("The item was a directory but now is a file");
uploadDeleteItem(item);
uploadNewFile(item.path);
} else {
if (verbose) writeln("The item has not changed");
}
break;
}
} else {
if (verbose) writeln("The item has been deleted");
if (verbose) writeln("The directory has been deleted");
uploadDeleteItem(item);
}
}
void uploadDifference(string path)
private void uploadFileDifferences(Item item)
{
try {
Item item;
if (itemdb.selectByPath(path, item)) {
uploadDifference(item);
} else {
if (isDir(path)) {
uploadCreateDir(path);
assert(item.type == ItemType.file);
if (exists(item.path)) {
if (isFile(item.path)) {
SysTime localModifiedTime = timeLastModified(item.path);
import core.time: Duration;
item.mtime.fracSecs = Duration.zero; // HACK
if (localModifiedTime != item.mtime) {
if (verbose) writeln("The file last modified time has changed");
string id = item.id;
string eTag = item.eTag;
if (!testCrc32(item.path, item.crc32)) {
if (verbose) writeln("The file content has changed");
writeln("Uploading: ", item.path);
auto res = onedrive.simpleUpload(item.path, item.path, item.eTag);
saveItem(res);
id = res["id"].str;
eTag = res["eTag"].str;
}
uploadLastModifiedTime(id, eTag, localModifiedTime.toUTC());
} else {
uploadNewFile(path);
}
if (verbose) writeln("The file has not changed");
}
} else {
if (verbose) writeln("The item was a file but now is a directory");
uploadDeleteItem(item);
uploadCreateDir(item.path);
}
} catch (FileException e) {
throw new SyncException(e.msg, e);
} else {
if (verbose) writeln("The file has been deleted");
uploadDeleteItem(item);
}
}
void uploadCreateDir(const(char)[] path)
private void uploadNewItems(string path)
{
if (isDir(path)) {
if (matchFirst(baseName(path), skipDir).empty) {
import std.string: chompPrefix;
path = chompPrefix(path, "./");
Item item;
if (!itemdb.selectByPath(path, item)) {
uploadCreateDir(path);
}
auto entries = dirEntries(path, SpanMode.shallow, false);
foreach (DirEntry entry; entries) {
uploadNewItems(entry.name);
}
}
} else {
if (matchFirst(baseName(path), skipFile).empty) {
Item item;
if (!itemdb.selectByPath(path, item)) {
uploadNewFile(path);
}
}
}
}
private void uploadCreateDir(const(char)[] path)
{
writeln("Creating remote directory: ", path);
JSONValue item = ["name": baseName(path).idup];