From 97a9d5391416f1bec51a80c7e154f11d0b2e203d Mon Sep 17 00:00:00 2001 From: skilion Date: Fri, 24 Mar 2017 22:30:03 +0100 Subject: [PATCH] refactored selective sync code fixed bug in selective sync --- Makefile | 1 + src/config.d | 1 + src/main.d | 9 ++++--- src/monitor.d | 39 +++++++++++---------------- src/selective.d | 72 +++++++++++++++++++++++++++++++++++++++++++++++++ src/sync.d | 39 ++++++++++----------------- src/util.d | 26 +----------------- 7 files changed, 111 insertions(+), 76 deletions(-) create mode 100644 src/selective.d diff --git a/Makefile b/Makefile index 3d8df009..363ff7fe 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ SOURCES = \ src/main.d \ src/monitor.d \ src/onedrive.d \ + src/selective.d \ src/sqlite.d \ src/sync.d \ src/upload.d \ diff --git a/src/config.d b/src/config.d index 9c00638e..653982d9 100644 --- a/src/config.d +++ b/src/config.d @@ -1,4 +1,5 @@ import std.file, std.string, std.regex, std.stdio; +import selective; static import log; final class Config diff --git a/src/main.d b/src/main.d index 4e657081..0de12d25 100644 --- a/src/main.d +++ b/src/main.d @@ -1,7 +1,7 @@ import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE; import core.memory, core.time, core.thread; import std.getopt, std.file, std.path, std.process; -import config, itemdb, monitor, onedrive, sync, util; +import config, itemdb, monitor, onedrive, selective, sync, util; static import log; int main(string[] args) @@ -87,13 +87,16 @@ int main(string[] args) chdir(syncDir); log.vlog("Initializing the Synchronization Engine ..."); - auto sync = new SyncEngine(cfg, onedrive, itemdb); + auto selectiveSync = new SelectiveSync(); + selectiveSync.load(cfg.syncListFilePath); + selectiveSync.setMask(cfg.getValue("skip_file")); + auto sync = new SyncEngine(cfg, onedrive, itemdb, selectiveSync); sync.init(); if (online) performSync(sync); if (monitor) { log.vlog("Initializing monitor ..."); - Monitor m; + Monitor m = new Monitor(selectiveSync); m.onDirCreated = delegate(string path) { log.vlog("[M] Directory created: ", path); try { diff --git a/src/monitor.d b/src/monitor.d index 2419025b..5dec10d4 100644 --- a/src/monitor.d +++ b/src/monitor.d @@ -1,8 +1,10 @@ import core.sys.linux.sys.inotify; import core.stdc.errno; import core.sys.posix.poll, core.sys.posix.unistd; -import std.algorithm, std.exception, std.file, std.path, std.regex, std.stdio, std.string; -import config, util; +import std.exception, std.file, std.path, std.regex, std.stdio, std.string; +import config; +import selective; +import util; static import log; // relevant inotify events @@ -17,13 +19,9 @@ class MonitorException: ErrnoException } } -struct Monitor +final class Monitor { bool verbose; - // regex that match files to skip - private Regex!char skipFile; - // list of paths to sync - private string[] selectiveSyncPaths; // inotify file descriptor private int fd; // map every inotify watch descriptor to its directory @@ -33,27 +31,22 @@ struct Monitor // buffer to receive the inotify events private void[] buffer; + private SelectiveSync selectiveSync; + void delegate(string path) onDirCreated; void delegate(string path) onFileChanged; void delegate(string path) onDelete; void delegate(string from, string to) onMove; - @disable this(this); + this(SelectiveSync selectiveSync) + { + assert(selectiveSync); + this.selectiveSync = selectiveSync; + } void init(Config cfg, bool verbose) { this.verbose = verbose; - skipFile = wild2regex(cfg.getValue("skip_file")); - // read the selective sync list - if (exists(cfg.syncListFilePath)) { - import std.array; - auto file = File(cfg.syncListFilePath); - selectiveSyncPaths = file - .byLine() - .map!(a => buildNormalizedPath(a)) - .filter!(a => a.length > 0) - .array; - } fd = inotify_init(); if (fd == -1) throw new MonitorException("inotify_init failed"); @@ -71,10 +64,10 @@ struct Monitor { // skip filtered items if (dirname != ".") { - if (!baseName(dirname).matchFirst(skipFile).empty) { + if (selectiveSync.isNameExcluded(baseName(dirname))) { return; } - if (isPathExcluded(buildNormalizedPath(dirname), selectiveSyncPaths)) { + if (selectiveSync.isPathExcluded(buildNormalizedPath(dirname))) { return; } } @@ -172,10 +165,10 @@ struct Monitor // skip filtered items path = getPath(event); - if (!baseName(path).matchFirst(skipFile).empty) { + if (selectiveSync.isNameExcluded(baseName(path))) { goto skip; } - if (isPathExcluded(path, selectiveSyncPaths)) { + if (selectiveSync.isPathExcluded(path)) { goto skip; } diff --git a/src/selective.d b/src/selective.d new file mode 100644 index 00000000..733a6aa6 --- /dev/null +++ b/src/selective.d @@ -0,0 +1,72 @@ +import std.algorithm; +import std.array; +import std.file; +import std.path; +import std.regex; +import std.stdio; +import util; + +final class SelectiveSync +{ + private string[] paths; + private Regex!char mask; + + void load(string filepath) + { + if (exists(filepath)) { + paths = File(filepath) + .byLine() + .map!(a => buildNormalizedPath(a)) + .filter!(a => a.length > 0) + .array; + } + } + + void setMask(const(char)[] mask) + { + this.mask = wild2regex(mask); + } + + bool isNameExcluded(string name) + { + return !name.matchFirst(mask).empty; + } + + bool isPathExcluded(string path) + { + return .isPathExcluded(path, paths); + } +} + +// test if the given path is not included in the allowed paths +// if there are no allowed paths always return false +private bool isPathExcluded(string path, string[] allowedPaths) +{ + // always allow the root + if (path == ".") return false; + // if there are no allowed paths always return false + if (allowedPaths.empty) return false; + + path = buildNormalizedPath(path); + foreach (allowed; allowedPaths) { + auto comm = commonPrefix(path, allowed); + if (comm.length == path.length) { + // the given path is contained in an allowed path + return false; + } + if (comm.length == allowed.length && path[comm.length] == '/') { + // the given path is a subitem of an allowed path + return false; + } + } + return true; +} + +unittest +{ + assert(isPathExcluded("Documents2", ["Documents"])); + assert(!isPathExcluded("Documents", ["Documents"])); + assert(!isPathExcluded("Documents/a.txt", ["Documents"])); + assert(isPathExcluded("Hello/World", ["Hello/John"])); + assert(!isPathExcluded(".", ["Documents"])); +} diff --git a/src/sync.d b/src/sync.d index d214e415..0554b4f6 100644 --- a/src/sync.d +++ b/src/sync.d @@ -1,8 +1,10 @@ +import std.algorithm; import std.net.curl: CurlTimeoutException; import std.exception: ErrnoException; -import std.algorithm, std.datetime, std.file, std.json, std.path, std.regex; +import std.datetime, std.file, std.json, std.path; +import std.regex; import std.stdio, std.string; -import config, itemdb, onedrive, upload, util; +import config, itemdb, onedrive, selective, upload, util; static import log; // threshold after which files will be uploaded using an upload session @@ -57,9 +59,7 @@ final class SyncEngine private OneDriveApi onedrive; private ItemDatabase itemdb; private UploadSession session; - private Regex!char skipFile; - // list of paths to sync - private string[] selectiveSyncPaths; + private SelectiveSync selectiveSync; // token representing the last status correctly synced private string statusToken; // list of items to skip while applying the changes @@ -67,25 +67,14 @@ final class SyncEngine // list of items to delete after the changes has been downloaded private string[] idsToDelete; - this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb) + this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync) { - assert(onedrive && itemdb); + assert(onedrive && itemdb && selectiveSync); this.cfg = cfg; this.onedrive = onedrive; this.itemdb = itemdb; - skipFile = wild2regex(cfg.getValue("skip_file")); + this.selectiveSync = selectiveSync; session = UploadSession(onedrive, cfg.uploadStateFilePath); - - // read the selective sync list - if (exists(cfg.syncListFilePath)) { - import std.array; - auto file = File(cfg.syncListFilePath); - selectiveSyncPaths = file - .byLine() - .map!(a => buildNormalizedPath(a)) - .filter!(a => a.length > 0) - .array; - } } void init() @@ -171,7 +160,7 @@ final class SyncEngine skippedItems ~= id; return; } - if (!name.matchFirst(skipFile).empty) { + if (selectiveSync.isNameExcluded(name)) { log.vlog("Filtered out"); skippedItems ~= id; return; @@ -202,7 +191,7 @@ final class SyncEngine if (parentId) { path = itemdb.computePath(parentId) ~ "/" ~ name; // selective sync - if (isPathExcluded(path, selectiveSyncPaths)) { + if (selectiveSync.isPathExcluded(path)) { log.vlog("Filtered out: ", path); skippedItems ~= id; return; @@ -390,12 +379,12 @@ final class SyncEngine log.vlog(item.id, " ", item.name); // skip filtered items - if (!item.name.matchFirst(skipFile).empty) { + if (selectiveSync.isNameExcluded(item.name)) { log.vlog("Filtered out"); return; } string path = itemdb.computePath(item.id); - if (isPathExcluded(path, selectiveSyncPaths)) { + if (selectiveSync.isPathExcluded(path)) { log.vlog("Filtered out: ", path); return; } @@ -484,10 +473,10 @@ final class SyncEngine // skip filtered items if (path != ".") { - if (!baseName(path).matchFirst(skipFile).empty) { + if (selectiveSync.isNameExcluded(baseName(path))) { return; } - if (isPathExcluded(path, selectiveSyncPaths)) { + if (selectiveSync.isPathExcluded(path)) { return; } } diff --git a/src/util.d b/src/util.d index 21f1edbe..f7512f22 100644 --- a/src/util.d +++ b/src/util.d @@ -1,4 +1,3 @@ -import std.algorithm; import std.conv; import std.digest.crc; import std.file; @@ -83,7 +82,7 @@ Regex!char wild2regex(const(char)[] pattern) // return true if the network connection is available bool testNetwork() { - HTTP http = HTTP("https://login.live.com"); + HTTP http = HTTP("https://login.microsoftonline.com"); http.method = HTTP.Method.head; return http.perform(ThrowOnError.no) == 0; } @@ -99,32 +98,9 @@ bool multiGlobMatch(const(char)[] path, const(char)[] pattern) return false; } -// test if the given path is not included in the allowed paths -// if there are no allowed paths always return false -bool isPathExcluded(string path, string[] allowedPaths) -{ - // always allow the root - if (path == ".") return false; - // if there are no allowed paths always return false - if (allowedPaths.empty) return false; - - path = buildNormalizedPath(path); - foreach (allowed; allowedPaths) { - auto comm = commonPrefix(path, allowed); - if (comm.length == path.length || comm.length == allowed.length) { - return false; - } - } - return true; -} - unittest { assert(multiGlobMatch(".hidden", ".*")); assert(multiGlobMatch(".hidden", "file|.*")); assert(!multiGlobMatch("foo.bar", "foo|bar")); - assert(isPathExcluded("Documents2", ["Documents"])); - assert(isPathExcluded("Hello/World", ["Hello/John"])); - assert(!isPathExcluded("Documents", ["Documents"])); - assert(!isPathExcluded("Documents/a.txt", ["Documents"])); }