From 9dd944286f381b30b0fe656e66d69900782732a2 Mon Sep 17 00:00:00 2001 From: skilion Date: Sun, 12 Mar 2017 16:07:45 +0100 Subject: [PATCH] new feature selective sync removed system wide config file hardcoded sane default config updated README --- Makefile | 5 +-- README.md | 34 +++++++++++------- config | 4 +++ onedrive.conf | 4 --- src/config.d | 25 +++++++++----- src/itemdb.d | 6 ++-- src/sync.d | 95 ++++++++++++++++++++++++++++++++------------------- src/util.d | 28 +++++++++++++-- 8 files changed, 133 insertions(+), 68 deletions(-) create mode 100644 config delete mode 100644 onedrive.conf diff --git a/Makefile b/Makefile index 006d1a0e..3d8df009 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ DC = dmd DFLAGS = -ofonedrive -L-lcurl -L-lsqlite3 -L-ldl DESTDIR = /usr/local/bin -CONFDIR = /usr/local/etc SOURCES = \ src/config.d \ @@ -27,12 +26,10 @@ unittest: $(SOURCES) clean: rm -f onedrive.o onedrive -install: onedrive onedrive.conf +install: onedrive onedrive.service install onedrive $(DESTDIR)/onedrive - install -m 644 onedrive.conf $(CONFDIR)/onedrive.conf install -m 644 onedrive.service /usr/lib/systemd/user uninstall: rm -f $(DESTDIR)/onedrive - rm -f $(CONFDIR)/onedrive.conf rm -f /usr/lib/systemd/user/onedrive.service diff --git a/README.md b/README.md index d208fc15..882c0225 100644 --- a/README.md +++ b/README.md @@ -37,21 +37,30 @@ sudo make install You should copy the default config file into your home directory before making changes: ``` mkdir -p ~/.config/onedrive -cp /usr/local/etc/onedrive.conf ~/.config/onedrive/config +cp ./config ~/.config/onedrive/config ``` Available options: -* `client_id`: application identifier necessary for the [authentication][2] * `sync_dir`: directory where the files will be synced -* `skip_file`: any files that match this pattern will be skipped during sync -* `skip_dir`: any directories that match this pattern will be skipped during sync +* `skip_file`: any files or directories that match this pattern will be skipped during sync Pattern are case insensitive. -`*` and `?` [wildcards characters][3] are supported. +`*` and `?` [wildcards characters][1] are supported. Use `|` to separate multiple patterns. -[2]: https://dev.onedrive.com/auth/msa_oauth.htm -[3]: https://technet.microsoft.com/en-us/library/bb490639.aspx +[1]: https://technet.microsoft.com/en-us/library/bb490639.aspx + +### Selective sync +Selective sync allows you to sync only specific files and directories. +To enable selective sync create a file named `sync_list` in `~/.config/onedrive`. +Each line represents a path to a file or directory relative from your `sync_dir`. +``` +$ cat ~/.config/onedrive/sync_list +Backup +Documents/report.odt +Work/ProjectX +notes.txt +``` ### First run The first time you run the program you will be asked to sign in. The procedure requires a web browser. @@ -72,16 +81,17 @@ journalctl --user-unit onedrive -f ``` onedrive [OPTION]... -no option Sync and exit. -m --monitor Keep monitoring for local and remote changes. - --resync Forget the last saved state, perform a full sync. + --resync Forget the last saved state, perform a full sync. + --logout Logout the current user. + --confdir Set the directory to use to store the configuration files. -v --verbose Print more details, useful for debugging. -h --help This help information. ``` ### Notes: -* After changing the filters (`skip_file` or `skip_dir` in your configs) you must execute `onedrive --resync` -* [Windows naming conventions][4] apply +* After changing `skip_file` in your configs or the sync list, you must execute `onedrive --resync` +* [Windows naming conventions][2] apply * Use `make debug` to generate an executable for debugging -[4]: https://msdn.microsoft.com/en-us/library/aa365247 +[2]: https://msdn.microsoft.com/en-us/library/aa365247 diff --git a/config b/config new file mode 100644 index 00000000..cc4b5b5a --- /dev/null +++ b/config @@ -0,0 +1,4 @@ +# Directory where the files will be synced +sync_dir = "~/OneDrive" +# Skip files and directories that match this pattern +skip_file = ".*|~*" diff --git a/onedrive.conf b/onedrive.conf deleted file mode 100644 index 8b73f21b..00000000 --- a/onedrive.conf +++ /dev/null @@ -1,4 +0,0 @@ -client_id = "22c49a0d-d21c-4792-aed1-8f163c982546" -sync_dir = "~/OneDrive" -skip_file = ".*|~*" -skip_dir = ".*" diff --git a/src/config.d b/src/config.d index 66ed19a1..9c00638e 100644 --- a/src/config.d +++ b/src/config.d @@ -1,4 +1,4 @@ -import std.file, std.regex, std.stdio; +import std.file, std.string, std.regex, std.stdio; static import log; final class Config @@ -7,6 +7,7 @@ final class Config public string statusTokenFilePath; public string databaseFilePath; public string uploadStateFilePath; + public string syncListFilePath; private string userConfigFilePath; // hashmap for the values found in the user config file @@ -19,15 +20,16 @@ final class Config databaseFilePath = configDirName ~ "/items.sqlite3"; uploadStateFilePath = configDirName ~ "/resume_upload"; userConfigFilePath = configDirName ~ "/config"; + syncListFilePath = configDirName ~ "/sync_list"; } void init() { - bool found = false; - found |= load("/etc/onedrive.conf"); - found |= load("/usr/local/etc/onedrive.conf"); - found |= load(userConfigFilePath); - if (!found) throw new Exception("No config file found"); + setValue("sync_dir", "~/OneDrive"); + setValue("skip_file", ".*|~*"); + if (!load(userConfigFilePath)) { + log.vlog("No config file found, using defaults"); + } } string getValue(string key) @@ -50,12 +52,19 @@ final class Config } } + void setValue(string key, string value) + { + values[key] = value; + } + private bool load(string filename) { scope(failure) return false; auto file = File(filename, "r"); - auto r = regex(`^\s*(\w+)\s*=\s*"(.*)"\s*$`); + auto r = regex(`^(\w+)\s*=\s*"(.*)"\s*$`); foreach (line; file.byLine()) { + line = stripLeft(line); + if (line.length == 0 || line[0] == ';' || line[0] == '#') continue; auto c = line.matchFirst(r); if (!c.empty) { c.popFront(); // skip the whole match @@ -73,7 +82,7 @@ final class Config unittest { auto cfg = new Config(""); - cfg.load("onedrive.conf"); + cfg.load("config"); assert(cfg.getValue("sync_dir") == "~/OneDrive"); assert(cfg.getValue("empty", "default") == "default"); } diff --git a/src/itemdb.d b/src/itemdb.d index 4f5cd7f5..6b77734f 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -114,7 +114,8 @@ final class ItemDatabase bool selectByPath(const(char)[] path, out Item item) { - path = "root/" ~ path.chompPrefix("."); // HACK + // prefix with the root dir + path = "root/" ~ path.chompPrefix("."); // initialize the search string[2][] candidates; // [id, parentId] @@ -240,8 +241,7 @@ final class ItemDatabase else path = r.front[0].idup; } else { // root - if (path) path = "./" ~ path; - else path = "."; + if (!path) path = "."; break; } id = r.front[1].dup; diff --git a/src/sync.d b/src/sync.d index 5bfac586..7efb3337 100644 --- a/src/sync.d +++ b/src/sync.d @@ -51,8 +51,10 @@ final class SyncEngine private Config cfg; private OneDriveApi onedrive; private ItemDatabase itemdb; - private Regex!char skipDir, skipFile; private UploadSession session; + private Regex!char skipFile; + // list of paths to sync + private string[] selectiveSyncPaths; // token representing the last status correctly synced private string statusToken; // list of items to skip while applying the changes @@ -66,9 +68,19 @@ final class SyncEngine this.cfg = cfg; this.onedrive = onedrive; this.itemdb = itemdb; - skipDir = wild2regex(cfg.getValue("skip_dir")); skipFile = wild2regex(cfg.getValue("skip_file")); session = UploadSession(onedrive, cfg.uploadStateFilePath); + + // read the selective sync list + if (exists(cfg.syncListFilePath)) { + import std.array; + auto file = File(cfg.syncListFilePath); + selectiveSyncPaths = file + .byLine() + .map!(a => buildNormalizedPath(a)) + .filter!(a => a.length > 0) + .array; + } } void init() @@ -146,13 +158,19 @@ final class SyncEngine parentId = null; } + log.vlog(id, " ", name); + // skip unwanted items early if (skippedItems.find(parentId).length != 0) { + log.vlog("Filtered out"); + skippedItems ~= id; + return; + } + if (!name.matchFirst(skipFile).empty) { + log.vlog("Filtered out"); skippedItems ~= id; return; } - - log.vlog(id, " ", name); // rename the local item if it is unsynced and there is a new version of it Item oldItem; @@ -178,22 +196,19 @@ final class SyncEngine string path = "."; if (parentId) { path = itemdb.computePath(parentId) ~ "/" ~ name; + // selective sync + if (isPathExcluded(path, selectiveSyncPaths)) { + log.vlog("Filtered out: ", path); + skippedItems ~= id; + return; + } } ItemType type; if (isItemFile(item)) { type = ItemType.file; - if (!path.matchFirst(skipFile).empty) { - log.vlog("Filtered out"); - return; - } } else if (isItemFolder(item)) { type = ItemType.dir; - if (!path.matchFirst(skipDir).empty) { - log.vlog("Filtered out"); - skippedItems ~= id; - return; - } } else { log.vlog("The item is neither a file nor a directory, skipping"); skippedItems ~= id; @@ -365,20 +380,23 @@ final class SyncEngine private void uploadDifferences(Item item) { log.vlog(item.id, " ", item.name); + + // skip filtered items + if (!item.name.matchFirst(skipFile).empty) { + log.vlog("Filtered out"); + return; + } string path = itemdb.computePath(item.id); + if (isPathExcluded(path, selectiveSyncPaths)) { + log.vlog("Filtered out: ", path); + return; + } + final switch (item.type) { case ItemType.dir: - if (!path.matchFirst(skipDir).empty) { - log.vlog("Filtered out"); - break; - } uploadDirDifferences(item, path); break; case ItemType.file: - if (!path.matchFirst(skipFile).empty) { - log.vlog("Filtered out"); - break; - } uploadFileDifferences(item, path); break; } @@ -448,26 +466,33 @@ final class SyncEngine private void uploadNewItems(string path) { + // skip unexisting symbolic links if (isSymlink(path) && !exists(readLink(path))) { return; } + + // skip filtered items + if (!baseName(path).matchFirst(skipFile).empty) { + return; + } + if (isPathExcluded(path, selectiveSyncPaths)) { + return; + } + if (isDir(path)) { - if (path.matchFirst(skipDir).empty) { - Item item; - if (!itemdb.selectByPath(path, item)) { - uploadCreateDir(path); - } - auto entries = dirEntries(path, SpanMode.shallow, false); - foreach (DirEntry entry; entries) { - uploadNewItems(entry.name); - } + Item item; + if (!itemdb.selectByPath(path, item)) { + uploadCreateDir(path); + } + // recursively traverse children + auto entries = dirEntries(path, SpanMode.shallow, false); + foreach (DirEntry entry; entries) { + uploadNewItems(entry.name); } } else { - if (path.matchFirst(skipFile).empty) { - Item item; - if (!itemdb.selectByPath(path, item)) { - uploadNewFile(path); - } + Item item; + if (!itemdb.selectByPath(path, item)) { + uploadNewFile(path); } } } diff --git a/src/util.d b/src/util.d index b9515905..21f1edbe 100644 --- a/src/util.d +++ b/src/util.d @@ -1,3 +1,4 @@ +import std.algorithm; import std.conv; import std.digest.crc; import std.file; @@ -55,7 +56,7 @@ Regex!char wild2regex(const(char)[] pattern) { string str; str.reserve(pattern.length + 2); - str ~= "/"; + str ~= "^"; foreach (c; pattern) { switch (c) { case '*': @@ -68,7 +69,7 @@ Regex!char wild2regex(const(char)[] pattern) str ~= "[^/]"; break; case '|': - str ~= "$|/"; + str ~= "$|^"; break; default: str ~= c; @@ -98,9 +99,32 @@ bool multiGlobMatch(const(char)[] path, const(char)[] pattern) return false; } +// test if the given path is not included in the allowed paths +// if there are no allowed paths always return false +bool isPathExcluded(string path, string[] allowedPaths) +{ + // always allow the root + if (path == ".") return false; + // if there are no allowed paths always return false + if (allowedPaths.empty) return false; + + path = buildNormalizedPath(path); + foreach (allowed; allowedPaths) { + auto comm = commonPrefix(path, allowed); + if (comm.length == path.length || comm.length == allowed.length) { + return false; + } + } + return true; +} + unittest { assert(multiGlobMatch(".hidden", ".*")); assert(multiGlobMatch(".hidden", "file|.*")); assert(!multiGlobMatch("foo.bar", "foo|bar")); + assert(isPathExcluded("Documents2", ["Documents"])); + assert(isPathExcluded("Hello/World", ["Hello/John"])); + assert(!isPathExcluded("Documents", ["Documents"])); + assert(!isPathExcluded("Documents/a.txt", ["Documents"])); }