mirror of
https://github.com/abraunegg/onedrive
synced 2024-05-20 06:36:40 +02:00
refactored selective sync code
fixed bug in selective sync
This commit is contained in:
parent
0d69ed805d
commit
97a9d53914
1
Makefile
1
Makefile
|
@ -9,6 +9,7 @@ SOURCES = \
|
||||||
src/main.d \
|
src/main.d \
|
||||||
src/monitor.d \
|
src/monitor.d \
|
||||||
src/onedrive.d \
|
src/onedrive.d \
|
||||||
|
src/selective.d \
|
||||||
src/sqlite.d \
|
src/sqlite.d \
|
||||||
src/sync.d \
|
src/sync.d \
|
||||||
src/upload.d \
|
src/upload.d \
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import std.file, std.string, std.regex, std.stdio;
|
import std.file, std.string, std.regex, std.stdio;
|
||||||
|
import selective;
|
||||||
static import log;
|
static import log;
|
||||||
|
|
||||||
final class Config
|
final class Config
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE;
|
import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE;
|
||||||
import core.memory, core.time, core.thread;
|
import core.memory, core.time, core.thread;
|
||||||
import std.getopt, std.file, std.path, std.process;
|
import std.getopt, std.file, std.path, std.process;
|
||||||
import config, itemdb, monitor, onedrive, sync, util;
|
import config, itemdb, monitor, onedrive, selective, sync, util;
|
||||||
static import log;
|
static import log;
|
||||||
|
|
||||||
int main(string[] args)
|
int main(string[] args)
|
||||||
|
@ -87,13 +87,16 @@ int main(string[] args)
|
||||||
chdir(syncDir);
|
chdir(syncDir);
|
||||||
|
|
||||||
log.vlog("Initializing the Synchronization Engine ...");
|
log.vlog("Initializing the Synchronization Engine ...");
|
||||||
auto sync = new SyncEngine(cfg, onedrive, itemdb);
|
auto selectiveSync = new SelectiveSync();
|
||||||
|
selectiveSync.load(cfg.syncListFilePath);
|
||||||
|
selectiveSync.setMask(cfg.getValue("skip_file"));
|
||||||
|
auto sync = new SyncEngine(cfg, onedrive, itemdb, selectiveSync);
|
||||||
sync.init();
|
sync.init();
|
||||||
if (online) performSync(sync);
|
if (online) performSync(sync);
|
||||||
|
|
||||||
if (monitor) {
|
if (monitor) {
|
||||||
log.vlog("Initializing monitor ...");
|
log.vlog("Initializing monitor ...");
|
||||||
Monitor m;
|
Monitor m = new Monitor(selectiveSync);
|
||||||
m.onDirCreated = delegate(string path) {
|
m.onDirCreated = delegate(string path) {
|
||||||
log.vlog("[M] Directory created: ", path);
|
log.vlog("[M] Directory created: ", path);
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -1,8 +1,10 @@
|
||||||
import core.sys.linux.sys.inotify;
|
import core.sys.linux.sys.inotify;
|
||||||
import core.stdc.errno;
|
import core.stdc.errno;
|
||||||
import core.sys.posix.poll, core.sys.posix.unistd;
|
import core.sys.posix.poll, core.sys.posix.unistd;
|
||||||
import std.algorithm, std.exception, std.file, std.path, std.regex, std.stdio, std.string;
|
import std.exception, std.file, std.path, std.regex, std.stdio, std.string;
|
||||||
import config, util;
|
import config;
|
||||||
|
import selective;
|
||||||
|
import util;
|
||||||
static import log;
|
static import log;
|
||||||
|
|
||||||
// relevant inotify events
|
// relevant inotify events
|
||||||
|
@ -17,13 +19,9 @@ class MonitorException: ErrnoException
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct Monitor
|
final class Monitor
|
||||||
{
|
{
|
||||||
bool verbose;
|
bool verbose;
|
||||||
// regex that match files to skip
|
|
||||||
private Regex!char skipFile;
|
|
||||||
// list of paths to sync
|
|
||||||
private string[] selectiveSyncPaths;
|
|
||||||
// inotify file descriptor
|
// inotify file descriptor
|
||||||
private int fd;
|
private int fd;
|
||||||
// map every inotify watch descriptor to its directory
|
// map every inotify watch descriptor to its directory
|
||||||
|
@ -33,27 +31,22 @@ struct Monitor
|
||||||
// buffer to receive the inotify events
|
// buffer to receive the inotify events
|
||||||
private void[] buffer;
|
private void[] buffer;
|
||||||
|
|
||||||
|
private SelectiveSync selectiveSync;
|
||||||
|
|
||||||
void delegate(string path) onDirCreated;
|
void delegate(string path) onDirCreated;
|
||||||
void delegate(string path) onFileChanged;
|
void delegate(string path) onFileChanged;
|
||||||
void delegate(string path) onDelete;
|
void delegate(string path) onDelete;
|
||||||
void delegate(string from, string to) onMove;
|
void delegate(string from, string to) onMove;
|
||||||
|
|
||||||
@disable this(this);
|
this(SelectiveSync selectiveSync)
|
||||||
|
{
|
||||||
|
assert(selectiveSync);
|
||||||
|
this.selectiveSync = selectiveSync;
|
||||||
|
}
|
||||||
|
|
||||||
void init(Config cfg, bool verbose)
|
void init(Config cfg, bool verbose)
|
||||||
{
|
{
|
||||||
this.verbose = verbose;
|
this.verbose = verbose;
|
||||||
skipFile = wild2regex(cfg.getValue("skip_file"));
|
|
||||||
// read the selective sync list
|
|
||||||
if (exists(cfg.syncListFilePath)) {
|
|
||||||
import std.array;
|
|
||||||
auto file = File(cfg.syncListFilePath);
|
|
||||||
selectiveSyncPaths = file
|
|
||||||
.byLine()
|
|
||||||
.map!(a => buildNormalizedPath(a))
|
|
||||||
.filter!(a => a.length > 0)
|
|
||||||
.array;
|
|
||||||
}
|
|
||||||
|
|
||||||
fd = inotify_init();
|
fd = inotify_init();
|
||||||
if (fd == -1) throw new MonitorException("inotify_init failed");
|
if (fd == -1) throw new MonitorException("inotify_init failed");
|
||||||
|
@ -71,10 +64,10 @@ struct Monitor
|
||||||
{
|
{
|
||||||
// skip filtered items
|
// skip filtered items
|
||||||
if (dirname != ".") {
|
if (dirname != ".") {
|
||||||
if (!baseName(dirname).matchFirst(skipFile).empty) {
|
if (selectiveSync.isNameExcluded(baseName(dirname))) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (isPathExcluded(buildNormalizedPath(dirname), selectiveSyncPaths)) {
|
if (selectiveSync.isPathExcluded(buildNormalizedPath(dirname))) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -172,10 +165,10 @@ struct Monitor
|
||||||
|
|
||||||
// skip filtered items
|
// skip filtered items
|
||||||
path = getPath(event);
|
path = getPath(event);
|
||||||
if (!baseName(path).matchFirst(skipFile).empty) {
|
if (selectiveSync.isNameExcluded(baseName(path))) {
|
||||||
goto skip;
|
goto skip;
|
||||||
}
|
}
|
||||||
if (isPathExcluded(path, selectiveSyncPaths)) {
|
if (selectiveSync.isPathExcluded(path)) {
|
||||||
goto skip;
|
goto skip;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
72
src/selective.d
Normal file
72
src/selective.d
Normal file
|
@ -0,0 +1,72 @@
|
||||||
|
import std.algorithm;
|
||||||
|
import std.array;
|
||||||
|
import std.file;
|
||||||
|
import std.path;
|
||||||
|
import std.regex;
|
||||||
|
import std.stdio;
|
||||||
|
import util;
|
||||||
|
|
||||||
|
final class SelectiveSync
|
||||||
|
{
|
||||||
|
private string[] paths;
|
||||||
|
private Regex!char mask;
|
||||||
|
|
||||||
|
void load(string filepath)
|
||||||
|
{
|
||||||
|
if (exists(filepath)) {
|
||||||
|
paths = File(filepath)
|
||||||
|
.byLine()
|
||||||
|
.map!(a => buildNormalizedPath(a))
|
||||||
|
.filter!(a => a.length > 0)
|
||||||
|
.array;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void setMask(const(char)[] mask)
|
||||||
|
{
|
||||||
|
this.mask = wild2regex(mask);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool isNameExcluded(string name)
|
||||||
|
{
|
||||||
|
return !name.matchFirst(mask).empty;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool isPathExcluded(string path)
|
||||||
|
{
|
||||||
|
return .isPathExcluded(path, paths);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// test if the given path is not included in the allowed paths
|
||||||
|
// if there are no allowed paths always return false
|
||||||
|
private bool isPathExcluded(string path, string[] allowedPaths)
|
||||||
|
{
|
||||||
|
// always allow the root
|
||||||
|
if (path == ".") return false;
|
||||||
|
// if there are no allowed paths always return false
|
||||||
|
if (allowedPaths.empty) return false;
|
||||||
|
|
||||||
|
path = buildNormalizedPath(path);
|
||||||
|
foreach (allowed; allowedPaths) {
|
||||||
|
auto comm = commonPrefix(path, allowed);
|
||||||
|
if (comm.length == path.length) {
|
||||||
|
// the given path is contained in an allowed path
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (comm.length == allowed.length && path[comm.length] == '/') {
|
||||||
|
// the given path is a subitem of an allowed path
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
unittest
|
||||||
|
{
|
||||||
|
assert(isPathExcluded("Documents2", ["Documents"]));
|
||||||
|
assert(!isPathExcluded("Documents", ["Documents"]));
|
||||||
|
assert(!isPathExcluded("Documents/a.txt", ["Documents"]));
|
||||||
|
assert(isPathExcluded("Hello/World", ["Hello/John"]));
|
||||||
|
assert(!isPathExcluded(".", ["Documents"]));
|
||||||
|
}
|
39
src/sync.d
39
src/sync.d
|
@ -1,8 +1,10 @@
|
||||||
|
import std.algorithm;
|
||||||
import std.net.curl: CurlTimeoutException;
|
import std.net.curl: CurlTimeoutException;
|
||||||
import std.exception: ErrnoException;
|
import std.exception: ErrnoException;
|
||||||
import std.algorithm, std.datetime, std.file, std.json, std.path, std.regex;
|
import std.datetime, std.file, std.json, std.path;
|
||||||
|
import std.regex;
|
||||||
import std.stdio, std.string;
|
import std.stdio, std.string;
|
||||||
import config, itemdb, onedrive, upload, util;
|
import config, itemdb, onedrive, selective, upload, util;
|
||||||
static import log;
|
static import log;
|
||||||
|
|
||||||
// threshold after which files will be uploaded using an upload session
|
// threshold after which files will be uploaded using an upload session
|
||||||
|
@ -57,9 +59,7 @@ final class SyncEngine
|
||||||
private OneDriveApi onedrive;
|
private OneDriveApi onedrive;
|
||||||
private ItemDatabase itemdb;
|
private ItemDatabase itemdb;
|
||||||
private UploadSession session;
|
private UploadSession session;
|
||||||
private Regex!char skipFile;
|
private SelectiveSync selectiveSync;
|
||||||
// list of paths to sync
|
|
||||||
private string[] selectiveSyncPaths;
|
|
||||||
// token representing the last status correctly synced
|
// token representing the last status correctly synced
|
||||||
private string statusToken;
|
private string statusToken;
|
||||||
// list of items to skip while applying the changes
|
// list of items to skip while applying the changes
|
||||||
|
@ -67,25 +67,14 @@ final class SyncEngine
|
||||||
// list of items to delete after the changes has been downloaded
|
// list of items to delete after the changes has been downloaded
|
||||||
private string[] idsToDelete;
|
private string[] idsToDelete;
|
||||||
|
|
||||||
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb)
|
this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
|
||||||
{
|
{
|
||||||
assert(onedrive && itemdb);
|
assert(onedrive && itemdb && selectiveSync);
|
||||||
this.cfg = cfg;
|
this.cfg = cfg;
|
||||||
this.onedrive = onedrive;
|
this.onedrive = onedrive;
|
||||||
this.itemdb = itemdb;
|
this.itemdb = itemdb;
|
||||||
skipFile = wild2regex(cfg.getValue("skip_file"));
|
this.selectiveSync = selectiveSync;
|
||||||
session = UploadSession(onedrive, cfg.uploadStateFilePath);
|
session = UploadSession(onedrive, cfg.uploadStateFilePath);
|
||||||
|
|
||||||
// read the selective sync list
|
|
||||||
if (exists(cfg.syncListFilePath)) {
|
|
||||||
import std.array;
|
|
||||||
auto file = File(cfg.syncListFilePath);
|
|
||||||
selectiveSyncPaths = file
|
|
||||||
.byLine()
|
|
||||||
.map!(a => buildNormalizedPath(a))
|
|
||||||
.filter!(a => a.length > 0)
|
|
||||||
.array;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void init()
|
void init()
|
||||||
|
@ -171,7 +160,7 @@ final class SyncEngine
|
||||||
skippedItems ~= id;
|
skippedItems ~= id;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!name.matchFirst(skipFile).empty) {
|
if (selectiveSync.isNameExcluded(name)) {
|
||||||
log.vlog("Filtered out");
|
log.vlog("Filtered out");
|
||||||
skippedItems ~= id;
|
skippedItems ~= id;
|
||||||
return;
|
return;
|
||||||
|
@ -202,7 +191,7 @@ final class SyncEngine
|
||||||
if (parentId) {
|
if (parentId) {
|
||||||
path = itemdb.computePath(parentId) ~ "/" ~ name;
|
path = itemdb.computePath(parentId) ~ "/" ~ name;
|
||||||
// selective sync
|
// selective sync
|
||||||
if (isPathExcluded(path, selectiveSyncPaths)) {
|
if (selectiveSync.isPathExcluded(path)) {
|
||||||
log.vlog("Filtered out: ", path);
|
log.vlog("Filtered out: ", path);
|
||||||
skippedItems ~= id;
|
skippedItems ~= id;
|
||||||
return;
|
return;
|
||||||
|
@ -390,12 +379,12 @@ final class SyncEngine
|
||||||
log.vlog(item.id, " ", item.name);
|
log.vlog(item.id, " ", item.name);
|
||||||
|
|
||||||
// skip filtered items
|
// skip filtered items
|
||||||
if (!item.name.matchFirst(skipFile).empty) {
|
if (selectiveSync.isNameExcluded(item.name)) {
|
||||||
log.vlog("Filtered out");
|
log.vlog("Filtered out");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
string path = itemdb.computePath(item.id);
|
string path = itemdb.computePath(item.id);
|
||||||
if (isPathExcluded(path, selectiveSyncPaths)) {
|
if (selectiveSync.isPathExcluded(path)) {
|
||||||
log.vlog("Filtered out: ", path);
|
log.vlog("Filtered out: ", path);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -484,10 +473,10 @@ final class SyncEngine
|
||||||
|
|
||||||
// skip filtered items
|
// skip filtered items
|
||||||
if (path != ".") {
|
if (path != ".") {
|
||||||
if (!baseName(path).matchFirst(skipFile).empty) {
|
if (selectiveSync.isNameExcluded(baseName(path))) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (isPathExcluded(path, selectiveSyncPaths)) {
|
if (selectiveSync.isPathExcluded(path)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
26
src/util.d
26
src/util.d
|
@ -1,4 +1,3 @@
|
||||||
import std.algorithm;
|
|
||||||
import std.conv;
|
import std.conv;
|
||||||
import std.digest.crc;
|
import std.digest.crc;
|
||||||
import std.file;
|
import std.file;
|
||||||
|
@ -83,7 +82,7 @@ Regex!char wild2regex(const(char)[] pattern)
|
||||||
// return true if the network connection is available
|
// return true if the network connection is available
|
||||||
bool testNetwork()
|
bool testNetwork()
|
||||||
{
|
{
|
||||||
HTTP http = HTTP("https://login.live.com");
|
HTTP http = HTTP("https://login.microsoftonline.com");
|
||||||
http.method = HTTP.Method.head;
|
http.method = HTTP.Method.head;
|
||||||
return http.perform(ThrowOnError.no) == 0;
|
return http.perform(ThrowOnError.no) == 0;
|
||||||
}
|
}
|
||||||
|
@ -99,32 +98,9 @@ bool multiGlobMatch(const(char)[] path, const(char)[] pattern)
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// test if the given path is not included in the allowed paths
|
|
||||||
// if there are no allowed paths always return false
|
|
||||||
bool isPathExcluded(string path, string[] allowedPaths)
|
|
||||||
{
|
|
||||||
// always allow the root
|
|
||||||
if (path == ".") return false;
|
|
||||||
// if there are no allowed paths always return false
|
|
||||||
if (allowedPaths.empty) return false;
|
|
||||||
|
|
||||||
path = buildNormalizedPath(path);
|
|
||||||
foreach (allowed; allowedPaths) {
|
|
||||||
auto comm = commonPrefix(path, allowed);
|
|
||||||
if (comm.length == path.length || comm.length == allowed.length) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
unittest
|
unittest
|
||||||
{
|
{
|
||||||
assert(multiGlobMatch(".hidden", ".*"));
|
assert(multiGlobMatch(".hidden", ".*"));
|
||||||
assert(multiGlobMatch(".hidden", "file|.*"));
|
assert(multiGlobMatch(".hidden", "file|.*"));
|
||||||
assert(!multiGlobMatch("foo.bar", "foo|bar"));
|
assert(!multiGlobMatch("foo.bar", "foo|bar"));
|
||||||
assert(isPathExcluded("Documents2", ["Documents"]));
|
|
||||||
assert(isPathExcluded("Hello/World", ["Hello/John"]));
|
|
||||||
assert(!isPathExcluded("Documents", ["Documents"]));
|
|
||||||
assert(!isPathExcluded("Documents/a.txt", ["Documents"]));
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue