abraunegg-onedrive/src/monitor.d

// What is this module called?
module monitor;

// What does this module require to function?
import core.stdc.errno;
import core.stdc.stdlib;
import core.sys.linux.sys.inotify;
import core.sys.posix.poll;
import core.sys.posix.unistd;
import core.sys.posix.sys.select;
import core.thread;
import core.time;
import std.algorithm;
import std.concurrency;
import std.exception;
import std.file;
import std.path;
import std.process;
import std.regex;
import std.stdio;
import std.string;
import std.conv;

// What other modules that we have created do we need to import?
import config;
import util;
import log;
import clientSideFiltering;

// Relevant inotify events
private immutable uint32_t mask = IN_CLOSE_WRITE | IN_CREATE | IN_DELETE | IN_MOVE | IN_IGNORED | IN_Q_OVERFLOW;

class MonitorException: ErrnoException {
    @safe this(string msg, string file = __FILE__, size_t line = __LINE__) {
        super(msg, file, line);
    }
}

class MonitorBackgroundWorker {
	// inotify file descriptor
	int fd;
	Pipe p;
	bool isAlive;

	this() {
		isAlive = true;
		p = pipe();
	}

	shared void initialise() {
		fd = inotify_init();
		if (fd < 0) throw new MonitorException("inotify_init failed");
	}

	// Add this path to be monitored
	shared int addInotifyWatch(string pathname) {
		int wd = inotify_add_watch(fd, toStringz(pathname), mask);
		if (wd < 0) {
			if (errno() == ENOSPC) {
				// Get the current value
				ulong maxInotifyWatches = to!int(strip(readText("/proc/sys/fs/inotify/max_user_watches")));
				addLogEntry("The user limit on the total number of inotify watches has been reached.");
				addLogEntry("Your current limit of inotify watches is: " ~ to!string(maxInotifyWatches));
				addLogEntry("It is recommended that you change the max number of inotify watches to at least double your existing value.");
				addLogEntry("To change the current max number of watches to " ~ to!string((maxInotifyWatches * 2)) ~ " run:");
				addLogEntry("EXAMPLE: sudo sysctl fs.inotify.max_user_watches=" ~ to!string((maxInotifyWatches * 2)));
			}
			if (errno() == 13) {
				addLogEntry("WARNING: inotify_add_watch failed - permission denied: " ~ pathname, ["verbose"]);
			}
			// Flag any other errors
			addLogEntry("ERROR: inotify_add_watch failed: " ~ pathname);
			return wd;
		}
		
		// Add path to inotify watch - required regardless if a '.folder' or 'folder'
		addLogEntry("inotify_add_watch successfully added for: " ~ pathname, ["debug"]);
		
		// Do we log that we are monitoring this directory?
		if (isDir(pathname)) {
			// Log that this is directory is being monitored
			addLogEntry("Monitoring directory: " ~ pathname, ["verbose"]);
		}
		return wd;
	}

	shared int removeInotifyWatch(int wd) {
		return inotify_rm_watch(fd, wd);
	}

	shared void watch(Tid callerTid) {
		// On failure, send -1 to caller
		int res;

		// wait for the caller to be ready
		receiveOnly!int();

		while (isAlive) {
			fd_set fds;
			FD_ZERO (&fds);
			FD_SET(fd, &fds);
			// Listen for messages from the caller
			FD_SET((cast()p).readEnd.fileno, &fds);
			
			res = select(FD_SETSIZE, &fds, null, null, null);

			if(res == -1) {
				if(errno() == EINTR) {
					// Received an interrupt signal but no events are available
					// directly watch again
				} else {
					// Error occurred, tell caller to terminate.
					callerTid.send(-1);
					break;
				}
			} else {
				// Wake up caller
				callerTid.send(1);

				// wait for the caller to be ready
				if (isAlive)
					isAlive = receiveOnly!bool();
			}
		}
	}

	shared void interrupt() {
		isAlive = false;
		(cast()p).writeEnd.writeln("done");
		(cast()p).writeEnd.flush();
	}

	shared void shutdown() {
		isAlive = false;
		if (fd > 0) {
			close(fd);
			fd = 0;
			(cast()p).close();
		}
	}
}

void startMonitorJob(shared(MonitorBackgroundWorker) worker, Tid callerTid)
{
	try {
    	worker.watch(callerTid);
	} catch (OwnerTerminated error) {
		// caller is terminated
		worker.shutdown();
	}
}

enum ActionType {
	moved,
	deleted, 
	changed,
	createDir
}

struct Action {
	ActionType type;
	bool skipped;
	string src;
	string dst;
}

struct ActionHolder {
	Action[] actions;
	ulong[string] srcMap;

	void append(ActionType type, string src, string dst=null) {
		ulong[] pendingTargets;
		switch (type) {
			case ActionType.changed:
				if (src in srcMap && actions[srcMap[src]].type == ActionType.changed) {
					// skip duplicate operations
					return;
				}
				break;
			case ActionType.createDir:
				break;
			case ActionType.deleted:
				if (src in srcMap) {
					ulong pendingTarget = srcMap[src];
					// Skip operations require reading local file that is gone
					switch (actions[pendingTarget].type) {
						case ActionType.changed:
						case ActionType.createDir:
							actions[srcMap[src]].skipped = true;
							srcMap.remove(src);
							break;
						default:
							break;
					}
				}
				break;
			case ActionType.moved:
				for(int i = 0; i < actions.length; i++) {
					// Only match for latest operation
					if (actions[i].src in srcMap) {
						switch (actions[i].type) {
							case ActionType.changed:
							case ActionType.createDir:
								// check if the source is the prefix of the target
								string prefix = src ~ "/";
								string target = actions[i].src;
								if (prefix[0] != '.')
									prefix = "./" ~ prefix;
								if (target[0] != '.')
									target = "./" ~ target;
								string comm = commonPrefix(prefix, target);
								if (src == actions[i].src || comm.length == prefix.length) {
									// Hold operations require reading local file that is moved after the target is moved online
									pendingTargets ~= i;
									actions[i].skipped = true;
									srcMap.remove(actions[i].src);
									if (comm.length == target.length)
										actions[i].src = dst;
									else
										actions[i].src = dst ~ target[comm.length - 1 .. target.length];
								}
								break;
							default:
								break;
						}
					}
				}
				break;
			default:
				break;
		}
		actions ~= Action(type, false, src, dst);
		srcMap[src] = actions.length - 1;
		
		foreach (pendingTarget; pendingTargets) {
			actions ~= actions[pendingTarget];
			actions[$-1].skipped = false;
			srcMap[actions[$-1].src] = actions.length - 1;
		}
	}
}

final class Monitor {
	// Class variables
	ApplicationConfig appConfig;
	ClientSideFiltering selectiveSync;

	// Are we verbose in logging output
	bool verbose = false;
	// skip symbolic links
	bool skip_symlinks = false;
	// check for .nosync if enabled
	bool check_nosync = false;
	// check if initialised
	bool initialised = false;
	// Worker Tid
	Tid workerTid;
	
	// Configure Private Class Variables
	shared(MonitorBackgroundWorker) worker;
	// map every inotify watch descriptor to its directory
	private string[int] wdToDirName;
	// map the inotify cookies of move_from events to their path
	private string[int] cookieToPath;
	// buffer to receive the inotify events
	private void[] buffer;

	// Configure function delegates
	void delegate(string path) onDirCreated;
	void delegate(string[] path) onFileChanged;
	void delegate(string path) onDelete;
	void delegate(string from, string to) onMove;
	
	// List of paths that were moved, not deleted
	bool[string] movedNotDeleted;

	ActionHolder actionHolder;
	
	// Configure the class varaible to consume the application configuration including selective sync
	this(ApplicationConfig appConfig, ClientSideFiltering selectiveSync) {
		this.appConfig = appConfig;
		this.selectiveSync = selectiveSync;
	}
	
	// Initialise the monitor class
	void initialise() {
		// Configure the variables
		skip_symlinks = appConfig.getValueBool("skip_symlinks");
		check_nosync = appConfig.getValueBool("check_nosync");
		if (appConfig.getValueLong("verbose") > 0) {
			verbose = true;
		}
		
		assert(onDirCreated && onFileChanged && onDelete && onMove);
		if (!buffer) buffer = new void[4096];
		worker = cast(shared) new MonitorBackgroundWorker;
		worker.initialise();

		// from which point do we start watching for changes?
		string monitorPath;
		if (appConfig.getValueString("single_directory") != ""){
			// single directory in use, monitor only this path
			monitorPath = "./" ~ appConfig.getValueString("single_directory");
		} else {
			// default 
			monitorPath = ".";
		}
		addRecursive(monitorPath);
		
		// Start monitoring
		workerTid = spawn(&startMonitorJob, worker, thisTid);

		initialised = true;
	}

	// Communication with worker
	void send(bool isAlive) {
		workerTid.send(isAlive);
	}

	// Shutdown the monitor class
	void shutdown() {
		if(!initialised)
			return;
		initialised = false;
		// Release all resources
		removeAll();
		// Notify the worker that the monitor has been shutdown
		worker.interrupt();
		send(false);
		wdToDirName = null;
	}

	// Recursivly add this path to be monitored
	private void addRecursive(string dirname) {
		// skip non existing/disappeared items
		if (!exists(dirname)) {
			addLogEntry("Not adding non-existing/disappeared directory: " ~ dirname, ["verbose"]);
			return;
		}

		// Skip the monitoring of any user filtered items
		if (dirname != ".") {
			// Is the directory name a match to a skip_dir entry?
			// The path that needs to be checked needs to include the '/'
			// This due to if the user has specified in skip_dir an exclusive path: '/path' - that is what must be matched
			if (isDir(dirname)) {
				if (selectiveSync.isDirNameExcluded(dirname.strip('.'))) {
					// dont add a watch for this item
					addLogEntry("Skipping monitoring due to skip_dir match: " ~ dirname, ["debug"]);
					return;
				}
			}
			if (isFile(dirname)) {
				// Is the filename a match to a skip_file entry?
				// The path that needs to be checked needs to include the '/'
				// This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched
				if (selectiveSync.isFileNameExcluded(dirname.strip('.'))) {
					// dont add a watch for this item
					addLogEntry("Skipping monitoring due to skip_file match: " ~ dirname, ["debug"]);
					return;
				}
			}
			// is the path exluded by sync_list?
			if (selectiveSync.isPathExcludedViaSyncList(buildNormalizedPath(dirname))) {
				// dont add a watch for this item
				addLogEntry("Skipping monitoring due to sync_list match: " ~ dirname, ["debug"]);
				return;
			}
		}
		
		// skip symlinks if configured
		if (isSymlink(dirname)) {
			// if config says so we skip all symlinked items
			if (skip_symlinks) {
				// dont add a watch for this directory
				return;
			}
		}
		
		// Do we need to check for .nosync? Only if check_nosync is true
		if (check_nosync) {
			if (exists(buildNormalizedPath(dirname) ~ "/.nosync")) {
				addLogEntry("Skipping watching path - .nosync found & --check-for-nosync enabled: " ~ buildNormalizedPath(dirname), ["verbose"]);
				return;
			}
		}

		if (isDir(dirname)) {
			// This is a directory			
			// is the path exluded if skip_dotfiles configured and path is a .folder?
			if ((selectiveSync.getSkipDotfiles()) && (isDotFile(dirname))) {
				// dont add a watch for this directory
				return;
			}
		}
		
		// passed all potential exclusions
		// add inotify watch for this path / directory / file
		addLogEntry("Calling worker.addInotifyWatch() for this dirname: " ~ dirname, ["debug"]);
		int wd = worker.addInotifyWatch(dirname);
		if (wd > 0) {
			wdToDirName[wd] = buildNormalizedPath(dirname) ~ "/";
		}
		
		// if this is a directory, recursivly add this path
		if (isDir(dirname)) {
			// try and get all the directory entities for this path
			try {
				auto pathList = dirEntries(dirname, SpanMode.shallow, false);
				foreach(DirEntry entry; pathList) {
					if (entry.isDir) {
						addLogEntry("Calling addRecursive() for this directory: " ~ entry.name, ["debug"]);
						addRecursive(entry.name);
					}
				}
			// catch any error which is generated
			} catch (std.file.FileException e) {
				// Standard filesystem error
				displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
				return;
			} catch (Exception e) {
				// Issue #1154 handling
				// Need to check for: Failed to stat file in error message
				if (canFind(e.msg, "Failed to stat file")) {
					// File system access issue
					addLogEntry("ERROR: The local file system returned an error with the following message:");
					addLogEntry("  Error Message: " ~ e.msg);
					addLogEntry("ACCESS ERROR: Please check your UID and GID access to this file, as the permissions on this file is preventing this application to read it");
					addLogEntry("\nFATAL: Forcing exiting application to avoid deleting data due to local file system access issues\n");
					// Must exit here
					exit(-1);
				} else {
					// some other error
					displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
					return;
				}
			}
		}
	}

	// Remove a watch descriptor
	private void removeAll() {
		string[int] copy = wdToDirName.dup;
		foreach (wd, path; copy) {
			remove(wd);
		}
	}

	private void remove(int wd) {
		assert(wd in wdToDirName);
		int ret = worker.removeInotifyWatch(wd);
		if (ret < 0) throw new MonitorException("inotify_rm_watch failed");
		addLogEntry("Monitored directory removed: " ~ to!string(wdToDirName[wd]), ["verbose"]);
		wdToDirName.remove(wd);
	}

	// Remove the watch descriptors associated to the given path
	private void remove(const(char)[] path) {
		path ~= "/";
		foreach (wd, dirname; wdToDirName) {
			if (dirname.startsWith(path)) {
				int ret = worker.removeInotifyWatch(wd);
				if (ret < 0) throw new MonitorException("inotify_rm_watch failed");
				wdToDirName.remove(wd);
				addLogEntry("Monitored directory removed: " ~ dirname, ["verbose"]);
			}
		}
	}

	// Return the file path from an inotify event
	private string getPath(const(inotify_event)* event) {
		string path = wdToDirName[event.wd];
		if (event.len > 0) path ~= fromStringz(event.name.ptr);
		addLogEntry("inotify path event for: " ~ path, ["debug"]);
		return path;
	}

	// Update
	void update(bool useCallbacks = true) {
		if(!initialised)
			return;
	
		pollfd fds = {
			fd: worker.fd,
			events: POLLIN
		};

		while (true) {
			bool hasNotification = false;
			int sleep_counter = 0;
			// Batch events up to 5 seconds
			while (sleep_counter < 5) {
				int ret = poll(&fds, 1, 0);
				if (ret == -1) throw new MonitorException("poll failed");
				else if (ret == 0) break; // no events available
				hasNotification = true;
				size_t length = read(worker.fd, buffer.ptr, buffer.length);
				if (length == -1) throw new MonitorException("read failed");

				int i = 0;
				while (i < length) {
					inotify_event *event = cast(inotify_event*) &buffer[i];
					string path;
					string evalPath;
					
					// inotify event debug
					addLogEntry("inotify event wd: " ~ to!string(event.wd), ["debug"]);
					addLogEntry("inotify event mask: " ~ to!string(event.mask), ["debug"]);
					addLogEntry("inotify event cookie: " ~ to!string(event.cookie), ["debug"]);
					addLogEntry("inotify event len: " ~ to!string(event.len), ["debug"]);
					addLogEntry("inotify event name: " ~ to!string(event.name), ["debug"]);
					
					// inotify event handling
					if (event.mask & IN_ACCESS) addLogEntry("inotify event flag: IN_ACCESS", ["debug"]);
					if (event.mask & IN_MODIFY) addLogEntry("inotify event flag: IN_MODIFY", ["debug"]);
					if (event.mask & IN_ATTRIB) addLogEntry("inotify event flag: IN_ATTRIB", ["debug"]);
					if (event.mask & IN_CLOSE_WRITE) addLogEntry("inotify event flag: IN_CLOSE_WRITE", ["debug"]);
					if (event.mask & IN_CLOSE_NOWRITE) addLogEntry("inotify event flag: IN_CLOSE_NOWRITE", ["debug"]);
					if (event.mask & IN_MOVED_FROM) addLogEntry("inotify event flag: IN_MOVED_FROM", ["debug"]);
					if (event.mask & IN_MOVED_TO) addLogEntry("inotify event flag: IN_MOVED_TO", ["debug"]);
					if (event.mask & IN_CREATE) addLogEntry("inotify event flag: IN_CREATE", ["debug"]);
					if (event.mask & IN_DELETE) addLogEntry("inotify event flag: IN_DELETE", ["debug"]);
					if (event.mask & IN_DELETE_SELF) addLogEntry("inotify event flag: IN_DELETE_SELF", ["debug"]);
					if (event.mask & IN_MOVE_SELF) addLogEntry("inotify event flag: IN_MOVE_SELF", ["debug"]);
					if (event.mask & IN_UNMOUNT) addLogEntry("inotify event flag: IN_UNMOUNT", ["debug"]);
					if (event.mask & IN_Q_OVERFLOW) addLogEntry("inotify event flag: IN_Q_OVERFLOW", ["debug"]);
					if (event.mask & IN_IGNORED) addLogEntry("inotify event flag: IN_IGNORED", ["debug"]);
					if (event.mask & IN_CLOSE) addLogEntry("inotify event flag: IN_CLOSE", ["debug"]);
					if (event.mask & IN_MOVE) addLogEntry("inotify event flag: IN_MOVE", ["debug"]);
					if (event.mask & IN_ONLYDIR) addLogEntry("inotify event flag: IN_ONLYDIR", ["debug"]);
					if (event.mask & IN_DONT_FOLLOW) addLogEntry("inotify event flag: IN_DONT_FOLLOW", ["debug"]);
					if (event.mask & IN_EXCL_UNLINK) addLogEntry("inotify event flag: IN_EXCL_UNLINK", ["debug"]);
					if (event.mask & IN_MASK_ADD) addLogEntry("inotify event flag: IN_MASK_ADD", ["debug"]);
					if (event.mask & IN_ISDIR) addLogEntry("inotify event flag: IN_ISDIR", ["debug"]);
					if (event.mask & IN_ONESHOT) addLogEntry("inotify event flag: IN_ONESHOT", ["debug"]);
					if (event.mask & IN_ALL_EVENTS) addLogEntry("inotify event flag: IN_ALL_EVENTS", ["debug"]);
					
					// skip events that need to be ignored
					if (event.mask & IN_IGNORED) {
						// forget the directory associated to the watch descriptor
						wdToDirName.remove(event.wd);
						goto skip;
					} else if (event.mask & IN_Q_OVERFLOW) {
						throw new MonitorException("inotify overflow, inotify events will be missing");
					}

					// if the event is not to be ignored, obtain path
					path = getPath(event);
					// configure the skip_dir & skip skip_file comparison item
					evalPath = path.strip('.');
					
					// Skip events that should be excluded based on application configuration
					// We cant use isDir or isFile as this information is missing from the inotify event itself
					// Thus this causes a segfault when attempting to query this - https://github.com/abraunegg/onedrive/issues/995
					
					// Based on the 'type' of event & object type (directory or file) check that path against the 'right' user exclusions
					// Directory events should only be compared against skip_dir and file events should only be compared against skip_file
					if (event.mask & IN_ISDIR) {
						// The event in question contains IN_ISDIR event mask, thus highly likely this is an event on a directory
						// This due to if the user has specified in skip_dir an exclusive path: '/path' - that is what must be matched
						if (selectiveSync.isDirNameExcluded(evalPath)) {
							// The path to evaluate matches a path that the user has configured to skip
							goto skip;
						}
					} else {
						// The event in question missing the IN_ISDIR event mask, thus highly likely this is an event on a file
						// This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched
						if (selectiveSync.isFileNameExcluded(evalPath)) {
							// The path to evaluate matches a file that the user has configured to skip
							goto skip;
						}
					}
					
					// is the path, excluded via sync_list
					if (selectiveSync.isPathExcludedViaSyncList(path)) {
						// The path to evaluate matches a directory or file that the user has configured not to include in the sync
						goto skip;
					}
					
					// handle the inotify events
					if (event.mask & IN_MOVED_FROM) {
						addLogEntry("event IN_MOVED_FROM: " ~ path, ["debug"]);
						cookieToPath[event.cookie] = path;
						movedNotDeleted[path] = true; // Mark as moved, not deleted
					} else if (event.mask & IN_MOVED_TO) {
						addLogEntry("event IN_MOVED_TO: " ~ path, ["debug"]);
						if (event.mask & IN_ISDIR) addRecursive(path);
						auto from = event.cookie in cookieToPath;
						if (from) {
							cookieToPath.remove(event.cookie);
							if (useCallbacks) actionHolder.append(ActionType.moved, *from, path);
							movedNotDeleted.remove(*from); // Clear moved status
						} else {
							// Handle file moved in from outside
							if (event.mask & IN_ISDIR) {
								if (useCallbacks) actionHolder.append(ActionType.createDir, path);
							} else {
								if (useCallbacks) actionHolder.append(ActionType.changed, path);
							}
						}
					} else if (event.mask & IN_CREATE) {
						addLogEntry("event IN_CREATE: " ~ path, ["debug"]);
						if (event.mask & IN_ISDIR) {
							addRecursive(path);
							if (useCallbacks) actionHolder.append(ActionType.createDir, path);
						}
					} else if (event.mask & IN_DELETE) {
						if (path in movedNotDeleted) {
							movedNotDeleted.remove(path); // Ignore delete for moved files
						} else {
							addLogEntry("event IN_DELETE: " ~ path, ["debug"]);
							if (useCallbacks) actionHolder.append(ActionType.deleted, path);
						}
					} else if ((event.mask & IN_CLOSE_WRITE) && !(event.mask & IN_ISDIR)) {
						addLogEntry("event IN_CLOSE_WRITE and not IN_ISDIR: " ~ path, ["debug"]);
						if (useCallbacks) actionHolder.append(ActionType.changed, path);
					} else {
						addLogEntry("event unhandled: " ~ path, ["debug"]);
						assert(0);
					}

					skip:
					i += inotify_event.sizeof + event.len;
				}

				// Sleep for one second to prevent missing fast-changing events.
				if (poll(&fds, 1, 0) == 0) {
					sleep_counter += 1;
					Thread.sleep(dur!"seconds"(1));
				}
			}
			if (!hasNotification) break;
			processChanges();

			// Assume that the items moved outside the watched directory have been deleted
			foreach (cookie, path; cookieToPath) {
				addLogEntry("Deleting cookie|watch (post loop): " ~ path, ["debug"]);
				if (useCallbacks) onDelete(path);
				remove(path);
				cookieToPath.remove(cookie);
			}
			// Debug Log that all inotify events are flushed
			addLogEntry("inotify events flushed", ["debug"]);
		}
	}
  
	private void processChanges() {
		string[] changes;

		foreach(action; actionHolder.actions) {
			if (action.skipped)
				continue;
			switch (action.type) {
				case ActionType.changed:
					changes ~= action.src;
					break;
				case ActionType.deleted:
					onDelete(action.src);
					break;
				case ActionType.createDir:
					onDirCreated(action.src);
					break;
				case ActionType.moved:
					onMove(action.src, action.dst);
					break;
				default:
					break;
			}
		}
		if (!changes.empty)
			onFileChanged(changes);

		object.destroy(actionHolder);
	}
}