abraunegg-onedrive/src/config.d

import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit;
import std.file, std.string, std.regex, std.stdio, std.process, std.algorithm.searching, std.getopt, std.conv, std.path;
import std.algorithm.sorting: sort;
import selective;
static import log;

final class Config
{
	// application defaults
	public string defaultSyncDir = "~/OneDrive";
	public string defaultSkipFile = "~*|.~*|*.tmp";
	public string defaultSkipDir = "";
	// application set items
	public string refreshTokenFilePath = "";
	public string deltaLinkFilePath = "";
	public string databaseFilePath = "";
	public string databaseFilePathDryRun = "";
	public string uploadStateFilePath = "";
	public string syncListFilePath = "";
	public string homePath = "";
	public string configDirName = "";
	public string configFileSyncDir = "";
	public string configFileSkipFile = "";
	public string configFileSkipDir = "";
	private string userConfigFilePath = "";
	// was the application just authorised - paste of response uri
	public bool applicationAuthorizeResponseUri = false;
	// hashmap for the values found in the user config file
	// ARGGGG D is stupid and cannot make hashmap initializations!!!
	// private string[string] foobar = [ "aa": "bb" ] does NOT work!!!
	private string[string] stringValues;
	private bool[string] boolValues;
	private long[string] longValues;
	public auto configRegex = ctRegex!(`^(\w+)\s*=\s*"(.*)"\s*$`);

	this(string confdirOption)
	{
		// default configuration - entries in config file ~/.config/onedrive/config
		// an entry here means it can be set via the config file if there is a coresponding read and set in update_from_args()
		stringValues["sync_dir"] = defaultSyncDir;
		stringValues["skip_file"] = defaultSkipFile;
		stringValues["skip_dir"] = defaultSkipDir;
		stringValues["log_dir"] = "/var/log/onedrive/";
		stringValues["drive_id"] = "";
		stringValues["user_agent"] = "";
		boolValues["upload_only"] = false;
		boolValues["check_nomount"] = false;
		boolValues["check_nosync"] = false;
		boolValues["download_only"] = false;
		boolValues["disable_notifications"] = false;
		boolValues["disable_upload_validation"] = false;
		boolValues["enable_logging"] = false;
		boolValues["force_http_11"] = false;
		boolValues["force_http_2"] = false;
		boolValues["local_first"] = false;
		boolValues["no_remote_delete"] = false;
		boolValues["skip_symlinks"] = false;
		boolValues["debug_https"] = false;
		boolValues["skip_dotfiles"] = false;
		boolValues["dry_run"] = false;
		boolValues["sync_root_files"] = false;
		longValues["verbose"] = log.verbose; // might be initialized by the first getopt call!
		longValues["monitor_interval"] = 45;
		longValues["skip_size"] = 0;
		longValues["min_notify_changes"] = 5;
		longValues["monitor_log_frequency"] = 5;
		// Number of n sync runs before performing a full local scan of sync_dir
		// By default 10 which means every ~7.5 minutes a full disk scan of sync_dir will occur
		longValues["monitor_fullscan_frequency"] = 10;
		// Number of children in a path that is locally removed which will be classified as a 'big data delete'
		longValues["classify_as_big_delete"] = 1000;
		// Delete source after successful transfer
		boolValues["remove_source_files"] = false;
		// Strict matching for skip_dir
		boolValues["skip_dir_strict_match"] = false;
		// Allow for a custom Client ID / Application ID to be used to replace the inbuilt default
		// This is a config file option ONLY
		stringValues["application_id"] = "";
		// allow for resync to be set via config file
		boolValues["resync"] = false;
		// Ignore data safety checks and overwrite local data rather than preserve & rename
		// This is a config file option ONLY
		boolValues["bypass_data_preservation"] = false;
		
		// DEVELOPER OPTIONS 
		// display_memory = true | false
		//  - It may be desirable to display the memory usage of the application to assist with diagnosing memory issues with the application
		//  - This is especially beneficial when debugging or performing memory tests with Valgrind
		boolValues["display_memory"] = false;
		// monitor_max_loop = long value
		//  - It may be desirable to, when running in monitor mode, force monitor mode to 'quit' after X number of loops
		//  - This is especially beneficial when debugging or performing memory tests with Valgrind
		longValues["monitor_max_loop"] = 0;

		// Determine the users home directory. 
		// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
		// Check for HOME environment variable
		if (environment.get("HOME") != ""){
			// Use HOME environment variable
			log.vdebug("homePath: HOME environment variable set");
			homePath = environment.get("HOME");
		} else {
			if ((environment.get("SHELL") == "") && (environment.get("USER") == "")){
				// No shell is set or username - observed case when running as systemd service under CentOS 7.x
				log.vdebug("homePath: WARNING - no HOME environment variable set");
				log.vdebug("homePath: WARNING - no SHELL environment variable set");
				log.vdebug("homePath: WARNING - no USER environment variable set");
				homePath = "/root";
			} else {
				// A shell & valid user is set, but no HOME is set, use ~ which can be expanded
				log.vdebug("homePath: WARNING - no HOME environment variable set");
				homePath = "~";
			}
		}
	
		// Output homePath calculation
		log.vdebug("homePath: ", homePath);
		
		// Determine the correct configuration directory to use
		string configDirBase;
		if (confdirOption != "") {
			// A CLI 'confdir' was passed in
			log.vdebug("configDirName: CLI override to set configDirName to: ", confdirOption);
			if (canFind(confdirOption,"~")) {
				// A ~ was found
				log.vdebug("configDirName: A '~' was found in configDirName, using the calculated 'homePath' to replace '~'");
				configDirName = homePath ~ strip(confdirOption,"~","~");
			} else {
				configDirName = confdirOption;
			}
		} else {
			// Determine the base directory relative to which user specific configuration files should be stored.
			if (environment.get("XDG_CONFIG_HOME") != ""){
				log.vdebug("configDirBase: XDG_CONFIG_HOME environment variable set");
				configDirBase = environment.get("XDG_CONFIG_HOME");
			} else {
				// XDG_CONFIG_HOME does not exist on systems where X11 is not present - ie - headless systems / servers
				log.vdebug("configDirBase: WARNING - no XDG_CONFIG_HOME environment variable set");
				configDirBase = homePath ~ "/.config";
			}
	
			// Output configDirBase calculation
			log.vdebug("configDirBase: ", configDirBase);
			// Set the default application configuration directory
			log.vdebug("configDirName: Configuring application to use default config path");
			// configDirBase contains the correct path so we do not need to check for presence of '~'
			configDirName = configDirBase ~ "/onedrive";
		}
		
		// Config directory options all determined
		// configDirName has a trailing /
		log.vlog("Using Config Dir: ", configDirName);
		if (!exists(configDirName)) mkdirRecurse(configDirName);
		
		// Update application set variables based on configDirName
		refreshTokenFilePath = buildNormalizedPath(configDirName ~ "/refresh_token");
		deltaLinkFilePath = buildNormalizedPath(configDirName ~ "/delta_link");
		databaseFilePath = buildNormalizedPath(configDirName ~ "/items.sqlite3");
		databaseFilePathDryRun = buildNormalizedPath(configDirName ~ "/items-dryrun.sqlite3");
		uploadStateFilePath = buildNormalizedPath(configDirName ~ "/resume_upload");
		userConfigFilePath = buildNormalizedPath(configDirName ~ "/config");
		syncListFilePath = buildNormalizedPath(configDirName ~ "/sync_list");
		
		// Debug Output for application set variables based on configDirName
		log.vdebug("refreshTokenFilePath = ", refreshTokenFilePath);
		log.vdebug("deltaLinkFilePath = ", deltaLinkFilePath);
		log.vdebug("databaseFilePath = ", databaseFilePath);
		log.vdebug("databaseFilePathDryRun = ", databaseFilePathDryRun);
		log.vdebug("uploadStateFilePath = ", uploadStateFilePath);
		log.vdebug("userConfigFilePath = ", userConfigFilePath);
		log.vdebug("syncListFilePath = ", syncListFilePath);
	}

	bool initialize()
	{
		// Initialise the application
		if (!exists(userConfigFilePath)) {
			// configuration file does not exist
			log.vlog("No config file found, using application defaults");
			return true;
		} else {
			// configuration file exists
			// can we load the configuration file without error?
			if (load(userConfigFilePath)) {
				// configuration file loaded without error
				log.log("Configuration file successfully loaded");
				return true;
			} else {
				// there was a problem loading the configuration file
				log.log("Configuration file has errors - please check your configuration");
				return false;
			}
		}
	}

	void update_from_args(string[] args)
	{
		// Add additional options that are NOT configurable via config file
		stringValues["create_directory"]  = "";
		stringValues["destination_directory"] = "";
		stringValues["get_file_link"]     = "";
		stringValues["get_o365_drive_id"] = "";
		stringValues["remove_directory"]  = "";
		stringValues["single_directory"]  = "";
		stringValues["source_directory"]  = "";
		stringValues["auth_files"]        = "";
		boolValues["display_config"]      = false;
		boolValues["display_sync_status"] = false;
		boolValues["print_token"]         = false;
		boolValues["logout"]              = false;
		boolValues["monitor"]             = false;
		boolValues["synchronize"]         = false;
		boolValues["force"]               = false;
		boolValues["remove_source_files"] = false;
		boolValues["skip_dir_strict_match"] = false;

		// Application Startup option validation
		try {
			string tmpStr;
			bool tmpBol;
			long tmpVerb;
			auto opt = getopt(
				args,
				std.getopt.config.bundling,
				std.getopt.config.caseSensitive,
				"auth-files",
					"Perform authentication not via interactive dialog but via files read/writes to these files.",
					&stringValues["auth_files"],
				"check-for-nomount",
					"Check for the presence of .nosync in the syncdir root. If found, do not perform sync.", 
					&boolValues["check_nomount"],
				"check-for-nosync",
					"Check for the presence of .nosync in each directory. If found, skip directory from sync.",
					&boolValues["check_nosync"],
				"classify-as-big-delete",
					"Number of children in a path that is locally removed which will be classified as a 'big data delete'",
					&longValues["classify_as_big_delete"],
				"create-directory",
					"Create a directory on OneDrive - no sync will be performed.",
					&stringValues["create_directory"],
				"debug-https", 
					"Debug OneDrive HTTPS communication.", 
					&boolValues["debug_https"],
				"destination-directory",
					"Destination directory for renamed or move on OneDrive - no sync will be performed.",
					&stringValues["destination_directory"],
				"disable-notifications",
					"Do not use desktop notifications in monitor mode.",
					&boolValues["disable_notifications"],
				"disable-upload-validation",
					"Disable upload validation when uploading to OneDrive",
					&boolValues["disable_upload_validation"],
				"display-config",
					"Display what options the client will use as currently configured - no sync will be performed.",
					&boolValues["display_config"],
				"display-sync-status",
					"Display the sync status of the client - no sync will be performed.",
					&boolValues["display_sync_status"],
				"download-only",
					"Only download remote changes",
					&boolValues["download_only"],
				"dry-run",
					"Perform a trial sync with no changes made",
					&boolValues["dry_run"],
				"enable-logging",
					"Enable client activity to a separate log file",
					&boolValues["enable_logging"],
				"force-http-1.1",
					"Force the use of HTTP/1.1 for all operations (DEPRECIATED)",
					&boolValues["force_http_11"],
				"force-http-2",
					"Force the use of HTTP/2 for all operations where applicable",
					&boolValues["force_http_2"],
				"force",
					"Force the deletion of data when a 'big delete' is detected",
					&boolValues["force"],
				"get-file-link",
					"Display the file link of a synced file",
					&stringValues["get_file_link"],
				"get-O365-drive-id",
					"Query and return the Office 365 Drive ID for a given Office 365 SharePoint Shared Library",
					&stringValues["get_o365_drive_id"],
				"local-first",
					"Synchronize from the local directory source first, before downloading changes from OneDrive.",
					&boolValues["local_first"],
				"log-dir",
					"Directory where logging output is saved to, needs to end with a slash.",
					&stringValues["log_dir"],
				"logout",
					"Logout the current user",
					&boolValues["logout"],
				"min-notify-changes",
					"Minimum number of pending incoming changes necessary to trigger a desktop notification",
					&longValues["min_notify_changes"],
				"monitor|m",
					"Keep monitoring for local and remote changes",
					&boolValues["monitor"],
				"monitor-interval",
					"Number of seconds by which each sync operation is undertaken when idle under monitor mode.",
					&longValues["monitor_interval"],
				"monitor-fullscan-frequency",
					"Number of sync runs before performing a full local scan of the synced directory",
					&longValues["monitor_fullscan_frequency"],
				"monitor-log-frequency",
					"Frequency of logging in monitor mode",
					&longValues["monitor_log_frequency"],
				"no-remote-delete",
					"Do not delete local file 'deletes' from OneDrive when using --upload-only",
					&boolValues["no_remote_delete"],
				"print-token",
					"Print the access token, useful for debugging",
					&boolValues["print_token"],
				"resync",
					"Forget the last saved state, perform a full sync",
					&boolValues["resync"],
				"remove-directory",
					"Remove a directory on OneDrive - no sync will be performed.",
					&stringValues["remove_directory"],
				"remove-source-files",
					"Remove source file after successful transfer to OneDrive when using --upload-only",
					&boolValues["remove_source_files"],
				"single-directory",
					"Specify a single local directory within the OneDrive root to sync.",
					&stringValues["single_directory"],
				"skip-dot-files",
					"Skip dot files and folders from syncing",
					&boolValues["skip_dotfiles"],
				"skip-file",
					"Skip any files that match this pattern from syncing",
					&stringValues["skip_file"],
				"skip-dir",
					"Skip any directories that match this pattern from syncing",
					&stringValues["skip_dir"],
				"skip-size",
					"Skip new files larger than this size (in MB)",
					&longValues["skip_size"],
				"skip-dir-strict-match",
					"When matching skip_dir directories, only match explicit matches",
					&boolValues["skip_dir_strict_match"],
				"skip-symlinks",
					"Skip syncing of symlinks",
					&boolValues["skip_symlinks"],
				"source-directory",
					"Source directory to rename or move on OneDrive - no sync will be performed.",
					&stringValues["source_directory"],
				"syncdir",
					"Specify the local directory used for synchronization to OneDrive",
					&stringValues["sync_dir"],
				"synchronize",
					"Perform a synchronization",
					&boolValues["synchronize"],
				"sync-root-files",
					"Sync all files in sync_dir root when using sync_list.",
					&boolValues["sync_root_files"],
				"upload-only",
					"Only upload to OneDrive, do not sync changes from OneDrive locally",
					&boolValues["upload_only"],
				"user-agent",
					"Specify a User Agent string to the http client",
					&stringValues["user_agent"],
				// duplicated from main.d to get full help output!
				"confdir",
					"Set the directory used to store the configuration files",
					&tmpStr,
				"verbose|v+",
					"Print more details, useful for debugging (repeat for extra debugging)",
					&tmpVerb,
				"version",
					"Print the version and exit",
					&tmpBol
			);
			if (opt.helpWanted) {
				outputLongHelp(opt.options);
				exit(EXIT_SUCCESS);
			}
		} catch (GetOptException e) {
			log.error(e.msg);
			log.error("Try 'onedrive -h' for more information");
			exit(EXIT_FAILURE);
		} catch (Exception e) {
			// error
			log.error(e.msg);
			log.error("Try 'onedrive -h' for more information");
			exit(EXIT_FAILURE);
		}
	}

	string getValueString(string key)
	{
		auto p = key in stringValues;
		if (p) {
			return *p;
		} else {
			throw new Exception("Missing config value: " ~ key);
		}
	}

	long getValueLong(string key)
	{
		auto p = key in longValues;
		if (p) {
			return *p;
		} else {
			throw new Exception("Missing config value: " ~ key);
		}
	}

	bool getValueBool(string key)
	{
		auto p = key in boolValues;
		if (p) {
			return *p;
		} else {
			throw new Exception("Missing config value: " ~ key);
		}
	}

	void setValueBool(string key, bool value)
	{
		boolValues[key] = value;
	}

	void setValueString(string key, string value)
	{
		stringValues[key] = value;
	}

	void setValueLong(string key, long value)
	{
		longValues[key] = value;
	}

	// load a configuration file
	private bool load(string filename)
	{
		// configure function variables
		auto file = File(filename, "r");
		string lineBuffer;
		
		// configure scopes
		// - failure
		scope(failure) {
			// close file if open
			if (file.isOpen()){
				// close open file
				file.close();
			}
			return false;
		}
		// - exit
		scope(exit) {
			// close file if open
			if (file.isOpen()){
				// close open file
				file.close();
			}
		}
	
		// read file line by line
		auto range = file.byLine();
		foreach (line; range) {
			lineBuffer = stripLeft(line).to!string;
			if (lineBuffer.length == 0 || lineBuffer[0] == ';' || lineBuffer[0] == '#') continue;
			auto c = lineBuffer.matchFirst(configRegex);
			if (!c.empty) {
				c.popFront(); // skip the whole match
				string key = c.front.dup;
				auto p = key in boolValues;
				if (p) {
					c.popFront();
					// only accept "true" as true value. TODO Should we support other formats?
					setValueBool(key, c.front.dup == "true" ? true : false);
				} else {
					auto pp = key in stringValues;
					if (pp) {
						c.popFront();
						setValueString(key, c.front.dup);
						// detect need for --resync for these:
						//  --syncdir ARG
						//  --skip-file ARG
						//  --skip-dir ARG
						if (key == "sync_dir") configFileSyncDir = c.front.dup;
						if (key == "skip_file") configFileSkipFile = c.front.dup;
						if (key == "skip_dir") configFileSkipDir = c.front.dup;
					} else {
						auto ppp = key in longValues;
						if (ppp) {
							c.popFront();
							setValueLong(key, to!long(c.front.dup));
						} else {
							log.log("Unknown key in config file: ", key);
							return false;
						}
					}
				}
			} else {
				log.log("Malformed config line: ", lineBuffer);
				return false;
			}
		}
		return true;
	}
}

void outputLongHelp(Option[] opt)
{
	auto argsNeedingOptions = [
		"--confdir",
		"--create-directory",
		"--destination-directory",
		"--get-O365-drive-id",
		"--log-dir",
		"--min-notify-changes",
		"--monitor-interval",
		"--monitor-log-frequency",
		"--monitor-fullscan-frequency",
		"--remove-directory",
		"--single-directory",
		"--skip-file",
		"--source-directory",
		"--syncdir",
		"--user-agent" ];
	writeln(`OneDrive - a client for OneDrive Cloud Services

Usage:
  onedrive [options] --synchronize
      Do a one time synchronization
  onedrive [options] --monitor
      Monitor filesystem and sync regularly
  onedrive [options] --display-config
      Display the currently used configuration
  onedrive [options] --display-sync-status
      Query OneDrive service and report on pending changes
  onedrive -h | --help
      Show this help screen
  onedrive --version
      Show version

Options:
`);
	foreach (it; opt.sort!("a.optLong < b.optLong")) {
		writefln("  %s%s%s%s\n      %s",
				it.optLong,
				it.optShort == "" ? "" : " " ~ it.optShort,
				argsNeedingOptions.canFind(it.optLong) ? " ARG" : "",
				it.required ? " (required)" : "", it.help);
	}
}

unittest
{
	auto cfg = new Config("");
	cfg.load("config");
	assert(cfg.getValueString("sync_dir") == "~/OneDrive");
}