deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-02 22:13:16 +02:00
Code Issues Projects Releases Wiki Activity

Detect the need for --resync when config changes (#617)

Browse source 
* Detect the need for --resync when config changes either via config file or cli override
This commit is contained in:
abraunegg 2019-08-24 17:18:58 +10:00 committed by GitHub
parent bc3853bc4f
commit ad0daf2df5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 233 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/USAGE.md
View file

@ -236,6 +236,8 @@ Proceed with caution here when changing the default sync dir from ~/OneDrive to
The issue here is around how the client stores the sync_dir path in the database. If the config file is missing, or you don't use the `--syncdir` parameter - what will happen is the client will default back to `~/OneDrive` and 'think' that either all your data has been deleted - thus delete the content on OneDrive, or will start downloading all data from OneDrive into the default location.
**Note:** After changing `sync_dir`, you must perform a full re-synchronization by adding `--resync` to your existing command line - for example: `onedrive --synchronize --resync`
### skip_dir
Example: `skip_dir = "Desktop|Documents/IISExpress|Documents/SQL Server Management Studio|Documents/Visual Studio*|Documents/WindowsPowerShell"`

34
src/config.d
View file

@ -14,7 +14,13 @@ final class Config
public string syncListFilePath;
public string homePath;
public string configDirName;
public string defaultSyncDir = "~/OneDrive";
public string defaultSkipFile = "~*|.~*|*.tmp";
public string defaultSkipDir = "";
public string configFileSyncDir;
public string configFileSkipFile;
public string configFileSkipDir;
private string userConfigFilePath;
// hashmap for the values found in the user config file
// ARGGGG D is stupid and cannot make hashmap initializations!!!
@ -23,13 +29,12 @@ final class Config
private bool[string] boolValues;
private long[string] longValues;
this(string confdirOption)
{
// default configuration
stringValues["sync_dir"] = "~/OneDrive";
stringValues["skip_file"] = "~*|.~*|*.tmp";
stringValues["skip_dir"] = "";
stringValues["sync_dir"] = defaultSyncDir;
stringValues["skip_file"] = defaultSkipFile;
stringValues["skip_dir"] = defaultSkipDir;
stringValues["log_dir"] = "/var/log/onedrive/";
stringValues["drive_id"] = "";
boolValues["upload_only"] = false;
@ -80,8 +85,7 @@ final class Config
// Output homePath calculation
log.vdebug("homePath: ", homePath);
// Determine the correct configuration directory to use
string configDirBase;
if (confdirOption != "") {
@ -113,7 +117,6 @@ final class Config
configDirName = configDirBase ~ "/onedrive";
}
log.vlog("Using Config Dir: ", configDirName);
if (!exists(configDirName)) mkdirRecurse(configDirName);
@ -141,10 +144,8 @@ final class Config
return true;
}
void update_from_args(string[] args)
{
// Add additional options that are NOT configurable via config file
stringValues["create_directory"] = "";
stringValues["destination_directory"] = "";
@ -162,7 +163,6 @@ final class Config
boolValues["monitor"] = false;
boolValues["synchronize"] = false;
// Application Startup option validation
try {
string tmpStr;
@ -268,6 +268,9 @@ final class Config
"skip-file",
"Skip any files that match this pattern from syncing",
&stringValues["skip_file"],
"skip-dir",
"Skip any directories that match this pattern from syncing",
&stringValues["skip_dir"],
"skip-size",
"Skip new files larger than this size (in MB)",
&longValues["skip_size"],
@ -299,7 +302,6 @@ final class Config
"version",
"Print the version and exit",
&tmpBol
);
if (opt.helpWanted) {
outputLongHelp(opt.options);
@ -317,7 +319,6 @@ final class Config
}
}
string getValueString(string key)
{
auto p = key in stringValues;
@ -385,6 +386,13 @@ final class Config
if (pp) {
c.popFront();
setValueString(key, c.front.dup);
// detect need for --resync for these:
// --syncdir ARG
// --skip-file ARG
// --skip-dir ARG
if (key == "sync_dir") configFileSyncDir = c.front.dup;
if (key == "skip_file") configFileSkipFile = c.front.dup;
if (key == "skip_dir") configFileSkipDir = c.front.dup;
} else {
auto ppp = key in longValues;
if (ppp) {

217
src/main.d
View file

@ -1,6 +1,6 @@
import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit;
import core.memory, core.time, core.thread;
import std.getopt, std.file, std.path, std.process, std.stdio, std.conv, std.algorithm.searching, std.string;
import std.getopt, std.file, std.path, std.process, std.stdio, std.conv, std.algorithm.searching, std.string, std.regex;
import config, itemdb, monitor, onedrive, selective, sync, util;
import std.net.curl: CurlException;
import core.stdc.signal;
@ -53,8 +53,7 @@ int main(string[] args)
log.error("Try 'onedrive -h' for more information");
return EXIT_FAILURE;
}
// load configuration file if available
auto cfg = new config.Config(confdirOption);
if (!cfg.initialize()) {
@ -62,15 +61,216 @@ int main(string[] args)
// Error message already printed
return EXIT_FAILURE;
}
// update configuration from command line args
cfg.update_from_args(args);
// Has any of our configuration that would require a --resync been changed?
// 1. sync_list file modification
// 2. config file modification - but only if sync_dir, skip_dir, skip_file or drive_id was modified
// 3. CLI input overriding configured config file option
string currentConfigHash;
string currentSyncListHash;
string previousConfigHash;
string previousSyncListHash;
string configHashFile = cfg.configDirName ~ "/.config.hash";
string syncListHashFile = cfg.configDirName ~ "/.sync_list.hash";
string configBackupFile = cfg.configDirName ~ "/.config.backup";
bool configOptionsDifferent = false;
bool syncListDifferent = false;
bool syncDirDifferent = false;
bool skipFileDifferent = false;
bool skipDirDifferent = false;
if ((exists(cfg.configDirName ~ "/config")) && (!exists(configHashFile))) {
// Hash of config file needs to be created
std.file.write(configHashFile, computeQuickXorHash(cfg.configDirName ~ "/config"));
}
if ((exists(cfg.configDirName ~ "/sync_list")) && (!exists(syncListHashFile))) {
// Hash of sync_list file needs to be created
std.file.write(syncListHashFile, computeQuickXorHash(cfg.configDirName ~ "/sync_list"));
}
// If hash files exist, but config files do not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different
if ((!exists(cfg.configDirName ~ "/config")) && (exists(configHashFile))) {
// if --resync safe remove config.hash and config.backup
if (cfg.getValueBool("resync")) {
safeRemove(configHashFile);
safeRemove(configBackupFile);
}
}
if ((!exists(cfg.configDirName ~ "/sync_list")) && (exists(syncListHashFile))) {
// if --resync safe remove sync_list.hash
if (cfg.getValueBool("resync")) safeRemove(syncListHashFile);
}
// Read config hashes if they exist
if (exists(cfg.configDirName ~ "/config")) currentConfigHash = computeQuickXorHash(cfg.configDirName ~ "/config");
if (exists(cfg.configDirName ~ "/sync_list")) currentSyncListHash = computeQuickXorHash(cfg.configDirName ~ "/sync_list");
if (exists(configHashFile)) previousConfigHash = readText(configHashFile);
if (exists(syncListHashFile)) previousSyncListHash = readText(syncListHashFile);
// Was sync_list updated?
if (currentSyncListHash != previousSyncListHash) {
// Debugging output to assist what changed
log.vdebug("sync_list file has been updated, --resync needed");
syncListDifferent = true;
}
// Was config updated?
if (currentConfigHash != previousConfigHash) {
// config file was updated, however we only want to trigger a --resync requirement if sync_dir, skip_dir, skip_file or drive_id was modified
log.vdebug("config file has been updated, checking if --resync needed");
if (exists(configBackupFile)) {
// check backup config what has changed for these configuration options if anything
// # sync_dir = "~/OneDrive"
// # skip_file = "~*|.~*|*.tmp"
// # skip_dir = ""
// # drive_id = ""
string[string] stringValues;
stringValues["sync_dir"] = "";
stringValues["skip_file"] = "";
stringValues["skip_dir"] = "";
stringValues["drive_id"] = "";
auto file = File(configBackupFile, "r");
auto r = regex(`^(\w+)\s*=\s*"(.*)"\s*$`);
foreach (line; file.byLine()) {
line = stripLeft(line);
if (line.length == 0 || line[0] == ';' || line[0] == '#') continue;
auto c = line.matchFirst(r);
if (!c.empty) {
c.popFront(); // skip the whole match
string key = c.front.dup;
auto p = key in stringValues;
if (p) {
c.popFront();
// compare this key
if ((key == "sync_dir") && (c.front.dup != cfg.getValueString("sync_dir"))) {
log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed");
configOptionsDifferent = true;
}
if ((key == "skip_file") && (c.front.dup != cfg.getValueString("skip_file"))){
log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed");
configOptionsDifferent = true;
}
if ((key == "skip_dir") && (c.front.dup != cfg.getValueString("skip_dir"))){
log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed");
configOptionsDifferent = true;
}
if ((key == "drive_id") && (c.front.dup != cfg.getValueString("drive_id"))){
log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed");
configOptionsDifferent = true;
}
}
}
}
} else {
// no backup to check
log.vdebug("WARNING: no backup config file was found, unable to validate if any changes made");
}
// If there was a backup, any modified values we need to worry about would been detected
if (!cfg.getValueBool("display_config")) {
// we are not testing the configuration
if (!configOptionsDifferent) {
// no options are different
if (!cfg.getValueBool("dry_run")) {
// we are not in a dry-run scenario
// update config hash
log.vdebug("updating config hash as it is out of date");
std.file.write(configHashFile, computeQuickXorHash(cfg.configDirName ~ "/config"));
// create backup copy of current config file
log.vdebug("making backup of config file as it is out of date");
std.file.copy(cfg.configDirName ~ "/config", configBackupFile);
}
}
}
}
// Is there a backup of the config file if the config file exists?
if ((exists(cfg.configDirName ~ "/config")) && (!exists(configBackupFile))) {
// create backup copy of current config file
std.file.copy(cfg.configDirName ~ "/config", configBackupFile);
}
// config file set options can be changed via CLI input, specifically these will impact sync and --resync will be needed:
// --syncdir ARG
// --skip-file ARG
// --skip-dir ARG
if (exists(cfg.configDirName ~ "/config")) {
// config file exists
// was the sync_dir updated by CLI?
if (cfg.configFileSyncDir != "") {
// sync_dir was set in config file
if (cfg.configFileSyncDir != cfg.getValueString("sync_dir")) {
// config file was set and CLI input changed this
log.vdebug("sync_dir: CLI override of config file option, --resync needed");
syncDirDifferent = true;
}
}
// was the skip_file updated by CLI?
if (cfg.configFileSkipFile != "") {
// skip_file was set in config file
if (cfg.configFileSkipFile != cfg.getValueString("skip_file")) {
// config file was set and CLI input changed this
log.vdebug("skip_file: CLI override of config file option, --resync needed");
skipFileDifferent = true;
}
}
// was the skip_dir updated by CLI?
if (cfg.configFileSkipDir != "") {
// skip_dir was set in config file
if (cfg.configFileSkipDir != cfg.getValueString("skip_dir")) {
// config file was set and CLI input changed this
log.vdebug("skip_dir: CLI override of config file option, --resync needed");
skipDirDifferent = true;
}
}
}
// Has anything triggered a --resync requirement?
if (configOptionsDifferent || syncListDifferent || syncDirDifferent || skipFileDifferent || skipDirDifferent) {
// --resync needed, is the user just testing configuration changes?
if (!cfg.getValueBool("display_config")){
// not testing configuration changes
if (!cfg.getValueBool("resync")) {
// --resync not issued, fail fast
log.error("An application configuration change has been detected where a --resync is required");
return EXIT_FAILURE;
} else {
// --resync issued, update hashes of config files if they exist
if (!cfg.getValueBool("dry_run")) {
// not doing a dry run, update hash files if config & sync_list exist
if (exists(cfg.configDirName ~ "/config")) {
// update hash
log.vdebug("updating config hash as --resync issued");
std.file.write(configHashFile, computeQuickXorHash(cfg.configDirName ~ "/config"));
// create backup copy of current config file
log.vdebug("making backup of config file as --resync issued");
std.file.copy(cfg.configDirName ~ "/config", configBackupFile);
}
if (exists(cfg.configDirName ~ "/sync_list")) {
// update sync_list hash
log.vdebug("updating sync_list hash as --resync issued");
std.file.write(syncListHashFile, computeQuickXorHash(cfg.configDirName ~ "/sync_list"));
}
}
}
}
}
// dry-run notification
if (cfg.getValueBool("dry_run")) {
log.log("DRY-RUN Configured. Output below shows what 'would' have occurred.");
}
// Are we able to reach the OneDrive Service
bool online = false;
@ -134,6 +334,7 @@ int main(string[] args)
}
if (cfg.getValueBool("resync") || cfg.getValueBool("logout")) {
if (cfg.getValueBool("resync")) log.vdebug("--resync requested");
log.vlog("Deleting the saved status ...");
if (!cfg.getValueBool("dry_run")) {
safeRemove(cfg.databaseFilePath);
@ -141,6 +342,7 @@ int main(string[] args)
safeRemove(cfg.uploadStateFilePath);
}
if (cfg.getValueBool("logout")) {
log.vdebug("--logout requested");
if (!cfg.getValueBool("dry_run")) {
safeRemove(cfg.refreshTokenFilePath);
}
@ -196,6 +398,7 @@ int main(string[] args)
writeln("Selective sync configured = false");
}
// exit
return EXIT_SUCCESS;
}
@ -257,11 +460,11 @@ int main(string[] args)
log.vlog("Opening the item database ...");
if (!cfg.getValueBool("dry_run")) {
// Load the items.sqlite3 file as the database
log.vdebug("Using database file: ", cfg.databaseFilePath);
log.vdebug("Using database file: ", asNormalizedPath(cfg.databaseFilePath));
itemDb = new ItemDatabase(cfg.databaseFilePath);
} else {
// Load the items-dryrun.sqlite3 file as the database
log.vdebug("Using database file: ", cfg.databaseFilePathDryRun);
log.vdebug("Using database file: ", asNormalizedPath(cfg.databaseFilePathDryRun));
itemDb = new ItemDatabase(cfg.databaseFilePathDryRun);
}