2018-12-28 03:19:20 +01:00
import core.stdc.stdlib : EXIT_SUCCESS , EXIT_FAILURE , exit ;
2015-09-20 21:21:51 +02:00
import core.memory , core . time , core . thread ;
2019-08-24 09:18:58 +02:00
import std.getopt , std . file , std . path , std . process , std . stdio , std . conv , std . algorithm . searching , std . string , std . regex ;
2017-03-24 22:30:03 +01:00
import config , itemdb , monitor , onedrive , selective , sync , util ;
2018-08-13 23:21:11 +02:00
import std.net.curl : CurlException ;
2018-12-28 03:19:20 +01:00
import core.stdc.signal ;
import std.traits ;
2016-08-04 23:35:58 +02:00
static import log ;
2015-09-01 20:45:34 +02:00
2018-12-28 03:19:20 +01:00
OneDriveApi oneDrive ;
ItemDatabase itemDb ;
2019-07-06 00:01:04 +02:00
const int EXIT_UNAUTHORIZED = 3 ;
2019-02-26 21:21:23 +01:00
enum MONITOR_LOG_SILENT = 2 ;
enum MONITOR_LOG_QUIET = 1 ;
enum LOG_NORMAL = 0 ;
2016-08-04 23:35:58 +02:00
int main ( string [ ] args )
2015-09-01 20:45:34 +02:00
{
2018-12-23 01:15:10 +01:00
// Disable buffering on stdout
stdout . setvbuf ( 0 , _IONBF ) ;
2018-04-13 01:33:16 +02:00
2020-05-20 03:37:11 +02:00
// main function variables
2019-04-11 04:26:20 +02:00
string confdirOption ;
2020-06-21 02:01:24 +02:00
string configFilePath ;
string syncListFilePath ;
string databaseFilePath ;
2020-06-27 11:10:37 +02:00
string businessSharedFolderFilePath ;
2020-05-20 03:37:11 +02:00
string currentConfigHash ;
string currentSyncListHash ;
string previousConfigHash ;
string previousSyncListHash ;
string configHashFile ;
string syncListHashFile ;
string configBackupFile ;
string syncDir ;
2020-05-25 03:30:04 +02:00
string logOutputMessage ;
2020-06-27 11:10:37 +02:00
string currentBusinessSharedFoldersHash ;
string previousBusinessSharedFoldersHash ;
string businessSharedFoldersHashFile ;
2020-05-20 03:37:11 +02:00
bool configOptionsDifferent = false ;
2020-06-27 11:10:37 +02:00
bool businessSharedFoldersDifferent = false ;
2020-05-20 03:37:11 +02:00
bool syncListConfigured = false ;
bool syncListDifferent = false ;
bool syncDirDifferent = false ;
bool skipFileDifferent = false ;
bool skipDirDifferent = false ;
bool online = false ;
bool performSyncOK = false ;
bool onedriveInitialised = false ;
bool displayMemoryUsage = false ;
2020-05-25 03:30:04 +02:00
bool displaySyncOptions = false ;
2020-05-20 03:37:11 +02:00
// Define scopes
scope ( exit ) {
// Display memory details
if ( displayMemoryUsage ) {
log . displayMemoryUsagePreGC ( ) ;
}
// if initialised, shut down the HTTP instance
if ( onedriveInitialised ) {
oneDrive . shutdown ( ) ;
}
2020-11-06 00:28:15 +01:00
// was itemDb initialised?
if ( itemDb ! is null ) {
// Make sure the .wal file is incorporated into the main db before we exit
itemDb . performVacuum ( ) ;
destroy ( itemDb ) ;
}
2020-05-20 03:37:11 +02:00
// free API instance
2020-11-06 00:28:15 +01:00
if ( oneDrive ! is null ) {
destroy ( oneDrive ) ;
}
2020-05-20 03:37:11 +02:00
// Perform Garbage Cleanup
GC . collect ( ) ;
// Display memory details
if ( displayMemoryUsage ) {
log . displayMemoryUsagePostGC ( ) ;
}
}
scope ( failure ) {
// Display memory details
if ( displayMemoryUsage ) {
log . displayMemoryUsagePreGC ( ) ;
}
// if initialised, shut down the HTTP instance
if ( onedriveInitialised ) {
oneDrive . shutdown ( ) ;
}
2020-11-06 00:28:15 +01:00
// was itemDb initialised?
if ( itemDb ! is null ) {
// Make sure the .wal file is incorporated into the main db before we exit
itemDb . performVacuum ( ) ;
destroy ( itemDb ) ;
}
2020-05-20 03:37:11 +02:00
// free API instance
2020-11-06 00:28:15 +01:00
if ( oneDrive ! is null ) {
destroy ( oneDrive ) ;
}
2020-05-20 03:37:11 +02:00
// Perform Garbage Cleanup
GC . collect ( ) ;
// Display memory details
if ( displayMemoryUsage ) {
log . displayMemoryUsagePostGC ( ) ;
}
}
2019-04-11 04:26:20 +02:00
2020-05-20 03:37:11 +02:00
// read in application options as passed in
2019-01-26 01:03:00 +01:00
try {
2019-04-11 04:26:20 +02:00
bool printVersion = false ;
2019-01-26 01:03:00 +01:00
auto opt = getopt (
args ,
2019-04-11 04:26:20 +02:00
std . getopt . config . passThrough ,
2019-01-26 01:03:00 +01:00
std . getopt . config . bundling ,
std . getopt . config . caseSensitive ,
2019-04-11 04:26:20 +02:00
"confdir" , "Set the directory used to store the configuration files" , & confdirOption ,
2019-03-17 05:10:41 +01:00
"verbose|v+" , "Print more details, useful for debugging (repeat for extra debugging)" , & log . verbose ,
2019-01-26 01:03:00 +01:00
"version" , "Print the version and exit" , & printVersion
) ;
2020-05-20 03:37:11 +02:00
// print help and exit
2019-01-26 01:03:00 +01:00
if ( opt . helpWanted ) {
2019-04-11 04:26:20 +02:00
args ~ = "--help" ;
}
2020-05-20 03:37:11 +02:00
// print the version and exit
2019-04-11 04:26:20 +02:00
if ( printVersion ) {
2020-05-20 03:37:11 +02:00
writeln ( "onedrive " , strip ( import ( "version" ) ) ) ;
2019-01-26 01:03:00 +01:00
return EXIT_SUCCESS ;
}
} catch ( GetOptException e ) {
2020-05-20 03:37:11 +02:00
// option errors
2019-01-26 01:03:00 +01:00
log . error ( e . msg ) ;
log . error ( "Try 'onedrive -h' for more information" ) ;
return EXIT_FAILURE ;
} catch ( Exception e ) {
2020-05-20 03:37:11 +02:00
// generic error
2019-01-26 01:03:00 +01:00
log . error ( e . msg ) ;
log . error ( "Try 'onedrive -h' for more information" ) ;
return EXIT_FAILURE ;
}
2019-08-24 09:18:58 +02:00
2019-04-11 04:26:20 +02:00
// load configuration file if available
auto cfg = new config . Config ( confdirOption ) ;
if ( ! cfg . initialize ( ) ) {
// There was an error loading the configuration
// Error message already printed
return EXIT_FAILURE ;
2019-04-01 19:51:25 +02:00
}
2019-08-24 09:18:58 +02:00
2020-05-20 03:37:11 +02:00
// set memory display
displayMemoryUsage = cfg . getValueBool ( "display_memory" ) ;
2020-05-25 03:30:04 +02:00
// set display sync options
displaySyncOptions = cfg . getValueBool ( "display_sync_options" ) ;
2019-04-11 04:26:20 +02:00
// update configuration from command line args
cfg . update_from_args ( args ) ;
2019-08-24 09:18:58 +02:00
2020-06-21 02:01:24 +02:00
// Initialise normalised file paths
configFilePath = buildNormalizedPath ( cfg . configDirName ~ "/config" ) ;
syncListFilePath = buildNormalizedPath ( cfg . configDirName ~ "/sync_list" ) ;
databaseFilePath = buildNormalizedPath ( cfg . configDirName ~ "/items.db" ) ;
2020-06-27 11:10:37 +02:00
businessSharedFolderFilePath = buildNormalizedPath ( cfg . configDirName ~ "/business_shared_folders" ) ;
2020-06-21 02:01:24 +02:00
2019-08-24 09:18:58 +02:00
// Has any of our configuration that would require a --resync been changed?
// 1. sync_list file modification
// 2. config file modification - but only if sync_dir, skip_dir, skip_file or drive_id was modified
// 3. CLI input overriding configured config file option
2020-05-20 03:37:11 +02:00
configHashFile = buildNormalizedPath ( cfg . configDirName ~ "/.config.hash" ) ;
syncListHashFile = buildNormalizedPath ( cfg . configDirName ~ "/.sync_list.hash" ) ;
configBackupFile = buildNormalizedPath ( cfg . configDirName ~ "/.config.backup" ) ;
2020-06-27 11:10:37 +02:00
businessSharedFoldersHashFile = buildNormalizedPath ( cfg . configDirName ~ "/.business_shared_folders.hash" ) ;
2019-08-24 09:18:58 +02:00
2020-05-20 03:37:11 +02:00
// Does a config file exist with a valid hash file
2020-06-21 02:01:24 +02:00
if ( ( exists ( configFilePath ) ) & & ( ! exists ( configHashFile ) ) ) {
2019-08-24 09:18:58 +02:00
// Hash of config file needs to be created
2020-06-21 02:01:24 +02:00
std . file . write ( configHashFile , computeQuickXorHash ( configFilePath ) ) ;
2019-08-24 09:18:58 +02:00
}
2020-05-20 03:37:11 +02:00
// Does a sync_list file exist with a valid hash file
2020-06-21 02:01:24 +02:00
if ( ( exists ( syncListFilePath ) ) & & ( ! exists ( syncListHashFile ) ) ) {
2019-08-24 09:18:58 +02:00
// Hash of sync_list file needs to be created
2020-06-21 02:01:24 +02:00
std . file . write ( syncListHashFile , computeQuickXorHash ( syncListFilePath ) ) ;
2019-08-24 09:18:58 +02:00
}
2020-06-27 11:10:37 +02:00
// check if business_shared_folders & business_shared_folders hash exists
if ( ( exists ( businessSharedFolderFilePath ) ) & & ( ! exists ( businessSharedFoldersHashFile ) ) ) {
// Hash of business_shared_folders file needs to be created
std . file . write ( businessSharedFoldersHashFile , computeQuickXorHash ( businessSharedFolderFilePath ) ) ;
}
2019-08-24 09:18:58 +02:00
// If hash files exist, but config files do not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different
2020-06-21 02:01:24 +02:00
if ( ( ! exists ( configFilePath ) ) & & ( exists ( configHashFile ) ) ) {
2019-08-24 09:18:58 +02:00
// if --resync safe remove config.hash and config.backup
if ( cfg . getValueBool ( "resync" ) ) {
safeRemove ( configHashFile ) ;
safeRemove ( configBackupFile ) ;
}
}
2020-05-20 03:37:11 +02:00
// If sync_list hash file exists, but sync_list file does not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different
2020-06-21 02:01:24 +02:00
if ( ( ! exists ( syncListFilePath ) ) & & ( exists ( syncListHashFile ) ) ) {
2019-08-24 09:18:58 +02:00
// if --resync safe remove sync_list.hash
if ( cfg . getValueBool ( "resync" ) ) safeRemove ( syncListHashFile ) ;
}
2020-06-27 11:10:37 +02:00
if ( ( ! exists ( businessSharedFolderFilePath ) ) & & ( exists ( businessSharedFoldersHashFile ) ) ) {
// if --resync safe remove business_shared_folders.hash
if ( cfg . getValueBool ( "resync" ) ) safeRemove ( businessSharedFoldersHashFile ) ;
}
2019-08-24 09:18:58 +02:00
// Read config hashes if they exist
2020-06-21 02:01:24 +02:00
if ( exists ( configFilePath ) ) currentConfigHash = computeQuickXorHash ( configFilePath ) ;
if ( exists ( syncListFilePath ) ) currentSyncListHash = computeQuickXorHash ( syncListFilePath ) ;
2020-06-27 11:10:37 +02:00
if ( exists ( businessSharedFolderFilePath ) ) currentBusinessSharedFoldersHash = computeQuickXorHash ( businessSharedFolderFilePath ) ;
2019-08-24 09:18:58 +02:00
if ( exists ( configHashFile ) ) previousConfigHash = readText ( configHashFile ) ;
if ( exists ( syncListHashFile ) ) previousSyncListHash = readText ( syncListHashFile ) ;
2020-06-27 11:10:37 +02:00
if ( exists ( businessSharedFoldersHashFile ) ) previousBusinessSharedFoldersHash = readText ( businessSharedFoldersHashFile ) ;
2019-08-24 09:18:58 +02:00
2020-05-20 03:37:11 +02:00
// Was sync_list file updated?
2019-08-24 09:18:58 +02:00
if ( currentSyncListHash ! = previousSyncListHash ) {
// Debugging output to assist what changed
log . vdebug ( "sync_list file has been updated, --resync needed" ) ;
syncListDifferent = true ;
}
2020-06-27 11:10:37 +02:00
// Was business_shared_folders updated?
if ( currentBusinessSharedFoldersHash ! = previousBusinessSharedFoldersHash ) {
// Debugging output to assist what changed
log . vdebug ( "business_shared_folders file has been updated, --resync needed" ) ;
businessSharedFoldersDifferent = true ;
}
2020-05-20 03:37:11 +02:00
// Was config file updated between last execution ang this execution?
2019-08-24 09:18:58 +02:00
if ( currentConfigHash ! = previousConfigHash ) {
// config file was updated, however we only want to trigger a --resync requirement if sync_dir, skip_dir, skip_file or drive_id was modified
2020-05-20 03:37:11 +02:00
log . log ( "config file has been updated, checking if --resync needed" ) ;
2019-08-24 09:18:58 +02:00
if ( exists ( configBackupFile ) ) {
// check backup config what has changed for these configuration options if anything
// # sync_dir = "~/OneDrive"
// # skip_file = "~*|.~*|*.tmp"
// # skip_dir = ""
// # drive_id = ""
string [ string ] stringValues ;
stringValues [ "sync_dir" ] = "" ;
stringValues [ "skip_file" ] = "" ;
stringValues [ "skip_dir" ] = "" ;
stringValues [ "drive_id" ] = "" ;
2020-05-20 03:37:11 +02:00
auto configBackupFileHandle = File ( configBackupFile , "r" ) ;
string lineBuffer ;
auto range = configBackupFileHandle . byLine ( ) ;
// read configBackupFile line by line
foreach ( line ; range ) {
lineBuffer = stripLeft ( line ) . to ! string ;
if ( lineBuffer . length = = 0 | | lineBuffer [ 0 ] = = ';' | | lineBuffer [ 0 ] = = '#' ) continue ;
auto c = lineBuffer . matchFirst ( cfg . configRegex ) ;
2019-08-24 09:18:58 +02:00
if ( ! c . empty ) {
c . popFront ( ) ; // skip the whole match
string key = c . front . dup ;
auto p = key in stringValues ;
if ( p ) {
c . popFront ( ) ;
// compare this key
if ( ( key = = "sync_dir" ) & & ( c . front . dup ! = cfg . getValueString ( "sync_dir" ) ) ) {
log . vdebug ( key , " was modified since the last time the application was successfully run, --resync needed" ) ;
configOptionsDifferent = true ;
}
if ( ( key = = "skip_file" ) & & ( c . front . dup ! = cfg . getValueString ( "skip_file" ) ) ) {
log . vdebug ( key , " was modified since the last time the application was successfully run, --resync needed" ) ;
configOptionsDifferent = true ;
}
if ( ( key = = "skip_dir" ) & & ( c . front . dup ! = cfg . getValueString ( "skip_dir" ) ) ) {
log . vdebug ( key , " was modified since the last time the application was successfully run, --resync needed" ) ;
configOptionsDifferent = true ;
}
if ( ( key = = "drive_id" ) & & ( c . front . dup ! = cfg . getValueString ( "drive_id" ) ) ) {
log . vdebug ( key , " was modified since the last time the application was successfully run, --resync needed" ) ;
configOptionsDifferent = true ;
}
}
}
}
2020-05-20 03:37:11 +02:00
// close file if open
if ( configBackupFileHandle . isOpen ( ) ) {
// close open file
configBackupFileHandle . close ( ) ;
}
2019-08-24 09:18:58 +02:00
} else {
// no backup to check
log . vdebug ( "WARNING: no backup config file was found, unable to validate if any changes made" ) ;
}
// If there was a backup, any modified values we need to worry about would been detected
if ( ! cfg . getValueBool ( "display_config" ) ) {
// we are not testing the configuration
if ( ! configOptionsDifferent ) {
// no options are different
if ( ! cfg . getValueBool ( "dry_run" ) ) {
// we are not in a dry-run scenario
// update config hash
log . vdebug ( "updating config hash as it is out of date" ) ;
2020-06-21 02:01:24 +02:00
std . file . write ( configHashFile , computeQuickXorHash ( configFilePath ) ) ;
2019-08-24 09:18:58 +02:00
// create backup copy of current config file
log . vdebug ( "making backup of config file as it is out of date" ) ;
2020-06-21 02:01:24 +02:00
std . file . copy ( configFilePath , configBackupFile ) ;
2019-08-24 09:18:58 +02:00
}
}
}
}
// Is there a backup of the config file if the config file exists?
2020-06-21 02:01:24 +02:00
if ( ( exists ( configFilePath ) ) & & ( ! exists ( configBackupFile ) ) ) {
2019-08-24 09:18:58 +02:00
// create backup copy of current config file
2020-06-21 02:01:24 +02:00
std . file . copy ( configFilePath , configBackupFile ) ;
2019-08-24 09:18:58 +02:00
}
// config file set options can be changed via CLI input, specifically these will impact sync and --resync will be needed:
// --syncdir ARG
// --skip-file ARG
// --skip-dir ARG
2020-06-21 02:01:24 +02:00
if ( exists ( configFilePath ) ) {
2019-08-24 09:18:58 +02:00
// config file exists
// was the sync_dir updated by CLI?
if ( cfg . configFileSyncDir ! = "" ) {
// sync_dir was set in config file
if ( cfg . configFileSyncDir ! = cfg . getValueString ( "sync_dir" ) ) {
// config file was set and CLI input changed this
log . vdebug ( "sync_dir: CLI override of config file option, --resync needed" ) ;
syncDirDifferent = true ;
}
}
// was the skip_file updated by CLI?
if ( cfg . configFileSkipFile ! = "" ) {
// skip_file was set in config file
if ( cfg . configFileSkipFile ! = cfg . getValueString ( "skip_file" ) ) {
// config file was set and CLI input changed this
log . vdebug ( "skip_file: CLI override of config file option, --resync needed" ) ;
skipFileDifferent = true ;
}
}
// was the skip_dir updated by CLI?
if ( cfg . configFileSkipDir ! = "" ) {
// skip_dir was set in config file
if ( cfg . configFileSkipDir ! = cfg . getValueString ( "skip_dir" ) ) {
// config file was set and CLI input changed this
log . vdebug ( "skip_dir: CLI override of config file option, --resync needed" ) ;
skipDirDifferent = true ;
}
}
}
// Has anything triggered a --resync requirement?
2020-06-27 11:10:37 +02:00
if ( configOptionsDifferent | | syncListDifferent | | syncDirDifferent | | skipFileDifferent | | skipDirDifferent | | businessSharedFoldersDifferent ) {
2019-08-24 09:18:58 +02:00
// --resync needed, is the user just testing configuration changes?
if ( ! cfg . getValueBool ( "display_config" ) ) {
// not testing configuration changes
if ( ! cfg . getValueBool ( "resync" ) ) {
// --resync not issued, fail fast
log . error ( "An application configuration change has been detected where a --resync is required" ) ;
return EXIT_FAILURE ;
} else {
// --resync issued, update hashes of config files if they exist
if ( ! cfg . getValueBool ( "dry_run" ) ) {
// not doing a dry run, update hash files if config & sync_list exist
2020-06-21 02:01:24 +02:00
if ( exists ( configFilePath ) ) {
2019-08-24 09:18:58 +02:00
// update hash
log . vdebug ( "updating config hash as --resync issued" ) ;
2020-06-21 02:01:24 +02:00
std . file . write ( configHashFile , computeQuickXorHash ( configFilePath ) ) ;
2019-08-24 09:18:58 +02:00
// create backup copy of current config file
log . vdebug ( "making backup of config file as --resync issued" ) ;
2020-06-21 02:01:24 +02:00
std . file . copy ( configFilePath , configBackupFile ) ;
2019-08-24 09:18:58 +02:00
}
2020-06-21 02:01:24 +02:00
if ( exists ( syncListFilePath ) ) {
2019-08-24 09:18:58 +02:00
// update sync_list hash
log . vdebug ( "updating sync_list hash as --resync issued" ) ;
2020-06-21 02:01:24 +02:00
std . file . write ( syncListHashFile , computeQuickXorHash ( syncListFilePath ) ) ;
2019-08-24 09:18:58 +02:00
}
2020-06-27 11:10:37 +02:00
if ( exists ( businessSharedFolderFilePath ) ) {
// update business_shared_folders hash
log . vdebug ( "updating business_shared_folders hash as --resync issued" ) ;
std . file . write ( businessSharedFoldersHashFile , computeQuickXorHash ( businessSharedFolderFilePath ) ) ;
}
2019-08-24 09:18:58 +02:00
}
}
}
}
2020-05-25 03:30:04 +02:00
// dry-run notification and database setup
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "dry_run" ) ) {
2019-03-17 04:41:26 +01:00
log . log ( "DRY-RUN Configured. Output below shows what 'would' have occurred." ) ;
2020-10-31 20:55:17 +01:00
string dryRunShmFile = cfg . databaseFilePathDryRun ~ "-shm" ;
string dryRunWalFile = cfg . databaseFilePathDryRun ~ "-wal" ;
2020-03-29 21:55:27 +02:00
// If the dry run database exists, clean this up
if ( exists ( cfg . databaseFilePathDryRun ) ) {
// remove the existing file
log . vdebug ( "Removing items-dryrun.sqlite3 as it still exists for some reason" ) ;
safeRemove ( cfg . databaseFilePathDryRun ) ;
}
2020-10-31 20:55:17 +01:00
// silent cleanup of shm and wal files if they exist
if ( exists ( dryRunShmFile ) ) {
// remove items-dryrun.sqlite3-shm
safeRemove ( dryRunShmFile ) ;
}
if ( exists ( dryRunWalFile ) ) {
// remove items-dryrun.sqlite3-wal
safeRemove ( dryRunWalFile ) ;
}
2020-03-29 21:55:27 +02:00
2019-03-11 07:57:47 +01:00
// Make a copy of the original items.sqlite3 for use as the dry run copy if it exists
if ( exists ( cfg . databaseFilePath ) ) {
2020-03-29 21:55:27 +02:00
// in a --dry-run --resync scenario, we should not copy the existing database file
if ( ! cfg . getValueBool ( "resync" ) ) {
// copy the existing DB file to the dry-run copy
log . vdebug ( "Copying items.sqlite3 to items-dryrun.sqlite3 to use for dry run operations" ) ;
copy ( cfg . databaseFilePath , cfg . databaseFilePathDryRun ) ;
} else {
// no database copy due to --resync
log . vdebug ( "No database copy created for --dry-run due to --resync also being used" ) ;
}
2019-03-11 07:57:47 +01:00
}
}
2018-12-23 01:15:10 +01:00
// sync_dir environment handling to handle ~ expansion properly
2021-02-06 10:46:56 +01:00
bool shellEnvSet = false ;
2018-12-19 19:42:28 +01:00
if ( ( environment . get ( "SHELL" ) = = "" ) & & ( environment . get ( "USER" ) = = "" ) ) {
2018-12-23 01:15:10 +01:00
log . vdebug ( "sync_dir: No SHELL or USER environment variable configuration detected" ) ;
// No shell or user set, so expandTilde() will fail - usually headless system running under init.d / systemd or potentially Docker
// Does the 'currently configured' sync_dir include a ~
2019-04-11 04:26:20 +02:00
if ( canFind ( cfg . getValueString ( "sync_dir" ) , "~" ) ) {
2021-02-06 10:46:56 +01:00
// A ~ was found in sync_dir
log . vdebug ( "sync_dir: A '~' was found in sync_dir, using the calculated 'homePath' to replace '~' as no SHELL or USER environment variable set" ) ;
2019-04-11 04:26:20 +02:00
syncDir = cfg . homePath ~ strip ( cfg . getValueString ( "sync_dir" ) , "~" ) ;
2018-12-19 19:42:28 +01:00
} else {
2018-12-23 01:15:10 +01:00
// No ~ found in sync_dir, use as is
log . vdebug ( "sync_dir: Getting syncDir from config value sync_dir" ) ;
2019-04-11 04:26:20 +02:00
syncDir = cfg . getValueString ( "sync_dir" ) ;
2018-12-19 19:42:28 +01:00
}
} else {
2018-12-23 01:15:10 +01:00
// A shell and user is set, expand any ~ as this will be expanded correctly if present
2021-02-06 10:46:56 +01:00
shellEnvSet = true ;
2018-12-23 01:15:10 +01:00
log . vdebug ( "sync_dir: Getting syncDir from config value sync_dir" ) ;
2019-04-11 04:26:20 +02:00
if ( canFind ( cfg . getValueString ( "sync_dir" ) , "~" ) ) {
2018-12-23 01:15:10 +01:00
log . vdebug ( "sync_dir: A '~' was found in configured sync_dir, automatically expanding as SHELL and USER environment variable is set" ) ;
2019-04-11 04:26:20 +02:00
syncDir = expandTilde ( cfg . getValueString ( "sync_dir" ) ) ;
2018-12-23 01:15:10 +01:00
} else {
2019-04-11 04:26:20 +02:00
syncDir = cfg . getValueString ( "sync_dir" ) ;
2018-12-23 01:15:10 +01:00
}
2018-12-19 19:42:28 +01:00
}
2018-12-23 01:15:10 +01:00
// vdebug syncDir as set and calculated
log . vdebug ( "syncDir: " , syncDir ) ;
2021-02-06 10:46:56 +01:00
// Configure the logging directory if different from application default
// log_dir environment handling to handle ~ expansion properly
string logDir = cfg . getValueString ( "log_dir" ) ;
if ( logDir ! = cfg . defaultLogFileDir ) {
// user modified log_dir entry
// if 'log_dir' contains a '~' this needs to be expanded correctly
if ( canFind ( cfg . getValueString ( "log_dir" ) , "~" ) ) {
// ~ needs to be expanded correctly
if ( ! shellEnvSet ) {
// No shell or user set, so expandTilde() will fail - usually headless system running under init.d / systemd or potentially Docker
log . vdebug ( "log_dir: A '~' was found in log_dir, using the calculated 'homePath' to replace '~' as no SHELL or USER environment variable set" ) ;
logDir = cfg . homePath ~ strip ( cfg . getValueString ( "log_dir" ) , "~" ) ;
} else {
// A shell and user is set, expand any ~ as this will be expanded correctly if present
log . vdebug ( "log_dir: A '~' was found in log_dir, using SHELL or USER environment variable to expand '~'" ) ;
logDir = expandTilde ( cfg . getValueString ( "log_dir" ) ) ;
}
} else {
// '~' not found in log_dir entry, use as is
logDir = cfg . getValueString ( "log_dir" ) ;
}
// update log_dir with normalised path, with '~' expanded correctly
cfg . setValueString ( "log_dir" , logDir ) ;
}
// Configure logging only if enabled
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "enable_logging" ) ) {
2021-02-06 10:46:56 +01:00
// Initialise using the configured logging directory
2018-11-23 21:13:16 +01:00
log . vlog ( "Using logfile dir: " , logDir ) ;
log . init ( logDir ) ;
}
2018-12-05 20:19:00 +01:00
// Configure whether notifications are used
2019-04-11 04:26:20 +02:00
log . setNotifications ( cfg . getValueBool ( "monitor" ) & & ! cfg . getValueBool ( "disable_notifications" ) ) ;
2018-11-23 21:13:16 +01:00
2020-05-20 03:37:11 +02:00
// Application upgrades - skilion version etc
2020-06-21 02:01:24 +02:00
if ( exists ( databaseFilePath ) ) {
2019-04-11 04:26:20 +02:00
if ( ! cfg . getValueBool ( "dry_run" ) ) {
2020-06-21 02:01:24 +02:00
safeRemove ( databaseFilePath ) ;
2019-03-11 07:57:47 +01:00
}
2018-12-05 20:19:00 +01:00
log . logAndNotify ( "Database schema changed, resync needed" ) ;
2019-04-11 04:26:20 +02:00
cfg . setValueBool ( "resync" , true ) ;
2016-12-25 19:23:33 +01:00
}
2020-05-20 03:37:11 +02:00
// Handle --resync and --logout to remove local files
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "resync" ) | | cfg . getValueBool ( "logout" ) ) {
2019-08-24 09:18:58 +02:00
if ( cfg . getValueBool ( "resync" ) ) log . vdebug ( "--resync requested" ) ;
2017-05-28 23:14:37 +02:00
log . vlog ( "Deleting the saved status ..." ) ;
2019-04-11 04:26:20 +02:00
if ( ! cfg . getValueBool ( "dry_run" ) ) {
2019-03-11 07:57:47 +01:00
safeRemove ( cfg . databaseFilePath ) ;
safeRemove ( cfg . deltaLinkFilePath ) ;
safeRemove ( cfg . uploadStateFilePath ) ;
}
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "logout" ) ) {
2019-08-24 09:18:58 +02:00
log . vdebug ( "--logout requested" ) ;
2019-04-11 04:26:20 +02:00
if ( ! cfg . getValueBool ( "dry_run" ) ) {
2019-03-11 07:57:47 +01:00
safeRemove ( cfg . refreshTokenFilePath ) ;
}
2016-08-05 00:12:58 +02:00
}
2015-09-14 19:21:06 +02:00
}
2020-05-20 03:37:11 +02:00
2018-12-19 19:42:28 +01:00
// Display current application configuration, no application initialisation
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "display_config" ) ) {
2019-01-05 19:43:44 +01:00
// Display application version
2020-01-02 21:46:58 +01:00
writeln ( "onedrive version = " , strip ( import ( "version" ) ) ) ;
2018-12-19 19:42:28 +01:00
// Display all of the pertinent configuration options
2020-01-02 21:46:58 +01:00
writeln ( "Config path = " , cfg . configDirName ) ;
2018-12-19 19:42:28 +01:00
// Does a config file exist or are we using application defaults
2020-06-21 02:01:24 +02:00
writeln ( "Config file found in config path = " , exists ( configFilePath ) ) ;
2018-12-19 19:42:28 +01:00
// Config Options
2020-01-02 21:46:58 +01:00
writeln ( "Config option 'check_nosync' = " , cfg . getValueBool ( "check_nosync" ) ) ;
writeln ( "Config option 'sync_dir' = " , syncDir ) ;
writeln ( "Config option 'skip_dir' = " , cfg . getValueString ( "skip_dir" ) ) ;
writeln ( "Config option 'skip_file' = " , cfg . getValueString ( "skip_file" ) ) ;
writeln ( "Config option 'skip_dotfiles' = " , cfg . getValueBool ( "skip_dotfiles" ) ) ;
writeln ( "Config option 'skip_symlinks' = " , cfg . getValueBool ( "skip_symlinks" ) ) ;
writeln ( "Config option 'monitor_interval' = " , cfg . getValueLong ( "monitor_interval" ) ) ;
writeln ( "Config option 'min_notify_changes' = " , cfg . getValueLong ( "min_notify_changes" ) ) ;
writeln ( "Config option 'log_dir' = " , cfg . getValueString ( "log_dir" ) ) ;
writeln ( "Config option 'classify_as_big_delete' = " , cfg . getValueLong ( "classify_as_big_delete" ) ) ;
2020-11-13 20:17:53 +01:00
writeln ( "Config option 'upload_only' = " , cfg . getValueBool ( "upload_only" ) ) ;
writeln ( "Config option 'no_remote_delete' = " , cfg . getValueBool ( "no_remote_delete" ) ) ;
writeln ( "Config option 'remove_source_files' = " , cfg . getValueBool ( "remove_source_files" ) ) ;
2020-01-02 21:46:58 +01:00
2018-12-19 19:42:28 +01:00
// Is config option drive_id configured?
2019-04-11 04:26:20 +02:00
if ( cfg . getValueString ( "drive_id" ) ! = "" ) {
2020-01-02 21:46:58 +01:00
writeln ( "Config option 'drive_id' = " , cfg . getValueString ( "drive_id" ) ) ;
2018-12-19 19:42:28 +01:00
}
// Is sync_list configured?
2020-06-21 02:01:24 +02:00
if ( exists ( syncListFilePath ) ) {
2020-01-02 21:46:58 +01:00
writeln ( "Config option 'sync_root_files' = " , cfg . getValueBool ( "sync_root_files" ) ) ;
2020-06-27 11:10:37 +02:00
writeln ( "Selective sync 'sync_list' configured = true" ) ;
2019-01-05 19:43:44 +01:00
writeln ( "sync_list contents:" ) ;
// Output the sync_list contents
2020-06-21 02:01:24 +02:00
auto syncListFile = File ( syncListFilePath ) ;
2019-01-05 19:43:44 +01:00
auto range = syncListFile . byLine ( ) ;
foreach ( line ; range )
{
writeln ( line ) ;
}
2018-12-19 19:42:28 +01:00
} else {
2020-01-02 21:46:58 +01:00
writeln ( "Config option 'sync_root_files' = " , cfg . getValueBool ( "sync_root_files" ) ) ;
2020-06-27 11:10:37 +02:00
writeln ( "Selective sync 'sync_list' configured = false" ) ;
2018-12-19 19:42:28 +01:00
}
2020-06-27 11:10:37 +02:00
// Is business_shared_folders configured
if ( exists ( businessSharedFolderFilePath ) ) {
writeln ( "Business Shared Folders configured = true" ) ;
writeln ( "business_shared_folders contents:" ) ;
// Output the business_shared_folders contents
auto businessSharedFolderFileList = File ( businessSharedFolderFilePath ) ;
auto range = businessSharedFolderFileList . byLine ( ) ;
foreach ( line ; range )
{
writeln ( line ) ;
}
} else {
writeln ( "Business Shared Folders configured = false" ) ;
}
// Exit
2018-12-19 19:42:28 +01:00
return EXIT_SUCCESS ;
}
2020-12-17 20:17:52 +01:00
// Test if OneDrive service can be reached, exit if it cant be reached
2020-03-14 20:29:44 +01:00
log . vdebug ( "Testing network to ensure network connectivity to Microsoft OneDrive Service" ) ;
2020-12-17 20:17:52 +01:00
online = testNetwork ( ) ;
if ( ! online ) {
// Cant initialise the API as we are not online
2019-04-11 04:26:20 +02:00
if ( ! cfg . getValueBool ( "monitor" ) ) {
2020-12-17 20:17:52 +01:00
// Running as --synchronize
log . error ( "Unable to reach Microsoft OneDrive API service, unable to initialize application\n" ) ;
2018-12-04 01:15:44 +01:00
return EXIT_FAILURE ;
2020-12-17 20:17:52 +01:00
} else {
// Running as --monitor
log . error ( "Unable to reach Microsoft OneDrive API service at this point in time, re-trying network tests\n" ) ;
// re-try network connection to OneDrive
// https://github.com/abraunegg/onedrive/issues/1184
// Back off & retry with incremental delay
int retryCount = 10000 ;
int retryAttempts = 1 ;
int backoffInterval = 1 ;
int maxBackoffInterval = 3600 ;
bool retrySuccess = false ;
while ( ! retrySuccess ) {
// retry to access OneDrive API
backoffInterval + + ;
int thisBackOffInterval = retryAttempts * backoffInterval ;
log . vdebug ( " Retry Attempt: " , retryAttempts ) ;
if ( thisBackOffInterval < = maxBackoffInterval ) {
log . vdebug ( " Retry In (seconds): " , thisBackOffInterval ) ;
Thread . sleep ( dur ! "seconds" ( thisBackOffInterval ) ) ;
} else {
log . vdebug ( " Retry In (seconds): " , maxBackoffInterval ) ;
Thread . sleep ( dur ! "seconds" ( maxBackoffInterval ) ) ;
}
// perform the re-rty
online = testNetwork ( ) ;
if ( online ) {
// We are now online
log . log ( "Internet connectivity to Microsoft OneDrive service has been restored" ) ;
retrySuccess = true ;
} else {
// We are still offline
if ( retryAttempts = = retryCount ) {
// we have attempted to re-connect X number of times
// false set this to true to break out of while loop
retrySuccess = true ;
}
}
// Increment & loop around
retryAttempts + + ;
}
if ( ! online ) {
// Not online after 1.2 years of trying
log . error ( "ERROR: Was unable to reconnect to the Microsoft OneDrive service after 10000 attempts lasting over 1.2 years!" ) ;
return EXIT_FAILURE ;
}
2018-12-04 01:15:44 +01:00
}
}
2020-03-14 20:29:44 +01:00
2018-05-16 11:19:43 +02:00
// Initialize OneDrive, check for authorization
2020-12-17 20:17:52 +01:00
if ( online ) {
// we can only initialise if we are online
log . vlog ( "Initializing the OneDrive API ..." ) ;
oneDrive = new OneDriveApi ( cfg ) ;
onedriveInitialised = oneDrive . init ( ) ;
oneDrive . printAccessToken = cfg . getValueBool ( "print_token" ) ;
}
2020-05-20 03:37:11 +02:00
if ( ! onedriveInitialised ) {
2017-12-28 15:21:41 +01:00
log . error ( "Could not initialize the OneDrive API" ) ;
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2019-07-06 00:01:04 +02:00
return EXIT_UNAUTHORIZED ;
2015-09-01 20:45:34 +02:00
}
2018-12-19 19:42:28 +01:00
2020-03-14 20:29:44 +01:00
// if --synchronize or --monitor not passed in, configure the flag to display help & exit
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "synchronize" ) | | cfg . getValueBool ( "monitor" ) ) {
2018-04-24 04:14:36 +02:00
performSyncOK = true ;
}
2020-06-27 11:10:37 +02:00
// create-directory, remove-directory, source-directory, destination-directory
2020-05-20 03:37:11 +02:00
// these are activities that dont perform a sync, so to not generate an error message for these items either
2020-09-14 09:49:50 +02:00
if ( ( ( cfg . getValueString ( "create_directory" ) ! = "" ) | | ( cfg . getValueString ( "remove_directory" ) ! = "" ) ) | | ( ( cfg . getValueString ( "source_directory" ) ! = "" ) & & ( cfg . getValueString ( "destination_directory" ) ! = "" ) ) | | ( cfg . getValueString ( "get_file_link" ) ! = "" ) | | ( cfg . getValueString ( "create_share_link" ) ! = "" ) | | ( cfg . getValueString ( "get_o365_drive_id" ) ! = "" ) | | cfg . getValueBool ( "display_sync_status" ) | | cfg . getValueBool ( "list_business_shared_folders" ) ) {
2018-05-16 11:19:43 +02:00
performSyncOK = true ;
}
2020-05-20 03:37:11 +02:00
// Were acceptable sync operations provided? Was --synchronize or --monitor passed in
2018-04-24 04:14:36 +02:00
if ( ! performSyncOK ) {
2020-03-14 20:29:44 +01:00
// was the application just authorised?
if ( cfg . applicationAuthorizeResponseUri ) {
// Application was just authorised
2020-11-20 20:46:05 +01:00
if ( exists ( cfg . refreshTokenFilePath ) ) {
// OneDrive refresh token exists
log . log ( "\nApplication has been successfully authorised, however no additional command switches were provided.\n" ) ;
log . log ( "Please use --help for further assistance in regards to running this application.\n" ) ;
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
} else {
// we just authorised, but refresh_token does not exist .. probably an auth error
log . log ( "\nApplication has not been successfully authorised. Please check your URI response entry and try again.\n" ) ;
return EXIT_FAILURE ;
}
2020-03-14 20:29:44 +01:00
} else {
// Application was not just authorised
log . log ( "\n--synchronize or --monitor switches missing from your command line input. Please add one (not both) of these switches to your command line or use --help for further assistance.\n" ) ;
log . log ( "No OneDrive sync will be performed without one of these two arguments being present.\n" ) ;
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2020-03-14 20:29:44 +01:00
return EXIT_FAILURE ;
}
2018-04-24 04:14:36 +02:00
}
2019-03-23 00:20:27 +01:00
// if --synchronize && --monitor passed in, exit & display help as these conflict with each other
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "synchronize" ) & & cfg . getValueBool ( "monitor" ) ) {
2019-03-23 00:20:27 +01:00
writeln ( "\nERROR: --synchronize and --monitor cannot be used together\n" ) ;
writeln ( "Refer to --help to determine which command option you should use.\n" ) ;
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2019-03-23 00:20:27 +01:00
return EXIT_FAILURE ;
}
2019-03-11 07:57:47 +01:00
// Initialize the item database
2016-08-04 23:35:58 +02:00
log . vlog ( "Opening the item database ..." ) ;
2019-04-11 04:26:20 +02:00
if ( ! cfg . getValueBool ( "dry_run" ) ) {
2019-03-11 07:57:47 +01:00
// Load the items.sqlite3 file as the database
2019-08-24 09:18:58 +02:00
log . vdebug ( "Using database file: " , asNormalizedPath ( cfg . databaseFilePath ) ) ;
2019-03-11 07:57:47 +01:00
itemDb = new ItemDatabase ( cfg . databaseFilePath ) ;
} else {
// Load the items-dryrun.sqlite3 file as the database
2019-08-24 09:18:58 +02:00
log . vdebug ( "Using database file: " , asNormalizedPath ( cfg . databaseFilePathDryRun ) ) ;
2019-03-11 07:57:47 +01:00
itemDb = new ItemDatabase ( cfg . databaseFilePathDryRun ) ;
}
2018-03-14 05:43:40 +01:00
2020-10-29 22:00:26 +01:00
// What are the permission that have been set for the application?
// These are relevant for:
// - The ~/OneDrive parent folder or 'sync_dir' configured item
// - Any new folder created under ~/OneDrive or 'sync_dir'
// - Any new file created under ~/OneDrive or 'sync_dir'
// valid permissions are 000 -> 777 - anything else is invalid
if ( ( cfg . getValueLong ( "sync_dir_permissions" ) < 0 ) | | ( cfg . getValueLong ( "sync_file_permissions" ) < 0 ) | | ( cfg . getValueLong ( "sync_dir_permissions" ) > 777 ) | | ( cfg . getValueLong ( "sync_file_permissions" ) > 777 ) ) {
log . error ( "ERROR: Invalid 'User|Group|Other' permissions set within config file. Please check." ) ;
return EXIT_FAILURE ;
} else {
// debug log output what permissions are being set to
log . vdebug ( "Configuring default new folder permissions as: " , cfg . getValueLong ( "sync_dir_permissions" ) ) ;
cfg . configureRequiredDirectoryPermisions ( ) ;
log . vdebug ( "Configuring default new file permissions as: " , cfg . getValueLong ( "sync_file_permissions" ) ) ;
cfg . configureRequiredFilePermisions ( ) ;
}
2020-05-20 03:37:11 +02:00
// configure the sync direcory based on syncDir config option
2016-08-04 23:35:58 +02:00
log . vlog ( "All operations will be performed in: " , syncDir ) ;
2018-12-23 01:15:10 +01:00
if ( ! exists ( syncDir ) ) {
log . vdebug ( "syncDir: Configured syncDir is missing. Creating: " , syncDir ) ;
2019-07-30 23:20:26 +02:00
try {
// Attempt to create the sync dir we have been configured with
mkdirRecurse ( syncDir ) ;
2020-10-29 22:00:26 +01:00
// Configure the applicable permissions for the folder
2020-11-19 19:36:20 +01:00
log . vdebug ( "Setting directory permissions for: " , syncDir ) ;
2020-10-29 22:00:26 +01:00
syncDir . setAttributes ( cfg . returnRequiredDirectoryPermisions ( ) ) ;
2019-07-30 23:20:26 +02:00
} catch ( std . file . FileException e ) {
// Creating the sync directory failed
log . error ( "ERROR: Unable to create local OneDrive syncDir - " , e . msg ) ;
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2019-07-30 23:20:26 +02:00
return EXIT_FAILURE ;
}
2018-12-23 01:15:10 +01:00
}
2020-10-29 22:00:26 +01:00
// Change the working directory to the 'sync_dir' configured item
2015-09-27 18:47:41 +02:00
chdir ( syncDir ) ;
2018-03-14 05:43:40 +01:00
2018-05-16 11:19:43 +02:00
// Configure selective sync by parsing and getting a regex for skip_file config component
2017-03-24 22:30:03 +01:00
auto selectiveSync = new SelectiveSync ( ) ;
2020-06-27 11:10:37 +02:00
// load sync_list if it exists
if ( exists ( syncListFilePath ) ) {
2019-01-05 19:43:44 +01:00
log . vdebug ( "Loading user configured sync_list file ..." ) ;
2019-10-09 03:02:17 +02:00
syncListConfigured = true ;
2019-01-05 19:43:44 +01:00
// list what will be synced
2020-06-27 11:10:37 +02:00
auto syncListFile = File ( syncListFilePath ) ;
2019-01-05 19:43:44 +01:00
auto range = syncListFile . byLine ( ) ;
foreach ( line ; range )
{
log . vdebug ( "sync_list: " , line ) ;
}
2020-05-20 03:37:11 +02:00
// close syncListFile if open
if ( syncListFile . isOpen ( ) ) {
// close open file
syncListFile . close ( ) ;
}
2019-01-05 19:43:44 +01:00
}
2020-06-27 11:10:37 +02:00
selectiveSync . load ( syncListFilePath ) ;
// load business_shared_folders if it exists
if ( exists ( businessSharedFolderFilePath ) ) {
log . vdebug ( "Loading user configured business_shared_folders file ..." ) ;
// list what will be synced
auto businessSharedFolderFileList = File ( businessSharedFolderFilePath ) ;
auto range = businessSharedFolderFileList . byLine ( ) ;
foreach ( line ; range )
{
log . vdebug ( "business_shared_folders: " , line ) ;
}
}
selectiveSync . loadSharedFolders ( businessSharedFolderFilePath ) ;
2018-05-16 11:19:43 +02:00
2020-04-06 12:05:06 +02:00
// Configure skip_dir, skip_file, skip-dir-strict-match & skip_dotfiles from config entries
// Handle skip_dir configuration in config file
2019-03-14 20:55:05 +01:00
log . vdebug ( "Configuring skip_dir ..." ) ;
2019-04-11 04:26:20 +02:00
log . vdebug ( "skip_dir: " , cfg . getValueString ( "skip_dir" ) ) ;
selectiveSync . setDirMask ( cfg . getValueString ( "skip_dir" ) ) ;
2020-04-06 12:05:06 +02:00
2020-01-29 06:37:50 +01:00
// Was --skip-dir-strict-match configured?
2020-04-06 12:05:06 +02:00
log . vdebug ( "Configuring skip_dir_strict_match ..." ) ;
log . vdebug ( "skip_dir_strict_match: " , cfg . getValueBool ( "skip_dir_strict_match" ) ) ;
2020-01-29 06:37:50 +01:00
if ( cfg . getValueBool ( "skip_dir_strict_match" ) ) {
selectiveSync . setSkipDirStrictMatch ( ) ;
}
2020-04-06 12:05:06 +02:00
// Was --skip-dot-files configured?
log . vdebug ( "Configuring skip_dotfiles ..." ) ;
log . vdebug ( "skip_dotfiles: " , cfg . getValueBool ( "skip_dotfiles" ) ) ;
if ( cfg . getValueBool ( "skip_dotfiles" ) ) {
selectiveSync . setSkipDotfiles ( ) ;
}
// Handle skip_file configuration in config file
2019-03-14 20:55:05 +01:00
log . vdebug ( "Configuring skip_file ..." ) ;
2019-04-25 03:00:23 +02:00
// Validate skip_file to ensure that this does not contain an invalid configuration
// Do not use a skip_file entry of .* as this will prevent correct searching of local changes to process.
foreach ( entry ; cfg . getValueString ( "skip_file" ) . split ( "|" ) ) {
if ( entry = = ".*" ) {
// invalid entry element detected
log . logAndNotify ( "ERROR: Invalid skip_file entry '.*' detected" ) ;
return EXIT_FAILURE ;
}
}
2020-04-06 12:05:06 +02:00
// All skip_file entries are valid
2019-04-11 04:26:20 +02:00
log . vdebug ( "skip_file: " , cfg . getValueString ( "skip_file" ) ) ;
selectiveSync . setFileMask ( cfg . getValueString ( "skip_file" ) ) ;
2020-05-20 03:37:11 +02:00
2019-03-11 07:57:47 +01:00
// Initialize the sync engine
2019-04-11 04:26:20 +02:00
auto sync = new SyncEngine ( cfg , oneDrive , itemDb , selectiveSync ) ;
2018-07-16 01:58:36 +02:00
try {
2018-12-04 01:15:44 +01:00
if ( ! initSyncEngine ( sync ) ) {
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2018-07-16 01:58:36 +02:00
return EXIT_FAILURE ;
2019-09-11 12:26:21 +02:00
} else {
2020-09-14 09:49:50 +02:00
if ( ( cfg . getValueString ( "get_file_link" ) = = "" ) & & ( cfg . getValueString ( "create_share_link" ) = = "" ) ) {
// Print out that we are initializing the engine only if we are not grabbing the file link or creating a shareable link
2019-09-11 12:26:21 +02:00
log . logAndNotify ( "Initializing the Synchronization Engine ..." ) ;
}
2018-07-16 01:58:36 +02:00
}
2018-12-04 01:15:44 +01:00
} catch ( CurlException e ) {
2019-04-11 04:26:20 +02:00
if ( ! cfg . getValueBool ( "monitor" ) ) {
2019-09-11 12:26:21 +02:00
log . log ( "\nNo Internet connection." ) ;
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2018-09-24 21:25:40 +02:00
return EXIT_FAILURE ;
}
2018-07-16 01:58:36 +02:00
}
2018-12-04 01:15:44 +01:00
2020-03-14 20:58:57 +01:00
// if sync list is configured, set to true now that the sync engine is initialised
if ( syncListConfigured ) {
sync . setSyncListConfigured ( ) ;
}
2020-01-26 22:42:00 +01:00
// Do we need to configure specific --upload-only options?
if ( cfg . getValueBool ( "upload_only" ) ) {
// --upload-only was passed in or configured
2020-11-19 20:33:47 +01:00
log . vdebug ( "Configuring uploadOnly flag to TRUE as --upload-only passed in or configured" ) ;
2020-11-13 20:17:53 +01:00
sync . setUploadOnly ( ) ;
2020-01-26 22:42:00 +01:00
// was --no-remote-delete passed in or configured
if ( cfg . getValueBool ( "no_remote_delete" ) ) {
// Configure the noRemoteDelete flag
2020-11-19 20:33:47 +01:00
log . vdebug ( "Configuring noRemoteDelete flag to TRUE as --no-remote-delete passed in or configured" ) ;
2020-01-26 22:42:00 +01:00
sync . setNoRemoteDelete ( ) ;
}
// was --remove-source-files passed in or configured
if ( cfg . getValueBool ( "remove_source_files" ) ) {
// Configure the localDeleteAfterUpload flag
2020-11-19 20:33:47 +01:00
log . vdebug ( "Configuring localDeleteAfterUpload flag to TRUE as --remove-source-files passed in or configured" ) ;
2020-01-26 22:42:00 +01:00
sync . setLocalDeleteAfterUpload ( ) ;
}
}
2018-11-23 20:26:30 +01:00
// Do we configure to disable the upload validation routine
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "disable_upload_validation" ) ) sync . setDisableUploadValidation ( ) ;
2018-11-23 20:26:30 +01:00
2020-05-20 22:48:02 +02:00
// Has the user enabled to bypass data preservation of renaming local files when there is a conflict?
if ( cfg . getValueBool ( "bypass_data_preservation" ) ) {
log . log ( "WARNING: Application has been configured to bypass local data preservation in the event of file conflict." ) ;
log . log ( "WARNING: Local data loss MAY occur in this scenario." ) ;
2020-05-26 09:04:22 +02:00
sync . setBypassDataPreservation ( ) ;
2020-05-20 22:48:02 +02:00
}
2020-06-16 23:57:14 +02:00
// Are we configured to use a National Cloud Deployment
if ( cfg . getValueString ( "azure_ad_endpoint" ) ! = "" ) {
// value is configured, is it a valid value?
if ( ( cfg . getValueString ( "azure_ad_endpoint" ) = = "USL4" ) | | ( cfg . getValueString ( "azure_ad_endpoint" ) = = "USL5" ) | | ( cfg . getValueString ( "azure_ad_endpoint" ) = = "DE" ) | | ( cfg . getValueString ( "azure_ad_endpoint" ) = = "CN" ) ) {
// valid entries to flag we are using a National Cloud Deployment
sync . setNationalCloudDeployment ( ) ;
}
}
2018-06-17 00:27:43 +02:00
// Do we need to validate the syncDir to check for the presence of a '.nosync' file
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "check_nomount" ) ) {
2018-06-17 00:27:43 +02:00
// we were asked to check the mounts
if ( exists ( syncDir ~ "/.nosync" ) ) {
2018-12-05 20:19:00 +01:00
log . logAndNotify ( "ERROR: .nosync file found. Aborting synchronization process to safeguard data." ) ;
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2018-06-17 00:27:43 +02:00
return EXIT_FAILURE ;
}
}
2018-03-14 05:43:40 +01:00
// Do we need to create or remove a directory?
2019-04-11 04:26:20 +02:00
if ( ( cfg . getValueString ( "create_directory" ) ! = "" ) | | ( cfg . getValueString ( "remove_directory" ) ! = "" ) ) {
2018-03-14 05:43:40 +01:00
2019-04-11 04:26:20 +02:00
if ( cfg . getValueString ( "create_directory" ) ! = "" ) {
2018-03-14 05:43:40 +01:00
// create a directory on OneDrive
2019-04-11 04:26:20 +02:00
sync . createDirectoryNoSync ( cfg . getValueString ( "create_directory" ) ) ;
2018-03-14 05:43:40 +01:00
}
2019-04-11 04:26:20 +02:00
if ( cfg . getValueString ( "remove_directory" ) ! = "" ) {
2018-03-14 05:43:40 +01:00
// remove a directory on OneDrive
2019-04-11 04:26:20 +02:00
sync . deleteDirectoryNoSync ( cfg . getValueString ( "remove_directory" ) ) ;
2018-03-14 05:43:40 +01:00
}
}
// Are we renaming or moving a directory?
2019-04-11 04:26:20 +02:00
if ( ( cfg . getValueString ( "source_directory" ) ! = "" ) & & ( cfg . getValueString ( "destination_directory" ) ! = "" ) ) {
2018-03-14 05:43:40 +01:00
// We are renaming or moving a directory
2019-04-11 04:26:20 +02:00
sync . renameDirectoryNoSync ( cfg . getValueString ( "source_directory" ) , cfg . getValueString ( "destination_directory" ) ) ;
2018-03-14 05:43:40 +01:00
}
2018-12-04 00:59:23 +01:00
// Are we obtaining the Office 365 Drive ID for a given Office 365 SharePoint Shared Library?
2019-08-02 10:43:31 +02:00
if ( cfg . getValueString ( "get_o365_drive_id" ) ! = "" ) {
2019-04-11 04:26:20 +02:00
sync . querySiteCollectionForDriveID ( cfg . getValueString ( "get_o365_drive_id" ) ) ;
2020-06-27 11:10:37 +02:00
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
2018-12-04 00:59:23 +01:00
}
2020-09-14 09:49:50 +02:00
// Are we createing an anonymous read-only shareable link for an existing file on OneDrive?
if ( cfg . getValueString ( "create_share_link" ) ! = "" ) {
// Query OneDrive for the file, and if valid, create a shareable link for the file
sync . createShareableLinkForFile ( cfg . getValueString ( "create_share_link" ) ) ;
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
2019-08-02 10:43:31 +02:00
// Are we obtaining the URL path for a synced file?
if ( cfg . getValueString ( "get_file_link" ) ! = "" ) {
2020-09-14 09:49:50 +02:00
// Query OneDrive for the file link
2019-08-02 10:43:31 +02:00
sync . queryOneDriveForFileURL ( cfg . getValueString ( "get_file_link" ) , syncDir ) ;
2020-06-27 11:10:37 +02:00
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
// Are we listing OneDrive Business Shared Folders
if ( cfg . getValueBool ( "list_business_shared_folders" ) ) {
// Is this a business account type?
if ( sync . getAccountType ( ) = = "business" ) {
// List OneDrive Business Shared Folders
sync . listOneDriveBusinessSharedFolders ( ) ;
} else {
log . error ( "ERROR: Unsupported account type for listing OneDrive Business Shared Folders" ) ;
}
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
// Are we going to sync OneDrive Business Shared Folders
if ( cfg . getValueBool ( "sync_business_shared_folders" ) ) {
// Is this a business account type?
if ( sync . getAccountType ( ) = = "business" ) {
// Configure flag to sync business folders
sync . setSyncBusinessFolders ( ) ;
} else {
log . error ( "ERROR: Unsupported account type for syncing OneDrive Business Shared Folders" ) ;
}
2019-08-02 10:43:31 +02:00
}
2018-12-28 02:26:03 +01:00
// Are we displaying the sync status of the client?
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "display_sync_status" ) ) {
2018-12-28 02:26:03 +01:00
string remotePath = "/" ;
// Are we doing a single directory check?
2019-04-11 04:26:20 +02:00
if ( cfg . getValueString ( "single_directory" ) ! = "" ) {
2018-12-28 02:26:03 +01:00
// Need two different path strings here
2019-04-11 04:26:20 +02:00
remotePath = cfg . getValueString ( "single_directory" ) ;
2018-12-28 02:26:03 +01:00
}
sync . queryDriveForChanges ( remotePath ) ;
}
2020-05-20 05:15:43 +02:00
// Are we performing a sync, or monitor operation?
if ( ( cfg . getValueBool ( "synchronize" ) ) | | ( cfg . getValueBool ( "monitor" ) ) ) {
2020-06-11 22:46:59 +02:00
// Initialise the monitor class, so that we can do more granular inotify handling when performing the actual sync
// needed for --synchronize and --monitor handling
Monitor m = new Monitor ( selectiveSync ) ;
2015-09-01 20:45:34 +02:00
2020-05-20 05:15:43 +02:00
if ( cfg . getValueBool ( "synchronize" ) ) {
2018-03-14 05:43:40 +01:00
if ( online ) {
// Check user entry for local path - the above chdir means we are already in ~/OneDrive/ thus singleDirectory is local to this path
2020-06-27 11:10:37 +02:00
if ( cfg . getValueString ( "single_directory" ) ! = "" ) {
2018-03-14 05:43:40 +01:00
// Does the directory we want to sync actually exist?
2020-06-27 11:10:37 +02:00
if ( ! exists ( cfg . getValueString ( "single_directory" ) ) ) {
2020-08-20 23:06:56 +02:00
// The requested path to use with --single-directory does not exist locally within the configured 'sync_dir'
log . logAndNotify ( "WARNING: The requested path for --single-directory does not exist locally. Creating requested path within " , syncDir ) ;
2020-10-29 22:00:26 +01:00
// Make the required --single-directory path locally
string singleDirectoryPath = cfg . getValueString ( "single_directory" ) ;
mkdirRecurse ( singleDirectoryPath ) ;
// Configure the applicable permissions for the folder
2020-11-19 19:36:20 +01:00
log . vdebug ( "Setting directory permissions for: " , singleDirectoryPath ) ;
2020-10-29 22:00:26 +01:00
singleDirectoryPath . setAttributes ( cfg . returnRequiredDirectoryPermisions ( ) ) ;
2018-03-14 05:43:40 +01:00
}
}
2020-05-01 20:05:06 +02:00
// perform a --synchronize sync
// fullScanRequired = false, for final true-up
// but if we have sync_list configured, use syncListConfigured which = true
2020-06-11 22:46:59 +02:00
performSync ( sync , cfg . getValueString ( "single_directory" ) , cfg . getValueBool ( "download_only" ) , cfg . getValueBool ( "local_first" ) , cfg . getValueBool ( "upload_only" ) , LOG_NORMAL , false , syncListConfigured , displaySyncOptions , cfg . getValueBool ( "monitor" ) , m ) ;
2020-11-06 00:28:15 +01:00
// Write WAL and SHM data to file for this sync
log . vdebug ( "Merge contents of WAL and SHM files into main database file" ) ;
itemDb . performVacuum ( ) ;
2015-09-20 21:21:51 +02:00
}
2018-03-14 05:43:40 +01:00
}
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "monitor" ) ) {
2018-12-05 20:19:00 +01:00
log . logAndNotify ( "Initializing monitor ..." ) ;
2019-04-11 04:26:20 +02:00
log . log ( "OneDrive monitor interval (seconds): " , cfg . getValueLong ( "monitor_interval" ) ) ;
2020-06-11 22:46:59 +02:00
2018-03-14 05:43:40 +01:00
m . onDirCreated = delegate ( string path ) {
2020-04-06 12:05:06 +02:00
// Handle .folder creation if skip_dotfiles is enabled
if ( ( cfg . getValueBool ( "skip_dotfiles" ) ) & & ( selectiveSync . isDotFile ( path ) ) ) {
log . vlog ( "[M] Skipping watching path - .folder found & --skip-dot-files enabled: " , path ) ;
} else {
log . vlog ( "[M] Directory created: " , path ) ;
try {
sync . scanForDifferences ( path ) ;
} catch ( CurlException e ) {
log . vlog ( "Offline, cannot create remote dir!" ) ;
} catch ( Exception e ) {
log . logAndNotify ( "Cannot create remote directory: " , e . msg ) ;
}
2018-03-14 05:43:40 +01:00
}
} ;
m . onFileChanged = delegate ( string path ) {
log . vlog ( "[M] File changed: " , path ) ;
try {
sync . scanForDifferences ( path ) ;
2019-02-21 09:51:15 +01:00
} catch ( CurlException e ) {
log . vlog ( "Offline, cannot upload changed item!" ) ;
2018-03-14 05:43:40 +01:00
} catch ( Exception e ) {
2018-12-06 08:05:52 +01:00
log . logAndNotify ( "Cannot upload file changes/creation: " , e . msg ) ;
2018-03-14 05:43:40 +01:00
}
} ;
m . onDelete = delegate ( string path ) {
log . vlog ( "[M] Item deleted: " , path ) ;
try {
sync . deleteByPath ( path ) ;
2019-02-21 09:51:15 +01:00
} catch ( CurlException e ) {
log . vlog ( "Offline, cannot delete item!" ) ;
2018-12-06 00:50:46 +01:00
} catch ( SyncException e ) {
if ( e . msg = = "The item to delete is not in the local database" ) {
2020-04-06 12:05:06 +02:00
log . vlog ( "Item cannot be deleted from OneDrive because not found in the local database" ) ;
2018-12-06 00:50:46 +01:00
} else {
2018-12-06 08:05:52 +01:00
log . logAndNotify ( "Cannot delete remote item: " , e . msg ) ;
2018-12-06 00:50:46 +01:00
}
2018-03-14 05:43:40 +01:00
} catch ( Exception e ) {
2018-12-06 08:05:52 +01:00
log . logAndNotify ( "Cannot delete remote item: " , e . msg ) ;
2018-03-14 05:43:40 +01:00
}
} ;
m . onMove = delegate ( string from , string to ) {
log . vlog ( "[M] Item moved: " , from , " -> " , to ) ;
try {
2020-04-06 12:05:06 +02:00
// Handle .folder -> folder if skip_dotfiles is enabled
if ( ( cfg . getValueBool ( "skip_dotfiles" ) ) & & ( selectiveSync . isDotFile ( from ) ) ) {
// .folder -> folder handling - has to be handled as a new folder
sync . scanForDifferences ( to ) ;
} else {
sync . uploadMoveItem ( from , to ) ;
}
2019-02-21 09:51:15 +01:00
} catch ( CurlException e ) {
log . vlog ( "Offline, cannot move item!" ) ;
2018-03-14 05:43:40 +01:00
} catch ( Exception e ) {
2020-03-19 19:35:21 +01:00
log . logAndNotify ( "Cannot move item: " , e . msg ) ;
2018-03-14 05:43:40 +01:00
}
} ;
2018-12-28 03:19:20 +01:00
signal ( SIGINT , & exitHandler ) ;
signal ( SIGTERM , & exitHandler ) ;
2019-10-30 07:46:02 +01:00
// attempt to initialise monitor class
if ( ! cfg . getValueBool ( "download_only" ) ) {
2019-10-31 04:59:44 +01:00
try {
m . init ( cfg , cfg . getValueLong ( "verbose" ) > 0 , cfg . getValueBool ( "skip_symlinks" ) , cfg . getValueBool ( "check_nosync" ) ) ;
} catch ( MonitorException e ) {
// monitor initialisation failed
log . error ( "ERROR: " , e . msg ) ;
exit ( - 1 ) ;
}
}
// monitor loop
2020-05-20 03:37:11 +02:00
bool performMonitor = true ;
ulong monitorLoopFullCount = 0 ;
2019-10-31 04:59:44 +01:00
immutable auto checkInterval = dur ! "seconds" ( cfg . getValueLong ( "monitor_interval" ) ) ;
2020-05-20 03:37:11 +02:00
immutable long logInterval = cfg . getValueLong ( "monitor_log_frequency" ) ;
immutable long fullScanFrequency = cfg . getValueLong ( "monitor_fullscan_frequency" ) ;
MonoTime lastCheckTime = MonoTime . currTime ( ) ;
long logMonitorCounter = 0 ;
long fullScanCounter = 0 ;
2020-05-20 05:15:43 +02:00
// set fullScanRequired to true so that at application startup we perform a full walk
bool fullScanRequired = true ;
2020-05-01 20:05:06 +02:00
bool syncListConfiguredFullScanOverride = false ;
2019-12-16 19:32:38 +01:00
// if sync list is configured, set to true
if ( syncListConfigured ) {
2020-05-01 20:05:06 +02:00
// sync list is configured
syncListConfiguredFullScanOverride = true ;
2019-12-16 19:32:38 +01:00
}
2020-05-20 03:37:11 +02:00
while ( performMonitor ) {
2020-05-05 23:20:13 +02:00
if ( ! cfg . getValueBool ( "download_only" ) ) {
try {
m . update ( online ) ;
} catch ( MonitorException e ) {
// Catch any exceptions thrown by inotify / monitor engine
log . error ( "ERROR: The following inotify error was generated: " , e . msg ) ;
}
}
2019-10-31 04:59:44 +01:00
auto currTime = MonoTime . currTime ( ) ;
2020-06-04 22:49:23 +02:00
// has monitor_interval elapsed or are we at application startup / monitor startup?
// in a --resync scenario, if we have not 're-populated' the database, valid changes will get skipped:
// Monitor directory: ./target
// Monitor directory: target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
// [M] Item moved: random_files/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby -> target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
// Moving random_files/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby to target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
// Skipping uploading this new file as parent path is not in the database: target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
// 'target' should be in the DB, it should also exist online, but because of --resync, it does not exist in the database thus parent check fails
if ( ( currTime - lastCheckTime > checkInterval ) | | ( monitorLoopFullCount = = 0 ) ) {
2020-05-20 05:15:43 +02:00
// monitor sync loop
2020-05-25 03:30:04 +02:00
logOutputMessage = "################################################## NEW LOOP ##################################################" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
} else {
log . vdebug ( logOutputMessage ) ;
}
2020-05-20 03:37:11 +02:00
// Increment monitorLoopFullCount
monitorLoopFullCount + + ;
// Display memory details at start of loop
if ( displayMemoryUsage ) {
log . displayMemoryUsagePreGC ( ) ;
}
2020-05-20 05:15:43 +02:00
2019-10-31 04:59:44 +01:00
// log monitor output suppression
logMonitorCounter + = 1 ;
2019-12-16 19:32:38 +01:00
if ( logMonitorCounter > logInterval ) {
2019-10-31 04:59:44 +01:00
logMonitorCounter = 1 ;
2019-12-16 19:32:38 +01:00
}
2019-10-31 04:59:44 +01:00
2020-05-01 20:05:06 +02:00
// do we perform a full scan of sync_dir?
2019-10-31 04:59:44 +01:00
fullScanCounter + = 1 ;
if ( fullScanCounter > fullScanFrequency ) {
2020-05-01 20:05:06 +02:00
// loop counter has exceeded
2019-10-31 04:59:44 +01:00
fullScanCounter = 1 ;
if ( syncListConfigured ) {
2020-05-20 05:15:43 +02:00
// set fullScanRequired = true due to sync_list being used
fullScanRequired = true ;
2020-05-01 20:05:06 +02:00
// sync list is configured
syncListConfiguredFullScanOverride = true ;
2020-05-20 05:15:43 +02:00
} else {
// dont set fullScanRequired to true as this is excessive if sync_list is not being used
fullScanRequired = false ;
2019-10-31 04:59:44 +01:00
}
}
2019-11-21 09:58:34 +01:00
2020-05-25 03:30:04 +02:00
if ( displaySyncOptions ) {
// sync option handling per sync loop
log . log ( "fullScanCounter = " , fullScanCounter ) ;
log . log ( "syncListConfigured = " , syncListConfigured ) ;
log . log ( "fullScanRequired = " , fullScanRequired ) ;
log . log ( "syncListConfiguredFullScanOverride = " , syncListConfiguredFullScanOverride ) ;
} else {
// sync option handling per sync loop via debug
log . vdebug ( "fullScanCounter = " , fullScanCounter ) ;
log . vdebug ( "syncListConfigured = " , syncListConfigured ) ;
log . vdebug ( "fullScanRequired = " , fullScanRequired ) ;
log . vdebug ( "syncListConfiguredFullScanOverride = " , syncListConfiguredFullScanOverride ) ;
}
2019-10-31 04:59:44 +01:00
try {
if ( ! initSyncEngine ( sync ) ) {
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2019-10-31 04:59:44 +01:00
return EXIT_FAILURE ;
2018-12-04 01:15:44 +01:00
}
2018-12-12 21:08:18 +01:00
try {
2019-10-31 04:59:44 +01:00
// perform a --monitor sync
2020-07-22 07:56:20 +02:00
if ( ( cfg . getValueLong ( "verbose" ) > 0 ) | | ( logMonitorCounter = = logInterval ) ) log . log ( "Starting a sync with OneDrive" ) ;
2020-06-11 22:46:59 +02:00
performSync ( sync , cfg . getValueString ( "single_directory" ) , cfg . getValueBool ( "download_only" ) , cfg . getValueBool ( "local_first" ) , cfg . getValueBool ( "upload_only" ) , ( logMonitorCounter = = logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT ) , fullScanRequired , syncListConfiguredFullScanOverride , displaySyncOptions , cfg . getValueBool ( "monitor" ) , m ) ;
2019-10-31 04:59:44 +01:00
if ( ! cfg . getValueBool ( "download_only" ) ) {
2020-06-11 22:46:59 +02:00
// discard all events that may have been generated by the sync that have not already been handled
2020-05-05 23:20:13 +02:00
try {
m . update ( false ) ;
} catch ( MonitorException e ) {
// Catch any exceptions thrown by inotify / monitor engine
log . error ( "ERROR: The following inotify error was generated: " , e . msg ) ;
}
2018-12-12 21:08:18 +01:00
}
2020-07-22 07:56:20 +02:00
if ( ( cfg . getValueLong ( "verbose" ) > 0 ) | | ( logMonitorCounter = = logInterval ) ) log . log ( "Sync with OneDrive is complete" ) ;
2018-12-12 21:08:18 +01:00
} catch ( CurlException e ) {
2019-10-31 04:59:44 +01:00
// we already tried three times in the performSync routine
// if we still have problems, then the sync handle might have
// gone stale and we need to re-initialize the sync engine
log . log ( "Persistent connection errors, reinitializing connection" ) ;
sync . reset ( ) ;
2019-10-29 20:32:17 +01:00
}
2019-10-31 04:59:44 +01:00
} catch ( CurlException e ) {
log . log ( "Cannot initialize connection to OneDrive" ) ;
}
// performSync complete, set lastCheckTime to current time
fullScanRequired = false ;
if ( syncListConfigured ) {
2020-05-01 20:05:06 +02:00
syncListConfiguredFullScanOverride = false ;
2019-10-31 04:59:44 +01:00
}
lastCheckTime = MonoTime . currTime ( ) ;
2020-05-20 03:37:11 +02:00
// Display memory details before cleanup
if ( displayMemoryUsage ) {
log . displayMemoryUsagePreGC ( ) ;
}
// Perform Garbage Cleanup
2019-10-31 04:59:44 +01:00
GC . collect ( ) ;
2020-05-20 03:37:11 +02:00
// Display memory details after cleanup
if ( displayMemoryUsage ) {
log . displayMemoryUsagePostGC ( ) ;
}
2020-11-06 00:28:15 +01:00
// Write WAL and SHM data to file for this loop
log . vdebug ( "Merge contents of WAL and SHM files into main database file" ) ;
itemDb . performVacuum ( ) ;
2020-05-25 03:30:04 +02:00
// monitor loop complete
logOutputMessage = "################################################ LOOP COMPLETE ###############################################" ;
2020-11-06 00:28:15 +01:00
// Handle display options
2020-05-25 03:30:04 +02:00
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
} else {
log . vdebug ( logOutputMessage ) ;
}
2020-05-20 03:37:11 +02:00
// Developer break via config option
if ( cfg . getValueLong ( "monitor_max_loop" ) > 0 ) {
// developer set option to limit --monitor loops
if ( monitorLoopFullCount = = ( cfg . getValueLong ( "monitor_max_loop" ) ) ) {
performMonitor = false ;
log . log ( "Exiting after " , monitorLoopFullCount , " loops due to developer set option" ) ;
}
}
}
2019-10-31 04:59:44 +01:00
Thread . sleep ( dur ! "msecs" ( 500 ) ) ;
2015-09-17 00:16:23 +02:00
}
}
2015-09-14 19:21:06 +02:00
}
2016-02-24 17:07:59 +01:00
2019-03-11 07:57:47 +01:00
// --dry-run temp database cleanup
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "dry_run" ) ) {
2020-10-31 20:55:17 +01:00
string dryRunShmFile = cfg . databaseFilePathDryRun ~ "-shm" ;
string dryRunWalFile = cfg . databaseFilePathDryRun ~ "-wal" ;
2019-03-11 07:57:47 +01:00
if ( exists ( cfg . databaseFilePathDryRun ) ) {
// remove the file
log . vdebug ( "Removing items-dryrun.sqlite3 as dry run operations complete" ) ;
2020-10-31 20:55:17 +01:00
// remove items-dryrun.sqlite3
2019-03-11 07:57:47 +01:00
safeRemove ( cfg . databaseFilePathDryRun ) ;
}
2020-10-31 20:55:17 +01:00
// silent cleanup of shm and wal files if they exist
if ( exists ( dryRunShmFile ) ) {
// remove items-dryrun.sqlite3-shm
safeRemove ( dryRunShmFile ) ;
}
if ( exists ( dryRunWalFile ) ) {
// remove items-dryrun.sqlite3-wal
safeRemove ( dryRunWalFile ) ;
}
2019-03-11 07:57:47 +01:00
}
2020-05-20 03:37:11 +02:00
// Exit application
// Use exit scopes to shutdown API
2016-08-04 23:35:58 +02:00
return EXIT_SUCCESS ;
2015-09-01 20:45:34 +02:00
}
2015-09-20 21:21:51 +02:00
2018-12-04 01:15:44 +01:00
bool initSyncEngine ( SyncEngine sync )
{
try {
sync . init ( ) ;
} catch ( OneDriveException e ) {
if ( e . httpStatusCode = = 400 | | e . httpStatusCode = = 401 ) {
// Authorization is invalid
log . log ( "\nAuthorization token invalid, use --logout to authorize the client again\n" ) ;
return false ;
}
if ( e . httpStatusCode > = 500 ) {
// There was a HTTP 5xx Server Side Error, message already printed
return false ;
}
}
return true ;
}
2015-09-20 21:21:51 +02:00
// try to synchronize the folder three times
2020-06-11 22:46:59 +02:00
void performSync ( SyncEngine sync , string singleDirectory , bool downloadOnly , bool localFirst , bool uploadOnly , long logLevel , bool fullScanRequired , bool syncListConfiguredFullScanOverride , bool displaySyncOptions , bool monitorEnabled , Monitor m )
2015-09-20 21:21:51 +02:00
{
int count ;
2018-03-14 05:43:40 +01:00
string remotePath = "/" ;
string localPath = "." ;
2020-05-25 03:30:04 +02:00
string logOutputMessage ;
2018-03-14 05:43:40 +01:00
2020-05-01 20:05:06 +02:00
// performSync API scan triggers
log . vdebug ( "performSync API scan triggers" ) ;
log . vdebug ( "-----------------------------" ) ;
log . vdebug ( "fullScanRequired = " , fullScanRequired ) ;
log . vdebug ( "syncListConfiguredFullScanOverride = " , syncListConfiguredFullScanOverride ) ;
log . vdebug ( "-----------------------------" ) ;
2018-03-14 05:43:40 +01:00
// Are we doing a single directory sync?
if ( singleDirectory ! = "" ) {
// Need two different path strings here
remotePath = singleDirectory ;
localPath = singleDirectory ;
2019-09-09 04:30:59 +02:00
// Set flag for singleDirectoryScope for change handling
sync . setSingleDirectoryScope ( ) ;
2018-03-14 05:43:40 +01:00
}
2019-04-06 03:44:51 +02:00
// Due to Microsoft Sharepoint 'enrichment' of files, we try to download the Microsoft modified file automatically
// Set flag if we are in upload only state to handle this differently
// See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details
if ( uploadOnly ) sync . setUploadOnly ( ) ;
2015-09-20 21:21:51 +02:00
do {
try {
2020-05-25 03:30:04 +02:00
// starting a sync
logOutputMessage = "################################################## NEW SYNC ##################################################" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
} else {
log . vdebug ( logOutputMessage ) ;
}
2018-03-14 05:43:40 +01:00
if ( singleDirectory ! = "" ) {
// we were requested to sync a single directory
log . vlog ( "Syncing changes from this selected path: " , singleDirectory ) ;
2018-07-15 07:22:08 +02:00
if ( uploadOnly ) {
// Upload Only of selected single directory
2019-02-26 21:21:23 +01:00
if ( logLevel < MONITOR_LOG_QUIET ) log . log ( "Syncing changes from selected local path only - NOT syncing data changes from OneDrive ..." ) ;
2018-07-15 07:22:08 +02:00
sync . scanForDifferences ( localPath ) ;
} else {
// No upload only
if ( localFirst ) {
// Local First
2019-02-26 21:21:23 +01:00
if ( logLevel < MONITOR_LOG_QUIET ) log . log ( "Syncing changes from selected local path first before downloading changes from OneDrive ..." ) ;
2018-04-07 09:06:57 +02:00
sync . scanForDifferences ( localPath ) ;
sync . applyDifferencesSingleDirectory ( remotePath ) ;
2018-07-15 07:22:08 +02:00
} else {
// OneDrive First
2019-02-26 21:21:23 +01:00
if ( logLevel < MONITOR_LOG_QUIET ) log . log ( "Syncing changes from selected OneDrive path ..." ) ;
2018-07-15 07:22:08 +02:00
sync . applyDifferencesSingleDirectory ( remotePath ) ;
// is this a download only request?
if ( ! downloadOnly ) {
// process local changes
sync . scanForDifferences ( localPath ) ;
// ensure that the current remote state is updated locally
sync . applyDifferencesSingleDirectory ( remotePath ) ;
}
2018-04-07 09:06:57 +02:00
}
2018-03-14 05:43:40 +01:00
}
} else {
2018-07-15 07:22:08 +02:00
// no single directory sync
if ( uploadOnly ) {
// Upload Only of entire sync_dir
2019-02-26 21:21:23 +01:00
if ( logLevel < MONITOR_LOG_QUIET ) log . log ( "Syncing changes from local path only - NOT syncing data changes from OneDrive ..." ) ;
2018-07-15 07:22:08 +02:00
sync . scanForDifferences ( localPath ) ;
} else {
// No upload only
2020-05-25 03:30:04 +02:00
string syncCallLogOutput ;
2018-07-15 07:22:08 +02:00
if ( localFirst ) {
// sync local files first before downloading from OneDrive
2019-02-26 21:21:23 +01:00
if ( logLevel < MONITOR_LOG_QUIET ) log . log ( "Syncing changes from local path first before downloading changes from OneDrive ..." ) ;
2018-04-07 09:06:57 +02:00
sync . scanForDifferences ( localPath ) ;
2020-05-01 20:05:06 +02:00
// if syncListConfiguredFullScanOverride = true
if ( syncListConfiguredFullScanOverride ) {
// perform a full walk of OneDrive objects
sync . applyDifferences ( syncListConfiguredFullScanOverride ) ;
} else {
// perform a walk based on if a full scan is required
sync . applyDifferences ( fullScanRequired ) ;
}
2018-07-15 07:22:08 +02:00
} else {
// sync from OneDrive first before uploading files to OneDrive
2019-02-26 21:21:23 +01:00
if ( logLevel < MONITOR_LOG_SILENT ) log . log ( "Syncing changes from OneDrive ..." ) ;
2020-05-01 20:05:06 +02:00
2019-11-21 09:58:34 +01:00
// For the initial sync, always use the delta link so that we capture all the right delta changes including adds, moves & deletes
2020-05-25 03:30:04 +02:00
logOutputMessage = "Initial Scan: Call OneDrive Delta API for delta changes as compared to last successful sync." ;
syncCallLogOutput = "Calling sync.applyDifferences(false);" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
log . log ( syncCallLogOutput ) ;
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
2019-11-21 09:58:34 +01:00
sync . applyDifferences ( false ) ;
2020-05-01 20:05:06 +02:00
// is this a download only request?
if ( ! downloadOnly ) {
// process local changes walking the entire path checking for changes
// in monitor mode all local changes are captured via inotify
2020-05-25 03:30:04 +02:00
// thus scanning every 'monitor_interval' (default 300 seconds) for local changes is excessive and not required
logOutputMessage = "Process local filesystem (sync_dir) for file changes as compared to database entries" ;
syncCallLogOutput = "Calling sync.scanForDifferences(localPath);" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
log . log ( syncCallLogOutput ) ;
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
2020-06-11 22:46:59 +02:00
// What sort of local scan do we want to do?
// In --monitor mode, when performing the DB scan, a race condition occurs where by if a file or folder is moved during this process
// the inotify event is discarded once performSync() is finished (see m.update(false) above), so these events need to be handled
// This can be remediated by breaking the DB and file system scan into separate processes, and handing any applicable inotify events in between
if ( ! monitorEnabled ) {
// --synchronize in use
// standard process flow
sync . scanForDifferences ( localPath ) ;
} else {
// --monitor in use
// Use individual calls with inotify checks between to avoid a race condition between these 2 functions
// Database scan
sync . scanForDifferencesDatabaseScan ( localPath ) ;
// handle any inotify events that occured 'whilst' we were scanning the database
m . update ( true ) ;
// Filesystem walk to find new files not uploaded
sync . scanForDifferencesFilesystemScan ( localPath ) ;
// handle any inotify events that occured 'whilst' we were scanning the local filesystem
m . update ( true ) ;
}
2020-05-01 20:05:06 +02:00
// At this point, all OneDrive changes / local changes should be uploaded and in sync
// This MAY not be the case when using sync_list, thus a full walk of OneDrive ojects is required
// --synchronize & no sync_list : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// --synchronize & sync_list in use : fullScanRequired = false, syncListConfiguredFullScanOverride = true
2020-05-05 23:20:13 +02:00
// --monitor loops around 10 iterations. On the 1st loop, sets fullScanRequired = false, syncListConfiguredFullScanOverride = true if requried
2020-05-01 20:05:06 +02:00
// --monitor & no sync_list (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = false
// --monitor & no sync_list (loop #2 - #10) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// --monitor & sync_list in use (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = true
// --monitor & sync_list in use (loop #2 - #10) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// Do not perform a full walk of the OneDrive objects
if ( ( ! fullScanRequired ) & & ( ! syncListConfiguredFullScanOverride ) ) {
2020-05-25 03:30:04 +02:00
logOutputMessage = "Final True-Up: Do not perform a full walk of the OneDrive objects - not required" ;
syncCallLogOutput = "Calling sync.applyDifferences(false);" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
log . log ( syncCallLogOutput ) ;
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
2020-05-01 20:05:06 +02:00
sync . applyDifferences ( false ) ;
}
// Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop
if ( ( ! fullScanRequired ) & & ( syncListConfiguredFullScanOverride ) ) {
2020-05-25 03:30:04 +02:00
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop" ;
syncCallLogOutput = "Calling sync.applyDifferences(true);" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
log . log ( syncCallLogOutput ) ;
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
2020-05-01 20:05:06 +02:00
sync . applyDifferences ( true ) ;
}
// Perform a full walk of OneDrive objects because a full scan was required
if ( ( fullScanRequired ) & & ( ! syncListConfiguredFullScanOverride ) ) {
2020-05-25 03:30:04 +02:00
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required" ;
syncCallLogOutput = "Calling sync.applyDifferences(true);" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
log . log ( syncCallLogOutput ) ;
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
2020-05-01 20:05:06 +02:00
sync . applyDifferences ( true ) ;
}
// Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop
if ( ( fullScanRequired ) & & ( syncListConfiguredFullScanOverride ) ) {
2020-05-25 03:30:04 +02:00
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop" ;
syncCallLogOutput = "Calling sync.applyDifferences(true);" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
log . log ( syncCallLogOutput ) ;
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
2020-05-01 20:05:06 +02:00
sync . applyDifferences ( true ) ;
2019-04-01 20:21:02 +02:00
}
2018-07-15 07:22:08 +02:00
}
2018-04-07 09:06:57 +02:00
}
2018-03-14 05:43:40 +01:00
}
2017-12-31 17:07:21 +01:00
}
2020-05-25 03:30:04 +02:00
// sync is complete
logOutputMessage = "################################################ SYNC COMPLETE ###############################################" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
} else {
log . vdebug ( logOutputMessage ) ;
}
2015-09-20 21:21:51 +02:00
count = - 1 ;
2017-12-28 15:03:15 +01:00
} catch ( Exception e ) {
2018-12-12 21:08:18 +01:00
if ( + + count = = 3 ) {
log . log ( "Giving up on sync after three attempts: " , e . msg ) ;
throw e ;
} else
log . log ( "Retry sync count: " , count , ": " , e . msg ) ;
2015-09-20 21:21:51 +02:00
}
} while ( count ! = - 1 ) ;
}
2018-12-05 20:19:00 +01:00
2018-12-28 03:19:20 +01:00
// getting around the @nogc problem
// https://p0nce.github.io/d-idioms/#Bypassing-@nogc
auto assumeNoGC ( T ) ( T t ) if ( isFunctionPointer ! T | | isDelegate ! T )
{
enum attrs = functionAttributes ! T | FunctionAttribute . nogc ;
return cast ( SetFunctionAttributes ! ( T , functionLinkage ! T , attrs ) ) t ;
}
extern ( C ) nothrow @nogc @system void exitHandler ( int value ) {
try {
assumeNoGC ( ( ) {
log . log ( "Got termination signal, shutting down db connection" ) ;
2020-11-06 00:28:15 +01:00
// was itemDb initialised?
if ( itemDb ! is null ) {
// Make sure the .wal file is incorporated into the main db before we exit
itemDb . performVacuum ( ) ;
destroy ( itemDb ) ;
}
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown OneDrive API
2018-12-28 03:19:20 +01:00
} ) ( ) ;
} catch ( Exception e ) { }
exit ( 0 ) ;
}
2018-12-28 10:01:50 +01:00