2018-12-28 03:19:20 +01:00
import core.stdc.stdlib : EXIT_SUCCESS , EXIT_FAILURE , exit ;
2015-09-20 21:21:51 +02:00
import core.memory , core . time , core . thread ;
2019-08-24 09:18:58 +02:00
import std.getopt , std . file , std . path , std . process , std . stdio , std . conv , std . algorithm . searching , std . string , std . regex ;
2017-03-24 22:30:03 +01:00
import config , itemdb , monitor , onedrive , selective , sync , util ;
2018-08-13 23:21:11 +02:00
import std.net.curl : CurlException ;
2018-12-28 03:19:20 +01:00
import core.stdc.signal ;
2022-02-01 01:39:14 +01:00
import std.traits , std . format ;
2021-11-23 20:54:28 +01:00
import std.concurrency : receiveTimeout ;
2016-08-04 23:35:58 +02:00
static import log ;
2015-09-01 20:45:34 +02:00
2018-12-28 03:19:20 +01:00
OneDriveApi oneDrive ;
ItemDatabase itemDb ;
2021-11-23 20:54:28 +01:00
bool onedriveInitialised = false ;
2019-07-06 00:01:04 +02:00
const int EXIT_UNAUTHORIZED = 3 ;
2019-02-26 21:21:23 +01:00
enum MONITOR_LOG_SILENT = 2 ;
enum MONITOR_LOG_QUIET = 1 ;
enum LOG_NORMAL = 0 ;
2016-08-04 23:35:58 +02:00
int main ( string [ ] args )
2015-09-01 20:45:34 +02:00
{
2018-12-23 01:15:10 +01:00
// Disable buffering on stdout
stdout . setvbuf ( 0 , _IONBF ) ;
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
// main function variables
2019-04-11 04:26:20 +02:00
string confdirOption ;
2020-06-21 02:01:24 +02:00
string configFilePath ;
string syncListFilePath ;
string databaseFilePath ;
2020-06-27 11:10:37 +02:00
string businessSharedFolderFilePath ;
2020-05-20 03:37:11 +02:00
string currentConfigHash ;
string currentSyncListHash ;
string previousConfigHash ;
string previousSyncListHash ;
string configHashFile ;
string syncListHashFile ;
string configBackupFile ;
string syncDir ;
2020-05-25 03:30:04 +02:00
string logOutputMessage ;
2020-06-27 11:10:37 +02:00
string currentBusinessSharedFoldersHash ;
string previousBusinessSharedFoldersHash ;
string businessSharedFoldersHashFile ;
2020-05-20 03:37:11 +02:00
bool configOptionsDifferent = false ;
2020-06-27 11:10:37 +02:00
bool businessSharedFoldersDifferent = false ;
2020-05-20 03:37:11 +02:00
bool syncListConfigured = false ;
bool syncListDifferent = false ;
bool syncDirDifferent = false ;
bool skipFileDifferent = false ;
bool skipDirDifferent = false ;
bool online = false ;
bool performSyncOK = false ;
bool displayMemoryUsage = false ;
2020-05-25 03:30:04 +02:00
bool displaySyncOptions = false ;
2022-05-19 08:12:53 +02:00
// hash file permission values
string hashPermissionValue = "600" ;
auto convertedPermissionValue = parse ! long ( hashPermissionValue , 8 ) ;
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
// Define scopes
scope ( exit ) {
// Display memory details
if ( displayMemoryUsage ) {
log . displayMemoryUsagePreGC ( ) ;
}
// if initialised, shut down the HTTP instance
if ( onedriveInitialised ) {
oneDrive . shutdown ( ) ;
}
2020-11-06 00:28:15 +01:00
// was itemDb initialised?
if ( itemDb ! is null ) {
// Make sure the .wal file is incorporated into the main db before we exit
itemDb . performVacuum ( ) ;
destroy ( itemDb ) ;
}
2020-05-20 03:37:11 +02:00
// free API instance
2020-11-06 00:28:15 +01:00
if ( oneDrive ! is null ) {
destroy ( oneDrive ) ;
}
2020-05-20 03:37:11 +02:00
// Perform Garbage Cleanup
GC . collect ( ) ;
// Display memory details
if ( displayMemoryUsage ) {
log . displayMemoryUsagePostGC ( ) ;
}
}
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
scope ( failure ) {
// Display memory details
if ( displayMemoryUsage ) {
log . displayMemoryUsagePreGC ( ) ;
}
// if initialised, shut down the HTTP instance
if ( onedriveInitialised ) {
oneDrive . shutdown ( ) ;
}
2020-11-06 00:28:15 +01:00
// was itemDb initialised?
if ( itemDb ! is null ) {
// Make sure the .wal file is incorporated into the main db before we exit
itemDb . performVacuum ( ) ;
destroy ( itemDb ) ;
}
2020-05-20 03:37:11 +02:00
// free API instance
2020-11-06 00:28:15 +01:00
if ( oneDrive ! is null ) {
destroy ( oneDrive ) ;
}
2020-05-20 03:37:11 +02:00
// Perform Garbage Cleanup
GC . collect ( ) ;
// Display memory details
if ( displayMemoryUsage ) {
log . displayMemoryUsagePostGC ( ) ;
}
}
2019-04-11 04:26:20 +02:00
2020-05-20 03:37:11 +02:00
// read in application options as passed in
2019-01-26 01:03:00 +01:00
try {
2019-04-11 04:26:20 +02:00
bool printVersion = false ;
2019-01-26 01:03:00 +01:00
auto opt = getopt (
args ,
2019-04-11 04:26:20 +02:00
std . getopt . config . passThrough ,
2019-01-26 01:03:00 +01:00
std . getopt . config . bundling ,
std . getopt . config . caseSensitive ,
2019-04-11 04:26:20 +02:00
"confdir" , "Set the directory used to store the configuration files" , & confdirOption ,
2019-03-17 05:10:41 +01:00
"verbose|v+" , "Print more details, useful for debugging (repeat for extra debugging)" , & log . verbose ,
2019-01-26 01:03:00 +01:00
"version" , "Print the version and exit" , & printVersion
) ;
2022-01-27 22:58:38 +01:00
2020-05-20 03:37:11 +02:00
// print help and exit
2019-01-26 01:03:00 +01:00
if ( opt . helpWanted ) {
2019-04-11 04:26:20 +02:00
args ~ = "--help" ;
}
2020-05-20 03:37:11 +02:00
// print the version and exit
2019-04-11 04:26:20 +02:00
if ( printVersion ) {
2020-05-20 03:37:11 +02:00
writeln ( "onedrive " , strip ( import ( "version" ) ) ) ;
2019-01-26 01:03:00 +01:00
return EXIT_SUCCESS ;
}
} catch ( GetOptException e ) {
2020-05-20 03:37:11 +02:00
// option errors
2019-01-26 01:03:00 +01:00
log . error ( e . msg ) ;
2022-04-26 22:04:19 +02:00
log . error ( "Try 'onedrive --help' for more information" ) ;
2019-01-26 01:03:00 +01:00
return EXIT_FAILURE ;
} catch ( Exception e ) {
2020-05-20 03:37:11 +02:00
// generic error
2019-01-26 01:03:00 +01:00
log . error ( e . msg ) ;
2022-04-26 22:04:19 +02:00
log . error ( "Try 'onedrive --help' for more information" ) ;
2019-01-26 01:03:00 +01:00
return EXIT_FAILURE ;
}
2021-11-23 20:54:28 +01:00
2019-04-11 04:26:20 +02:00
// load configuration file if available
auto cfg = new config . Config ( confdirOption ) ;
if ( ! cfg . initialize ( ) ) {
// There was an error loading the configuration
// Error message already printed
return EXIT_FAILURE ;
2019-04-01 19:51:25 +02:00
}
2022-01-27 22:58:38 +01:00
// How was this application started - what options were passed in
log . vdebug ( "passed in options: " , args ) ;
log . vdebug ( "note --confdir and --verbose not listed in args" ) ;
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
// set memory display
displayMemoryUsage = cfg . getValueBool ( "display_memory" ) ;
2021-11-23 20:54:28 +01:00
2020-05-25 03:30:04 +02:00
// set display sync options
displaySyncOptions = cfg . getValueBool ( "display_sync_options" ) ;
2021-11-23 20:54:28 +01:00
2019-04-11 04:26:20 +02:00
// update configuration from command line args
cfg . update_from_args ( args ) ;
2022-05-03 07:09:08 +02:00
2022-01-27 22:58:38 +01:00
// --resync should be a 'last resort item' .. the user needs to 'accept' to proceed
2022-05-05 20:48:50 +02:00
if ( ( cfg . getValueBool ( "resync" ) ) & & ( ! cfg . getValueBool ( "display_config" ) ) ) {
2022-01-27 22:58:38 +01:00
// what is the risk acceptance?
bool resyncRiskAcceptance = false ;
if ( ! cfg . getValueBool ( "resync_auth" ) ) {
// need to prompt user
char response ;
// warning message
writeln ( "\nThe use of --resync will remove your local 'onedrive' client state, thus no record will exist regarding your current 'sync status'" ) ;
writeln ( "This has the potential to overwrite local versions of files with potentially older versions downloaded from OneDrive which can lead to data loss" ) ;
writeln ( "If in-doubt, backup your local data first before proceeding with --resync" ) ;
write ( "\nAre you sure you wish to proceed with --resync? [Y/N] " ) ;
try {
// Attempt to read user response
readf ( " %c\n" , & response ) ;
} catch ( std . format . FormatException e ) {
// Caught an error
return EXIT_FAILURE ;
}
// Evaluate user repsonse
if ( ( to ! string ( response ) = = "y" ) | | ( to ! string ( response ) = = "Y" ) ) {
// User has accepted --resync risk to proceed
resyncRiskAcceptance = true ;
// Are you sure you wish .. does not use writeln();
write ( "\n" ) ;
}
} else {
// resync_auth is true
resyncRiskAcceptance = true ;
}
// Action based on response
if ( ! resyncRiskAcceptance ) {
// --resync risk not accepted
return EXIT_FAILURE ;
}
}
2020-06-21 02:01:24 +02:00
// Initialise normalised file paths
configFilePath = buildNormalizedPath ( cfg . configDirName ~ "/config" ) ;
syncListFilePath = buildNormalizedPath ( cfg . configDirName ~ "/sync_list" ) ;
databaseFilePath = buildNormalizedPath ( cfg . configDirName ~ "/items.db" ) ;
2020-06-27 11:10:37 +02:00
businessSharedFolderFilePath = buildNormalizedPath ( cfg . configDirName ~ "/business_shared_folders" ) ;
2021-11-23 20:54:28 +01:00
2019-08-24 09:18:58 +02:00
// Has any of our configuration that would require a --resync been changed?
// 1. sync_list file modification
// 2. config file modification - but only if sync_dir, skip_dir, skip_file or drive_id was modified
// 3. CLI input overriding configured config file option
2020-05-20 03:37:11 +02:00
configHashFile = buildNormalizedPath ( cfg . configDirName ~ "/.config.hash" ) ;
syncListHashFile = buildNormalizedPath ( cfg . configDirName ~ "/.sync_list.hash" ) ;
configBackupFile = buildNormalizedPath ( cfg . configDirName ~ "/.config.backup" ) ;
2020-06-27 11:10:37 +02:00
businessSharedFoldersHashFile = buildNormalizedPath ( cfg . configDirName ~ "/.business_shared_folders.hash" ) ;
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
// Does a config file exist with a valid hash file
2020-06-21 02:01:24 +02:00
if ( ( exists ( configFilePath ) ) & & ( ! exists ( configHashFile ) ) ) {
2019-08-24 09:18:58 +02:00
// Hash of config file needs to be created
2020-06-21 02:01:24 +02:00
std . file . write ( configHashFile , computeQuickXorHash ( configFilePath ) ) ;
2022-05-19 08:12:53 +02:00
// Hash file should only be readable by the user who created it - 0600 permissions needed
configHashFile . setAttributes ( to ! int ( convertedPermissionValue ) ) ;
2019-08-24 09:18:58 +02:00
}
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
// Does a sync_list file exist with a valid hash file
2020-06-21 02:01:24 +02:00
if ( ( exists ( syncListFilePath ) ) & & ( ! exists ( syncListHashFile ) ) ) {
2019-08-24 09:18:58 +02:00
// Hash of sync_list file needs to be created
2020-06-21 02:01:24 +02:00
std . file . write ( syncListHashFile , computeQuickXorHash ( syncListFilePath ) ) ;
2022-05-19 08:12:53 +02:00
// Hash file should only be readable by the user who created it - 0600 permissions needed
syncListHashFile . setAttributes ( to ! int ( convertedPermissionValue ) ) ;
2019-08-24 09:18:58 +02:00
}
2021-11-23 20:54:28 +01:00
2020-06-27 11:10:37 +02:00
// check if business_shared_folders & business_shared_folders hash exists
if ( ( exists ( businessSharedFolderFilePath ) ) & & ( ! exists ( businessSharedFoldersHashFile ) ) ) {
// Hash of business_shared_folders file needs to be created
std . file . write ( businessSharedFoldersHashFile , computeQuickXorHash ( businessSharedFolderFilePath ) ) ;
2022-05-19 08:12:53 +02:00
// Hash file should only be readable by the user who created it - 0600 permissions needed
businessSharedFoldersHashFile . setAttributes ( to ! int ( convertedPermissionValue ) ) ;
2020-06-27 11:10:37 +02:00
}
2021-11-23 20:54:28 +01:00
2019-08-24 09:18:58 +02:00
// If hash files exist, but config files do not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different
2020-06-21 02:01:24 +02:00
if ( ( ! exists ( configFilePath ) ) & & ( exists ( configHashFile ) ) ) {
2019-08-24 09:18:58 +02:00
// if --resync safe remove config.hash and config.backup
if ( cfg . getValueBool ( "resync" ) ) {
safeRemove ( configHashFile ) ;
safeRemove ( configBackupFile ) ;
}
}
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
// If sync_list hash file exists, but sync_list file does not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different
2020-06-21 02:01:24 +02:00
if ( ( ! exists ( syncListFilePath ) ) & & ( exists ( syncListHashFile ) ) ) {
2019-08-24 09:18:58 +02:00
// if --resync safe remove sync_list.hash
if ( cfg . getValueBool ( "resync" ) ) safeRemove ( syncListHashFile ) ;
}
2021-11-23 20:54:28 +01:00
2020-06-27 11:10:37 +02:00
if ( ( ! exists ( businessSharedFolderFilePath ) ) & & ( exists ( businessSharedFoldersHashFile ) ) ) {
// if --resync safe remove business_shared_folders.hash
if ( cfg . getValueBool ( "resync" ) ) safeRemove ( businessSharedFoldersHashFile ) ;
}
2021-11-23 20:54:28 +01:00
2019-08-24 09:18:58 +02:00
// Read config hashes if they exist
2020-06-21 02:01:24 +02:00
if ( exists ( configFilePath ) ) currentConfigHash = computeQuickXorHash ( configFilePath ) ;
if ( exists ( syncListFilePath ) ) currentSyncListHash = computeQuickXorHash ( syncListFilePath ) ;
2020-06-27 11:10:37 +02:00
if ( exists ( businessSharedFolderFilePath ) ) currentBusinessSharedFoldersHash = computeQuickXorHash ( businessSharedFolderFilePath ) ;
2022-05-19 08:12:53 +02:00
if ( exists ( configHashFile ) ) {
try {
previousConfigHash = readText ( configHashFile ) ;
} catch ( std . file . FileException e ) {
// Unable to access required file
log . error ( "ERROR: Unable to access " , e . msg ) ;
// Use exit scopes to shutdown API
return EXIT_FAILURE ;
}
}
if ( exists ( syncListHashFile ) ) {
try {
previousSyncListHash = readText ( syncListHashFile ) ;
} catch ( std . file . FileException e ) {
// Unable to access required file
log . error ( "ERROR: Unable to access " , e . msg ) ;
// Use exit scopes to shutdown API
return EXIT_FAILURE ;
}
}
if ( exists ( businessSharedFoldersHashFile ) ) {
try {
previousBusinessSharedFoldersHash = readText ( businessSharedFoldersHashFile ) ;
} catch ( std . file . FileException e ) {
// Unable to access required file
log . error ( "ERROR: Unable to access " , e . msg ) ;
// Use exit scopes to shutdown API
return EXIT_FAILURE ;
}
}
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
// Was sync_list file updated?
2019-08-24 09:18:58 +02:00
if ( currentSyncListHash ! = previousSyncListHash ) {
// Debugging output to assist what changed
log . vdebug ( "sync_list file has been updated, --resync needed" ) ;
syncListDifferent = true ;
}
2021-11-23 20:54:28 +01:00
2020-06-27 11:10:37 +02:00
// Was business_shared_folders updated?
if ( currentBusinessSharedFoldersHash ! = previousBusinessSharedFoldersHash ) {
// Debugging output to assist what changed
log . vdebug ( "business_shared_folders file has been updated, --resync needed" ) ;
businessSharedFoldersDifferent = true ;
}
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
// Was config file updated between last execution ang this execution?
2019-08-24 09:18:58 +02:00
if ( currentConfigHash ! = previousConfigHash ) {
// config file was updated, however we only want to trigger a --resync requirement if sync_dir, skip_dir, skip_file or drive_id was modified
2022-05-05 20:48:50 +02:00
if ( ! cfg . getValueBool ( "display_config" ) ) {
// only print this message if we are not using --display-config
log . log ( "config file has been updated, checking if --resync needed" ) ;
}
2019-08-24 09:18:58 +02:00
if ( exists ( configBackupFile ) ) {
// check backup config what has changed for these configuration options if anything
// # sync_dir = "~/OneDrive"
// # skip_file = "~*|.~*|*.tmp"
// # skip_dir = ""
// # drive_id = ""
string [ string ] stringValues ;
stringValues [ "sync_dir" ] = "" ;
stringValues [ "skip_file" ] = "" ;
stringValues [ "skip_dir" ] = "" ;
stringValues [ "drive_id" ] = "" ;
2020-05-20 03:37:11 +02:00
auto configBackupFileHandle = File ( configBackupFile , "r" ) ;
string lineBuffer ;
auto range = configBackupFileHandle . byLine ( ) ;
// read configBackupFile line by line
foreach ( line ; range ) {
lineBuffer = stripLeft ( line ) . to ! string ;
if ( lineBuffer . length = = 0 | | lineBuffer [ 0 ] = = ';' | | lineBuffer [ 0 ] = = '#' ) continue ;
auto c = lineBuffer . matchFirst ( cfg . configRegex ) ;
2019-08-24 09:18:58 +02:00
if ( ! c . empty ) {
c . popFront ( ) ; // skip the whole match
string key = c . front . dup ;
auto p = key in stringValues ;
if ( p ) {
c . popFront ( ) ;
// compare this key
if ( ( key = = "sync_dir" ) & & ( c . front . dup ! = cfg . getValueString ( "sync_dir" ) ) ) {
log . vdebug ( key , " was modified since the last time the application was successfully run, --resync needed" ) ;
configOptionsDifferent = true ;
}
2021-11-23 20:54:28 +01:00
2019-08-24 09:18:58 +02:00
if ( ( key = = "skip_file" ) & & ( c . front . dup ! = cfg . getValueString ( "skip_file" ) ) ) {
log . vdebug ( key , " was modified since the last time the application was successfully run, --resync needed" ) ;
configOptionsDifferent = true ;
}
if ( ( key = = "skip_dir" ) & & ( c . front . dup ! = cfg . getValueString ( "skip_dir" ) ) ) {
log . vdebug ( key , " was modified since the last time the application was successfully run, --resync needed" ) ;
configOptionsDifferent = true ;
}
if ( ( key = = "drive_id" ) & & ( c . front . dup ! = cfg . getValueString ( "drive_id" ) ) ) {
log . vdebug ( key , " was modified since the last time the application was successfully run, --resync needed" ) ;
configOptionsDifferent = true ;
}
}
}
}
2020-05-20 03:37:11 +02:00
// close file if open
if ( configBackupFileHandle . isOpen ( ) ) {
// close open file
configBackupFileHandle . close ( ) ;
}
2019-08-24 09:18:58 +02:00
} else {
// no backup to check
log . vdebug ( "WARNING: no backup config file was found, unable to validate if any changes made" ) ;
}
2021-11-23 20:54:28 +01:00
2019-08-24 09:18:58 +02:00
// If there was a backup, any modified values we need to worry about would been detected
if ( ! cfg . getValueBool ( "display_config" ) ) {
// we are not testing the configuration
if ( ! configOptionsDifferent ) {
// no options are different
if ( ! cfg . getValueBool ( "dry_run" ) ) {
// we are not in a dry-run scenario
// update config hash
log . vdebug ( "updating config hash as it is out of date" ) ;
2020-06-21 02:01:24 +02:00
std . file . write ( configHashFile , computeQuickXorHash ( configFilePath ) ) ;
2022-05-19 08:12:53 +02:00
// Hash file should only be readable by the user who created it - 0600 permissions needed
configHashFile . setAttributes ( to ! int ( convertedPermissionValue ) ) ;
2019-08-24 09:18:58 +02:00
// create backup copy of current config file
log . vdebug ( "making backup of config file as it is out of date" ) ;
2020-06-21 02:01:24 +02:00
std . file . copy ( configFilePath , configBackupFile ) ;
2022-05-19 08:12:53 +02:00
// File Copy should only be readable by the user who created it - 0600 permissions needed
configBackupFile . setAttributes ( to ! int ( convertedPermissionValue ) ) ;
2019-08-24 09:18:58 +02:00
}
}
}
}
2021-11-23 20:54:28 +01:00
2019-08-24 09:18:58 +02:00
// Is there a backup of the config file if the config file exists?
2020-06-21 02:01:24 +02:00
if ( ( exists ( configFilePath ) ) & & ( ! exists ( configBackupFile ) ) ) {
2019-08-24 09:18:58 +02:00
// create backup copy of current config file
2020-06-21 02:01:24 +02:00
std . file . copy ( configFilePath , configBackupFile ) ;
2022-05-19 08:12:53 +02:00
// File Copy should only be readable by the user who created it - 0600 permissions needed
configBackupFile . setAttributes ( to ! int ( convertedPermissionValue ) ) ;
2019-08-24 09:18:58 +02:00
}
2021-11-23 20:54:28 +01:00
2019-08-24 09:18:58 +02:00
// config file set options can be changed via CLI input, specifically these will impact sync and --resync will be needed:
// --syncdir ARG
// --skip-file ARG
// --skip-dir ARG
2020-06-21 02:01:24 +02:00
if ( exists ( configFilePath ) ) {
2019-08-24 09:18:58 +02:00
// config file exists
// was the sync_dir updated by CLI?
if ( cfg . configFileSyncDir ! = "" ) {
// sync_dir was set in config file
if ( cfg . configFileSyncDir ! = cfg . getValueString ( "sync_dir" ) ) {
// config file was set and CLI input changed this
log . vdebug ( "sync_dir: CLI override of config file option, --resync needed" ) ;
syncDirDifferent = true ;
}
}
2021-11-23 20:54:28 +01:00
2019-08-24 09:18:58 +02:00
// was the skip_file updated by CLI?
if ( cfg . configFileSkipFile ! = "" ) {
// skip_file was set in config file
if ( cfg . configFileSkipFile ! = cfg . getValueString ( "skip_file" ) ) {
// config file was set and CLI input changed this
log . vdebug ( "skip_file: CLI override of config file option, --resync needed" ) ;
skipFileDifferent = true ;
}
2021-11-23 20:54:28 +01:00
}
2019-08-24 09:18:58 +02:00
// was the skip_dir updated by CLI?
if ( cfg . configFileSkipDir ! = "" ) {
// skip_dir was set in config file
if ( cfg . configFileSkipDir ! = cfg . getValueString ( "skip_dir" ) ) {
// config file was set and CLI input changed this
log . vdebug ( "skip_dir: CLI override of config file option, --resync needed" ) ;
skipDirDifferent = true ;
}
}
}
2021-11-23 20:54:28 +01:00
2019-08-24 09:18:58 +02:00
// Has anything triggered a --resync requirement?
2020-06-27 11:10:37 +02:00
if ( configOptionsDifferent | | syncListDifferent | | syncDirDifferent | | skipFileDifferent | | skipDirDifferent | | businessSharedFoldersDifferent ) {
2019-08-24 09:18:58 +02:00
// --resync needed, is the user just testing configuration changes?
if ( ! cfg . getValueBool ( "display_config" ) ) {
// not testing configuration changes
if ( ! cfg . getValueBool ( "resync" ) ) {
// --resync not issued, fail fast
log . error ( "An application configuration change has been detected where a --resync is required" ) ;
return EXIT_FAILURE ;
} else {
// --resync issued, update hashes of config files if they exist
if ( ! cfg . getValueBool ( "dry_run" ) ) {
// not doing a dry run, update hash files if config & sync_list exist
2020-06-21 02:01:24 +02:00
if ( exists ( configFilePath ) ) {
2019-08-24 09:18:58 +02:00
// update hash
log . vdebug ( "updating config hash as --resync issued" ) ;
2020-06-21 02:01:24 +02:00
std . file . write ( configHashFile , computeQuickXorHash ( configFilePath ) ) ;
2022-05-19 08:12:53 +02:00
// Hash file should only be readable by the user who created it - 0600 permissions needed
configHashFile . setAttributes ( to ! int ( convertedPermissionValue ) ) ;
2019-08-24 09:18:58 +02:00
// create backup copy of current config file
log . vdebug ( "making backup of config file as --resync issued" ) ;
2020-06-21 02:01:24 +02:00
std . file . copy ( configFilePath , configBackupFile ) ;
2022-05-19 08:12:53 +02:00
// File copy should only be readable by the user who created it - 0600 permissions needed
configBackupFile . setAttributes ( to ! int ( convertedPermissionValue ) ) ;
2019-08-24 09:18:58 +02:00
}
2020-06-21 02:01:24 +02:00
if ( exists ( syncListFilePath ) ) {
2019-08-24 09:18:58 +02:00
// update sync_list hash
log . vdebug ( "updating sync_list hash as --resync issued" ) ;
2020-06-21 02:01:24 +02:00
std . file . write ( syncListHashFile , computeQuickXorHash ( syncListFilePath ) ) ;
2022-05-19 08:12:53 +02:00
// Hash file should only be readable by the user who created it - 0600 permissions needed
syncListHashFile . setAttributes ( to ! int ( convertedPermissionValue ) ) ;
2019-08-24 09:18:58 +02:00
}
2020-06-27 11:10:37 +02:00
if ( exists ( businessSharedFolderFilePath ) ) {
// update business_shared_folders hash
log . vdebug ( "updating business_shared_folders hash as --resync issued" ) ;
std . file . write ( businessSharedFoldersHashFile , computeQuickXorHash ( businessSharedFolderFilePath ) ) ;
2022-05-19 08:12:53 +02:00
// Hash file should only be readable by the user who created it - 0600 permissions needed
businessSharedFoldersHashFile . setAttributes ( to ! int ( convertedPermissionValue ) ) ;
2020-06-27 11:10:37 +02:00
}
2019-08-24 09:18:58 +02:00
}
}
}
}
2021-11-23 20:54:28 +01:00
2020-05-25 03:30:04 +02:00
// dry-run notification and database setup
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "dry_run" ) ) {
2019-03-17 04:41:26 +01:00
log . log ( "DRY-RUN Configured. Output below shows what 'would' have occurred." ) ;
2020-10-31 20:55:17 +01:00
string dryRunShmFile = cfg . databaseFilePathDryRun ~ "-shm" ;
string dryRunWalFile = cfg . databaseFilePathDryRun ~ "-wal" ;
2020-03-29 21:55:27 +02:00
// If the dry run database exists, clean this up
if ( exists ( cfg . databaseFilePathDryRun ) ) {
// remove the existing file
log . vdebug ( "Removing items-dryrun.sqlite3 as it still exists for some reason" ) ;
2021-11-23 20:54:28 +01:00
safeRemove ( cfg . databaseFilePathDryRun ) ;
2020-03-29 21:55:27 +02:00
}
2020-10-31 20:55:17 +01:00
// silent cleanup of shm and wal files if they exist
if ( exists ( dryRunShmFile ) ) {
// remove items-dryrun.sqlite3-shm
2021-11-23 20:54:28 +01:00
safeRemove ( dryRunShmFile ) ;
2020-10-31 20:55:17 +01:00
}
if ( exists ( dryRunWalFile ) ) {
// remove items-dryrun.sqlite3-wal
2021-11-23 20:54:28 +01:00
safeRemove ( dryRunWalFile ) ;
2020-10-31 20:55:17 +01:00
}
2021-11-23 20:54:28 +01:00
2019-03-11 07:57:47 +01:00
// Make a copy of the original items.sqlite3 for use as the dry run copy if it exists
if ( exists ( cfg . databaseFilePath ) ) {
2020-03-29 21:55:27 +02:00
// in a --dry-run --resync scenario, we should not copy the existing database file
if ( ! cfg . getValueBool ( "resync" ) ) {
// copy the existing DB file to the dry-run copy
log . vdebug ( "Copying items.sqlite3 to items-dryrun.sqlite3 to use for dry run operations" ) ;
copy ( cfg . databaseFilePath , cfg . databaseFilePathDryRun ) ;
} else {
// no database copy due to --resync
log . vdebug ( "No database copy created for --dry-run due to --resync also being used" ) ;
}
2019-03-11 07:57:47 +01:00
}
}
2021-11-23 20:54:28 +01:00
2018-12-23 01:15:10 +01:00
// sync_dir environment handling to handle ~ expansion properly
2021-02-06 10:46:56 +01:00
bool shellEnvSet = false ;
2018-12-19 19:42:28 +01:00
if ( ( environment . get ( "SHELL" ) = = "" ) & & ( environment . get ( "USER" ) = = "" ) ) {
2018-12-23 01:15:10 +01:00
log . vdebug ( "sync_dir: No SHELL or USER environment variable configuration detected" ) ;
// No shell or user set, so expandTilde() will fail - usually headless system running under init.d / systemd or potentially Docker
// Does the 'currently configured' sync_dir include a ~
2019-04-11 04:26:20 +02:00
if ( canFind ( cfg . getValueString ( "sync_dir" ) , "~" ) ) {
2021-02-06 10:46:56 +01:00
// A ~ was found in sync_dir
log . vdebug ( "sync_dir: A '~' was found in sync_dir, using the calculated 'homePath' to replace '~' as no SHELL or USER environment variable set" ) ;
2019-04-11 04:26:20 +02:00
syncDir = cfg . homePath ~ strip ( cfg . getValueString ( "sync_dir" ) , "~" ) ;
2018-12-19 19:42:28 +01:00
} else {
2018-12-23 01:15:10 +01:00
// No ~ found in sync_dir, use as is
log . vdebug ( "sync_dir: Getting syncDir from config value sync_dir" ) ;
2019-04-11 04:26:20 +02:00
syncDir = cfg . getValueString ( "sync_dir" ) ;
2018-12-19 19:42:28 +01:00
}
} else {
2018-12-23 01:15:10 +01:00
// A shell and user is set, expand any ~ as this will be expanded correctly if present
2021-02-06 10:46:56 +01:00
shellEnvSet = true ;
2018-12-23 01:15:10 +01:00
log . vdebug ( "sync_dir: Getting syncDir from config value sync_dir" ) ;
2019-04-11 04:26:20 +02:00
if ( canFind ( cfg . getValueString ( "sync_dir" ) , "~" ) ) {
2018-12-23 01:15:10 +01:00
log . vdebug ( "sync_dir: A '~' was found in configured sync_dir, automatically expanding as SHELL and USER environment variable is set" ) ;
2019-04-11 04:26:20 +02:00
syncDir = expandTilde ( cfg . getValueString ( "sync_dir" ) ) ;
2018-12-23 01:15:10 +01:00
} else {
2019-04-11 04:26:20 +02:00
syncDir = cfg . getValueString ( "sync_dir" ) ;
2018-12-23 01:15:10 +01:00
}
2018-12-19 19:42:28 +01:00
}
2021-11-23 20:54:28 +01:00
2018-12-23 01:15:10 +01:00
// vdebug syncDir as set and calculated
log . vdebug ( "syncDir: " , syncDir ) ;
2021-11-23 20:54:28 +01:00
2021-02-06 10:46:56 +01:00
// Configure the logging directory if different from application default
// log_dir environment handling to handle ~ expansion properly
string logDir = cfg . getValueString ( "log_dir" ) ;
if ( logDir ! = cfg . defaultLogFileDir ) {
// user modified log_dir entry
// if 'log_dir' contains a '~' this needs to be expanded correctly
if ( canFind ( cfg . getValueString ( "log_dir" ) , "~" ) ) {
// ~ needs to be expanded correctly
if ( ! shellEnvSet ) {
// No shell or user set, so expandTilde() will fail - usually headless system running under init.d / systemd or potentially Docker
log . vdebug ( "log_dir: A '~' was found in log_dir, using the calculated 'homePath' to replace '~' as no SHELL or USER environment variable set" ) ;
logDir = cfg . homePath ~ strip ( cfg . getValueString ( "log_dir" ) , "~" ) ;
} else {
// A shell and user is set, expand any ~ as this will be expanded correctly if present
log . vdebug ( "log_dir: A '~' was found in log_dir, using SHELL or USER environment variable to expand '~'" ) ;
logDir = expandTilde ( cfg . getValueString ( "log_dir" ) ) ;
}
} else {
// '~' not found in log_dir entry, use as is
logDir = cfg . getValueString ( "log_dir" ) ;
}
// update log_dir with normalised path, with '~' expanded correctly
cfg . setValueString ( "log_dir" , logDir ) ;
}
2021-11-23 20:54:28 +01:00
2021-02-06 10:46:56 +01:00
// Configure logging only if enabled
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "enable_logging" ) ) {
2021-02-06 10:46:56 +01:00
// Initialise using the configured logging directory
2018-11-23 21:13:16 +01:00
log . vlog ( "Using logfile dir: " , logDir ) ;
log . init ( logDir ) ;
}
2018-12-05 20:19:00 +01:00
// Configure whether notifications are used
2019-04-11 04:26:20 +02:00
log . setNotifications ( cfg . getValueBool ( "monitor" ) & & ! cfg . getValueBool ( "disable_notifications" ) ) ;
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
// Application upgrades - skilion version etc
2020-06-21 02:01:24 +02:00
if ( exists ( databaseFilePath ) ) {
2019-04-11 04:26:20 +02:00
if ( ! cfg . getValueBool ( "dry_run" ) ) {
2020-06-21 02:01:24 +02:00
safeRemove ( databaseFilePath ) ;
2019-03-11 07:57:47 +01:00
}
2018-12-05 20:19:00 +01:00
log . logAndNotify ( "Database schema changed, resync needed" ) ;
2019-04-11 04:26:20 +02:00
cfg . setValueBool ( "resync" , true ) ;
2016-12-25 19:23:33 +01:00
}
2021-11-23 20:54:28 +01:00
2022-03-07 19:35:00 +01:00
// Handle --logout as separate item, do not 'resync' on a --logout
2021-11-16 20:05:11 +01:00
if ( cfg . getValueBool ( "logout" ) ) {
log . vdebug ( "--logout requested" ) ;
log . log ( "Deleting the saved authentication status ..." ) ;
if ( ! cfg . getValueBool ( "dry_run" ) ) {
safeRemove ( cfg . refreshTokenFilePath ) ;
}
// Exit
return EXIT_SUCCESS ;
}
2022-03-07 19:35:00 +01:00
// Handle --reauth to re-authenticate the client
if ( cfg . getValueBool ( "reauth" ) ) {
log . vdebug ( "--reauth requested" ) ;
log . log ( "Deleting the saved authentication status ... re-authentication requested" ) ;
if ( ! cfg . getValueBool ( "dry_run" ) ) {
safeRemove ( cfg . refreshTokenFilePath ) ;
}
}
2018-12-19 19:42:28 +01:00
// Display current application configuration, no application initialisation
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "display_config" ) ) {
2019-01-05 19:43:44 +01:00
// Display application version
2022-05-10 05:43:32 +02:00
writeln ( "onedrive version = " , strip ( import ( "version" ) ) ) ;
2018-12-19 19:42:28 +01:00
// Display all of the pertinent configuration options
2022-05-10 05:43:32 +02:00
writeln ( "Config path = " , cfg . configDirName ) ;
2018-12-19 19:42:28 +01:00
// Does a config file exist or are we using application defaults
2022-05-10 05:43:32 +02:00
writeln ( "Config file found in config path = " , exists ( configFilePath ) ) ;
2018-12-19 19:42:28 +01:00
// Is config option drive_id configured?
2019-04-11 04:26:20 +02:00
if ( cfg . getValueString ( "drive_id" ) ! = "" ) {
2022-05-10 05:43:32 +02:00
writeln ( "Config option 'drive_id' = " , cfg . getValueString ( "drive_id" ) ) ;
2018-12-19 19:42:28 +01:00
}
2021-11-23 20:54:28 +01:00
2022-05-10 05:43:32 +02:00
// Config Options as per 'config' file
writeln ( "Config option 'sync_dir' = " , syncDir ) ;
// logging and notifications
writeln ( "Config option 'enable_logging' = " , cfg . getValueBool ( "enable_logging" ) ) ;
writeln ( "Config option 'log_dir' = " , cfg . getValueString ( "log_dir" ) ) ;
writeln ( "Config option 'disable_notifications' = " , cfg . getValueBool ( "disable_notifications" ) ) ;
writeln ( "Config option 'min_notify_changes' = " , cfg . getValueLong ( "min_notify_changes" ) ) ;
// skip files and directory and 'matching' policy
writeln ( "Config option 'skip_dir' = " , cfg . getValueString ( "skip_dir" ) ) ;
writeln ( "Config option 'skip_dir_strict_match' = " , cfg . getValueBool ( "skip_dir_strict_match" ) ) ;
writeln ( "Config option 'skip_file' = " , cfg . getValueString ( "skip_file" ) ) ;
writeln ( "Config option 'skip_dotfiles' = " , cfg . getValueBool ( "skip_dotfiles" ) ) ;
writeln ( "Config option 'skip_symlinks' = " , cfg . getValueBool ( "skip_symlinks" ) ) ;
// --monitor sync process options
writeln ( "Config option 'monitor_interval' = " , cfg . getValueLong ( "monitor_interval" ) ) ;
writeln ( "Config option 'monitor_log_frequency' = " , cfg . getValueLong ( "monitor_log_frequency" ) ) ;
writeln ( "Config option 'monitor_fullscan_frequency' = " , cfg . getValueLong ( "monitor_fullscan_frequency" ) ) ;
// sync process and method
writeln ( "Config option 'dry_run' = " , cfg . getValueBool ( "dry_run" ) ) ;
writeln ( "Config option 'upload_only' = " , cfg . getValueBool ( "upload_only" ) ) ;
writeln ( "Config option 'download_only' = " , cfg . getValueBool ( "download_only" ) ) ;
writeln ( "Config option 'local_first' = " , cfg . getValueBool ( "local_first" ) ) ;
writeln ( "Config option 'check_nosync' = " , cfg . getValueBool ( "check_nosync" ) ) ;
writeln ( "Config option 'check_nomount' = " , cfg . getValueBool ( "check_nomount" ) ) ;
writeln ( "Config option 'resync' = " , cfg . getValueBool ( "resync" ) ) ;
writeln ( "Config option 'resync_auth' = " , cfg . getValueBool ( "resync_auth" ) ) ;
// data integrity
writeln ( "Config option 'classify_as_big_delete' = " , cfg . getValueLong ( "classify_as_big_delete" ) ) ;
writeln ( "Config option 'disable_upload_validation' = " , cfg . getValueBool ( "disable_upload_validation" ) ) ;
writeln ( "Config option 'bypass_data_preservation' = " , cfg . getValueBool ( "bypass_data_preservation" ) ) ;
writeln ( "Config option 'no_remote_delete' = " , cfg . getValueBool ( "no_remote_delete" ) ) ;
writeln ( "Config option 'remove_source_files' = " , cfg . getValueBool ( "remove_source_files" ) ) ;
writeln ( "Config option 'sync_dir_permissions' = " , cfg . getValueLong ( "sync_dir_permissions" ) ) ;
writeln ( "Config option 'sync_file_permissions' = " , cfg . getValueLong ( "sync_file_permissions" ) ) ;
2022-05-31 21:57:05 +02:00
writeln ( "Config option 'space_reservation' = " , cfg . getValueLong ( "space_reservation" ) ) ;
2022-05-10 05:43:32 +02:00
// curl operations
writeln ( "Config option 'application_id' = " , cfg . getValueString ( "application_id" ) ) ;
writeln ( "Config option 'azure_ad_endpoint' = " , cfg . getValueString ( "azure_ad_endpoint" ) ) ;
writeln ( "Config option 'azure_tenant_id' = " , cfg . getValueString ( "azure_tenant_id" ) ) ;
writeln ( "Config option 'user_agent' = " , cfg . getValueString ( "user_agent" ) ) ;
writeln ( "Config option 'force_http_2' = " , cfg . getValueBool ( "force_http_2" ) ) ;
writeln ( "Config option 'debug_https' = " , cfg . getValueBool ( "debug_https" ) ) ;
writeln ( "Config option 'rate_limit' = " , cfg . getValueLong ( "rate_limit" ) ) ;
writeln ( "Config option 'operation_timeout' = " , cfg . getValueLong ( "operation_timeout" ) ) ;
// Is sync_list configured ?
writeln ( "Config option 'sync_root_files' = " , cfg . getValueBool ( "sync_root_files" ) ) ;
2020-06-21 02:01:24 +02:00
if ( exists ( syncListFilePath ) ) {
2022-05-10 05:43:32 +02:00
writeln ( "Selective sync 'sync_list' configured = true" ) ;
2019-01-05 19:43:44 +01:00
writeln ( "sync_list contents:" ) ;
// Output the sync_list contents
2022-05-19 08:12:53 +02:00
auto syncListFile = File ( syncListFilePath , "r" ) ;
2019-01-05 19:43:44 +01:00
auto range = syncListFile . byLine ( ) ;
foreach ( line ; range )
{
writeln ( line ) ;
}
2018-12-19 19:42:28 +01:00
} else {
2022-05-10 05:43:32 +02:00
writeln ( "Selective sync 'sync_list' configured = false" ) ;
2018-12-19 19:42:28 +01:00
}
2021-11-23 20:54:28 +01:00
2022-05-10 05:43:32 +02:00
// Is business_shared_folders enabled and configured ?
writeln ( "Config option 'sync_business_shared_folders' = " , cfg . getValueBool ( "sync_business_shared_folders" ) ) ;
2020-06-27 11:10:37 +02:00
if ( exists ( businessSharedFolderFilePath ) ) {
2022-05-10 05:43:32 +02:00
writeln ( "Business Shared Folders configured = true" ) ;
2020-06-27 11:10:37 +02:00
writeln ( "business_shared_folders contents:" ) ;
// Output the business_shared_folders contents
2022-05-19 08:12:53 +02:00
auto businessSharedFolderFileList = File ( businessSharedFolderFilePath , "r" ) ;
2020-06-27 11:10:37 +02:00
auto range = businessSharedFolderFileList . byLine ( ) ;
foreach ( line ; range )
{
writeln ( line ) ;
}
} else {
2022-05-10 05:43:32 +02:00
writeln ( "Business Shared Folders configured = false" ) ;
2020-06-27 11:10:37 +02:00
}
2022-05-10 05:43:32 +02:00
// Are webhooks enabled?
writeln ( "Config option 'webhook_enabled' = " , cfg . getValueBool ( "webhook_enabled" ) ) ;
if ( cfg . getValueBool ( "webhook_enabled" ) ) {
writeln ( "Config option 'webhook_public_url' = " , cfg . getValueString ( "webhook_public_url" ) ) ;
writeln ( "Config option 'webhook_listening_host' = " , cfg . getValueString ( "webhook_listening_host" ) ) ;
writeln ( "Config option 'webhook_listening_port' = " , cfg . getValueLong ( "webhook_listening_port" ) ) ;
writeln ( "Config option 'webhook_expiration_interval' = " , cfg . getValueLong ( "webhook_expiration_interval" ) ) ;
writeln ( "Config option 'webhook_renewal_interval' = " , cfg . getValueLong ( "webhook_renewal_interval" ) ) ;
}
2020-06-27 11:10:37 +02:00
// Exit
2018-12-19 19:42:28 +01:00
return EXIT_SUCCESS ;
}
2021-11-23 20:54:28 +01:00
2022-05-13 03:06:59 +02:00
// --upload-only and --download-only are mutually exclusive and cannot be used together
if ( ( cfg . getValueBool ( "upload_only" ) ) & & ( cfg . getValueBool ( "download_only" ) ) ) {
// both cannot be true at the same time
2022-05-16 02:16:50 +02:00
writeln ( "ERROR: --upload-only and --download-only are mutually exclusive and cannot be used together.\n" ) ;
2022-05-13 03:06:59 +02:00
return EXIT_FAILURE ;
}
2022-05-05 20:48:50 +02:00
// Handle --resync to remove local files
if ( cfg . getValueBool ( "resync" ) ) {
log . vdebug ( "--resync requested" ) ;
log . log ( "Deleting the saved application sync status ..." ) ;
if ( ! cfg . getValueBool ( "dry_run" ) ) {
safeRemove ( cfg . databaseFilePath ) ;
safeRemove ( cfg . deltaLinkFilePath ) ;
safeRemove ( cfg . uploadStateFilePath ) ;
}
}
2020-12-17 20:17:52 +01:00
// Test if OneDrive service can be reached, exit if it cant be reached
2020-03-14 20:29:44 +01:00
log . vdebug ( "Testing network to ensure network connectivity to Microsoft OneDrive Service" ) ;
2020-12-17 20:17:52 +01:00
online = testNetwork ( ) ;
if ( ! online ) {
2022-03-07 00:57:20 +01:00
// Cant initialise the API as we are not online
2019-04-11 04:26:20 +02:00
if ( ! cfg . getValueBool ( "monitor" ) ) {
2020-12-17 20:17:52 +01:00
// Running as --synchronize
log . error ( "Unable to reach Microsoft OneDrive API service, unable to initialize application\n" ) ;
2018-12-04 01:15:44 +01:00
return EXIT_FAILURE ;
2020-12-17 20:17:52 +01:00
} else {
// Running as --monitor
log . error ( "Unable to reach Microsoft OneDrive API service at this point in time, re-trying network tests\n" ) ;
2021-11-23 20:54:28 +01:00
2020-12-17 20:17:52 +01:00
// re-try network connection to OneDrive
// https://github.com/abraunegg/onedrive/issues/1184
// Back off & retry with incremental delay
int retryCount = 10000 ;
int retryAttempts = 1 ;
int backoffInterval = 1 ;
int maxBackoffInterval = 3600 ;
2021-11-23 20:54:28 +01:00
2020-12-17 20:17:52 +01:00
bool retrySuccess = false ;
while ( ! retrySuccess ) {
// retry to access OneDrive API
backoffInterval + + ;
int thisBackOffInterval = retryAttempts * backoffInterval ;
2021-11-23 20:54:28 +01:00
log . vdebug ( " Retry Attempt: " , retryAttempts ) ;
2020-12-17 20:17:52 +01:00
if ( thisBackOffInterval < = maxBackoffInterval ) {
log . vdebug ( " Retry In (seconds): " , thisBackOffInterval ) ;
Thread . sleep ( dur ! "seconds" ( thisBackOffInterval ) ) ;
} else {
log . vdebug ( " Retry In (seconds): " , maxBackoffInterval ) ;
Thread . sleep ( dur ! "seconds" ( maxBackoffInterval ) ) ;
}
// perform the re-rty
online = testNetwork ( ) ;
if ( online ) {
// We are now online
log . log ( "Internet connectivity to Microsoft OneDrive service has been restored" ) ;
retrySuccess = true ;
} else {
// We are still offline
if ( retryAttempts = = retryCount ) {
// we have attempted to re-connect X number of times
// false set this to true to break out of while loop
retrySuccess = true ;
2021-11-23 20:54:28 +01:00
}
2020-12-17 20:17:52 +01:00
}
// Increment & loop around
retryAttempts + + ;
}
if ( ! online ) {
// Not online after 1.2 years of trying
log . error ( "ERROR: Was unable to reconnect to the Microsoft OneDrive service after 10000 attempts lasting over 1.2 years!" ) ;
return EXIT_FAILURE ;
}
2018-12-04 01:15:44 +01:00
}
2022-03-07 04:02:50 +01:00
}
// Check application version and Initialize OneDrive API, check for authorization
if ( online ) {
2022-03-07 00:57:20 +01:00
// Check Application Version
log . vlog ( "Checking Application Version ..." ) ;
checkApplicationVersion ( ) ;
2020-12-17 20:17:52 +01:00
// we can only initialise if we are online
log . vlog ( "Initializing the OneDrive API ..." ) ;
oneDrive = new OneDriveApi ( cfg ) ;
onedriveInitialised = oneDrive . init ( ) ;
oneDrive . printAccessToken = cfg . getValueBool ( "print_token" ) ;
}
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
if ( ! onedriveInitialised ) {
2017-12-28 15:21:41 +01:00
log . error ( "Could not initialize the OneDrive API" ) ;
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2019-07-06 00:01:04 +02:00
return EXIT_UNAUTHORIZED ;
2015-09-01 20:45:34 +02:00
}
2021-11-23 20:54:28 +01:00
2020-03-14 20:29:44 +01:00
// if --synchronize or --monitor not passed in, configure the flag to display help & exit
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "synchronize" ) | | cfg . getValueBool ( "monitor" ) ) {
2018-04-24 04:14:36 +02:00
performSyncOK = true ;
}
2021-11-23 20:54:28 +01:00
2020-06-27 11:10:37 +02:00
// create-directory, remove-directory, source-directory, destination-directory
2020-05-20 03:37:11 +02:00
// these are activities that dont perform a sync, so to not generate an error message for these items either
2022-03-09 21:00:07 +01:00
if ( ( ( cfg . getValueString ( "create_directory" ) ! = "" ) | | ( cfg . getValueString ( "remove_directory" ) ! = "" ) ) | | ( ( cfg . getValueString ( "source_directory" ) ! = "" ) & & ( cfg . getValueString ( "destination_directory" ) ! = "" ) ) | | ( cfg . getValueString ( "get_file_link" ) ! = "" ) | | ( cfg . getValueString ( "modified_by" ) ! = "" ) | | ( cfg . getValueString ( "create_share_link" ) ! = "" ) | | ( cfg . getValueString ( "get_o365_drive_id" ) ! = "" ) | | cfg . getValueBool ( "display_sync_status" ) | | cfg . getValueBool ( "list_business_shared_folders" ) ) {
2018-05-16 11:19:43 +02:00
performSyncOK = true ;
}
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
// Were acceptable sync operations provided? Was --synchronize or --monitor passed in
2018-04-24 04:14:36 +02:00
if ( ! performSyncOK ) {
2020-03-14 20:29:44 +01:00
// was the application just authorised?
if ( cfg . applicationAuthorizeResponseUri ) {
// Application was just authorised
2020-11-20 20:46:05 +01:00
if ( exists ( cfg . refreshTokenFilePath ) ) {
// OneDrive refresh token exists
log . log ( "\nApplication has been successfully authorised, however no additional command switches were provided.\n" ) ;
2022-04-26 22:04:19 +02:00
log . log ( "Please use 'onedrive --help' for further assistance in regards to running this application.\n" ) ;
2020-11-20 20:46:05 +01:00
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
} else {
// we just authorised, but refresh_token does not exist .. probably an auth error
log . log ( "\nApplication has not been successfully authorised. Please check your URI response entry and try again.\n" ) ;
return EXIT_FAILURE ;
}
2020-03-14 20:29:44 +01:00
} else {
// Application was not just authorised
2022-04-26 22:04:19 +02:00
log . log ( "\n--synchronize or --monitor switches missing from your command line input. Please add one (not both) of these switches to your command line or use 'onedrive --help' for further assistance.\n" ) ;
2020-03-14 20:29:44 +01:00
log . log ( "No OneDrive sync will be performed without one of these two arguments being present.\n" ) ;
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2020-03-14 20:29:44 +01:00
return EXIT_FAILURE ;
}
2018-04-24 04:14:36 +02:00
}
2021-11-23 20:54:28 +01:00
2019-03-23 00:20:27 +01:00
// if --synchronize && --monitor passed in, exit & display help as these conflict with each other
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "synchronize" ) & & cfg . getValueBool ( "monitor" ) ) {
2019-03-23 00:20:27 +01:00
writeln ( "\nERROR: --synchronize and --monitor cannot be used together\n" ) ;
2022-04-26 22:04:19 +02:00
writeln ( "Please use 'onedrive --help' for further assistance in regards to running this application.\n" ) ;
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2019-03-23 00:20:27 +01:00
return EXIT_FAILURE ;
}
2021-11-23 20:54:28 +01:00
2019-03-11 07:57:47 +01:00
// Initialize the item database
2016-08-04 23:35:58 +02:00
log . vlog ( "Opening the item database ..." ) ;
2019-04-11 04:26:20 +02:00
if ( ! cfg . getValueBool ( "dry_run" ) ) {
2019-03-11 07:57:47 +01:00
// Load the items.sqlite3 file as the database
2019-08-24 09:18:58 +02:00
log . vdebug ( "Using database file: " , asNormalizedPath ( cfg . databaseFilePath ) ) ;
2019-03-11 07:57:47 +01:00
itemDb = new ItemDatabase ( cfg . databaseFilePath ) ;
} else {
// Load the items-dryrun.sqlite3 file as the database
2019-08-24 09:18:58 +02:00
log . vdebug ( "Using database file: " , asNormalizedPath ( cfg . databaseFilePathDryRun ) ) ;
2019-03-11 07:57:47 +01:00
itemDb = new ItemDatabase ( cfg . databaseFilePathDryRun ) ;
}
2021-11-23 20:54:28 +01:00
2020-10-29 22:00:26 +01:00
// What are the permission that have been set for the application?
// These are relevant for:
// - The ~/OneDrive parent folder or 'sync_dir' configured item
// - Any new folder created under ~/OneDrive or 'sync_dir'
// - Any new file created under ~/OneDrive or 'sync_dir'
// valid permissions are 000 -> 777 - anything else is invalid
if ( ( cfg . getValueLong ( "sync_dir_permissions" ) < 0 ) | | ( cfg . getValueLong ( "sync_file_permissions" ) < 0 ) | | ( cfg . getValueLong ( "sync_dir_permissions" ) > 777 ) | | ( cfg . getValueLong ( "sync_file_permissions" ) > 777 ) ) {
log . error ( "ERROR: Invalid 'User|Group|Other' permissions set within config file. Please check." ) ;
return EXIT_FAILURE ;
} else {
// debug log output what permissions are being set to
log . vdebug ( "Configuring default new folder permissions as: " , cfg . getValueLong ( "sync_dir_permissions" ) ) ;
cfg . configureRequiredDirectoryPermisions ( ) ;
log . vdebug ( "Configuring default new file permissions as: " , cfg . getValueLong ( "sync_file_permissions" ) ) ;
cfg . configureRequiredFilePermisions ( ) ;
}
2021-11-23 20:54:28 +01:00
2020-05-20 03:37:11 +02:00
// configure the sync direcory based on syncDir config option
2016-08-04 23:35:58 +02:00
log . vlog ( "All operations will be performed in: " , syncDir ) ;
2018-12-23 01:15:10 +01:00
if ( ! exists ( syncDir ) ) {
log . vdebug ( "syncDir: Configured syncDir is missing. Creating: " , syncDir ) ;
2019-07-30 23:20:26 +02:00
try {
// Attempt to create the sync dir we have been configured with
mkdirRecurse ( syncDir ) ;
2020-10-29 22:00:26 +01:00
// Configure the applicable permissions for the folder
2020-11-19 19:36:20 +01:00
log . vdebug ( "Setting directory permissions for: " , syncDir ) ;
2020-10-29 22:00:26 +01:00
syncDir . setAttributes ( cfg . returnRequiredDirectoryPermisions ( ) ) ;
2019-07-30 23:20:26 +02:00
} catch ( std . file . FileException e ) {
// Creating the sync directory failed
log . error ( "ERROR: Unable to create local OneDrive syncDir - " , e . msg ) ;
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2019-07-30 23:20:26 +02:00
return EXIT_FAILURE ;
}
2018-12-23 01:15:10 +01:00
}
2021-11-23 20:54:28 +01:00
2020-10-29 22:00:26 +01:00
// Change the working directory to the 'sync_dir' configured item
2015-09-27 18:47:41 +02:00
chdir ( syncDir ) ;
2021-11-23 20:54:28 +01:00
2018-05-16 11:19:43 +02:00
// Configure selective sync by parsing and getting a regex for skip_file config component
2017-03-24 22:30:03 +01:00
auto selectiveSync = new SelectiveSync ( ) ;
2021-11-23 20:54:28 +01:00
2020-06-27 11:10:37 +02:00
// load sync_list if it exists
if ( exists ( syncListFilePath ) ) {
2019-01-05 19:43:44 +01:00
log . vdebug ( "Loading user configured sync_list file ..." ) ;
2019-10-09 03:02:17 +02:00
syncListConfigured = true ;
2019-01-05 19:43:44 +01:00
// list what will be synced
2022-05-19 08:12:53 +02:00
auto syncListFile = File ( syncListFilePath , "r" ) ;
2019-01-05 19:43:44 +01:00
auto range = syncListFile . byLine ( ) ;
foreach ( line ; range )
{
log . vdebug ( "sync_list: " , line ) ;
}
2020-05-20 03:37:11 +02:00
// close syncListFile if open
if ( syncListFile . isOpen ( ) ) {
// close open file
syncListFile . close ( ) ;
}
2019-01-05 19:43:44 +01:00
}
2020-06-27 11:10:37 +02:00
selectiveSync . load ( syncListFilePath ) ;
2021-11-23 20:54:28 +01:00
2020-06-27 11:10:37 +02:00
// load business_shared_folders if it exists
if ( exists ( businessSharedFolderFilePath ) ) {
log . vdebug ( "Loading user configured business_shared_folders file ..." ) ;
// list what will be synced
2022-05-19 08:12:53 +02:00
auto businessSharedFolderFileList = File ( businessSharedFolderFilePath , "r" ) ;
2020-06-27 11:10:37 +02:00
auto range = businessSharedFolderFileList . byLine ( ) ;
foreach ( line ; range )
{
log . vdebug ( "business_shared_folders: " , line ) ;
}
}
selectiveSync . loadSharedFolders ( businessSharedFolderFilePath ) ;
2021-11-23 20:54:28 +01:00
2020-04-06 12:05:06 +02:00
// Configure skip_dir, skip_file, skip-dir-strict-match & skip_dotfiles from config entries
// Handle skip_dir configuration in config file
2019-03-14 20:55:05 +01:00
log . vdebug ( "Configuring skip_dir ..." ) ;
2019-04-11 04:26:20 +02:00
log . vdebug ( "skip_dir: " , cfg . getValueString ( "skip_dir" ) ) ;
selectiveSync . setDirMask ( cfg . getValueString ( "skip_dir" ) ) ;
2021-11-23 20:54:28 +01:00
2020-01-29 06:37:50 +01:00
// Was --skip-dir-strict-match configured?
2020-04-06 12:05:06 +02:00
log . vdebug ( "Configuring skip_dir_strict_match ..." ) ;
log . vdebug ( "skip_dir_strict_match: " , cfg . getValueBool ( "skip_dir_strict_match" ) ) ;
2020-01-29 06:37:50 +01:00
if ( cfg . getValueBool ( "skip_dir_strict_match" ) ) {
selectiveSync . setSkipDirStrictMatch ( ) ;
}
2021-11-23 20:54:28 +01:00
2020-04-06 12:05:06 +02:00
// Was --skip-dot-files configured?
log . vdebug ( "Configuring skip_dotfiles ..." ) ;
log . vdebug ( "skip_dotfiles: " , cfg . getValueBool ( "skip_dotfiles" ) ) ;
if ( cfg . getValueBool ( "skip_dotfiles" ) ) {
selectiveSync . setSkipDotfiles ( ) ;
}
2021-11-23 20:54:28 +01:00
2020-04-06 12:05:06 +02:00
// Handle skip_file configuration in config file
2019-03-14 20:55:05 +01:00
log . vdebug ( "Configuring skip_file ..." ) ;
2019-04-25 03:00:23 +02:00
// Validate skip_file to ensure that this does not contain an invalid configuration
// Do not use a skip_file entry of .* as this will prevent correct searching of local changes to process.
foreach ( entry ; cfg . getValueString ( "skip_file" ) . split ( "|" ) ) {
if ( entry = = ".*" ) {
// invalid entry element detected
log . logAndNotify ( "ERROR: Invalid skip_file entry '.*' detected" ) ;
return EXIT_FAILURE ;
}
}
2020-04-06 12:05:06 +02:00
// All skip_file entries are valid
2019-04-11 04:26:20 +02:00
log . vdebug ( "skip_file: " , cfg . getValueString ( "skip_file" ) ) ;
selectiveSync . setFileMask ( cfg . getValueString ( "skip_file" ) ) ;
2021-11-23 20:54:28 +01:00
2022-05-16 02:16:50 +02:00
// Implement https://github.com/abraunegg/onedrive/issues/1129
// Force a synchronization of a specific folder, only when using --synchronize --single-directory and ignoring all non-default skip_dir and skip_file rules
if ( ( cfg . getValueBool ( "synchronize" ) ) & & ( cfg . getValueString ( "single_directory" ) ! = "" ) & & ( cfg . getValueBool ( "force_sync" ) ) ) {
log . log ( "\nWARNING: Overriding application configuration to use application defaults for skip_dir and skip_file due to --synchronize --single-directory --force-sync being used" ) ;
// performing this action could have undesirable effects .. the user must accept this risk
// what is the risk acceptance?
bool resyncRiskAcceptance = false ;
// need to prompt user
char response ;
// warning message
writeln ( "\nThe use of --force-sync will reconfigure the application to use defaults. This may have untold and unknown future impacts." ) ;
writeln ( "By proceeding in using this option you accept any impacts including any data loss that may occur as a result of using --force-sync." ) ;
write ( "\nAre you sure you wish to proceed with --force-sync [Y/N] " ) ;
try {
// Attempt to read user response
readf ( " %c\n" , & response ) ;
} catch ( std . format . FormatException e ) {
// Caught an error
return EXIT_FAILURE ;
}
// Evaluate user repsonse
if ( ( to ! string ( response ) = = "y" ) | | ( to ! string ( response ) = = "Y" ) ) {
// User has accepted --force-sync risk to proceed
resyncRiskAcceptance = true ;
// Are you sure you wish .. does not use writeln();
write ( "\n" ) ;
}
// Action based on response
if ( ! resyncRiskAcceptance ) {
// --force-sync not accepted
return EXIT_FAILURE ;
} else {
// --force-sync risk accepted
// reset set config using function to use application defaults
cfg . resetSkipToDefaults ( ) ;
// update sync engine regex with reset defaults
selectiveSync . setDirMask ( cfg . getValueString ( "skip_dir" ) ) ;
selectiveSync . setFileMask ( cfg . getValueString ( "skip_file" ) ) ;
}
}
2019-03-11 07:57:47 +01:00
// Initialize the sync engine
2019-04-11 04:26:20 +02:00
auto sync = new SyncEngine ( cfg , oneDrive , itemDb , selectiveSync ) ;
2018-07-16 01:58:36 +02:00
try {
2018-12-04 01:15:44 +01:00
if ( ! initSyncEngine ( sync ) ) {
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2018-07-16 01:58:36 +02:00
return EXIT_FAILURE ;
2019-09-11 12:26:21 +02:00
} else {
2020-09-14 09:49:50 +02:00
if ( ( cfg . getValueString ( "get_file_link" ) = = "" ) & & ( cfg . getValueString ( "create_share_link" ) = = "" ) ) {
// Print out that we are initializing the engine only if we are not grabbing the file link or creating a shareable link
2019-09-11 12:26:21 +02:00
log . logAndNotify ( "Initializing the Synchronization Engine ..." ) ;
}
2018-07-16 01:58:36 +02:00
}
2018-12-04 01:15:44 +01:00
} catch ( CurlException e ) {
2019-04-11 04:26:20 +02:00
if ( ! cfg . getValueBool ( "monitor" ) ) {
2019-09-11 12:26:21 +02:00
log . log ( "\nNo Internet connection." ) ;
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2018-09-24 21:25:40 +02:00
return EXIT_FAILURE ;
}
2018-07-16 01:58:36 +02:00
}
2018-12-04 01:15:44 +01:00
2020-03-14 20:58:57 +01:00
// if sync list is configured, set to true now that the sync engine is initialised
if ( syncListConfigured ) {
sync . setSyncListConfigured ( ) ;
}
2021-11-23 20:54:28 +01:00
2020-01-26 22:42:00 +01:00
// Do we need to configure specific --upload-only options?
if ( cfg . getValueBool ( "upload_only" ) ) {
// --upload-only was passed in or configured
2020-11-19 20:33:47 +01:00
log . vdebug ( "Configuring uploadOnly flag to TRUE as --upload-only passed in or configured" ) ;
2020-11-13 20:17:53 +01:00
sync . setUploadOnly ( ) ;
2020-01-26 22:42:00 +01:00
// was --no-remote-delete passed in or configured
if ( cfg . getValueBool ( "no_remote_delete" ) ) {
// Configure the noRemoteDelete flag
2020-11-19 20:33:47 +01:00
log . vdebug ( "Configuring noRemoteDelete flag to TRUE as --no-remote-delete passed in or configured" ) ;
2020-01-26 22:42:00 +01:00
sync . setNoRemoteDelete ( ) ;
}
// was --remove-source-files passed in or configured
if ( cfg . getValueBool ( "remove_source_files" ) ) {
// Configure the localDeleteAfterUpload flag
2020-11-19 20:33:47 +01:00
log . vdebug ( "Configuring localDeleteAfterUpload flag to TRUE as --remove-source-files passed in or configured" ) ;
2020-01-26 22:42:00 +01:00
sync . setLocalDeleteAfterUpload ( ) ;
}
}
2021-11-23 20:54:28 +01:00
2018-11-23 20:26:30 +01:00
// Do we configure to disable the upload validation routine
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "disable_upload_validation" ) ) sync . setDisableUploadValidation ( ) ;
2021-11-23 20:54:28 +01:00
2021-11-22 21:34:16 +01:00
// Do we configure to disable the download validation routine
2021-11-23 20:54:28 +01:00
if ( cfg . getValueBool ( "disable_download_validation" ) ) sync . setDisableDownloadValidation ( ) ;
2021-11-22 21:06:13 +01:00
2020-05-20 22:48:02 +02:00
// Has the user enabled to bypass data preservation of renaming local files when there is a conflict?
if ( cfg . getValueBool ( "bypass_data_preservation" ) ) {
log . log ( "WARNING: Application has been configured to bypass local data preservation in the event of file conflict." ) ;
log . log ( "WARNING: Local data loss MAY occur in this scenario." ) ;
2020-05-26 09:04:22 +02:00
sync . setBypassDataPreservation ( ) ;
2020-05-20 22:48:02 +02:00
}
2021-11-23 20:54:28 +01:00
2020-06-16 23:57:14 +02:00
// Are we configured to use a National Cloud Deployment
if ( cfg . getValueString ( "azure_ad_endpoint" ) ! = "" ) {
// value is configured, is it a valid value?
if ( ( cfg . getValueString ( "azure_ad_endpoint" ) = = "USL4" ) | | ( cfg . getValueString ( "azure_ad_endpoint" ) = = "USL5" ) | | ( cfg . getValueString ( "azure_ad_endpoint" ) = = "DE" ) | | ( cfg . getValueString ( "azure_ad_endpoint" ) = = "CN" ) ) {
// valid entries to flag we are using a National Cloud Deployment
sync . setNationalCloudDeployment ( ) ;
}
}
2021-11-23 20:54:28 +01:00
2018-06-17 00:27:43 +02:00
// Do we need to validate the syncDir to check for the presence of a '.nosync' file
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "check_nomount" ) ) {
2018-06-17 00:27:43 +02:00
// we were asked to check the mounts
if ( exists ( syncDir ~ "/.nosync" ) ) {
2018-12-05 20:19:00 +01:00
log . logAndNotify ( "ERROR: .nosync file found. Aborting synchronization process to safeguard data." ) ;
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2018-06-17 00:27:43 +02:00
return EXIT_FAILURE ;
}
}
2021-11-23 20:54:28 +01:00
2018-03-14 05:43:40 +01:00
// Do we need to create or remove a directory?
2019-04-11 04:26:20 +02:00
if ( ( cfg . getValueString ( "create_directory" ) ! = "" ) | | ( cfg . getValueString ( "remove_directory" ) ! = "" ) ) {
2021-11-23 20:54:28 +01:00
2019-04-11 04:26:20 +02:00
if ( cfg . getValueString ( "create_directory" ) ! = "" ) {
2018-03-14 05:43:40 +01:00
// create a directory on OneDrive
2019-04-11 04:26:20 +02:00
sync . createDirectoryNoSync ( cfg . getValueString ( "create_directory" ) ) ;
2018-03-14 05:43:40 +01:00
}
2021-11-23 20:54:28 +01:00
2019-04-11 04:26:20 +02:00
if ( cfg . getValueString ( "remove_directory" ) ! = "" ) {
2018-03-14 05:43:40 +01:00
// remove a directory on OneDrive
2019-04-11 04:26:20 +02:00
sync . deleteDirectoryNoSync ( cfg . getValueString ( "remove_directory" ) ) ;
2018-03-14 05:43:40 +01:00
}
}
2021-11-23 20:54:28 +01:00
2018-03-14 05:43:40 +01:00
// Are we renaming or moving a directory?
2019-04-11 04:26:20 +02:00
if ( ( cfg . getValueString ( "source_directory" ) ! = "" ) & & ( cfg . getValueString ( "destination_directory" ) ! = "" ) ) {
2018-03-14 05:43:40 +01:00
// We are renaming or moving a directory
2019-04-11 04:26:20 +02:00
sync . renameDirectoryNoSync ( cfg . getValueString ( "source_directory" ) , cfg . getValueString ( "destination_directory" ) ) ;
2018-03-14 05:43:40 +01:00
}
2021-11-23 20:54:28 +01:00
2018-12-04 00:59:23 +01:00
// Are we obtaining the Office 365 Drive ID for a given Office 365 SharePoint Shared Library?
2019-08-02 10:43:31 +02:00
if ( cfg . getValueString ( "get_o365_drive_id" ) ! = "" ) {
2019-04-11 04:26:20 +02:00
sync . querySiteCollectionForDriveID ( cfg . getValueString ( "get_o365_drive_id" ) ) ;
2021-11-23 20:54:28 +01:00
// Exit application
2020-06-27 11:10:37 +02:00
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
2018-12-04 00:59:23 +01:00
}
2021-11-23 20:54:28 +01:00
2020-09-14 09:49:50 +02:00
// Are we createing an anonymous read-only shareable link for an existing file on OneDrive?
if ( cfg . getValueString ( "create_share_link" ) ! = "" ) {
// Query OneDrive for the file, and if valid, create a shareable link for the file
sync . createShareableLinkForFile ( cfg . getValueString ( "create_share_link" ) ) ;
2021-11-23 20:54:28 +01:00
// Exit application
2020-09-14 09:49:50 +02:00
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
2021-11-23 20:54:28 +01:00
2022-03-09 21:00:07 +01:00
// --get-file-link - Are we obtaining the URL path for a synced file?
2019-08-02 10:43:31 +02:00
if ( cfg . getValueString ( "get_file_link" ) ! = "" ) {
2020-09-14 09:49:50 +02:00
// Query OneDrive for the file link
2022-03-09 21:00:07 +01:00
sync . queryOneDriveForFileDetails ( cfg . getValueString ( "get_file_link" ) , syncDir , "URL" ) ;
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
// --modified-by - Are we listing the modified-by details of a provided path?
if ( cfg . getValueString ( "modified_by" ) ! = "" ) {
// Query OneDrive for the file link
sync . queryOneDriveForFileDetails ( cfg . getValueString ( "modified_by" ) , syncDir , "ModifiedBy" ) ;
2021-11-23 20:54:28 +01:00
// Exit application
2020-06-27 11:10:37 +02:00
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
2021-11-23 20:54:28 +01:00
2020-06-27 11:10:37 +02:00
// Are we listing OneDrive Business Shared Folders
if ( cfg . getValueBool ( "list_business_shared_folders" ) ) {
// Is this a business account type?
if ( sync . getAccountType ( ) = = "business" ) {
// List OneDrive Business Shared Folders
sync . listOneDriveBusinessSharedFolders ( ) ;
} else {
log . error ( "ERROR: Unsupported account type for listing OneDrive Business Shared Folders" ) ;
}
2021-11-23 20:54:28 +01:00
// Exit application
2020-06-27 11:10:37 +02:00
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
2021-11-23 20:54:28 +01:00
2020-06-27 11:10:37 +02:00
// Are we going to sync OneDrive Business Shared Folders
if ( cfg . getValueBool ( "sync_business_shared_folders" ) ) {
// Is this a business account type?
if ( sync . getAccountType ( ) = = "business" ) {
// Configure flag to sync business folders
sync . setSyncBusinessFolders ( ) ;
} else {
log . error ( "ERROR: Unsupported account type for syncing OneDrive Business Shared Folders" ) ;
}
2019-08-02 10:43:31 +02:00
}
2021-11-23 20:54:28 +01:00
2018-12-28 02:26:03 +01:00
// Are we displaying the sync status of the client?
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "display_sync_status" ) ) {
2018-12-28 02:26:03 +01:00
string remotePath = "/" ;
// Are we doing a single directory check?
2019-04-11 04:26:20 +02:00
if ( cfg . getValueString ( "single_directory" ) ! = "" ) {
2018-12-28 02:26:03 +01:00
// Need two different path strings here
2019-04-11 04:26:20 +02:00
remotePath = cfg . getValueString ( "single_directory" ) ;
2018-12-28 02:26:03 +01:00
}
sync . queryDriveForChanges ( remotePath ) ;
}
2021-11-23 20:54:28 +01:00
2020-05-20 05:15:43 +02:00
// Are we performing a sync, or monitor operation?
if ( ( cfg . getValueBool ( "synchronize" ) ) | | ( cfg . getValueBool ( "monitor" ) ) ) {
2020-06-11 22:46:59 +02:00
// Initialise the monitor class, so that we can do more granular inotify handling when performing the actual sync
// needed for --synchronize and --monitor handling
Monitor m = new Monitor ( selectiveSync ) ;
2015-09-01 20:45:34 +02:00
2020-05-20 05:15:43 +02:00
if ( cfg . getValueBool ( "synchronize" ) ) {
2018-03-14 05:43:40 +01:00
if ( online ) {
// Check user entry for local path - the above chdir means we are already in ~/OneDrive/ thus singleDirectory is local to this path
2020-06-27 11:10:37 +02:00
if ( cfg . getValueString ( "single_directory" ) ! = "" ) {
2018-03-14 05:43:40 +01:00
// Does the directory we want to sync actually exist?
2020-06-27 11:10:37 +02:00
if ( ! exists ( cfg . getValueString ( "single_directory" ) ) ) {
2020-08-20 23:06:56 +02:00
// The requested path to use with --single-directory does not exist locally within the configured 'sync_dir'
log . logAndNotify ( "WARNING: The requested path for --single-directory does not exist locally. Creating requested path within " , syncDir ) ;
2020-10-29 22:00:26 +01:00
// Make the required --single-directory path locally
string singleDirectoryPath = cfg . getValueString ( "single_directory" ) ;
mkdirRecurse ( singleDirectoryPath ) ;
// Configure the applicable permissions for the folder
2020-11-19 19:36:20 +01:00
log . vdebug ( "Setting directory permissions for: " , singleDirectoryPath ) ;
2020-10-29 22:00:26 +01:00
singleDirectoryPath . setAttributes ( cfg . returnRequiredDirectoryPermisions ( ) ) ;
2018-03-14 05:43:40 +01:00
}
}
2020-05-01 20:05:06 +02:00
// perform a --synchronize sync
// fullScanRequired = false, for final true-up
// but if we have sync_list configured, use syncListConfigured which = true
2020-06-11 22:46:59 +02:00
performSync ( sync , cfg . getValueString ( "single_directory" ) , cfg . getValueBool ( "download_only" ) , cfg . getValueBool ( "local_first" ) , cfg . getValueBool ( "upload_only" ) , LOG_NORMAL , false , syncListConfigured , displaySyncOptions , cfg . getValueBool ( "monitor" ) , m ) ;
2021-11-23 20:54:28 +01:00
2020-11-06 00:28:15 +01:00
// Write WAL and SHM data to file for this sync
log . vdebug ( "Merge contents of WAL and SHM files into main database file" ) ;
itemDb . performVacuum ( ) ;
2015-09-20 21:21:51 +02:00
}
2018-03-14 05:43:40 +01:00
}
2021-11-23 20:54:28 +01:00
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "monitor" ) ) {
2018-12-05 20:19:00 +01:00
log . logAndNotify ( "Initializing monitor ..." ) ;
2019-04-11 04:26:20 +02:00
log . log ( "OneDrive monitor interval (seconds): " , cfg . getValueLong ( "monitor_interval" ) ) ;
2021-11-23 20:54:28 +01:00
2018-03-14 05:43:40 +01:00
m . onDirCreated = delegate ( string path ) {
2020-04-06 12:05:06 +02:00
// Handle .folder creation if skip_dotfiles is enabled
if ( ( cfg . getValueBool ( "skip_dotfiles" ) ) & & ( selectiveSync . isDotFile ( path ) ) ) {
2022-04-21 00:19:24 +02:00
log . vlog ( "[M] Skipping watching local path - .folder found & --skip-dot-files enabled: " , path ) ;
2020-04-06 12:05:06 +02:00
} else {
2022-04-21 00:19:24 +02:00
log . vlog ( "[M] Local directory created: " , path ) ;
2020-04-06 12:05:06 +02:00
try {
sync . scanForDifferences ( path ) ;
} catch ( CurlException e ) {
log . vlog ( "Offline, cannot create remote dir!" ) ;
} catch ( Exception e ) {
log . logAndNotify ( "Cannot create remote directory: " , e . msg ) ;
}
2018-03-14 05:43:40 +01:00
}
} ;
m . onFileChanged = delegate ( string path ) {
2022-04-21 00:19:24 +02:00
log . vlog ( "[M] Local file changed: " , path ) ;
2018-03-14 05:43:40 +01:00
try {
sync . scanForDifferences ( path ) ;
2019-02-21 09:51:15 +01:00
} catch ( CurlException e ) {
log . vlog ( "Offline, cannot upload changed item!" ) ;
2018-03-14 05:43:40 +01:00
} catch ( Exception e ) {
2018-12-06 08:05:52 +01:00
log . logAndNotify ( "Cannot upload file changes/creation: " , e . msg ) ;
2018-03-14 05:43:40 +01:00
}
} ;
m . onDelete = delegate ( string path ) {
2022-04-21 00:19:24 +02:00
log . log ( "Received inotify delete event from operating system .. attempting item deletion as requested" ) ;
log . vlog ( "[M] Local item deleted: " , path ) ;
2018-03-14 05:43:40 +01:00
try {
sync . deleteByPath ( path ) ;
2019-02-21 09:51:15 +01:00
} catch ( CurlException e ) {
log . vlog ( "Offline, cannot delete item!" ) ;
2018-12-06 00:50:46 +01:00
} catch ( SyncException e ) {
if ( e . msg = = "The item to delete is not in the local database" ) {
2021-03-19 07:29:55 +01:00
log . vlog ( "Item cannot be deleted from OneDrive because it was not found in the local database" ) ;
2018-12-06 00:50:46 +01:00
} else {
2018-12-06 08:05:52 +01:00
log . logAndNotify ( "Cannot delete remote item: " , e . msg ) ;
2018-12-06 00:50:46 +01:00
}
2018-03-14 05:43:40 +01:00
} catch ( Exception e ) {
2018-12-06 08:05:52 +01:00
log . logAndNotify ( "Cannot delete remote item: " , e . msg ) ;
2018-03-14 05:43:40 +01:00
}
} ;
m . onMove = delegate ( string from , string to ) {
2022-04-21 00:19:24 +02:00
log . vlog ( "[M] Local item moved: " , from , " -> " , to ) ;
2018-03-14 05:43:40 +01:00
try {
2020-04-06 12:05:06 +02:00
// Handle .folder -> folder if skip_dotfiles is enabled
if ( ( cfg . getValueBool ( "skip_dotfiles" ) ) & & ( selectiveSync . isDotFile ( from ) ) ) {
// .folder -> folder handling - has to be handled as a new folder
sync . scanForDifferences ( to ) ;
} else {
sync . uploadMoveItem ( from , to ) ;
}
2019-02-21 09:51:15 +01:00
} catch ( CurlException e ) {
log . vlog ( "Offline, cannot move item!" ) ;
2018-03-14 05:43:40 +01:00
} catch ( Exception e ) {
2020-03-19 19:35:21 +01:00
log . logAndNotify ( "Cannot move item: " , e . msg ) ;
2018-03-14 05:43:40 +01:00
}
} ;
2018-12-28 03:19:20 +01:00
signal ( SIGINT , & exitHandler ) ;
signal ( SIGTERM , & exitHandler ) ;
2019-10-30 07:46:02 +01:00
// attempt to initialise monitor class
if ( ! cfg . getValueBool ( "download_only" ) ) {
2019-10-31 04:59:44 +01:00
try {
m . init ( cfg , cfg . getValueLong ( "verbose" ) > 0 , cfg . getValueBool ( "skip_symlinks" ) , cfg . getValueBool ( "check_nosync" ) ) ;
} catch ( MonitorException e ) {
// monitor initialisation failed
log . error ( "ERROR: " , e . msg ) ;
exit ( - 1 ) ;
}
}
// monitor loop
2020-05-20 03:37:11 +02:00
bool performMonitor = true ;
ulong monitorLoopFullCount = 0 ;
2019-10-31 04:59:44 +01:00
immutable auto checkInterval = dur ! "seconds" ( cfg . getValueLong ( "monitor_interval" ) ) ;
2022-03-07 04:02:50 +01:00
immutable auto githubCheckInterval = dur ! "seconds" ( 86400 ) ;
2020-05-20 03:37:11 +02:00
immutable long logInterval = cfg . getValueLong ( "monitor_log_frequency" ) ;
immutable long fullScanFrequency = cfg . getValueLong ( "monitor_fullscan_frequency" ) ;
MonoTime lastCheckTime = MonoTime . currTime ( ) ;
2022-03-07 04:02:50 +01:00
MonoTime lastGitHubCheckTime = MonoTime . currTime ( ) ;
2020-05-20 03:37:11 +02:00
long logMonitorCounter = 0 ;
long fullScanCounter = 0 ;
2020-05-20 05:15:43 +02:00
// set fullScanRequired to true so that at application startup we perform a full walk
bool fullScanRequired = true ;
2020-05-01 20:05:06 +02:00
bool syncListConfiguredFullScanOverride = false ;
2019-12-16 19:32:38 +01:00
// if sync list is configured, set to true
if ( syncListConfigured ) {
2020-05-01 20:05:06 +02:00
// sync list is configured
syncListConfiguredFullScanOverride = true ;
2019-12-16 19:32:38 +01:00
}
2021-11-23 20:54:28 +01:00
immutable bool webhookEnabled = cfg . getValueBool ( "webhook_enabled" ) ;
2020-05-20 03:37:11 +02:00
while ( performMonitor ) {
2020-05-05 23:20:13 +02:00
if ( ! cfg . getValueBool ( "download_only" ) ) {
try {
m . update ( online ) ;
} catch ( MonitorException e ) {
// Catch any exceptions thrown by inotify / monitor engine
log . error ( "ERROR: The following inotify error was generated: " , e . msg ) ;
}
}
2021-11-23 20:54:28 +01:00
// Check for notifications pushed from Microsoft to the webhook
bool notificationReceived = false ;
if ( webhookEnabled ) {
// Create a subscription on the first run, or renew the subscription
// on subsequent runs when it is about to expire.
oneDrive . createOrRenewSubscription ( ) ;
// Process incoming notifications if any.
// Empirical evidence shows that Microsoft often sends multiple
// notifications for one single change, so we need a loop to exhaust
// all signals that were queued up by the webhook. The notifications
// do not contain any actual changes, and we will always rely do the
// delta endpoint to sync to latest. Therefore, only one sync run is
// good enough to catch up for multiple notifications.
for ( int signalCount = 0 ; ; signalCount + + ) {
const auto signalExists = receiveTimeout ( dur ! "seconds" ( - 1 ) , ( ulong _ ) { } ) ;
if ( signalExists ) {
notificationReceived = true ;
} else {
if ( notificationReceived ) {
log . log ( "Received " , signalCount , " refresh signals from the webhook" ) ;
}
break ;
}
}
}
2019-10-31 04:59:44 +01:00
auto currTime = MonoTime . currTime ( ) ;
2020-06-04 22:49:23 +02:00
// has monitor_interval elapsed or are we at application startup / monitor startup?
// in a --resync scenario, if we have not 're-populated' the database, valid changes will get skipped:
// Monitor directory: ./target
// Monitor directory: target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
// [M] Item moved: random_files/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby -> target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
// Moving random_files/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby to target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
// Skipping uploading this new file as parent path is not in the database: target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
// 'target' should be in the DB, it should also exist online, but because of --resync, it does not exist in the database thus parent check fails
2021-11-23 20:54:28 +01:00
if ( notificationReceived | | ( currTime - lastCheckTime > checkInterval ) | | ( monitorLoopFullCount = = 0 ) ) {
2022-03-07 04:02:50 +01:00
// Check Application Version against GitHub once per day
if ( currTime - lastGitHubCheckTime > githubCheckInterval ) {
// --monitor GitHub Application Version Check time expired
checkApplicationVersion ( ) ;
// update when we have performed this check
lastGitHubCheckTime = MonoTime . currTime ( ) ;
}
2020-05-20 05:15:43 +02:00
// monitor sync loop
2020-05-25 03:30:04 +02:00
logOutputMessage = "################################################## NEW LOOP ##################################################" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
} else {
log . vdebug ( logOutputMessage ) ;
}
2020-05-20 03:37:11 +02:00
// Increment monitorLoopFullCount
monitorLoopFullCount + + ;
// Display memory details at start of loop
if ( displayMemoryUsage ) {
log . displayMemoryUsagePreGC ( ) ;
}
2021-11-23 20:54:28 +01:00
2019-10-31 04:59:44 +01:00
// log monitor output suppression
logMonitorCounter + = 1 ;
2019-12-16 19:32:38 +01:00
if ( logMonitorCounter > logInterval ) {
2019-10-31 04:59:44 +01:00
logMonitorCounter = 1 ;
2019-12-16 19:32:38 +01:00
}
2019-10-31 04:59:44 +01:00
2022-04-28 03:51:50 +02:00
// do we perform a full scan of sync_dir and database integrity check?
2019-10-31 04:59:44 +01:00
fullScanCounter + = 1 ;
2022-04-28 03:51:50 +02:00
// fullScanFrequency = 'monitor_fullscan_frequency' from config
2019-10-31 04:59:44 +01:00
if ( fullScanCounter > fullScanFrequency ) {
2022-04-28 03:51:50 +02:00
// 'monitor_fullscan_frequency' counter has exceeded
2019-10-31 04:59:44 +01:00
fullScanCounter = 1 ;
2022-04-28 03:51:50 +02:00
// set fullScanRequired = true due to 'monitor_fullscan_frequency' counter has been exceeded
fullScanRequired = true ;
// are we using sync_list?
2019-10-31 04:59:44 +01:00
if ( syncListConfigured ) {
2020-05-01 20:05:06 +02:00
// sync list is configured
syncListConfiguredFullScanOverride = true ;
2019-10-31 04:59:44 +01:00
}
}
2021-11-23 20:54:28 +01:00
2020-05-25 03:30:04 +02:00
if ( displaySyncOptions ) {
// sync option handling per sync loop
log . log ( "fullScanCounter = " , fullScanCounter ) ;
log . log ( "syncListConfigured = " , syncListConfigured ) ;
log . log ( "fullScanRequired = " , fullScanRequired ) ;
log . log ( "syncListConfiguredFullScanOverride = " , syncListConfiguredFullScanOverride ) ;
} else {
// sync option handling per sync loop via debug
log . vdebug ( "fullScanCounter = " , fullScanCounter ) ;
log . vdebug ( "syncListConfigured = " , syncListConfigured ) ;
log . vdebug ( "fullScanRequired = " , fullScanRequired ) ;
log . vdebug ( "syncListConfiguredFullScanOverride = " , syncListConfiguredFullScanOverride ) ;
}
2019-10-31 04:59:44 +01:00
try {
if ( ! initSyncEngine ( sync ) ) {
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2019-10-31 04:59:44 +01:00
return EXIT_FAILURE ;
2018-12-04 01:15:44 +01:00
}
2018-12-12 21:08:18 +01:00
try {
2022-05-03 22:03:45 +02:00
string startMessage = "Starting a sync with OneDrive" ;
string finishMessage = "Sync with OneDrive is complete" ;
2019-10-31 04:59:44 +01:00
// perform a --monitor sync
2022-05-03 22:03:45 +02:00
if ( ( cfg . getValueLong ( "verbose" ) > 0 ) | | ( logMonitorCounter = = logInterval ) ) {
// log to console and log file if enabled
log . log ( startMessage ) ;
} else {
// log file only if enabled so we know when a sync started when not using --verbose
log . fileOnly ( startMessage ) ;
}
2020-06-11 22:46:59 +02:00
performSync ( sync , cfg . getValueString ( "single_directory" ) , cfg . getValueBool ( "download_only" ) , cfg . getValueBool ( "local_first" ) , cfg . getValueBool ( "upload_only" ) , ( logMonitorCounter = = logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT ) , fullScanRequired , syncListConfiguredFullScanOverride , displaySyncOptions , cfg . getValueBool ( "monitor" ) , m ) ;
2019-10-31 04:59:44 +01:00
if ( ! cfg . getValueBool ( "download_only" ) ) {
2020-06-11 22:46:59 +02:00
// discard all events that may have been generated by the sync that have not already been handled
2020-05-05 23:20:13 +02:00
try {
m . update ( false ) ;
} catch ( MonitorException e ) {
// Catch any exceptions thrown by inotify / monitor engine
log . error ( "ERROR: The following inotify error was generated: " , e . msg ) ;
}
2018-12-12 21:08:18 +01:00
}
2022-05-03 22:03:45 +02:00
if ( ( cfg . getValueLong ( "verbose" ) > 0 ) | | ( logMonitorCounter = = logInterval ) ) {
// log to console and log file if enabled
log . log ( finishMessage ) ;
} else {
// log file only if enabled so we know when a sync completed when not using --verbose
log . fileOnly ( finishMessage ) ;
}
2018-12-12 21:08:18 +01:00
} catch ( CurlException e ) {
2019-10-31 04:59:44 +01:00
// we already tried three times in the performSync routine
// if we still have problems, then the sync handle might have
// gone stale and we need to re-initialize the sync engine
log . log ( "Persistent connection errors, reinitializing connection" ) ;
sync . reset ( ) ;
2019-10-29 20:32:17 +01:00
}
2019-10-31 04:59:44 +01:00
} catch ( CurlException e ) {
log . log ( "Cannot initialize connection to OneDrive" ) ;
}
// performSync complete, set lastCheckTime to current time
lastCheckTime = MonoTime . currTime ( ) ;
2022-06-15 01:16:06 +02:00
2020-05-20 03:37:11 +02:00
// Display memory details before cleanup
2022-06-15 01:16:06 +02:00
if ( displayMemoryUsage ) log . displayMemoryUsagePreGC ( ) ;
2020-05-20 03:37:11 +02:00
// Perform Garbage Cleanup
2019-10-31 04:59:44 +01:00
GC . collect ( ) ;
2020-05-20 03:37:11 +02:00
// Display memory details after cleanup
2022-06-15 01:16:06 +02:00
if ( displayMemoryUsage ) log . displayMemoryUsagePostGC ( ) ;
// If we did a full scan, make sure we merge the conents of the WAL and SHM to disk
if ( fullScanRequired ) {
// Write WAL and SHM data to file for this loop
log . vdebug ( "Merge contents of WAL and SHM files into main database file" ) ;
itemDb . performVacuum ( ) ;
2020-05-20 03:37:11 +02:00
}
2022-06-15 01:16:06 +02:00
// reset fullScanRequired and syncListConfiguredFullScanOverride
fullScanRequired = false ;
if ( syncListConfigured ) syncListConfiguredFullScanOverride = false ;
2020-05-25 03:30:04 +02:00
// monitor loop complete
logOutputMessage = "################################################ LOOP COMPLETE ###############################################" ;
2021-11-23 20:54:28 +01:00
2020-11-06 00:28:15 +01:00
// Handle display options
2020-05-25 03:30:04 +02:00
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
} else {
log . vdebug ( logOutputMessage ) ;
}
2020-05-20 03:37:11 +02:00
// Developer break via config option
if ( cfg . getValueLong ( "monitor_max_loop" ) > 0 ) {
// developer set option to limit --monitor loops
if ( monitorLoopFullCount = = ( cfg . getValueLong ( "monitor_max_loop" ) ) ) {
performMonitor = false ;
log . log ( "Exiting after " , monitorLoopFullCount , " loops due to developer set option" ) ;
}
}
}
2019-10-31 04:59:44 +01:00
Thread . sleep ( dur ! "msecs" ( 500 ) ) ;
2015-09-17 00:16:23 +02:00
}
}
2015-09-14 19:21:06 +02:00
}
2016-02-24 17:07:59 +01:00
2019-03-11 07:57:47 +01:00
// --dry-run temp database cleanup
2019-04-11 04:26:20 +02:00
if ( cfg . getValueBool ( "dry_run" ) ) {
2020-10-31 20:55:17 +01:00
string dryRunShmFile = cfg . databaseFilePathDryRun ~ "-shm" ;
string dryRunWalFile = cfg . databaseFilePathDryRun ~ "-wal" ;
2019-03-11 07:57:47 +01:00
if ( exists ( cfg . databaseFilePathDryRun ) ) {
// remove the file
log . vdebug ( "Removing items-dryrun.sqlite3 as dry run operations complete" ) ;
2020-10-31 20:55:17 +01:00
// remove items-dryrun.sqlite3
2021-11-23 20:54:28 +01:00
safeRemove ( cfg . databaseFilePathDryRun ) ;
2019-03-11 07:57:47 +01:00
}
2020-10-31 20:55:17 +01:00
// silent cleanup of shm and wal files if they exist
if ( exists ( dryRunShmFile ) ) {
// remove items-dryrun.sqlite3-shm
2021-11-23 20:54:28 +01:00
safeRemove ( dryRunShmFile ) ;
2020-10-31 20:55:17 +01:00
}
if ( exists ( dryRunWalFile ) ) {
// remove items-dryrun.sqlite3-wal
2021-11-23 20:54:28 +01:00
safeRemove ( dryRunWalFile ) ;
2020-10-31 20:55:17 +01:00
}
2019-03-11 07:57:47 +01:00
}
2021-11-23 20:54:28 +01:00
// Exit application
2020-05-20 03:37:11 +02:00
// Use exit scopes to shutdown API
2016-08-04 23:35:58 +02:00
return EXIT_SUCCESS ;
2015-09-01 20:45:34 +02:00
}
2015-09-20 21:21:51 +02:00
2018-12-04 01:15:44 +01:00
bool initSyncEngine ( SyncEngine sync )
{
try {
sync . init ( ) ;
} catch ( OneDriveException e ) {
if ( e . httpStatusCode = = 400 | | e . httpStatusCode = = 401 ) {
// Authorization is invalid
2022-03-07 19:35:00 +01:00
log . log ( "\nAuthorization token invalid, use --reauth to authorize the client again\n" ) ;
2018-12-04 01:15:44 +01:00
return false ;
}
if ( e . httpStatusCode > = 500 ) {
// There was a HTTP 5xx Server Side Error, message already printed
return false ;
}
}
return true ;
}
2015-09-20 21:21:51 +02:00
// try to synchronize the folder three times
2020-06-11 22:46:59 +02:00
void performSync ( SyncEngine sync , string singleDirectory , bool downloadOnly , bool localFirst , bool uploadOnly , long logLevel , bool fullScanRequired , bool syncListConfiguredFullScanOverride , bool displaySyncOptions , bool monitorEnabled , Monitor m )
2015-09-20 21:21:51 +02:00
{
int count ;
2018-03-14 05:43:40 +01:00
string remotePath = "/" ;
string localPath = "." ;
2020-05-25 03:30:04 +02:00
string logOutputMessage ;
2021-11-23 20:54:28 +01:00
2020-05-01 20:05:06 +02:00
// performSync API scan triggers
log . vdebug ( "performSync API scan triggers" ) ;
log . vdebug ( "-----------------------------" ) ;
log . vdebug ( "fullScanRequired = " , fullScanRequired ) ;
log . vdebug ( "syncListConfiguredFullScanOverride = " , syncListConfiguredFullScanOverride ) ;
log . vdebug ( "-----------------------------" ) ;
2021-11-23 20:54:28 +01:00
2018-03-14 05:43:40 +01:00
// Are we doing a single directory sync?
if ( singleDirectory ! = "" ) {
// Need two different path strings here
remotePath = singleDirectory ;
localPath = singleDirectory ;
2019-09-09 04:30:59 +02:00
// Set flag for singleDirectoryScope for change handling
sync . setSingleDirectoryScope ( ) ;
2018-03-14 05:43:40 +01:00
}
2021-11-23 20:54:28 +01:00
2019-04-06 03:44:51 +02:00
// Due to Microsoft Sharepoint 'enrichment' of files, we try to download the Microsoft modified file automatically
// Set flag if we are in upload only state to handle this differently
2021-11-23 20:54:28 +01:00
// See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details
2019-04-06 03:44:51 +02:00
if ( uploadOnly ) sync . setUploadOnly ( ) ;
2021-11-23 20:54:28 +01:00
2015-09-20 21:21:51 +02:00
do {
try {
2020-05-25 03:30:04 +02:00
// starting a sync
logOutputMessage = "################################################## NEW SYNC ##################################################" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
} else {
log . vdebug ( logOutputMessage ) ;
}
2018-03-14 05:43:40 +01:00
if ( singleDirectory ! = "" ) {
// we were requested to sync a single directory
log . vlog ( "Syncing changes from this selected path: " , singleDirectory ) ;
2018-07-15 07:22:08 +02:00
if ( uploadOnly ) {
// Upload Only of selected single directory
2019-02-26 21:21:23 +01:00
if ( logLevel < MONITOR_LOG_QUIET ) log . log ( "Syncing changes from selected local path only - NOT syncing data changes from OneDrive ..." ) ;
2018-07-15 07:22:08 +02:00
sync . scanForDifferences ( localPath ) ;
} else {
// No upload only
if ( localFirst ) {
// Local First
2019-02-26 21:21:23 +01:00
if ( logLevel < MONITOR_LOG_QUIET ) log . log ( "Syncing changes from selected local path first before downloading changes from OneDrive ..." ) ;
2018-04-07 09:06:57 +02:00
sync . scanForDifferences ( localPath ) ;
sync . applyDifferencesSingleDirectory ( remotePath ) ;
2018-07-15 07:22:08 +02:00
} else {
// OneDrive First
2019-02-26 21:21:23 +01:00
if ( logLevel < MONITOR_LOG_QUIET ) log . log ( "Syncing changes from selected OneDrive path ..." ) ;
2018-07-15 07:22:08 +02:00
sync . applyDifferencesSingleDirectory ( remotePath ) ;
// is this a download only request?
if ( ! downloadOnly ) {
// process local changes
sync . scanForDifferences ( localPath ) ;
// ensure that the current remote state is updated locally
sync . applyDifferencesSingleDirectory ( remotePath ) ;
}
2018-04-07 09:06:57 +02:00
}
2018-03-14 05:43:40 +01:00
}
} else {
2018-07-15 07:22:08 +02:00
// no single directory sync
if ( uploadOnly ) {
// Upload Only of entire sync_dir
2019-02-26 21:21:23 +01:00
if ( logLevel < MONITOR_LOG_QUIET ) log . log ( "Syncing changes from local path only - NOT syncing data changes from OneDrive ..." ) ;
2018-07-15 07:22:08 +02:00
sync . scanForDifferences ( localPath ) ;
} else {
// No upload only
2020-05-25 03:30:04 +02:00
string syncCallLogOutput ;
2018-07-15 07:22:08 +02:00
if ( localFirst ) {
// sync local files first before downloading from OneDrive
2019-02-26 21:21:23 +01:00
if ( logLevel < MONITOR_LOG_QUIET ) log . log ( "Syncing changes from local path first before downloading changes from OneDrive ..." ) ;
2018-04-07 09:06:57 +02:00
sync . scanForDifferences ( localPath ) ;
2020-05-01 20:05:06 +02:00
// if syncListConfiguredFullScanOverride = true
if ( syncListConfiguredFullScanOverride ) {
// perform a full walk of OneDrive objects
sync . applyDifferences ( syncListConfiguredFullScanOverride ) ;
} else {
// perform a walk based on if a full scan is required
sync . applyDifferences ( fullScanRequired ) ;
}
2018-07-15 07:22:08 +02:00
} else {
// sync from OneDrive first before uploading files to OneDrive
2019-02-26 21:21:23 +01:00
if ( logLevel < MONITOR_LOG_SILENT ) log . log ( "Syncing changes from OneDrive ..." ) ;
2021-11-23 20:54:28 +01:00
2019-11-21 09:58:34 +01:00
// For the initial sync, always use the delta link so that we capture all the right delta changes including adds, moves & deletes
2020-05-25 03:30:04 +02:00
logOutputMessage = "Initial Scan: Call OneDrive Delta API for delta changes as compared to last successful sync." ;
syncCallLogOutput = "Calling sync.applyDifferences(false);" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
log . log ( syncCallLogOutput ) ;
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
2019-11-21 09:58:34 +01:00
sync . applyDifferences ( false ) ;
2021-11-23 20:54:28 +01:00
// is this a download only request?
2020-05-01 20:05:06 +02:00
if ( ! downloadOnly ) {
// process local changes walking the entire path checking for changes
// in monitor mode all local changes are captured via inotify
2020-05-25 03:30:04 +02:00
// thus scanning every 'monitor_interval' (default 300 seconds) for local changes is excessive and not required
logOutputMessage = "Process local filesystem (sync_dir) for file changes as compared to database entries" ;
syncCallLogOutput = "Calling sync.scanForDifferences(localPath);" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
log . log ( syncCallLogOutput ) ;
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
2021-11-23 20:54:28 +01:00
2020-06-11 22:46:59 +02:00
// What sort of local scan do we want to do?
// In --monitor mode, when performing the DB scan, a race condition occurs where by if a file or folder is moved during this process
// the inotify event is discarded once performSync() is finished (see m.update(false) above), so these events need to be handled
// This can be remediated by breaking the DB and file system scan into separate processes, and handing any applicable inotify events in between
if ( ! monitorEnabled ) {
// --synchronize in use
// standard process flow
sync . scanForDifferences ( localPath ) ;
} else {
// --monitor in use
// Use individual calls with inotify checks between to avoid a race condition between these 2 functions
2022-04-28 03:51:50 +02:00
// Database scan integrity check to compare DB data vs actual content on disk to ensure what we think is local, is local
// and that the data 'hash' as recorded in the DB equals the hash of the actual content
// This process can be extremely expensive time and CPU processing wise
//
// fullScanRequired is set to TRUE when the application starts up, or the config option 'monitor_fullscan_frequency' count is reached
// By default, 'monitor_fullscan_frequency' = 12, and 'monitor_interval' = 300, meaning that by default, a full database consistency check
// is done once an hour.
//
// To change this behaviour adjust 'monitor_interval' and 'monitor_fullscan_frequency' to desired values in the application config file
if ( fullScanRequired ) {
log . vlog ( "Performing Database Consistency Integrity Check .. " ) ;
sync . scanForDifferencesDatabaseScan ( localPath ) ;
// handle any inotify events that occured 'whilst' we were scanning the database
m . update ( true ) ;
} else {
log . vdebug ( "NOT performing Database Integrity Check .. fullScanRequired = FALSE" ) ;
m . update ( true ) ;
}
2020-06-11 22:46:59 +02:00
// Filesystem walk to find new files not uploaded
2022-04-28 03:51:50 +02:00
log . vdebug ( "Searching local filesystem for new data" ) ;
2020-06-11 22:46:59 +02:00
sync . scanForDifferencesFilesystemScan ( localPath ) ;
// handle any inotify events that occured 'whilst' we were scanning the local filesystem
m . update ( true ) ;
}
2021-11-23 20:54:28 +01:00
2020-05-01 20:05:06 +02:00
// At this point, all OneDrive changes / local changes should be uploaded and in sync
// This MAY not be the case when using sync_list, thus a full walk of OneDrive ojects is required
2021-11-23 20:54:28 +01:00
2020-05-01 20:05:06 +02:00
// --synchronize & no sync_list : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// --synchronize & sync_list in use : fullScanRequired = false, syncListConfiguredFullScanOverride = true
2021-11-23 20:54:28 +01:00
2022-04-28 03:51:50 +02:00
// --monitor loops around 12 iterations. On the 1st loop, sets fullScanRequired = true, syncListConfiguredFullScanOverride = true if requried
2021-11-23 20:54:28 +01:00
2020-05-01 20:05:06 +02:00
// --monitor & no sync_list (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = false
2022-04-28 03:51:50 +02:00
// --monitor & no sync_list (loop #2 - #12) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
2020-05-01 20:05:06 +02:00
// --monitor & sync_list in use (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = true
2022-04-28 03:51:50 +02:00
// --monitor & sync_list in use (loop #2 - #12) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
2021-11-23 20:54:28 +01:00
2020-05-01 20:05:06 +02:00
// Do not perform a full walk of the OneDrive objects
if ( ( ! fullScanRequired ) & & ( ! syncListConfiguredFullScanOverride ) ) {
2020-05-25 03:30:04 +02:00
logOutputMessage = "Final True-Up: Do not perform a full walk of the OneDrive objects - not required" ;
syncCallLogOutput = "Calling sync.applyDifferences(false);" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
log . log ( syncCallLogOutput ) ;
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
2020-05-01 20:05:06 +02:00
sync . applyDifferences ( false ) ;
}
2021-11-23 20:54:28 +01:00
2020-05-01 20:05:06 +02:00
// Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop
if ( ( ! fullScanRequired ) & & ( syncListConfiguredFullScanOverride ) ) {
2020-05-25 03:30:04 +02:00
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop" ;
syncCallLogOutput = "Calling sync.applyDifferences(true);" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
log . log ( syncCallLogOutput ) ;
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
2020-05-01 20:05:06 +02:00
sync . applyDifferences ( true ) ;
}
2021-11-23 20:54:28 +01:00
2020-05-01 20:05:06 +02:00
// Perform a full walk of OneDrive objects because a full scan was required
if ( ( fullScanRequired ) & & ( ! syncListConfiguredFullScanOverride ) ) {
2020-05-25 03:30:04 +02:00
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required" ;
syncCallLogOutput = "Calling sync.applyDifferences(true);" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
log . log ( syncCallLogOutput ) ;
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
2021-11-23 20:54:28 +01:00
}
2020-05-01 20:05:06 +02:00
sync . applyDifferences ( true ) ;
}
2021-11-23 20:54:28 +01:00
2020-05-01 20:05:06 +02:00
// Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop
if ( ( fullScanRequired ) & & ( syncListConfiguredFullScanOverride ) ) {
2020-05-25 03:30:04 +02:00
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop" ;
syncCallLogOutput = "Calling sync.applyDifferences(true);" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
log . log ( syncCallLogOutput ) ;
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
2020-05-01 20:05:06 +02:00
sync . applyDifferences ( true ) ;
2019-04-01 20:21:02 +02:00
}
2018-07-15 07:22:08 +02:00
}
2018-04-07 09:06:57 +02:00
}
2018-03-14 05:43:40 +01:00
}
2017-12-31 17:07:21 +01:00
}
2021-11-23 20:54:28 +01:00
2020-05-25 03:30:04 +02:00
// sync is complete
logOutputMessage = "################################################ SYNC COMPLETE ###############################################" ;
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
} else {
log . vdebug ( logOutputMessage ) ;
}
2021-11-23 20:54:28 +01:00
2015-09-20 21:21:51 +02:00
count = - 1 ;
2017-12-28 15:03:15 +01:00
} catch ( Exception e ) {
2018-12-12 21:08:18 +01:00
if ( + + count = = 3 ) {
log . log ( "Giving up on sync after three attempts: " , e . msg ) ;
throw e ;
2021-11-23 20:54:28 +01:00
} else
2018-12-12 21:08:18 +01:00
log . log ( "Retry sync count: " , count , ": " , e . msg ) ;
2015-09-20 21:21:51 +02:00
}
} while ( count ! = - 1 ) ;
}
2018-12-05 20:19:00 +01:00
2018-12-28 03:19:20 +01:00
// getting around the @nogc problem
// https://p0nce.github.io/d-idioms/#Bypassing-@nogc
auto assumeNoGC ( T ) ( T t ) if ( isFunctionPointer ! T | | isDelegate ! T )
{
enum attrs = functionAttributes ! T | FunctionAttribute . nogc ;
return cast ( SetFunctionAttributes ! ( T , functionLinkage ! T , attrs ) ) t ;
}
extern ( C ) nothrow @nogc @system void exitHandler ( int value ) {
try {
assumeNoGC ( ( ) {
2021-11-23 20:54:28 +01:00
log . log ( "Got termination signal, performing clean up" ) ;
// if initialised, shut down the HTTP instance
if ( onedriveInitialised ) {
log . log ( "Shutting down the HTTP instance" ) ;
oneDrive . shutdown ( ) ;
}
2020-11-06 00:28:15 +01:00
// was itemDb initialised?
if ( itemDb ! is null ) {
// Make sure the .wal file is incorporated into the main db before we exit
2022-06-15 01:16:06 +02:00
log . log ( "Shutting down db connection and merging temporary data" ) ;
2020-11-06 00:28:15 +01:00
itemDb . performVacuum ( ) ;
destroy ( itemDb ) ;
}
2018-12-28 03:19:20 +01:00
} ) ( ) ;
} catch ( Exception e ) { }
exit ( 0 ) ;
}
2018-12-28 10:01:50 +01:00