2023-10-18 20:31:50 +02:00
// What is this module called?
module main ;
// What does this module require to function?
2018-12-28 03:19:20 +01:00
import core.stdc.stdlib : EXIT_SUCCESS , EXIT_FAILURE , exit ;
import core.stdc.signal ;
2023-10-18 20:31:50 +02:00
import core.memory ;
import core.time ;
import core.thread ;
import std.stdio ;
import std.getopt ;
import std.string ;
import std.file ;
import std.process ;
import std.algorithm ;
import std.path ;
import std.concurrency ;
import std.parallelism ;
import std.conv ;
import std.traits ;
import std.net.curl : CurlException ;
2022-10-12 04:34:42 +02:00
import std.datetime ;
2015-09-01 20:45:34 +02:00
2023-10-18 20:31:50 +02:00
// What other modules that we have created do we need to import?
import config ;
import log ;
import curlEngine ;
import util ;
import onedrive ;
import syncEngine ;
import itemdb ;
import clientSideFiltering ;
import monitor ;
// What other constant variables do we require?
2022-08-17 05:19:51 +02:00
const int EXIT_RESYNC_REQUIRED = 126 ;
2019-07-06 00:01:04 +02:00
2023-10-18 20:31:50 +02:00
// Class objects
ApplicationConfig appConfig ;
OneDriveApi oneDriveApiInstance ;
SyncEngine syncEngineInstance ;
ItemDatabase itemDB ;
ClientSideFiltering selectiveSync ;
2023-10-25 06:03:15 +02:00
Monitor filesystemMonitor ;
2019-02-26 21:21:23 +01:00
2023-10-18 20:31:50 +02:00
int main ( string [ ] cliArgs ) {
// Disable buffering on stdout - this is needed so that when we are using plain write() it will go to the terminal
2018-12-23 01:15:10 +01:00
stdout . setvbuf ( 0 , _IONBF ) ;
2023-10-18 20:31:50 +02:00
// Required main function variables
string genericHelpMessage = "Try 'onedrive --help' for more information" ;
// If the user passes in --confdir we need to store this as a variable
string confdirOption = "" ;
// Are we online?
2020-05-20 03:37:11 +02:00
bool online = false ;
2023-10-18 20:31:50 +02:00
// Does the operating environment have shell environment variables set
bool shellEnvSet = false ;
// What is the runtime syncronisation directory that will be used
// Typically this will be '~/OneDrive' .. however tilde expansion is unreliable
string runtimeSyncDirectory = "" ;
// Configure the runtime database file path. Typically this will be the default, but in a --dry-run scenario, we use a separate database file
string runtimeDatabaseFile = "" ;
// Application Start Time - used during monitor loop to detail how long it has been running for
auto applicationStartTime = Clock . currTime ( ) ;
// DEVELOPER OPTIONS OUTPUT VARIABLES
2020-05-20 03:37:11 +02:00
bool displayMemoryUsage = false ;
2020-05-25 03:30:04 +02:00
bool displaySyncOptions = false ;
2022-05-19 08:12:53 +02:00
2023-10-18 20:31:50 +02:00
// Define 'exit' and 'failure' scopes
2020-05-20 03:37:11 +02:00
scope ( exit ) {
2023-10-25 06:03:15 +02:00
// Detail what scope was called
2023-10-18 20:31:50 +02:00
log . vdebug ( "Exit scope was called" ) ;
2023-10-25 06:03:15 +02:00
// Perform exit tasks
2023-10-25 21:23:05 +02:00
performStandardExitProcess ( "exitScope" ) ;
2020-05-20 03:37:11 +02:00
}
2023-10-18 20:31:50 +02:00
2020-05-20 03:37:11 +02:00
scope ( failure ) {
2023-10-25 06:03:15 +02:00
// Detail what scope was called
2023-10-18 20:31:50 +02:00
log . vdebug ( "Failure scope was called" ) ;
2023-10-25 06:03:15 +02:00
// Perform exit tasks
2023-10-25 21:23:05 +02:00
performStandardExitProcess ( "failureScope" ) ;
2020-05-20 03:37:11 +02:00
}
2023-10-18 20:31:50 +02:00
// Read in application options as passed in
2019-01-26 01:03:00 +01:00
try {
2019-04-11 04:26:20 +02:00
bool printVersion = false ;
2023-10-18 20:31:50 +02:00
auto cliOptions = getopt (
cliArgs ,
2019-04-11 04:26:20 +02:00
std . getopt . config . passThrough ,
2019-01-26 01:03:00 +01:00
std . getopt . config . bundling ,
std . getopt . config . caseSensitive ,
2019-04-11 04:26:20 +02:00
"confdir" , "Set the directory used to store the configuration files" , & confdirOption ,
2019-03-17 05:10:41 +01:00
"verbose|v+" , "Print more details, useful for debugging (repeat for extra debugging)" , & log . verbose ,
2019-01-26 01:03:00 +01:00
"version" , "Print the version and exit" , & printVersion
) ;
2022-01-27 22:58:38 +01:00
2023-10-18 20:31:50 +02:00
// Print help and exit
if ( cliOptions . helpWanted ) {
cliArgs ~ = "--help" ;
2019-04-11 04:26:20 +02:00
}
2023-10-18 20:31:50 +02:00
// Print the version and exit
2019-04-11 04:26:20 +02:00
if ( printVersion ) {
2023-10-18 20:31:50 +02:00
//writeln("onedrive ", strip(import("version")));
2023-10-18 20:39:33 +02:00
string tempVersion = "v2.5.0-alpha-3" ~ " GitHub version: " ~ strip ( import ( "version" ) ) ;
2023-10-18 20:31:50 +02:00
writeln ( tempVersion ) ;
2019-01-26 01:03:00 +01:00
return EXIT_SUCCESS ;
}
} catch ( GetOptException e ) {
2023-10-18 20:31:50 +02:00
// Option errors
2019-01-26 01:03:00 +01:00
log . error ( e . msg ) ;
2023-10-18 20:31:50 +02:00
log . error ( genericHelpMessage ) ;
2019-01-26 01:03:00 +01:00
return EXIT_FAILURE ;
} catch ( Exception e ) {
2023-10-18 20:31:50 +02:00
// Generic error
2019-01-26 01:03:00 +01:00
log . error ( e . msg ) ;
2023-10-18 20:31:50 +02:00
log . error ( genericHelpMessage ) ;
2019-01-26 01:03:00 +01:00
return EXIT_FAILURE ;
}
2023-10-18 20:31:50 +02:00
// How was this application started - what options were passed in
log . vdebug ( "passed in options: " , cliArgs ) ;
log . vdebug ( "note --confdir and --verbose not listed in 'cliArgs'" ) ;
// Create a new AppConfig object with default values,
appConfig = new ApplicationConfig ( ) ;
// Initialise the application configuration, utilising --confdir if it was passed in
// Otherwise application defaults will be used to configure the application
if ( ! appConfig . initialise ( confdirOption ) ) {
// There was an error loading the user specified application configuration
2019-04-11 04:26:20 +02:00
// Error message already printed
return EXIT_FAILURE ;
2019-04-01 19:51:25 +02:00
}
2022-01-27 22:58:38 +01:00
2023-10-18 20:31:50 +02:00
// Update the existing application configuration (default or 'config' file) from any passed in command line arguments
appConfig . updateFromArgs ( cliArgs ) ;
2022-05-03 07:09:08 +02:00
2023-10-18 20:31:50 +02:00
// Depreciated options check now that the config file (if present) and CLI options have all been parsed
appConfig . checkDepreciatedOptions ( cliArgs ) ;
2022-01-27 22:58:38 +01:00
2023-10-18 20:31:50 +02:00
// Configure GUI Notifications
// - This option is reverse action. If 'disable_notifications' is 'true', we need to send 'false'
if ( appConfig . getValueBool ( "disable_notifications" ) ) {
// disable_notifications is true, initialise with false
log . enableNotifications ( false ) ;
2022-12-05 20:02:41 +01:00
} else {
2023-10-18 20:31:50 +02:00
log . enableNotifications ( true ) ;
2019-08-24 09:18:58 +02:00
}
2022-12-05 20:02:41 +01:00
2023-10-18 20:31:50 +02:00
// Configure application logging to a log file only if enabled
// This is the earliest point to do so as the client configuration has been read in, CLI arguments have been processed.
// Either of those could be enabling logging
if ( appConfig . getValueBool ( "enable_logging" ) ) {
// configure the application logging directory
string initialisedLogDirPath = appConfig . initialiseLogDirectory ( ) ;
// Initialise using the configured logging directory
log . vlog ( "Using the following path to store the runtime application log: " , initialisedLogDirPath ) ;
log . initialise ( initialisedLogDirPath ) ;
2019-08-24 09:18:58 +02:00
}
2023-10-18 20:31:50 +02:00
// Configure Client Side Filtering (selective sync) by parsing and getting a usable regex for skip_file, skip_dir and sync_list config components
selectiveSync = new ClientSideFiltering ( appConfig ) ;
if ( ! selectiveSync . initialise ( ) ) {
// exit here as something triggered a selective sync configuration failure
return EXIT_FAILURE ;
2020-06-27 11:10:37 +02:00
}
2023-10-18 20:31:50 +02:00
// Set runtimeDatabaseFile, this will get updated if we are using --dry-run
runtimeDatabaseFile = appConfig . databaseFilePath ;
// Read in 'sync_dir' from appConfig with '~' if present expanded
runtimeSyncDirectory = appConfig . initialiseRuntimeSyncDirectory ( ) ;
// DEVELOPER OPTIONS OUTPUT
// Set to display memory details as early as possible
displayMemoryUsage = appConfig . getValueBool ( "display_memory" ) ;
// set to display sync options
displaySyncOptions = appConfig . getValueBool ( "display_sync_options" ) ;
// Display the current application configuration (based on all defaults, 'config' file parsing and/or options passed in via the CLI) and exit if --display-config has been used
if ( ( appConfig . getValueBool ( "display_config" ) ) | | ( appConfig . getValueBool ( "display_running_config" ) ) ) {
// Display the application configuration
appConfig . displayApplicationConfiguration ( ) ;
// Do we exit? We exit only if '--display-config' has been used
if ( appConfig . getValueBool ( "display_config" ) ) {
return EXIT_SUCCESS ;
2019-08-24 09:18:58 +02:00
}
}
2023-10-18 20:31:50 +02:00
// Check for basic application option conflicts - flags that should not be used together and/or flag combinations that conflict with each other, values that should be present and are not
if ( appConfig . checkForBasicOptionConflicts ) {
// Any error will have been printed by the function itself
return EXIT_FAILURE ;
2019-08-24 09:18:58 +02:00
}
2023-10-18 20:31:50 +02:00
// Check for --dry-run operation
// If this has been requested, we need to ensure that all actions are performed against the dry-run database copy, and,
// no actual action takes place - such as deleting files if deleted online, moving files if moved online or local, downloading new & changed files, uploading new & changed files
if ( appConfig . getValueBool ( "dry_run" ) ) {
// this is a --dry-run operation
log . log ( "DRY-RUN Configured. Output below shows what 'would' have occurred." ) ;
2022-08-26 01:11:10 +02:00
2023-10-18 20:31:50 +02:00
// Cleanup any existing dry-run elements ... these should never be left hanging around
cleanupDryRunDatabaseFiles ( appConfig . databaseFilePathDryRun ) ;
2022-08-05 01:16:54 +02:00
2019-03-11 07:57:47 +01:00
// Make a copy of the original items.sqlite3 for use as the dry run copy if it exists
2023-10-18 20:31:50 +02:00
if ( exists ( appConfig . databaseFilePath ) ) {
// In a --dry-run --resync scenario, we should not copy the existing database file
if ( ! appConfig . getValueBool ( "resync" ) ) {
// Copy the existing DB file to the dry-run copy
log . log ( "DRY-RUN: Copying items.sqlite3 to items-dryrun.sqlite3 to use for dry run operations" ) ;
copy ( appConfig . databaseFilePath , appConfig . databaseFilePathDryRun ) ;
2021-02-06 10:46:56 +01:00
} else {
2023-10-18 20:31:50 +02:00
// No database copy due to --resync
log . log ( "DRY-RUN: No database copy created for --dry-run due to --resync also being used" ) ;
2021-02-06 10:46:56 +01:00
}
}
2023-10-18 20:31:50 +02:00
// update runtimeDatabaseFile now that we are using the dry run path
runtimeDatabaseFile = appConfig . databaseFilePathDryRun ;
2021-02-06 10:46:56 +01:00
}
2023-10-18 20:31:50 +02:00
2022-03-07 19:35:00 +01:00
// Handle --logout as separate item, do not 'resync' on a --logout
2023-10-18 20:31:50 +02:00
if ( appConfig . getValueBool ( "logout" ) ) {
2021-11-16 20:05:11 +01:00
log . vdebug ( "--logout requested" ) ;
log . log ( "Deleting the saved authentication status ..." ) ;
2023-10-18 20:31:50 +02:00
if ( ! appConfig . getValueBool ( "dry_run" ) ) {
safeRemove ( appConfig . refreshTokenFilePath ) ;
} else {
// --dry-run scenario ... technically we should not be making any local file changes .......
log . log ( "DRY RUN: Not removing the saved authentication status" ) ;
2021-11-16 20:05:11 +01:00
}
// Exit
return EXIT_SUCCESS ;
}
2022-03-07 19:35:00 +01:00
// Handle --reauth to re-authenticate the client
2023-10-18 20:31:50 +02:00
if ( appConfig . getValueBool ( "reauth" ) ) {
2022-03-07 19:35:00 +01:00
log . vdebug ( "--reauth requested" ) ;
log . log ( "Deleting the saved authentication status ... re-authentication requested" ) ;
2023-10-18 20:31:50 +02:00
if ( ! appConfig . getValueBool ( "dry_run" ) ) {
safeRemove ( appConfig . refreshTokenFilePath ) ;
} else {
// --dry-run scenario ... technically we should not be making any local file changes .......
log . log ( "DRY RUN: Not removing the saved authentication status" ) ;
2022-03-07 19:35:00 +01:00
}
}
2023-10-18 20:31:50 +02:00
// --resync should be considered a 'last resort item' or if the application configuration has changed, where a resync is needed .. the user needs to 'accept' this warning to proceed
// If --resync has not been used (bool value is false), check the application configuration for 'changes' that require a --resync to ensure that the data locally reflects the users requested configuration
if ( appConfig . getValueBool ( "resync" ) ) {
// what is the risk acceptance for --resync?
bool resyncRiskAcceptance = appConfig . displayResyncRiskForAcceptance ( ) ;
log . vdebug ( "Returned --resync risk acceptance: " , resyncRiskAcceptance ) ;
// Action based on user response
if ( ! resyncRiskAcceptance ) {
// --resync risk not accepted
return EXIT_FAILURE ;
2018-12-19 19:42:28 +01:00
} else {
2023-10-18 20:31:50 +02:00
log . vdebug ( "--resync issued and risk accepted" ) ;
// --resync risk accepted, perform a cleanup of items that require a cleanup
appConfig . cleanupHashFilesDueToResync ( ) ;
// Make a backup of the applicable configuration file
appConfig . createBackupConfigFile ( ) ;
// Update hash files and generate a new config backup
appConfig . updateHashContentsForConfigFiles ( ) ;
// Remove the items database
processResyncDatabaseRemoval ( runtimeDatabaseFile ) ;
2018-12-19 19:42:28 +01:00
}
2023-10-18 20:31:50 +02:00
} else {
// Has any of our application configuration that would require a --resync been changed?
if ( appConfig . applicationChangeWhereResyncRequired ( ) ) {
// Application configuration has changed however --resync not issued, fail fast
log . error ( "\nAn application configuration change has been detected where a --resync is required\n" ) ;
return EXIT_RESYNC_REQUIRED ;
2020-06-27 11:10:37 +02:00
} else {
2023-10-18 20:31:50 +02:00
// No configuration change that requires a --resync to be issued
// Make a backup of the applicable configuration file
appConfig . createBackupConfigFile ( ) ;
// Update hash files and generate a new config backup
appConfig . updateHashContentsForConfigFiles ( ) ;
2022-08-05 22:25:58 +02:00
}
2018-12-19 19:42:28 +01:00
}
2023-10-18 20:31:50 +02:00
// Implement https://github.com/abraunegg/onedrive/issues/1129
// Force a synchronization of a specific folder, only when using --synchronize --single-directory and ignoring all non-default skip_dir and skip_file rules
if ( appConfig . getValueBool ( "force_sync" ) ) {
// appConfig.checkForBasicOptionConflicts() has already checked for the basic requirements for --force-sync
log . log ( "\nWARNING: Overriding application configuration to use application defaults for skip_dir and skip_file due to --synch --single-directory --force-sync being used" ) ;
bool forceSyncRiskAcceptance = appConfig . displayForceSyncRiskForAcceptance ( ) ;
log . vdebug ( "Returned --force-sync risk acceptance: " , forceSyncRiskAcceptance ) ;
// Action based on user response
if ( ! forceSyncRiskAcceptance ) {
// --force-sync risk not accepted
2022-09-26 03:49:48 +02:00
return EXIT_FAILURE ;
2023-10-18 20:31:50 +02:00
} else {
// --force-sync risk accepted
// reset set config using function to use application defaults
appConfig . resetSkipToDefaults ( ) ;
// update sync engine regex with reset defaults
selectiveSync . setDirMask ( appConfig . getValueString ( "skip_dir" ) ) ;
selectiveSync . setFileMask ( appConfig . getValueString ( "skip_file" ) ) ;
2022-05-05 20:48:50 +02:00
}
}
2020-12-17 20:17:52 +01:00
// Test if OneDrive service can be reached, exit if it cant be reached
2020-03-14 20:29:44 +01:00
log . vdebug ( "Testing network to ensure network connectivity to Microsoft OneDrive Service" ) ;
2023-10-18 20:31:50 +02:00
online = testInternetReachability ( appConfig ) ;
// If we are not 'online' - how do we handle this situation?
2020-12-17 20:17:52 +01:00
if ( ! online ) {
2023-10-18 20:31:50 +02:00
// We are unable to initialise the OneDrive API as we are not online
if ( ! appConfig . getValueBool ( "monitor" ) ) {
2020-12-17 20:17:52 +01:00
// Running as --synchronize
2023-10-18 20:31:50 +02:00
log . error ( "Unable to reach Microsoft OneDrive API service, unable to initialise application\n" ) ;
2018-12-04 01:15:44 +01:00
return EXIT_FAILURE ;
2020-12-17 20:17:52 +01:00
} else {
// Running as --monitor
2023-10-18 20:31:50 +02:00
log . error ( "Unable to reach the Microsoft OneDrive API service at this point in time, re-trying network tests based on applicable intervals\n" ) ;
if ( ! retryInternetConnectivtyTest ( appConfig ) ) {
2020-12-17 20:17:52 +01:00
return EXIT_FAILURE ;
}
2018-12-04 01:15:44 +01:00
}
2022-03-07 04:02:50 +01:00
}
2023-10-18 20:31:50 +02:00
// This needs to be a separate 'if' statement, as, if this was an 'if-else' from above, if we were originally offline and using --monitor, we would never get to this point
2022-03-07 04:02:50 +01:00
if ( online ) {
2022-03-07 00:57:20 +01:00
// Check Application Version
log . vlog ( "Checking Application Version ..." ) ;
checkApplicationVersion ( ) ;
2023-10-18 20:31:50 +02:00
// Initialise the OneDrive API
log . vlog ( "Attempting to initialise the OneDrive API ..." ) ;
oneDriveApiInstance = new OneDriveApi ( appConfig ) ;
appConfig . apiWasInitialised = oneDriveApiInstance . initialise ( ) ;
if ( appConfig . apiWasInitialised ) {
log . vlog ( "The OneDrive API was initialised successfully" ) ;
// Flag that we were able to initalise the API in the application config
oneDriveApiInstance . debugOutputConfiguredAPIItems ( ) ;
// Need to configure the itemDB and syncEngineInstance for 'sync' and 'non-sync' operations
log . vlog ( "Opening the item database ..." ) ;
// Configure the Item Database
itemDB = new ItemDatabase ( runtimeDatabaseFile ) ;
// Was the database successfully initialised?
if ( ! itemDB . isDatabaseInitialised ( ) ) {
// no .. destroy class
itemDB = null ;
// exit application
2020-11-20 20:46:05 +01:00
return EXIT_FAILURE ;
}
2023-10-18 20:31:50 +02:00
// Initialise the syncEngine
syncEngineInstance = new SyncEngine ( appConfig , itemDB , selectiveSync ) ;
appConfig . syncEngineWasInitialised = syncEngineInstance . initialise ( ) ;
// Are we not doing a --sync or a --monitor operation? Both of these will be false if they are not set
if ( ( ! appConfig . getValueBool ( "synchronize" ) ) & & ( ! appConfig . getValueBool ( "monitor" ) ) ) {
// Are we performing some sort of 'no-sync' task?
// - Are we obtaining the Office 365 Drive ID for a given Office 365 SharePoint Shared Library?
// - Are we displaying the sync satus?
// - Are we getting the URL for a file online
// - Are we listing who modified a file last online
// - Are we createing a shareable link for an existing file on OneDrive?
// - Are we just creating a directory online, without any sync being performed?
// - Are we just deleting a directory online, without any sync being performed?
// - Are we renaming or moving a directory?
2023-10-19 23:53:59 +02:00
// - Are we displaying the quota information?
2023-10-18 20:31:50 +02:00
// --get-sharepoint-drive-id - Get the SharePoint Library drive_id
if ( appConfig . getValueString ( "sharepoint_library_name" ) ! = "" ) {
// Get the SharePoint Library drive_id
syncEngineInstance . querySiteCollectionForDriveID ( appConfig . getValueString ( "sharepoint_library_name" ) ) ;
// Exit application
// Use exit scopes to shutdown API and cleanup data
return EXIT_SUCCESS ;
}
// --display-sync-status - Query the sync status
if ( appConfig . getValueBool ( "display_sync_status" ) ) {
// path to query variable
string pathToQueryStatusOn ;
// What path do we query?
if ( ! appConfig . getValueString ( "single_directory" ) . empty ) {
pathToQueryStatusOn = "/" ~ appConfig . getValueString ( "single_directory" ) ;
} else {
pathToQueryStatusOn = "/" ;
}
// Query the sync status
syncEngineInstance . queryOneDriveForSyncStatus ( pathToQueryStatusOn ) ;
// Exit application
// Use exit scopes to shutdown API and cleanup data
return EXIT_SUCCESS ;
}
// --get-file-link - Get the URL path for a synced file?
if ( appConfig . getValueString ( "get_file_link" ) ! = "" ) {
// Query the OneDrive API for the file link
syncEngineInstance . queryOneDriveForFileDetails ( appConfig . getValueString ( "get_file_link" ) , runtimeSyncDirectory , "URL" ) ;
// Exit application
// Use exit scopes to shutdown API and cleanup data
return EXIT_SUCCESS ;
}
// --modified-by - Are we listing the modified-by details of a provided path?
if ( appConfig . getValueString ( "modified_by" ) ! = "" ) {
// Query the OneDrive API for the last modified by details
syncEngineInstance . queryOneDriveForFileDetails ( appConfig . getValueString ( "modified_by" ) , runtimeSyncDirectory , "ModifiedBy" ) ;
// Exit application
// Use exit scopes to shutdown API and cleanup data
return EXIT_SUCCESS ;
}
// --create-share-link - Are we createing a shareable link for an existing file on OneDrive?
if ( appConfig . getValueString ( "create_share_link" ) ! = "" ) {
// Query OneDrive for the file, and if valid, create a shareable link for the file
// By default, the shareable link will be read-only.
// If the user adds:
// --with-editing-perms
// this will create a writeable link
syncEngineInstance . queryOneDriveForFileDetails ( appConfig . getValueString ( "create_share_link" ) , runtimeSyncDirectory , "ShareableLink" ) ;
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
// --create-directory - Are we just creating a directory online, without any sync being performed?
if ( ( appConfig . getValueString ( "create_directory" ) ! = "" ) ) {
// Handle the remote path creation and updating of the local database without performing a sync
syncEngineInstance . createDirectoryOnline ( appConfig . getValueString ( "create_directory" ) ) ;
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
// --remove-directory - Are we just deleting a directory online, without any sync being performed?
if ( ( appConfig . getValueString ( "remove_directory" ) ! = "" ) ) {
// Handle the remote path deletion without performing a sync
syncEngineInstance . deleteByPath ( appConfig . getValueString ( "remove_directory" ) ) ;
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
// Are we renaming or moving a directory?
// onedrive --source-directory 'path/as/source/' --destination-directory 'path/as/destination'
if ( ( appConfig . getValueString ( "source_directory" ) ! = "" ) & & ( appConfig . getValueString ( "destination_directory" ) ! = "" ) ) {
// We are renaming or moving a directory
syncEngineInstance . uploadMoveItem ( appConfig . getValueString ( "source_directory" ) , appConfig . getValueString ( "destination_directory" ) ) ;
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
2023-10-19 23:53:59 +02:00
// Are we displaying the quota information?
if ( appConfig . getValueBool ( "display_quota" ) ) {
// Query and respond with the quota details
syncEngineInstance . queryOneDriveForQuotaDetails ( ) ;
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
2023-10-18 20:31:50 +02:00
// If we get to this point, we have not performed a 'no-sync' task ..
log . error ( "\nYour command line input is missing either the '--sync' or '--monitor' switches. Please include one (but not both) of these switches in your command line, or refer to 'onedrive --help' for additional guidance.\n" ) ;
log . error ( "It is important to note that you must include one of these two arguments in your command line for the application to perform a synchronisation with Microsoft OneDrive\n" ) ;
2022-11-18 22:53:49 +01:00
// Use exit scopes to shutdown API
2023-10-18 20:31:50 +02:00
// invalidSyncExit = true;
2022-11-18 22:53:49 +01:00
return EXIT_FAILURE ;
}
2023-10-18 20:31:50 +02:00
} else {
// API could not be initialised
log . error ( "The OneDrive API could not be initialised" ) ;
return EXIT_FAILURE ;
2019-07-30 23:20:26 +02:00
}
2020-05-20 22:48:02 +02:00
}
2022-08-30 22:41:52 +02:00
2023-10-18 20:31:50 +02:00
// Configure the sync direcory based on the runtimeSyncDirectory configured directory
log . vlog ( "All application operations will be performed in the configured local 'sync_dir' directory: " , runtimeSyncDirectory ) ;
try {
if ( ! exists ( runtimeSyncDirectory ) ) {
log . vdebug ( "runtimeSyncDirectory: Configured 'sync_dir' is missing locally. Creating: " , runtimeSyncDirectory ) ;
try {
// Attempt to create the sync dir we have been configured with
mkdirRecurse ( runtimeSyncDirectory ) ;
// Configure the applicable permissions for the folder
log . vdebug ( "Setting directory permissions for: " , runtimeSyncDirectory ) ;
runtimeSyncDirectory . setAttributes ( appConfig . returnRequiredDirectoryPermisions ( ) ) ;
} catch ( std . file . FileException e ) {
// Creating the sync directory failed
log . error ( "ERROR: Unable to create the configured local 'sync_dir' directory: " , e . msg ) ;
// Use exit scopes to shutdown API
return EXIT_FAILURE ;
}
2020-06-16 23:57:14 +02:00
}
2023-10-18 20:31:50 +02:00
} catch ( std . file . FileException e ) {
// Creating the sync directory failed
log . error ( "ERROR: Unable to test for the existence of the configured local 'sync_dir' directory: " , e . msg ) ;
// Use exit scopes to shutdown API
return EXIT_FAILURE ;
2020-06-16 23:57:14 +02:00
}
2022-06-29 23:08:29 +02:00
2023-10-18 20:31:50 +02:00
// Change the working directory to the 'sync_dir' as configured
chdir ( runtimeSyncDirectory ) ;
2022-10-12 04:34:42 +02:00
2023-10-18 20:31:50 +02:00
// Do we need to validate the runtimeSyncDirectory to check for the presence of a '.nosync' file
checkForNoMountScenario ( ) ;
2022-03-09 21:00:07 +01:00
2023-10-18 20:31:50 +02:00
// Set the default thread pool value - hard coded to 16
defaultPoolThreads ( to ! int ( appConfig . concurrentThreads ) ) ;
2023-03-17 23:53:58 +01:00
2023-10-18 20:31:50 +02:00
// Is the sync engine initiallised correctly?
if ( appConfig . syncEngineWasInitialised ) {
// Configure some initial variables
string singleDirectoryPath ;
string localPath = "." ;
2018-12-28 02:26:03 +01:00
string remotePath = "/" ;
2023-10-18 20:31:50 +02:00
// Are we doing a single directory operation (--single-directory) ?
if ( ! appConfig . getValueString ( "single_directory" ) . empty ) {
// Set singleDirectoryPath
singleDirectoryPath = appConfig . getValueString ( "single_directory" ) ;
// Ensure that this is a normalised relative path to runtimeSyncDirectory
string normalisedRelativePath = replace ( buildNormalizedPath ( absolutePath ( singleDirectoryPath ) ) , buildNormalizedPath ( absolutePath ( runtimeSyncDirectory ) ) , "." ) ;
// The user provided a directory to sync within the configured 'sync_dir' path
// This also validates if the path being used exists online and/or does not have a 'case-insensitive match'
syncEngineInstance . setSingleDirectoryScope ( normalisedRelativePath ) ;
// Does the directory we want to sync actually exist locally?
if ( ! exists ( singleDirectoryPath ) ) {
// The requested path to use with --single-directory does not exist locally within the configured 'sync_dir'
log . logAndNotify ( "WARNING: The requested path for --single-directory does not exist locally. Creating requested path within " , runtimeSyncDirectory ) ;
// Make the required --single-directory path locally
mkdirRecurse ( singleDirectoryPath ) ;
// Configure the applicable permissions for the folder
log . vdebug ( "Setting directory permissions for: " , singleDirectoryPath ) ;
singleDirectoryPath . setAttributes ( appConfig . returnRequiredDirectoryPermisions ( ) ) ;
}
// Update the paths that we use to perform the sync actions
localPath = singleDirectoryPath ;
remotePath = singleDirectoryPath ;
// Display that we are syncing from a specific path due to --single-directory
log . vlog ( "Syncing changes from this selected path: " , singleDirectoryPath ) ;
2018-12-28 02:26:03 +01:00
}
2021-11-23 20:54:28 +01:00
2023-10-18 20:31:50 +02:00
// Are we doing a --sync operation? This includes doing any --single-directory operations
if ( appConfig . getValueBool ( "synchronize" ) ) {
// Did the user specify --upload-only?
if ( appConfig . getValueBool ( "upload_only" ) ) {
// Perform the --upload-only sync process
performUploadOnlySyncProcess ( localPath ) ;
}
// Did the user specify --download-only?
if ( appConfig . getValueBool ( "download_only" ) ) {
// Only download data from OneDrive
syncEngineInstance . syncOneDriveAccountToLocalDisk ( ) ;
// Perform the DB consistency check
// This will also delete any out-of-sync flagged items if configured to do so
syncEngineInstance . performDatabaseConsistencyAndIntegrityCheck ( ) ;
// Do we cleanup local files?
// - Deletes of data from online will already have been performed, but what we are now doing is searching the local filesystem
// for any new data locally, that usually would be uploaded to OneDrive, but instead, because of the options being
// used, will need to be deleted from the local filesystem
if ( appConfig . getValueBool ( "cleanup_local_files" ) ) {
// Perform the filesystem walk
syncEngineInstance . scanLocalFilesystemPathForNewData ( localPath ) ;
2018-03-14 05:43:40 +01:00
}
2015-09-20 21:21:51 +02:00
}
2023-10-18 20:31:50 +02:00
// If no use of --upload-only or --download-only
if ( ( ! appConfig . getValueBool ( "upload_only" ) ) & & ( ! appConfig . getValueBool ( "download_only" ) ) ) {
// Perform the standard sync process
performStandardSyncProcess ( localPath ) ;
}
// Detail the outcome of the sync process
displaySyncOutcome ( ) ;
2018-03-14 05:43:40 +01:00
}
2023-10-18 20:31:50 +02:00
// Are we doing a --monitor operation?
if ( appConfig . getValueBool ( "monitor" ) ) {
// What are the current values for the platform we are running on
// Max number of open files /proc/sys/fs/file-max
string maxOpenFiles = strip ( readText ( "/proc/sys/fs/file-max" ) ) ;
// What is the currently configured maximum inotify watches that can be used
// /proc/sys/fs/inotify/max_user_watches
string maxInotifyWatches = strip ( readText ( "/proc/sys/fs/inotify/max_user_watches" ) ) ;
// Start the monitor process
log . log ( "OneDrive synchronisation interval (seconds): " , appConfig . getValueLong ( "monitor_interval" ) ) ;
// If we are in a --download-only method of operation, the output of these is not required
if ( ! appConfig . getValueBool ( "download_only" ) ) {
log . vlog ( "Maximum allowed open files: " , maxOpenFiles ) ;
log . vlog ( "Maximum allowed inotify user watches: " , maxInotifyWatches ) ;
}
// Configure the monitor class
2023-10-25 06:03:15 +02:00
filesystemMonitor = new Monitor ( appConfig , selectiveSync ) ;
2023-10-18 20:31:50 +02:00
// Delegated function for when inotify detects a new local directory has been created
filesystemMonitor . onDirCreated = delegate ( string path ) {
2020-04-06 12:05:06 +02:00
// Handle .folder creation if skip_dotfiles is enabled
2023-10-18 20:31:50 +02:00
if ( ( appConfig . getValueBool ( "skip_dotfiles" ) ) & & ( isDotFile ( path ) ) ) {
2022-04-21 00:19:24 +02:00
log . vlog ( "[M] Skipping watching local path - .folder found & --skip-dot-files enabled: " , path ) ;
2020-04-06 12:05:06 +02:00
} else {
2022-04-21 00:19:24 +02:00
log . vlog ( "[M] Local directory created: " , path ) ;
2020-04-06 12:05:06 +02:00
try {
2023-10-18 20:31:50 +02:00
syncEngineInstance . scanLocalFilesystemPathForNewData ( path ) ;
2020-04-06 12:05:06 +02:00
} catch ( CurlException e ) {
log . vlog ( "Offline, cannot create remote dir!" ) ;
} catch ( Exception e ) {
log . logAndNotify ( "Cannot create remote directory: " , e . msg ) ;
}
2018-03-14 05:43:40 +01:00
}
} ;
2023-10-18 20:31:50 +02:00
// Delegated function for when inotify detects a local file has been changed
filesystemMonitor . onFileChanged = delegate ( string path ) {
2022-04-21 00:19:24 +02:00
log . vlog ( "[M] Local file changed: " , path ) ;
2018-03-14 05:43:40 +01:00
try {
2023-10-18 20:31:50 +02:00
syncEngineInstance . handleLocalFileTrigger ( path ) ;
2019-02-21 09:51:15 +01:00
} catch ( CurlException e ) {
log . vlog ( "Offline, cannot upload changed item!" ) ;
2018-03-14 05:43:40 +01:00
} catch ( Exception e ) {
2018-12-06 08:05:52 +01:00
log . logAndNotify ( "Cannot upload file changes/creation: " , e . msg ) ;
2018-03-14 05:43:40 +01:00
}
} ;
2023-10-18 20:31:50 +02:00
// Delegated function for when inotify detects a delete event
filesystemMonitor . onDelete = delegate ( string path ) {
2022-04-21 00:19:24 +02:00
log . vlog ( "[M] Local item deleted: " , path ) ;
2018-03-14 05:43:40 +01:00
try {
2023-10-18 20:31:50 +02:00
log . log ( "The operating system sent a deletion notification. Trying to delete the item as requested" ) ;
syncEngineInstance . deleteByPath ( path ) ;
2019-02-21 09:51:15 +01:00
} catch ( CurlException e ) {
log . vlog ( "Offline, cannot delete item!" ) ;
2018-12-06 00:50:46 +01:00
} catch ( SyncException e ) {
if ( e . msg = = "The item to delete is not in the local database" ) {
2023-10-18 20:31:50 +02:00
log . vlog ( "Item cannot be deleted from Microsoft OneDrive because it was not found in the local database" ) ;
2018-12-06 00:50:46 +01:00
} else {
2018-12-06 08:05:52 +01:00
log . logAndNotify ( "Cannot delete remote item: " , e . msg ) ;
2018-12-06 00:50:46 +01:00
}
2018-03-14 05:43:40 +01:00
} catch ( Exception e ) {
2018-12-06 08:05:52 +01:00
log . logAndNotify ( "Cannot delete remote item: " , e . msg ) ;
2018-03-14 05:43:40 +01:00
}
} ;
2023-10-18 20:31:50 +02:00
// Delegated function for when inotify detects a move event
filesystemMonitor . onMove = delegate ( string from , string to ) {
2022-04-21 00:19:24 +02:00
log . vlog ( "[M] Local item moved: " , from , " -> " , to ) ;
2018-03-14 05:43:40 +01:00
try {
2020-04-06 12:05:06 +02:00
// Handle .folder -> folder if skip_dotfiles is enabled
2023-10-18 20:31:50 +02:00
if ( ( appConfig . getValueBool ( "skip_dotfiles" ) ) & & ( isDotFile ( from ) ) ) {
2020-04-06 12:05:06 +02:00
// .folder -> folder handling - has to be handled as a new folder
2023-10-18 20:31:50 +02:00
syncEngineInstance . scanLocalFilesystemPathForNewData ( to ) ;
2020-04-06 12:05:06 +02:00
} else {
2023-10-18 20:31:50 +02:00
syncEngineInstance . uploadMoveItem ( from , to ) ;
2020-04-06 12:05:06 +02:00
}
2019-02-21 09:51:15 +01:00
} catch ( CurlException e ) {
log . vlog ( "Offline, cannot move item!" ) ;
2018-03-14 05:43:40 +01:00
} catch ( Exception e ) {
2020-03-19 19:35:21 +01:00
log . logAndNotify ( "Cannot move item: " , e . msg ) ;
2018-03-14 05:43:40 +01:00
}
} ;
2023-10-18 20:31:50 +02:00
// Handle SIGINT and SIGTERM
2018-12-28 03:19:20 +01:00
signal ( SIGINT , & exitHandler ) ;
signal ( SIGTERM , & exitHandler ) ;
2023-10-18 20:31:50 +02:00
// Initialise the local filesystem monitor class using inotify to monitor for local filesystem changes
// If we are in a --download-only method of operation, we do not enable local filesystem monitoring
if ( ! appConfig . getValueBool ( "download_only" ) ) {
// Not using --download-only
2019-10-31 04:59:44 +01:00
try {
2023-10-18 20:31:50 +02:00
log . log ( "Initialising filesystem inotify monitoring ..." ) ;
filesystemMonitor . initialise ( ) ;
log . log ( "Performing initial syncronisation to ensure consistent local state ..." ) ;
} catch ( MonitorException e ) {
// monitor class initialisation failed
2019-10-31 04:59:44 +01:00
log . error ( "ERROR: " , e . msg ) ;
2022-08-30 22:41:52 +02:00
return EXIT_FAILURE ;
2019-10-31 04:59:44 +01:00
}
}
2023-10-18 20:31:50 +02:00
2023-10-18 21:48:25 +02:00
// Filesystem monitor loop variables
// immutables
2023-10-18 20:31:50 +02:00
immutable auto checkOnlineInterval = dur ! "seconds" ( appConfig . getValueLong ( "monitor_interval" ) ) ;
2022-03-07 04:02:50 +01:00
immutable auto githubCheckInterval = dur ! "seconds" ( 86400 ) ;
2023-10-18 20:31:50 +02:00
immutable ulong fullScanFrequency = appConfig . getValueLong ( "monitor_fullscan_frequency" ) ;
immutable ulong logOutputSupressionInterval = appConfig . getValueLong ( "monitor_log_frequency" ) ;
2023-10-18 21:48:25 +02:00
immutable bool webhookEnabled = appConfig . getValueBool ( "webhook_enabled" ) ;
// changables
bool performMonitor = true ;
ulong monitorLoopFullCount = 0 ;
ulong fullScanFrequencyLoopCount = 0 ;
ulong monitorLogOutputLoopCount = 0 ;
2020-05-20 03:37:11 +02:00
MonoTime lastCheckTime = MonoTime . currTime ( ) ;
2022-03-07 04:02:50 +01:00
MonoTime lastGitHubCheckTime = MonoTime . currTime ( ) ;
2023-10-18 20:31:50 +02:00
string loopStartOutputMessage = "################################################## NEW LOOP ##################################################" ;
string loopStopOutputMessage = "################################################ LOOP COMPLETE ###############################################" ;
2022-03-07 04:02:50 +01:00
2020-05-20 03:37:11 +02:00
while ( performMonitor ) {
2023-10-18 20:31:50 +02:00
// Do we need to validate the runtimeSyncDirectory to check for the presence of a '.nosync' file - the disk may have been ejected ..
checkForNoMountScenario ( ) ;
// If we are in a --download-only method of operation, there is no filesystem monitoring, so no inotify events to check
if ( ! appConfig . getValueBool ( "download_only" ) ) {
2020-05-05 23:20:13 +02:00
try {
2023-10-18 20:31:50 +02:00
// Process any inotify events
filesystemMonitor . update ( true ) ;
2020-05-05 23:20:13 +02:00
} catch ( MonitorException e ) {
// Catch any exceptions thrown by inotify / monitor engine
log . error ( "ERROR: The following inotify error was generated: " , e . msg ) ;
}
}
2023-10-18 20:31:50 +02:00
2023-10-18 21:48:25 +02:00
// Webhook Notification Handling
2021-11-23 20:54:28 +01:00
bool notificationReceived = false ;
2023-10-18 20:31:50 +02:00
2023-10-18 21:48:25 +02:00
// Check for notifications pushed from Microsoft to the webhook
if ( webhookEnabled ) {
// Create a subscription on the first run, or renew the subscription
// on subsequent runs when it is about to expire.
oneDriveApiInstance . createOrRenewSubscription ( ) ;
// Process incoming notifications if any.
// Empirical evidence shows that Microsoft often sends multiple
// notifications for one single change, so we need a loop to exhaust
// all signals that were queued up by the webhook. The notifications
// do not contain any actual changes, and we will always rely do the
// delta endpoint to sync to latest. Therefore, only one sync run is
// good enough to catch up for multiple notifications.
for ( int signalCount = 0 ; ; signalCount + + ) {
const auto signalExists = receiveTimeout ( dur ! "seconds" ( - 1 ) , ( ulong _ ) { } ) ;
if ( signalExists ) {
notificationReceived = true ;
} else {
if ( notificationReceived ) {
log . log ( "Received " , signalCount , " refresh signals from the webhook" ) ;
}
break ;
}
}
}
2023-10-18 20:31:50 +02:00
// Get the current time this loop is starting
auto currentTime = MonoTime . currTime ( ) ;
// Do we perform a sync with OneDrive?
if ( notificationReceived | | ( currentTime - lastCheckTime > checkOnlineInterval ) | | ( monitorLoopFullCount = = 0 ) ) {
// Increment relevant counters
monitorLoopFullCount + + ;
fullScanFrequencyLoopCount + + ;
monitorLogOutputLoopCount + + ;
// If full scan at a specific frequency enabled?
if ( fullScanFrequency > 0 ) {
// Full Scan set for some 'frequency' - do we flag to perform a full scan of the online data?
if ( fullScanFrequencyLoopCount > fullScanFrequency ) {
// set full scan trigger for true up
log . vdebug ( "Enabling Full Scan True Up (fullScanFrequencyLoopCount > fullScanFrequency), resetting fullScanFrequencyLoopCount = 1" ) ;
fullScanFrequencyLoopCount = 1 ;
appConfig . fullScanTrueUpRequired = true ;
2021-11-23 20:54:28 +01:00
} else {
2023-10-18 20:31:50 +02:00
// unset full scan trigger for true up
log . vdebug ( "Disabling Full Scan True Up" ) ;
appConfig . fullScanTrueUpRequired = false ;
2021-11-23 20:54:28 +01:00
}
2023-10-18 20:31:50 +02:00
} else {
// No it is disabled - ensure this is false
appConfig . fullScanTrueUpRequired = false ;
2021-11-23 20:54:28 +01:00
}
2022-03-07 04:02:50 +01:00
2023-10-18 20:31:50 +02:00
log . vdebug ( loopStartOutputMessage ) ;
log . vdebug ( "Total Run-Time Loop Number: " , monitorLoopFullCount ) ;
log . vdebug ( "Full Scan Freqency Loop Number: " , fullScanFrequencyLoopCount ) ;
SysTime startFunctionProcessingTime = Clock . currTime ( ) ;
log . vdebug ( "Start Monitor Loop Time: " , startFunctionProcessingTime ) ;
// Do we perform any monitor console logging output surpression?
// 'monitor_log_frequency' controls how often, in a non-verbose application output mode, how often
// the full output of what is occuring is done. This is done to lessen the 'verbosity' of non-verbose
// logging, but only when running in --monitor
if ( monitorLogOutputLoopCount > logOutputSupressionInterval ) {
// unsurpress the logging output
monitorLogOutputLoopCount = 1 ;
log . vdebug ( "Unsuppressing log output" ) ;
appConfig . surpressLoggingOutput = false ;
2020-05-25 03:30:04 +02:00
} else {
2023-10-18 20:31:50 +02:00
// do we surpress the logging output to absolute minimal
if ( monitorLoopFullCount = = 1 ) {
// application startup with --monitor
log . vdebug ( "Unsuppressing initial sync log output" ) ;
appConfig . surpressLoggingOutput = false ;
} else {
// only surpress if we are not doing --verbose or higher
if ( log . verbose = = 0 ) {
log . vdebug ( "Suppressing --monitor log output" ) ;
appConfig . surpressLoggingOutput = true ;
} else {
log . vdebug ( "Unsuppressing log output" ) ;
appConfig . surpressLoggingOutput = false ;
}
2019-10-31 04:59:44 +01:00
}
}
2023-10-18 20:31:50 +02:00
// How long has the application been running for?
auto elapsedTime = Clock . currTime ( ) - applicationStartTime ;
log . vdebug ( "Application run-time thus far: " , elapsedTime ) ;
// Need to re-validate that the client is still online for this loop
if ( testInternetReachability ( appConfig ) ) {
// Starting a sync
log . log ( "Starting a sync with Microsoft OneDrive" ) ;
// Attempt to reset syncFailures
syncEngineInstance . resetSyncFailures ( ) ;
// Did the user specify --upload-only?
if ( appConfig . getValueBool ( "upload_only" ) ) {
// Perform the --upload-only sync process
performUploadOnlySyncProcess ( localPath , filesystemMonitor ) ;
} else {
// Perform the standard sync process
performStandardSyncProcess ( localPath , filesystemMonitor ) ;
2018-12-04 01:15:44 +01:00
}
2023-10-18 20:31:50 +02:00
// Discard any inotify events generated as part of any sync operation
filesystemMonitor . update ( false ) ;
// Detail the outcome of the sync process
displaySyncOutcome ( ) ;
if ( appConfig . fullScanTrueUpRequired ) {
// Write WAL and SHM data to file for this loop
log . vdebug ( "Merge contents of WAL and SHM files into main database file" ) ;
itemDB . performVacuum ( ) ;
2019-10-29 20:32:17 +01:00
}
2023-10-18 20:31:50 +02:00
} else {
// Not online
log . log ( "Microsoft OneDrive service is not reachable at this time. Will re-try on next loop attempt." ) ;
2019-10-31 04:59:44 +01:00
}
2023-10-18 20:31:50 +02:00
// Output end of loop processing times
SysTime endFunctionProcessingTime = Clock . currTime ( ) ;
log . vdebug ( "End Monitor Loop Time: " , endFunctionProcessingTime ) ;
log . vdebug ( "Elapsed Monitor Loop Processing Time: " , ( endFunctionProcessingTime - startFunctionProcessingTime ) ) ;
2022-06-15 01:16:06 +02:00
2020-05-20 03:37:11 +02:00
// Display memory details before cleanup
2022-06-15 01:16:06 +02:00
if ( displayMemoryUsage ) log . displayMemoryUsagePreGC ( ) ;
2020-05-20 03:37:11 +02:00
// Perform Garbage Cleanup
2019-10-31 04:59:44 +01:00
GC . collect ( ) ;
2023-10-18 20:31:50 +02:00
// Return free memory to the OS
GC . minimize ( ) ;
2020-05-20 03:37:11 +02:00
// Display memory details after cleanup
2022-06-15 01:16:06 +02:00
if ( displayMemoryUsage ) log . displayMemoryUsagePostGC ( ) ;
2023-10-18 20:31:50 +02:00
// Log that this loop is complete
log . vdebug ( loopStopOutputMessage ) ;
// performSync complete, set lastCheckTime to current time
lastCheckTime = MonoTime . currTime ( ) ;
2022-06-15 01:16:06 +02:00
2020-05-20 03:37:11 +02:00
// Developer break via config option
2023-10-18 20:31:50 +02:00
if ( appConfig . getValueLong ( "monitor_max_loop" ) > 0 ) {
2020-05-20 03:37:11 +02:00
// developer set option to limit --monitor loops
2023-10-18 20:31:50 +02:00
if ( monitorLoopFullCount = = ( appConfig . getValueLong ( "monitor_max_loop" ) ) ) {
2020-05-20 03:37:11 +02:00
performMonitor = false ;
log . log ( "Exiting after " , monitorLoopFullCount , " loops due to developer set option" ) ;
}
}
}
2022-07-18 05:09:41 +02:00
// Sleep the monitor thread for 1 second, loop around and pick up any inotify changes
Thread . sleep ( dur ! "seconds" ( 1 ) ) ;
2015-09-17 00:16:23 +02:00
}
}
2023-10-18 20:31:50 +02:00
} else {
// Exit application as the sync engine could not be initialised
log . error ( "Application Sync Engine could not be initialised correctly" ) ;
// Use exit scope
return EXIT_FAILURE ;
}
// Exit application using exit scope
if ( ! syncEngineInstance . syncFailures ) {
return EXIT_SUCCESS ;
} else {
return EXIT_FAILURE ;
2015-09-14 19:21:06 +02:00
}
2015-09-01 20:45:34 +02:00
}
2015-09-20 21:21:51 +02:00
2023-10-25 21:23:05 +02:00
void performStandardExitProcess ( string scopeCaller ) {
// Who called this function
log . vdebug ( "Running performStandardExitProcess due to: " , scopeCaller ) ;
2023-10-27 22:21:54 +02:00
2023-10-25 21:23:05 +02:00
// Shutdown the OneDrive API instance
if ( oneDriveApiInstance ! is null ) {
2023-10-27 22:21:54 +02:00
log . vdebug ( "Shutdown OneDrive API instance" ) ;
2023-10-25 21:23:05 +02:00
oneDriveApiInstance . shutdown ( ) ;
object . destroy ( oneDriveApiInstance ) ;
}
// Shutdown the sync engine
2023-10-27 22:21:54 +02:00
if ( syncEngineInstance ! is null ) {
log . vdebug ( "Shutdown Sync Engine instance" ) ;
object . destroy ( syncEngineInstance ) ;
}
2023-10-25 21:23:05 +02:00
// Shutdown the client side filtering objects
2023-10-27 22:21:54 +02:00
if ( selectiveSync ! is null ) {
log . vdebug ( "Shutdown Client Side Filtering instance" ) ;
object . destroy ( selectiveSync ) ;
}
2023-10-25 21:23:05 +02:00
// Shutdown the application configuration objects
2023-10-25 06:03:15 +02:00
if ( appConfig ! is null ) {
2023-10-27 22:21:54 +02:00
log . vdebug ( "Shutdown Application Configuration instance" ) ;
2023-10-25 06:03:15 +02:00
// Cleanup any existing dry-run elements ... these should never be left hanging around
cleanupDryRunDatabaseFiles ( appConfig . databaseFilePathDryRun ) ;
object . destroy ( appConfig ) ;
}
2023-10-25 21:23:05 +02:00
// Shutdown any local filesystem monitoring
2023-10-25 06:03:15 +02:00
if ( filesystemMonitor ! is null ) {
2023-10-27 22:21:54 +02:00
log . vdebug ( "Shutdown Filesystem Monitoring instance" ) ;
2023-10-25 06:03:15 +02:00
filesystemMonitor . shutdown ( ) ;
object . destroy ( filesystemMonitor ) ;
}
2023-10-25 21:23:05 +02:00
2023-10-27 22:21:54 +02:00
// Shutdown the database
if ( itemDB ! is null ) {
log . vdebug ( "Shutdown Database instance" ) ;
// Make sure the .wal file is incorporated into the main db before we exit
if ( itemDB . isDatabaseInitialised ( ) ) {
itemDB . performVacuum ( ) ;
}
object . destroy ( itemDB ) ;
}
// Set all objects to null
2023-10-25 21:23:05 +02:00
if ( scopeCaller = = "failureScope" ) {
// Set these to be null due to failure scope - prevent 'ERROR: Unable to perform a database vacuum: out of memory' when the exit scope is then called
log . vdebug ( "Setting Class Objects to null due to failure scope" ) ;
itemDB = null ;
appConfig = null ;
oneDriveApiInstance = null ;
selectiveSync = null ;
syncEngineInstance = null ;
}
2023-10-25 06:03:15 +02:00
}
2023-10-18 20:31:50 +02:00
void performUploadOnlySyncProcess ( string localPath , Monitor filesystemMonitor = null ) {
// Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive
syncEngineInstance . performDatabaseConsistencyAndIntegrityCheck ( ) ;
if ( appConfig . getValueBool ( "monitor" ) ) {
// Handle any inotify events whilst the DB was being scanned
filesystemMonitor . update ( true ) ;
2022-08-05 01:16:54 +02:00
}
2023-10-18 20:31:50 +02:00
// Scan the configured 'sync_dir' for new data to upload
syncEngineInstance . scanLocalFilesystemPathForNewData ( localPath ) ;
if ( appConfig . getValueBool ( "monitor" ) ) {
// Handle any new inotify events whilst the local filesystem was being scanned
filesystemMonitor . update ( true ) ;
2022-08-05 01:16:54 +02:00
}
}
2023-10-18 20:31:50 +02:00
void performStandardSyncProcess ( string localPath , Monitor filesystemMonitor = null ) {
// If we are performing log supression, output this message so the user knows what is happening
if ( appConfig . surpressLoggingOutput ) {
log . log ( "Syncing changes from Microsoft OneDrive ..." ) ;
}
// Zero out these arrays
syncEngineInstance . fileDownloadFailures = [ ] ;
syncEngineInstance . fileUploadFailures = [ ] ;
// Which way do we sync first?
// OneDrive first then local changes (normal operational process that uses OneDrive as the source of truth)
// Local First then OneDrive changes (alternate operation process to use local files as source of truth)
if ( appConfig . getValueBool ( "local_first" ) ) {
// Local data first
// Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive
syncEngineInstance . performDatabaseConsistencyAndIntegrityCheck ( ) ;
if ( appConfig . getValueBool ( "monitor" ) ) {
// Handle any inotify events whilst the DB was being scanned
filesystemMonitor . update ( true ) ;
}
// Scan the configured 'sync_dir' for new data to upload to OneDrive
syncEngineInstance . scanLocalFilesystemPathForNewData ( localPath ) ;
if ( appConfig . getValueBool ( "monitor" ) ) {
// Handle any new inotify events whilst the local filesystem was being scanned
filesystemMonitor . update ( true ) ;
2018-12-04 01:15:44 +01:00
}
2023-10-18 20:31:50 +02:00
// Download data from OneDrive last
syncEngineInstance . syncOneDriveAccountToLocalDisk ( ) ;
if ( appConfig . getValueBool ( "monitor" ) ) {
// Cancel out any inotify events from downloading data
filesystemMonitor . update ( false ) ;
}
} else {
// Normal sync
// Download data from OneDrive first
syncEngineInstance . syncOneDriveAccountToLocalDisk ( ) ;
if ( appConfig . getValueBool ( "monitor" ) ) {
// Cancel out any inotify events from downloading data
filesystemMonitor . update ( false ) ;
}
2023-10-19 12:27:03 +02:00
2023-10-18 20:31:50 +02:00
// Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive
syncEngineInstance . performDatabaseConsistencyAndIntegrityCheck ( ) ;
if ( appConfig . getValueBool ( "monitor" ) ) {
// Handle any inotify events whilst the DB was being scanned
filesystemMonitor . update ( true ) ;
}
2023-10-19 12:27:03 +02:00
// Is --download-only NOT configured?
if ( ! appConfig . getValueBool ( "download_only" ) ) {
2023-10-18 20:31:50 +02:00
2023-10-19 12:27:03 +02:00
// Scan the configured 'sync_dir' for new data to upload to OneDrive
syncEngineInstance . scanLocalFilesystemPathForNewData ( localPath ) ;
if ( appConfig . getValueBool ( "monitor" ) ) {
// Handle any new inotify events whilst the local filesystem was being scanned
filesystemMonitor . update ( true ) ;
}
// Make sure we sync any DB data to this point, but only if not in --monitor mode
// In --monitor mode, this is handled within the 'loop', based on when the full scan true up is being performed
if ( ! appConfig . getValueBool ( "monitor" ) ) {
itemDB . performVacuum ( ) ;
}
// Perform the final true up scan to ensure we have correctly replicated the current online state locally
if ( ! appConfig . surpressLoggingOutput ) {
log . log ( "Performing a last examination of the most recent online data within Microsoft OneDrive to complete the reconciliation process" ) ;
}
// We pass in the 'appConfig.fullScanTrueUpRequired' value which then flags do we use the configured 'deltaLink'
// If 'appConfig.fullScanTrueUpRequired' is true, we do not use the 'deltaLink' if we are in --monitor mode, thus forcing a full scan true up
syncEngineInstance . syncOneDriveAccountToLocalDisk ( ) ;
if ( appConfig . getValueBool ( "monitor" ) ) {
// Cancel out any inotify events from downloading data
filesystemMonitor . update ( false ) ;
}
2018-12-04 01:15:44 +01:00
}
}
}
2023-10-18 20:31:50 +02:00
void displaySyncOutcome ( ) {
2021-11-23 20:54:28 +01:00
2023-10-18 20:31:50 +02:00
// Detail any download or upload transfer failures
syncEngineInstance . displaySyncFailures ( ) ;
// Sync is either complete or partially complete
if ( ! syncEngineInstance . syncFailures ) {
// No download or upload issues
if ( ! appConfig . getValueBool ( "monitor" ) ) writeln ( ) ; // Add an additional line break so that this is clear when using --sync
log . log ( "Sync with Microsoft OneDrive is complete" ) ;
} else {
log . log ( "\nSync with Microsoft OneDrive has completed, however there are items that failed to sync." ) ;
// Due to how the OneDrive API works 'changes' such as add new files online, rename files online, delete files online are only sent once when using the /delta API call.
// That we failed to download it, we need to track that, and then issue a --resync to download any of these failed files .. unfortunate, but there is no easy way here
if ( ! syncEngineInstance . fileDownloadFailures . empty ) {
log . log ( "To fix any download failures you may need to perform a --resync to ensure this system is correctly synced with your Microsoft OneDrive Account" ) ;
}
if ( ! syncEngineInstance . fileUploadFailures . empty ) {
log . log ( "To fix any upload failures you may need to perform a --resync to ensure this system is correctly synced with your Microsoft OneDrive Account" ) ;
}
// So that from a logging perspective these messages are clear, add a line break in
writeln ( ) ;
2018-03-14 05:43:40 +01:00
}
2023-10-18 20:31:50 +02:00
}
2021-11-23 20:54:28 +01:00
2023-10-18 20:31:50 +02:00
void processResyncDatabaseRemoval ( string databaseFilePathToRemove ) {
log . vdebug ( "Testing if we have exclusive access to local database file" ) ;
// Are we the only running instance? Test that we can open the database file path
itemDB = new ItemDatabase ( databaseFilePathToRemove ) ;
// did we successfully initialise the database class?
if ( ! itemDB . isDatabaseInitialised ( ) ) {
// no .. destroy class
itemDB = null ;
// exit application - void function, force exit this way
exit ( - 1 ) ;
}
// If we have exclusive access we will not have exited
// destroy access test
destroy ( itemDB ) ;
// delete application sync state
log . log ( "Deleting the saved application sync status ..." ) ;
if ( ! appConfig . getValueBool ( "dry_run" ) ) {
safeRemove ( databaseFilePathToRemove ) ;
} else {
// --dry-run scenario ... technically we should not be making any local file changes .......
log . log ( "DRY RUN: Not removing the saved application sync status" ) ;
}
}
2022-08-30 22:41:52 +02:00
2023-10-18 20:31:50 +02:00
void cleanupDryRunDatabaseFiles ( string dryRunDatabaseFile ) {
// Temp variables
string dryRunShmFile = dryRunDatabaseFile ~ "-shm" ;
string dryRunWalFile = dryRunDatabaseFile ~ "-wal" ;
2021-11-23 20:54:28 +01:00
2023-10-18 20:31:50 +02:00
// If the dry run database exists, clean this up
if ( exists ( dryRunDatabaseFile ) ) {
// remove the existing file
log . vdebug ( "DRY-RUN: Removing items-dryrun.sqlite3 as it still exists for some reason" ) ;
safeRemove ( dryRunDatabaseFile ) ;
}
// silent cleanup of shm files if it exists
if ( exists ( dryRunShmFile ) ) {
// remove items-dryrun.sqlite3-shm
log . vdebug ( "DRY-RUN: Removing items-dryrun.sqlite3-shm as it still exists for some reason" ) ;
safeRemove ( dryRunShmFile ) ;
}
// silent cleanup of wal files if it exists
if ( exists ( dryRunWalFile ) ) {
// remove items-dryrun.sqlite3-wal
log . vdebug ( "DRY-RUN: Removing items-dryrun.sqlite3-wal as it still exists for some reason" ) ;
safeRemove ( dryRunWalFile ) ;
}
}
2021-11-23 20:54:28 +01:00
2023-10-18 20:31:50 +02:00
void checkForNoMountScenario ( ) {
// If this is a 'mounted' folder, the 'mount point' should have this file to help the application stop any action to preserve data because the drive to mount is not currently mounted
if ( appConfig . getValueBool ( "check_nomount" ) ) {
// we were asked to check the mount point for the presence of a '.nosync' file
if ( exists ( ".nosync" ) ) {
log . logAndNotify ( "ERROR: .nosync file found in directory mount point. Aborting application startup process to safeguard data." ) ;
exit ( EXIT_FAILURE ) ;
2015-09-20 21:21:51 +02:00
}
2023-10-18 20:31:50 +02:00
}
2015-09-20 21:21:51 +02:00
}
2018-12-05 20:19:00 +01:00
2023-10-18 20:31:50 +02:00
// Getting around the @nogc problem
2018-12-28 03:19:20 +01:00
// https://p0nce.github.io/d-idioms/#Bypassing-@nogc
2023-10-18 20:31:50 +02:00
auto assumeNoGC ( T ) ( T t ) if ( isFunctionPointer ! T | | isDelegate ! T ) {
2018-12-28 03:19:20 +01:00
enum attrs = functionAttributes ! T | FunctionAttribute . nogc ;
return cast ( SetFunctionAttributes ! ( T , functionLinkage ! T , attrs ) ) t ;
}
2023-10-18 20:31:50 +02:00
// Catch CTRL-C
2018-12-28 03:19:20 +01:00
extern ( C ) nothrow @nogc @system void exitHandler ( int value ) {
try {
assumeNoGC ( ( ) {
2021-11-23 20:54:28 +01:00
log . log ( "Got termination signal, performing clean up" ) ;
2023-10-24 23:36:16 +02:00
// Wait for all parallel jobs that depend on the database to complete
taskPool . finish ( true ) ;
// Was itemDb initialised?
2023-10-18 20:31:50 +02:00
if ( itemDB . isDatabaseInitialised ( ) ) {
2020-11-06 00:28:15 +01:00
// Make sure the .wal file is incorporated into the main db before we exit
2023-10-18 20:31:50 +02:00
log . log ( "Shutting down DB connection and merging temporary data" ) ;
itemDB . performVacuum ( ) ;
2023-10-24 23:36:16 +02:00
object . destroy ( itemDB ) ;
2020-11-06 00:28:15 +01:00
}
2018-12-28 03:19:20 +01:00
} ) ( ) ;
} catch ( Exception e ) { }
exit ( 0 ) ;
2023-10-18 20:31:50 +02:00
}