@ -5,11 +5,13 @@ import config, itemdb, monitor, onedrive, selective, sync, util;
import std.net.curl : CurlException ;
import core.stdc.signal ;
import std.traits ;
import std.concurrency : receiveTimeout ;
static import log ;
OneDriveApi oneDrive ;
ItemDatabase itemDb ;
bool onedriveInitialised = false ;
const int EXIT_UNAUTHORIZED = 3 ;
enum MONITOR_LOG_SILENT = 2 ;
@ -20,7 +22,7 @@ int main(string[] args)
{
// Disable buffering on stdout
stdout . setvbuf ( 0 , _IONBF ) ;
// main function variables
string confdirOption ;
string configFilePath ;
@ -48,10 +50,9 @@ int main(string[] args)
bool skipDirDifferent = false ;
bool online = false ;
bool performSyncOK = false ;
bool onedriveInitialised = false ;
bool displayMemoryUsage = false ;
bool displaySyncOptions = false ;
// Define scopes
scope ( exit ) {
// Display memory details
@ -79,7 +80,7 @@ int main(string[] args)
log . displayMemoryUsagePostGC ( ) ;
}
}
scope ( failure ) {
// Display memory details
if ( displayMemoryUsage ) {
@ -139,7 +140,7 @@ int main(string[] args)
log . error ( "Try 'onedrive -h' for more information" ) ;
return EXIT_FAILURE ;
}
// load configuration file if available
auto cfg = new config . Config ( confdirOption ) ;
if ( ! cfg . initialize ( ) ) {
@ -147,22 +148,22 @@ int main(string[] args)
// Error message already printed
return EXIT_FAILURE ;
}
// set memory display
displayMemoryUsage = cfg . getValueBool ( "display_memory" ) ;
// set display sync options
displaySyncOptions = cfg . getValueBool ( "display_sync_options" ) ;
// update configuration from command line args
cfg . update_from_args ( args ) ;
// Initialise normalised file paths
configFilePath = buildNormalizedPath ( cfg . configDirName ~ "/config" ) ;
syncListFilePath = buildNormalizedPath ( cfg . configDirName ~ "/sync_list" ) ;
databaseFilePath = buildNormalizedPath ( cfg . configDirName ~ "/items.db" ) ;
businessSharedFolderFilePath = buildNormalizedPath ( cfg . configDirName ~ "/business_shared_folders" ) ;
// Has any of our configuration that would require a --resync been changed?
// 1. sync_list file modification
// 2. config file modification - but only if sync_dir, skip_dir, skip_file or drive_id was modified
@ -171,25 +172,25 @@ int main(string[] args)
syncListHashFile = buildNormalizedPath ( cfg . configDirName ~ "/.sync_list.hash" ) ;
configBackupFile = buildNormalizedPath ( cfg . configDirName ~ "/.config.backup" ) ;
businessSharedFoldersHashFile = buildNormalizedPath ( cfg . configDirName ~ "/.business_shared_folders.hash" ) ;
// Does a config file exist with a valid hash file
if ( ( exists ( configFilePath ) ) & & ( ! exists ( configHashFile ) ) ) {
// Hash of config file needs to be created
std . file . write ( configHashFile , computeQuickXorHash ( configFilePath ) ) ;
}
// Does a sync_list file exist with a valid hash file
if ( ( exists ( syncListFilePath ) ) & & ( ! exists ( syncListHashFile ) ) ) {
// Hash of sync_list file needs to be created
std . file . write ( syncListHashFile , computeQuickXorHash ( syncListFilePath ) ) ;
}
// check if business_shared_folders & business_shared_folders hash exists
if ( ( exists ( businessSharedFolderFilePath ) ) & & ( ! exists ( businessSharedFoldersHashFile ) ) ) {
// Hash of business_shared_folders file needs to be created
std . file . write ( businessSharedFoldersHashFile , computeQuickXorHash ( businessSharedFolderFilePath ) ) ;
}
// If hash files exist, but config files do not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different
if ( ( ! exists ( configFilePath ) ) & & ( exists ( configHashFile ) ) ) {
// if --resync safe remove config.hash and config.backup
@ -198,18 +199,18 @@ int main(string[] args)
safeRemove ( configBackupFile ) ;
}
}
// If sync_list hash file exists, but sync_list file does not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different
if ( ( ! exists ( syncListFilePath ) ) & & ( exists ( syncListHashFile ) ) ) {
// if --resync safe remove sync_list.hash
if ( cfg . getValueBool ( "resync" ) ) safeRemove ( syncListHashFile ) ;
}
if ( ( ! exists ( businessSharedFolderFilePath ) ) & & ( exists ( businessSharedFoldersHashFile ) ) ) {
// if --resync safe remove business_shared_folders.hash
if ( cfg . getValueBool ( "resync" ) ) safeRemove ( businessSharedFoldersHashFile ) ;
}
// Read config hashes if they exist
if ( exists ( configFilePath ) ) currentConfigHash = computeQuickXorHash ( configFilePath ) ;
if ( exists ( syncListFilePath ) ) currentSyncListHash = computeQuickXorHash ( syncListFilePath ) ;
@ -217,21 +218,21 @@ int main(string[] args)
if ( exists ( configHashFile ) ) previousConfigHash = readText ( configHashFile ) ;
if ( exists ( syncListHashFile ) ) previousSyncListHash = readText ( syncListHashFile ) ;
if ( exists ( businessSharedFoldersHashFile ) ) previousBusinessSharedFoldersHash = readText ( businessSharedFoldersHashFile ) ;
// Was sync_list file updated?
if ( currentSyncListHash ! = previousSyncListHash ) {
// Debugging output to assist what changed
log . vdebug ( "sync_list file has been updated, --resync needed" ) ;
syncListDifferent = true ;
}
// Was business_shared_folders updated?
if ( currentBusinessSharedFoldersHash ! = previousBusinessSharedFoldersHash ) {
// Debugging output to assist what changed
log . vdebug ( "business_shared_folders file has been updated, --resync needed" ) ;
businessSharedFoldersDifferent = true ;
}
// Was config file updated between last execution ang this execution?
if ( currentConfigHash ! = previousConfigHash ) {
// config file was updated, however we only want to trigger a --resync requirement if sync_dir, skip_dir, skip_file or drive_id was modified
@ -266,7 +267,7 @@ int main(string[] args)
log . vdebug ( key , " was modified since the last time the application was successfully run, --resync needed" ) ;
configOptionsDifferent = true ;
}
if ( ( key = = "skip_file" ) & & ( c . front . dup ! = cfg . getValueString ( "skip_file" ) ) ) {
log . vdebug ( key , " was modified since the last time the application was successfully run, --resync needed" ) ;
configOptionsDifferent = true ;
@ -291,7 +292,7 @@ int main(string[] args)
// no backup to check
log . vdebug ( "WARNING: no backup config file was found, unable to validate if any changes made" ) ;
}
// If there was a backup, any modified values we need to worry about would been detected
if ( ! cfg . getValueBool ( "display_config" ) ) {
// we are not testing the configuration
@ -309,13 +310,13 @@ int main(string[] args)
}
}
}
// Is there a backup of the config file if the config file exists?
if ( ( exists ( configFilePath ) ) & & ( ! exists ( configBackupFile ) ) ) {
// create backup copy of current config file
std . file . copy ( configFilePath , configBackupFile ) ;
}
// config file set options can be changed via CLI input, specifically these will impact sync and --resync will be needed:
// --syncdir ARG
// --skip-file ARG
@ -331,7 +332,7 @@ int main(string[] args)
syncDirDifferent = true ;
}
}
// was the skip_file updated by CLI?
if ( cfg . configFileSkipFile ! = "" ) {
// skip_file was set in config file
@ -340,8 +341,8 @@ int main(string[] args)
log . vdebug ( "skip_file: CLI override of config file option, --resync needed" ) ;
skipFileDifferent = true ;
}
}
}
// was the skip_dir updated by CLI?
if ( cfg . configFileSkipDir ! = "" ) {
// skip_dir was set in config file
@ -352,7 +353,7 @@ int main(string[] args)
}
}
}
// Has anything triggered a --resync requirement?
if ( configOptionsDifferent | | syncListDifferent | | syncDirDifferent | | skipFileDifferent | | skipDirDifferent | | businessSharedFoldersDifferent ) {
// --resync needed, is the user just testing configuration changes?
@ -388,7 +389,7 @@ int main(string[] args)
}
}
}
// dry-run notification and database setup
if ( cfg . getValueBool ( "dry_run" ) ) {
log . log ( "DRY-RUN Configured. Output below shows what 'would' have occurred." ) ;
@ -398,18 +399,18 @@ int main(string[] args)
if ( exists ( cfg . databaseFilePathDryRun ) ) {
// remove the existing file
log . vdebug ( "Removing items-dryrun.sqlite3 as it still exists for some reason" ) ;
safeRemove ( cfg . databaseFilePathDryRun ) ;
safeRemove ( cfg . databaseFilePathDryRun ) ;
}
// silent cleanup of shm and wal files if they exist
if ( exists ( dryRunShmFile ) ) {
// remove items-dryrun.sqlite3-shm
safeRemove ( dryRunShmFile ) ;
safeRemove ( dryRunShmFile ) ;
}
if ( exists ( dryRunWalFile ) ) {
// remove items-dryrun.sqlite3-wal
safeRemove ( dryRunWalFile ) ;
safeRemove ( dryRunWalFile ) ;
}
// Make a copy of the original items.sqlite3 for use as the dry run copy if it exists
if ( exists ( cfg . databaseFilePath ) ) {
// in a --dry-run --resync scenario, we should not copy the existing database file
@ -423,7 +424,7 @@ int main(string[] args)
}
}
}
// sync_dir environment handling to handle ~ expansion properly
bool shellEnvSet = false ;
if ( ( environment . get ( "SHELL" ) = = "" ) & & ( environment . get ( "USER" ) = = "" ) ) {
@ -450,10 +451,10 @@ int main(string[] args)
syncDir = cfg . getValueString ( "sync_dir" ) ;
}
}
// vdebug syncDir as set and calculated
log . vdebug ( "syncDir: " , syncDir ) ;
// Configure the logging directory if different from application default
// log_dir environment handling to handle ~ expansion properly
string logDir = cfg . getValueString ( "log_dir" ) ;
@ -478,7 +479,7 @@ int main(string[] args)
// update log_dir with normalised path, with '~' expanded correctly
cfg . setValueString ( "log_dir" , logDir ) ;
}
// Configure logging only if enabled
if ( cfg . getValueBool ( "enable_logging" ) ) {
// Initialise using the configured logging directory
@ -488,7 +489,7 @@ int main(string[] args)
// Configure whether notifications are used
log . setNotifications ( cfg . getValueBool ( "monitor" ) & & ! cfg . getValueBool ( "disable_notifications" ) ) ;
// Application upgrades - skilion version etc
if ( exists ( databaseFilePath ) ) {
if ( ! cfg . getValueBool ( "dry_run" ) ) {
@ -497,7 +498,7 @@ int main(string[] args)
log . logAndNotify ( "Database schema changed, resync needed" ) ;
cfg . setValueBool ( "resync" , true ) ;
}
// Handle --logout as separate item, do not 'resync' on a --logout / reauth
if ( cfg . getValueBool ( "logout" ) ) {
log . vdebug ( "--logout requested" ) ;
@ -508,7 +509,7 @@ int main(string[] args)
// Exit
return EXIT_SUCCESS ;
}
// Handle --resync to remove local files
if ( cfg . getValueBool ( "resync" ) ) {
if ( cfg . getValueBool ( "resync" ) ) log . vdebug ( "--resync requested" ) ;
@ -519,7 +520,7 @@ int main(string[] args)
safeRemove ( cfg . uploadStateFilePath ) ;
}
}
// Display current application configuration, no application initialisation
if ( cfg . getValueBool ( "display_config" ) ) {
// Display application version
@ -528,7 +529,7 @@ int main(string[] args)
writeln ( "Config path = " , cfg . configDirName ) ;
// Does a config file exist or are we using application defaults
writeln ( "Config file found in config path = " , exists ( configFilePath ) ) ;
// Config Options
writeln ( "Config option 'check_nosync' = " , cfg . getValueBool ( "check_nosync" ) ) ;
writeln ( "Config option 'sync_dir' = " , syncDir ) ;
@ -543,12 +544,12 @@ int main(string[] args)
writeln ( "Config option 'upload_only' = " , cfg . getValueBool ( "upload_only" ) ) ;
writeln ( "Config option 'no_remote_delete' = " , cfg . getValueBool ( "no_remote_delete" ) ) ;
writeln ( "Config option 'remove_source_files' = " , cfg . getValueBool ( "remove_source_files" ) ) ;
// Is config option drive_id configured?
if ( cfg . getValueString ( "drive_id" ) ! = "" ) {
writeln ( "Config option 'drive_id' = " , cfg . getValueString ( "drive_id" ) ) ;
}
// Is sync_list configured?
if ( exists ( syncListFilePath ) ) {
writeln ( "Config option 'sync_root_files' = " , cfg . getValueBool ( "sync_root_files" ) ) ;
@ -565,7 +566,7 @@ int main(string[] args)
writeln ( "Config option 'sync_root_files' = " , cfg . getValueBool ( "sync_root_files" ) ) ;
writeln ( "Selective sync 'sync_list' configured = false" ) ;
}
// Is business_shared_folders configured
if ( exists ( businessSharedFolderFilePath ) ) {
writeln ( "Business Shared Folders configured = true" ) ;
@ -580,11 +581,11 @@ int main(string[] args)
} else {
writeln ( "Business Shared Folders configured = false" ) ;
}
// Exit
return EXIT_SUCCESS ;
}
// Test if OneDrive service can be reached, exit if it cant be reached
log . vdebug ( "Testing network to ensure network connectivity to Microsoft OneDrive Service" ) ;
online = testNetwork ( ) ;
@ -597,7 +598,7 @@ int main(string[] args)
} else {
// Running as --monitor
log . error ( "Unable to reach Microsoft OneDrive API service at this point in time, re-trying network tests\n" ) ;
// re-try network connection to OneDrive
// https://github.com/abraunegg/onedrive/issues/1184
// Back off & retry with incremental delay
@ -605,13 +606,13 @@ int main(string[] args)
int retryAttempts = 1 ;
int backoffInterval = 1 ;
int maxBackoffInterval = 3600 ;
bool retrySuccess = false ;
while ( ! retrySuccess ) {
// retry to access OneDrive API
backoffInterval + + ;
int thisBackOffInterval = retryAttempts * backoffInterval ;
log . vdebug ( " Retry Attempt: " , retryAttempts ) ;
log . vdebug ( " Retry Attempt: " , retryAttempts ) ;
if ( thisBackOffInterval < = maxBackoffInterval ) {
log . vdebug ( " Retry In (seconds): " , thisBackOffInterval ) ;
Thread . sleep ( dur ! "seconds" ( thisBackOffInterval ) ) ;
@ -631,7 +632,7 @@ int main(string[] args)
// we have attempted to re-connect X number of times
// false set this to true to break out of while loop
retrySuccess = true ;
}
}
}
// Increment & loop around
retryAttempts + + ;
@ -643,7 +644,7 @@ int main(string[] args)
}
}
}
// Initialize OneDrive, check for authorization
if ( online ) {
// we can only initialise if we are online
@ -652,24 +653,24 @@ int main(string[] args)
onedriveInitialised = oneDrive . init ( ) ;
oneDrive . printAccessToken = cfg . getValueBool ( "print_token" ) ;
}
if ( ! onedriveInitialised ) {
log . error ( "Could not initialize the OneDrive API" ) ;
// Use exit scopes to shutdown API
return EXIT_UNAUTHORIZED ;
}
// if --synchronize or --monitor not passed in, configure the flag to display help & exit
if ( cfg . getValueBool ( "synchronize" ) | | cfg . getValueBool ( "monitor" ) ) {
performSyncOK = true ;
}
// create-directory, remove-directory, source-directory, destination-directory
// these are activities that dont perform a sync, so to not generate an error message for these items either
if ( ( ( cfg . getValueString ( "create_directory" ) ! = "" ) | | ( cfg . getValueString ( "remove_directory" ) ! = "" ) ) | | ( ( cfg . getValueString ( "source_directory" ) ! = "" ) & & ( cfg . getValueString ( "destination_directory" ) ! = "" ) ) | | ( cfg . getValueString ( "get_file_link" ) ! = "" ) | | ( cfg . getValueString ( "create_share_link" ) ! = "" ) | | ( cfg . getValueString ( "get_o365_drive_id" ) ! = "" ) | | cfg . getValueBool ( "display_sync_status" ) | | cfg . getValueBool ( "list_business_shared_folders" ) ) {
performSyncOK = true ;
}
// Were acceptable sync operations provided? Was --synchronize or --monitor passed in
if ( ! performSyncOK ) {
// was the application just authorised?
@ -694,7 +695,7 @@ int main(string[] args)
return EXIT_FAILURE ;
}
}
// if --synchronize && --monitor passed in, exit & display help as these conflict with each other
if ( cfg . getValueBool ( "synchronize" ) & & cfg . getValueBool ( "monitor" ) ) {
writeln ( "\nERROR: --synchronize and --monitor cannot be used together\n" ) ;
@ -702,7 +703,7 @@ int main(string[] args)
// Use exit scopes to shutdown API
return EXIT_FAILURE ;
}
// Initialize the item database
log . vlog ( "Opening the item database ..." ) ;
if ( ! cfg . getValueBool ( "dry_run" ) ) {
@ -714,7 +715,7 @@ int main(string[] args)
log . vdebug ( "Using database file: " , asNormalizedPath ( cfg . databaseFilePathDryRun ) ) ;
itemDb = new ItemDatabase ( cfg . databaseFilePathDryRun ) ;
}
// What are the permission that have been set for the application?
// These are relevant for:
// - The ~/OneDrive parent folder or 'sync_dir' configured item
@ -731,7 +732,7 @@ int main(string[] args)
log . vdebug ( "Configuring default new file permissions as: " , cfg . getValueLong ( "sync_file_permissions" ) ) ;
cfg . configureRequiredFilePermisions ( ) ;
}
// configure the sync direcory based on syncDir config option
log . vlog ( "All operations will be performed in: " , syncDir ) ;
if ( ! exists ( syncDir ) ) {
@ -749,13 +750,13 @@ int main(string[] args)
return EXIT_FAILURE ;
}
}
// Change the working directory to the 'sync_dir' configured item
chdir ( syncDir ) ;
// Configure selective sync by parsing and getting a regex for skip_file config component
auto selectiveSync = new SelectiveSync ( ) ;
// load sync_list if it exists
if ( exists ( syncListFilePath ) ) {
log . vdebug ( "Loading user configured sync_list file ..." ) ;
@ -774,7 +775,7 @@ int main(string[] args)
}
}
selectiveSync . load ( syncListFilePath ) ;
// load business_shared_folders if it exists
if ( exists ( businessSharedFolderFilePath ) ) {
log . vdebug ( "Loading user configured business_shared_folders file ..." ) ;
@ -787,27 +788,27 @@ int main(string[] args)
}
}
selectiveSync . loadSharedFolders ( businessSharedFolderFilePath ) ;
// Configure skip_dir, skip_file, skip-dir-strict-match & skip_dotfiles from config entries
// Handle skip_dir configuration in config file
log . vdebug ( "Configuring skip_dir ..." ) ;
log . vdebug ( "skip_dir: " , cfg . getValueString ( "skip_dir" ) ) ;
selectiveSync . setDirMask ( cfg . getValueString ( "skip_dir" ) ) ;
// Was --skip-dir-strict-match configured?
log . vdebug ( "Configuring skip_dir_strict_match ..." ) ;
log . vdebug ( "skip_dir_strict_match: " , cfg . getValueBool ( "skip_dir_strict_match" ) ) ;
if ( cfg . getValueBool ( "skip_dir_strict_match" ) ) {
selectiveSync . setSkipDirStrictMatch ( ) ;
}
// Was --skip-dot-files configured?
log . vdebug ( "Configuring skip_dotfiles ..." ) ;
log . vdebug ( "skip_dotfiles: " , cfg . getValueBool ( "skip_dotfiles" ) ) ;
if ( cfg . getValueBool ( "skip_dotfiles" ) ) {
selectiveSync . setSkipDotfiles ( ) ;
}
// Handle skip_file configuration in config file
log . vdebug ( "Configuring skip_file ..." ) ;
// Validate skip_file to ensure that this does not contain an invalid configuration
@ -822,7 +823,7 @@ int main(string[] args)
// All skip_file entries are valid
log . vdebug ( "skip_file: " , cfg . getValueString ( "skip_file" ) ) ;
selectiveSync . setFileMask ( cfg . getValueString ( "skip_file" ) ) ;
// Initialize the sync engine
auto sync = new SyncEngine ( cfg , oneDrive , itemDb , selectiveSync ) ;
try {
@ -847,7 +848,7 @@ int main(string[] args)
if ( syncListConfigured ) {
sync . setSyncListConfigured ( ) ;
}
// Do we need to configure specific --upload-only options?
if ( cfg . getValueBool ( "upload_only" ) ) {
// --upload-only was passed in or configured
@ -866,12 +867,12 @@ int main(string[] args)
sync . setLocalDeleteAfterUpload ( ) ;
}
}
// Do we configure to disable the upload validation routine
if ( cfg . getValueBool ( "disable_upload_validation" ) ) sync . setDisableUploadValidation ( ) ;
// Do we configure to disable the download validation routine
if ( cfg . getValueBool ( "disable_download_validation" ) ) sync . setDisableDownloadValidation ( ) ;
if ( cfg . getValueBool ( "disable_download_validation" ) ) sync . setDisableDownloadValidation ( ) ;
// Has the user enabled to bypass data preservation of renaming local files when there is a conflict?
if ( cfg . getValueBool ( "bypass_data_preservation" ) ) {
@ -879,7 +880,7 @@ int main(string[] args)
log . log ( "WARNING: Local data loss MAY occur in this scenario." ) ;
sync . setBypassDataPreservation ( ) ;
}
// Are we configured to use a National Cloud Deployment
if ( cfg . getValueString ( "azure_ad_endpoint" ) ! = "" ) {
// value is configured, is it a valid value?
@ -888,7 +889,7 @@ int main(string[] args)
sync . setNationalCloudDeployment ( ) ;
}
}
// Do we need to validate the syncDir to check for the presence of a '.nosync' file
if ( cfg . getValueBool ( "check_nomount" ) ) {
// we were asked to check the mounts
@ -898,53 +899,53 @@ int main(string[] args)
return EXIT_FAILURE ;
}
}
// Do we need to create or remove a directory?
if ( ( cfg . getValueString ( "create_directory" ) ! = "" ) | | ( cfg . getValueString ( "remove_directory" ) ! = "" ) ) {
if ( cfg . getValueString ( "create_directory" ) ! = "" ) {
// create a directory on OneDrive
sync . createDirectoryNoSync ( cfg . getValueString ( "create_directory" ) ) ;
}
if ( cfg . getValueString ( "remove_directory" ) ! = "" ) {
// remove a directory on OneDrive
sync . deleteDirectoryNoSync ( cfg . getValueString ( "remove_directory" ) ) ;
}
}
// Are we renaming or moving a directory?
if ( ( cfg . getValueString ( "source_directory" ) ! = "" ) & & ( cfg . getValueString ( "destination_directory" ) ! = "" ) ) {
// We are renaming or moving a directory
sync . renameDirectoryNoSync ( cfg . getValueString ( "source_directory" ) , cfg . getValueString ( "destination_directory" ) ) ;
}
// Are we obtaining the Office 365 Drive ID for a given Office 365 SharePoint Shared Library?
if ( cfg . getValueString ( "get_o365_drive_id" ) ! = "" ) {
sync . querySiteCollectionForDriveID ( cfg . getValueString ( "get_o365_drive_id" ) ) ;
// Exit application
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
// Are we createing an anonymous read-only shareable link for an existing file on OneDrive?
if ( cfg . getValueString ( "create_share_link" ) ! = "" ) {
// Query OneDrive for the file, and if valid, create a shareable link for the file
sync . createShareableLinkForFile ( cfg . getValueString ( "create_share_link" ) ) ;
// Exit application
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
// Are we obtaining the URL path for a synced file?
if ( cfg . getValueString ( "get_file_link" ) ! = "" ) {
// Query OneDrive for the file link
sync . queryOneDriveForFileURL ( cfg . getValueString ( "get_file_link" ) , syncDir ) ;
// Exit application
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
// Are we listing OneDrive Business Shared Folders
if ( cfg . getValueBool ( "list_business_shared_folders" ) ) {
// Is this a business account type?
@ -954,11 +955,11 @@ int main(string[] args)
} else {
log . error ( "ERROR: Unsupported account type for listing OneDrive Business Shared Folders" ) ;
}
// Exit application
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
// Are we going to sync OneDrive Business Shared Folders
if ( cfg . getValueBool ( "sync_business_shared_folders" ) ) {
// Is this a business account type?
@ -969,7 +970,7 @@ int main(string[] args)
log . error ( "ERROR: Unsupported account type for syncing OneDrive Business Shared Folders" ) ;
}
}
// Are we displaying the sync status of the client?
if ( cfg . getValueBool ( "display_sync_status" ) ) {
string remotePath = "/" ;
@ -980,7 +981,7 @@ int main(string[] args)
}
sync . queryDriveForChanges ( remotePath ) ;
}
// Are we performing a sync, or monitor operation?
if ( ( cfg . getValueBool ( "synchronize" ) ) | | ( cfg . getValueBool ( "monitor" ) ) ) {
// Initialise the monitor class, so that we can do more granular inotify handling when performing the actual sync
@ -1007,17 +1008,17 @@ int main(string[] args)
// fullScanRequired = false, for final true-up
// but if we have sync_list configured, use syncListConfigured which = true
performSync ( sync , cfg . getValueString ( "single_directory" ) , cfg . getValueBool ( "download_only" ) , cfg . getValueBool ( "local_first" ) , cfg . getValueBool ( "upload_only" ) , LOG_NORMAL , false , syncListConfigured , displaySyncOptions , cfg . getValueBool ( "monitor" ) , m ) ;
// Write WAL and SHM data to file for this sync
log . vdebug ( "Merge contents of WAL and SHM files into main database file" ) ;
itemDb . performVacuum ( ) ;
}
}
if ( cfg . getValueBool ( "monitor" ) ) {
log . logAndNotify ( "Initializing monitor ..." ) ;
log . log ( "OneDrive monitor interval (seconds): " , cfg . getValueLong ( "monitor_interval" ) ) ;
m . onDirCreated = delegate ( string path ) {
// Handle .folder creation if skip_dotfiles is enabled
if ( ( cfg . getValueBool ( "skip_dotfiles" ) ) & & ( selectiveSync . isDotFile ( path ) ) ) {
@ -1106,7 +1107,8 @@ int main(string[] args)
// sync list is configured
syncListConfiguredFullScanOverride = true ;
}
immutable bool webhookEnabled = cfg . getValueBool ( "webhook_enabled" ) ;
while ( performMonitor ) {
if ( ! cfg . getValueBool ( "download_only" ) ) {
try {
@ -1116,7 +1118,35 @@ int main(string[] args)
log . error ( "ERROR: The following inotify error was generated: " , e . msg ) ;
}
}
// Check for notifications pushed from Microsoft to the webhook
bool notificationReceived = false ;
if ( webhookEnabled ) {
// Create a subscription on the first run, or renew the subscription
// on subsequent runs when it is about to expire.
oneDrive . createOrRenewSubscription ( ) ;
// Process incoming notifications if any.
// Empirical evidence shows that Microsoft often sends multiple
// notifications for one single change, so we need a loop to exhaust
// all signals that were queued up by the webhook. The notifications
// do not contain any actual changes, and we will always rely do the
// delta endpoint to sync to latest. Therefore, only one sync run is
// good enough to catch up for multiple notifications.
for ( int signalCount = 0 ; ; signalCount + + ) {
const auto signalExists = receiveTimeout ( dur ! "seconds" ( - 1 ) , ( ulong _ ) { } ) ;
if ( signalExists ) {
notificationReceived = true ;
} else {
if ( notificationReceived ) {
log . log ( "Received " , signalCount , " refresh signals from the webhook" ) ;
}
break ;
}
}
}
auto currTime = MonoTime . currTime ( ) ;
// has monitor_interval elapsed or are we at application startup / monitor startup?
// in a --resync scenario, if we have not 're-populated' the database, valid changes will get skipped:
@ -1126,7 +1156,7 @@ int main(string[] args)
// Moving random_files/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby to target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
// Skipping uploading this new file as parent path is not in the database: target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
// 'target' should be in the DB, it should also exist online, but because of --resync, it does not exist in the database thus parent check fails
if ( ( currTime - lastCheckTime > checkInterval ) | | ( monitorLoopFullCount = = 0 ) ) {
if ( notificationReceived | | ( currTime - lastCheckTime > checkInterval ) | | ( monitorLoopFullCount = = 0 ) ) {
// monitor sync loop
logOutputMessage = "################################################## NEW LOOP ##################################################" ;
if ( displaySyncOptions ) {
@ -1140,7 +1170,7 @@ int main(string[] args)
if ( displayMemoryUsage ) {
log . displayMemoryUsagePreGC ( ) ;
}
// log monitor output suppression
logMonitorCounter + = 1 ;
if ( logMonitorCounter > logInterval ) {
@ -1162,7 +1192,7 @@ int main(string[] args)
fullScanRequired = false ;
}
}
if ( displaySyncOptions ) {
// sync option handling per sync loop
log . log ( "fullScanCounter = " , fullScanCounter ) ;
@ -1222,14 +1252,14 @@ int main(string[] args)
if ( displayMemoryUsage ) {
log . displayMemoryUsagePostGC ( ) ;
}
// Write WAL and SHM data to file for this loop
log . vdebug ( "Merge contents of WAL and SHM files into main database file" ) ;
itemDb . performVacuum ( ) ;
// monitor loop complete
logOutputMessage = "################################################ LOOP COMPLETE ###############################################" ;
// Handle display options
if ( displaySyncOptions ) {
log . log ( logOutputMessage ) ;
@ -1258,20 +1288,20 @@ int main(string[] args)
// remove the file
log . vdebug ( "Removing items-dryrun.sqlite3 as dry run operations complete" ) ;
// remove items-dryrun.sqlite3
safeRemove ( cfg . databaseFilePathDryRun ) ;
safeRemove ( cfg . databaseFilePathDryRun ) ;
}
// silent cleanup of shm and wal files if they exist
if ( exists ( dryRunShmFile ) ) {
// remove items-dryrun.sqlite3-shm
safeRemove ( dryRunShmFile ) ;
safeRemove ( dryRunShmFile ) ;
}
if ( exists ( dryRunWalFile ) ) {
// remove items-dryrun.sqlite3-wal
safeRemove ( dryRunWalFile ) ;
safeRemove ( dryRunWalFile ) ;
}
}
// Exit application
// Exit application
// Use exit scopes to shutdown API
return EXIT_SUCCESS ;
}
@ -1301,14 +1331,14 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
string remotePath = "/" ;
string localPath = "." ;
string logOutputMessage ;
// performSync API scan triggers
log . vdebug ( "performSync API scan triggers" ) ;
log . vdebug ( "-----------------------------" ) ;
log . vdebug ( "fullScanRequired = " , fullScanRequired ) ;
log . vdebug ( "syncListConfiguredFullScanOverride = " , syncListConfiguredFullScanOverride ) ;
log . vdebug ( "-----------------------------" ) ;
// Are we doing a single directory sync?
if ( singleDirectory ! = "" ) {
// Need two different path strings here
@ -1317,12 +1347,12 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// Set flag for singleDirectoryScope for change handling
sync . setSingleDirectoryScope ( ) ;
}
// Due to Microsoft Sharepoint 'enrichment' of files, we try to download the Microsoft modified file automatically
// Set flag if we are in upload only state to handle this differently
// See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details
// See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details
if ( uploadOnly ) sync . setUploadOnly ( ) ;
do {
try {
// starting a sync
@ -1383,7 +1413,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
} else {
// sync from OneDrive first before uploading files to OneDrive
if ( logLevel < MONITOR_LOG_SILENT ) log . log ( "Syncing changes from OneDrive ..." ) ;
// For the initial sync, always use the delta link so that we capture all the right delta changes including adds, moves & deletes
logOutputMessage = "Initial Scan: Call OneDrive Delta API for delta changes as compared to last successful sync." ;
syncCallLogOutput = "Calling sync.applyDifferences(false);" ;
@ -1395,8 +1425,8 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
log . vdebug ( syncCallLogOutput ) ;
}
sync . applyDifferences ( false ) ;
// is this a download only request?
// is this a download only request?
if ( ! downloadOnly ) {
// process local changes walking the entire path checking for changes
// in monitor mode all local changes are captured via inotify
@ -1410,7 +1440,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
// What sort of local scan do we want to do?
// In --monitor mode, when performing the DB scan, a race condition occurs where by if a file or folder is moved during this process
// the inotify event is discarded once performSync() is finished (see m.update(false) above), so these events need to be handled
@ -1431,20 +1461,20 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// handle any inotify events that occured 'whilst' we were scanning the local filesystem
m . update ( true ) ;
}
// At this point, all OneDrive changes / local changes should be uploaded and in sync
// This MAY not be the case when using sync_list, thus a full walk of OneDrive ojects is required
// --synchronize & no sync_list : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// --synchronize & sync_list in use : fullScanRequired = false, syncListConfiguredFullScanOverride = true
// --monitor loops around 10 iterations. On the 1st loop, sets fullScanRequired = false, syncListConfiguredFullScanOverride = true if requried
// --monitor & no sync_list (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = false
// --monitor & no sync_list (loop #2 - #10) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// --monitor & sync_list in use (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = true
// --monitor & sync_list in use (loop #2 - #10) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// Do not perform a full walk of the OneDrive objects
if ( ( ! fullScanRequired ) & & ( ! syncListConfiguredFullScanOverride ) ) {
logOutputMessage = "Final True-Up: Do not perform a full walk of the OneDrive objects - not required" ;
@ -1458,7 +1488,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
}
sync . applyDifferences ( false ) ;
}
// Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop
if ( ( ! fullScanRequired ) & & ( syncListConfiguredFullScanOverride ) ) {
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop" ;
@ -1472,7 +1502,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
}
sync . applyDifferences ( true ) ;
}
// Perform a full walk of OneDrive objects because a full scan was required
if ( ( fullScanRequired ) & & ( ! syncListConfiguredFullScanOverride ) ) {
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required" ;
@ -1483,10 +1513,10 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
} else {
log . vdebug ( logOutputMessage ) ;
log . vdebug ( syncCallLogOutput ) ;
}
}
sync . applyDifferences ( true ) ;
}
// Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop
if ( ( fullScanRequired ) & & ( syncListConfiguredFullScanOverride ) ) {
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop" ;
@ -1504,7 +1534,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
}
}
}
// sync is complete
logOutputMessage = "################################################ SYNC COMPLETE ###############################################" ;
if ( displaySyncOptions ) {
@ -1512,13 +1542,13 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
} else {
log . vdebug ( logOutputMessage ) ;
}
count = - 1 ;
} catch ( Exception e ) {
if ( + + count = = 3 ) {
log . log ( "Giving up on sync after three attempts: " , e . msg ) ;
throw e ;
} else
} else
log . log ( "Retry sync count: " , count , ": " , e . msg ) ;
}
} while ( count ! = - 1 ) ;
@ -1535,14 +1565,19 @@ auto assumeNoGC(T) (T t) if (isFunctionPointer!T || isDelegate!T)
extern ( C ) nothrow @nogc @system void exitHandler ( int value ) {
try {
assumeNoGC ( ( ) {
log . log ( "Got termination signal, shutting down db connection" ) ;
log . log ( "Got termination signal, performing clean up" ) ;
// if initialised, shut down the HTTP instance
if ( onedriveInitialised ) {
log . log ( "Shutting down the HTTP instance" ) ;
oneDrive . shutdown ( ) ;
}
// was itemDb initialised?
if ( itemDb ! is null ) {
// Make sure the .wal file is incorporated into the main db before we exit
log . log ( "Shutting down db connection" ) ;
itemDb . performVacuum ( ) ;
destroy ( itemDb ) ;
}
// Use exit scopes to shutdown OneDrive API
} ) ( ) ;
} catch ( Exception e ) { }
exit ( 0 ) ;
Yuan Liu
2 months ago
GitHub