Add option to cleanup local files regardless of sync state when using --download-only (#2113)

* Add option to cleanup local files regardless of sync state when using --download-only
This commit is contained in:
abraunegg 2022-08-31 06:41:52 +10:00 committed by GitHub
parent 5288f94ac4
commit 738be2d150
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 236 additions and 134 deletions

1
config
View file

@ -53,3 +53,4 @@
# space_reservation = "50"
# display_running_config = "false"
# read_only_auth_scope = "false"
# cleanup_local_files = "false"

View file

@ -505,6 +505,7 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con
# space_reservation = "50"
# display_running_config = "false"
# read_only_auth_scope = "false"
# cleanup_local_files = "false"
```
### 'config' file configuration examples:
@ -1180,6 +1181,8 @@ Options:
Check for the presence of .nosync in each directory. If found, skip directory from sync.
--classify-as-big-delete
Number of children in a path that is locally removed which will be classified as a 'big data delete'
--cleanup-local-files
Cleanup additional local files when using --download-only. This will remove local data.
--confdir ARG
Set the directory used to store the configuration files
--create-directory ARG

View file

@ -41,6 +41,11 @@ Number of children in a path that is locally removed which will be classified as
.br
Configuration file key: \fBclassify_as_big_delete\fP (default: \fB1000\fP)
.TP
\fB\-\-cleanup\-local\-files\fP
Cleanup additional local files when using \-\-download-only. This will remove local data.
.br
Configuration file key: \fBcleanup_local_files\fP (default: \fBfalse\fP)
.TP
\fB\-\-confdir\fP ARG
Set the directory used to store the configuration files
.TP

View file

@ -138,6 +138,8 @@ final class Config
boolValues["display_running_config"] = false;
// Configure read-only authentication scope
boolValues["read_only_auth_scope"] = false;
// Flag to cleanup local files when using --download-only
boolValues["cleanup_local_files"] = false;
// DEVELOPER OPTIONS
// display_memory = true | false
@ -357,6 +359,9 @@ final class Config
"classify-as-big-delete",
"Number of children in a path that is locally removed which will be classified as a 'big data delete'",
&longValues["classify_as_big_delete"],
"cleanup-local-files",
"Cleanup additional local files when using --download-only. This will remove local data.",
&boolValues["cleanup_local_files"],
"create-directory",
"Create a directory on OneDrive - no sync will be performed.",
&stringValues["create_directory"],

View file

@ -54,6 +54,7 @@ int main(string[] args)
bool performSyncOK = false;
bool displayMemoryUsage = false;
bool displaySyncOptions = false;
bool cleanupLocalFilesGlobal = false;
// hash file permission values
string hashPermissionValue = "600";
@ -698,6 +699,7 @@ int main(string[] args)
writeln("Config option 'check_nomount' = ", cfg.getValueBool("check_nomount"));
writeln("Config option 'resync' = ", cfg.getValueBool("resync"));
writeln("Config option 'resync_auth' = ", cfg.getValueBool("resync_auth"));
writeln("Config option 'cleanup_local_files' = ", cfg.getValueBool("cleanup_local_files"));
// data integrity
writeln("Config option 'classify_as_big_delete' = ", cfg.getValueLong("classify_as_big_delete"));
@ -1154,6 +1156,16 @@ int main(string[] args)
log.log("WARNING: Local data loss MAY occur in this scenario.");
sync.setBypassDataPreservation();
}
// Do we configure to clean up local files if using --download-only ?
if ((cfg.getValueBool("download_only")) && (cfg.getValueBool("cleanup_local_files"))) {
// --download-only and --cleanup-local-files were passed in
log.log("WARNING: Application has been configured to cleanup local files that are not present online.");
log.log("WARNING: Local data loss MAY occur in this scenario if you are expecting data to remain archived locally.");
sync.setCleanupLocalFiles();
// Set the global flag as we will use this as thhe item to be passed into the sync function below
cleanupLocalFilesGlobal = true;
}
// Are we configured to use a National Cloud Deployment
if (cfg.getValueString("azure_ad_endpoint") != "") {
@ -1299,7 +1311,7 @@ int main(string[] args)
// perform a --synchronize sync
// fullScanRequired = false, for final true-up
// but if we have sync_list configured, use syncListConfigured which = true
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, false, syncListConfigured, displaySyncOptions, cfg.getValueBool("monitor"), m);
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, false, syncListConfigured, displaySyncOptions, cfg.getValueBool("monitor"), m, cleanupLocalFilesGlobal);
// Write WAL and SHM data to file for this sync
log.vdebug("Merge contents of WAL and SHM files into main database file");
@ -1380,7 +1392,7 @@ int main(string[] args)
// monitor initialisation failed
log.error("ERROR: ", e.msg);
oneDrive.shutdown();
exit(-1);
return EXIT_FAILURE;
}
}
@ -1527,7 +1539,7 @@ int main(string[] args)
// log file only if enabled so we know when a sync started when not using --verbose
log.fileOnly(startMessage);
}
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired, syncListConfiguredFullScanOverride, displaySyncOptions, cfg.getValueBool("monitor"), m);
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired, syncListConfiguredFullScanOverride, displaySyncOptions, cfg.getValueBool("monitor"), m, cleanupLocalFilesGlobal);
if (!cfg.getValueBool("download_only")) {
// discard all events that may have been generated by the sync that have not already been handled
try {
@ -1646,7 +1658,7 @@ bool initSyncEngine(SyncEngine sync)
}
// try to synchronize the folder three times
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired, bool syncListConfiguredFullScanOverride, bool displaySyncOptions, bool monitorEnabled, Monitor m)
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired, bool syncListConfiguredFullScanOverride, bool displaySyncOptions, bool monitorEnabled, Monitor m, bool cleanupLocalFiles)
{
int count;
string remotePath = "/";
@ -1701,12 +1713,21 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// OneDrive First
if (logLevel < MONITOR_LOG_QUIET) log.log("Syncing changes from selected OneDrive path ...");
sync.applyDifferencesSingleDirectory(remotePath);
// is this a download only request?
if (!downloadOnly) {
// process local changes
sync.scanForDifferences(localPath);
// ensure that the current remote state is updated locally
sync.applyDifferencesSingleDirectory(remotePath);
// Is this a --download-only --cleanup-local-files request?
// If yes, scan for local changes - but --cleanup-local-files is being used, a further flag will trigger local file deletes rather than attempt to upload files to OneDrive
if (cleanupLocalFiles) {
// --download-only and --cleanup-local-files were passed in
log.log("Searching local filesystem for extra files and folders which need to be removed");
sync.scanForDifferencesFilesystemScan(localPath);
} else {
// is this a --download-only request?
if (!downloadOnly) {
// process local changes
sync.scanForDifferences(localPath);
// ensure that the current remote state is updated locally
sync.applyDifferencesSingleDirectory(remotePath);
}
}
}
}
@ -1746,126 +1767,134 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
log.vdebug(syncCallLogOutput);
}
sync.applyDifferences(false);
// is this a download only request?
if (!downloadOnly) {
// process local changes walking the entire path checking for changes
// in monitor mode all local changes are captured via inotify
// thus scanning every 'monitor_interval' (default 300 seconds) for local changes is excessive and not required
logOutputMessage = "Process local filesystem (sync_dir) for file changes as compared to database entries";
syncCallLogOutput = "Calling sync.scanForDifferences(localPath);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
// What sort of local scan do we want to do?
// In --monitor mode, when performing the DB scan, a race condition occurs where by if a file or folder is moved during this process
// the inotify event is discarded once performSync() is finished (see m.update(false) above), so these events need to be handled
// This can be remediated by breaking the DB and file system scan into separate processes, and handing any applicable inotify events in between
if (!monitorEnabled) {
// --synchronize in use
// standard process flow
sync.scanForDifferences(localPath);
} else {
// --monitor in use
// Use individual calls with inotify checks between to avoid a race condition between these 2 functions
// Database scan integrity check to compare DB data vs actual content on disk to ensure what we think is local, is local
// and that the data 'hash' as recorded in the DB equals the hash of the actual content
// This process can be extremely expensive time and CPU processing wise
//
// fullScanRequired is set to TRUE when the application starts up, or the config option 'monitor_fullscan_frequency' count is reached
// By default, 'monitor_fullscan_frequency' = 12, and 'monitor_interval' = 300, meaning that by default, a full database consistency check
// is done once an hour.
//
// To change this behaviour adjust 'monitor_interval' and 'monitor_fullscan_frequency' to desired values in the application config file
if (fullScanRequired) {
log.vlog("Performing Database Consistency Integrity Check .. ");
sync.scanForDifferencesDatabaseScan(localPath);
// handle any inotify events that occured 'whilst' we were scanning the database
m.update(true);
// Is this a --download-only --cleanup-local-files request?
// If yes, scan for local changes - but --cleanup-local-files is being used, a further flag will trigger local file deletes rather than attempt to upload files to OneDrive
if (cleanupLocalFiles) {
// --download-only and --cleanup-local-files were passed in
log.log("Searching local filesystem for extra files and folders which need to be removed");
sync.scanForDifferencesFilesystemScan(localPath);
} else {
// is this a --download-only request?
if (!downloadOnly) {
// process local changes walking the entire path checking for changes
// in monitor mode all local changes are captured via inotify
// thus scanning every 'monitor_interval' (default 300 seconds) for local changes is excessive and not required
logOutputMessage = "Process local filesystem (sync_dir) for file changes as compared to database entries";
syncCallLogOutput = "Calling sync.scanForDifferences(localPath);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug("NOT performing Database Integrity Check .. fullScanRequired = FALSE");
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
// What sort of local scan do we want to do?
// In --monitor mode, when performing the DB scan, a race condition occurs where by if a file or folder is moved during this process
// the inotify event is discarded once performSync() is finished (see m.update(false) above), so these events need to be handled
// This can be remediated by breaking the DB and file system scan into separate processes, and handing any applicable inotify events in between
if (!monitorEnabled) {
// --synchronize in use
// standard process flow
sync.scanForDifferences(localPath);
} else {
// --monitor in use
// Use individual calls with inotify checks between to avoid a race condition between these 2 functions
// Database scan integrity check to compare DB data vs actual content on disk to ensure what we think is local, is local
// and that the data 'hash' as recorded in the DB equals the hash of the actual content
// This process can be extremely expensive time and CPU processing wise
//
// fullScanRequired is set to TRUE when the application starts up, or the config option 'monitor_fullscan_frequency' count is reached
// By default, 'monitor_fullscan_frequency' = 12, and 'monitor_interval' = 300, meaning that by default, a full database consistency check
// is done once an hour.
//
// To change this behaviour adjust 'monitor_interval' and 'monitor_fullscan_frequency' to desired values in the application config file
if (fullScanRequired) {
log.vlog("Performing Database Consistency Integrity Check .. ");
sync.scanForDifferencesDatabaseScan(localPath);
// handle any inotify events that occured 'whilst' we were scanning the database
m.update(true);
} else {
log.vdebug("NOT performing Database Integrity Check .. fullScanRequired = FALSE");
m.update(true);
}
// Filesystem walk to find new files not uploaded
log.vdebug("Searching local filesystem for new data");
sync.scanForDifferencesFilesystemScan(localPath);
// handle any inotify events that occured 'whilst' we were scanning the local filesystem
m.update(true);
}
// Filesystem walk to find new files not uploaded
log.vdebug("Searching local filesystem for new data");
sync.scanForDifferencesFilesystemScan(localPath);
// handle any inotify events that occured 'whilst' we were scanning the local filesystem
m.update(true);
}
// At this point, all OneDrive changes / local changes should be uploaded and in sync
// This MAY not be the case when using sync_list, thus a full walk of OneDrive ojects is required
// At this point, all OneDrive changes / local changes should be uploaded and in sync
// This MAY not be the case when using sync_list, thus a full walk of OneDrive ojects is required
// --synchronize & no sync_list : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// --synchronize & sync_list in use : fullScanRequired = false, syncListConfiguredFullScanOverride = true
// --synchronize & no sync_list : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// --synchronize & sync_list in use : fullScanRequired = false, syncListConfiguredFullScanOverride = true
// --monitor loops around 12 iterations. On the 1st loop, sets fullScanRequired = true, syncListConfiguredFullScanOverride = true if requried
// --monitor loops around 12 iterations. On the 1st loop, sets fullScanRequired = true, syncListConfiguredFullScanOverride = true if requried
// --monitor & no sync_list (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = false
// --monitor & no sync_list (loop #2 - #12) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// --monitor & sync_list in use (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = true
// --monitor & sync_list in use (loop #2 - #12) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// --monitor & no sync_list (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = false
// --monitor & no sync_list (loop #2 - #12) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// --monitor & sync_list in use (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = true
// --monitor & sync_list in use (loop #2 - #12) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// Do not perform a full walk of the OneDrive objects
if ((!fullScanRequired) && (!syncListConfiguredFullScanOverride)){
logOutputMessage = "Final True-Up: Do not perform a full walk of the OneDrive objects - not required";
syncCallLogOutput = "Calling sync.applyDifferences(false);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
// Do not perform a full walk of the OneDrive objects
if ((!fullScanRequired) && (!syncListConfiguredFullScanOverride)){
logOutputMessage = "Final True-Up: Do not perform a full walk of the OneDrive objects - not required";
syncCallLogOutput = "Calling sync.applyDifferences(false);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
sync.applyDifferences(false);
}
sync.applyDifferences(false);
}
// Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop
if ((!fullScanRequired) && (syncListConfiguredFullScanOverride)){
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop";
syncCallLogOutput = "Calling sync.applyDifferences(true);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
// Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop
if ((!fullScanRequired) && (syncListConfiguredFullScanOverride)){
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop";
syncCallLogOutput = "Calling sync.applyDifferences(true);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
sync.applyDifferences(true);
}
sync.applyDifferences(true);
}
// Perform a full walk of OneDrive objects because a full scan was required
if ((fullScanRequired) && (!syncListConfiguredFullScanOverride)){
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required";
syncCallLogOutput = "Calling sync.applyDifferences(true);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
// Perform a full walk of OneDrive objects because a full scan was required
if ((fullScanRequired) && (!syncListConfiguredFullScanOverride)){
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required";
syncCallLogOutput = "Calling sync.applyDifferences(true);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
sync.applyDifferences(true);
}
sync.applyDifferences(true);
}
// Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop
if ((fullScanRequired) && (syncListConfiguredFullScanOverride)){
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop";
syncCallLogOutput = "Calling sync.applyDifferences(true);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
// Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop
if ((fullScanRequired) && (syncListConfiguredFullScanOverride)){
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop";
syncCallLogOutput = "Calling sync.applyDifferences(true);";
if (displaySyncOptions) {
log.log(logOutputMessage);
log.log(syncCallLogOutput);
} else {
log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput);
}
sync.applyDifferences(true);
}
sync.applyDifferences(true);
}
}
}

View file

@ -30,6 +30,9 @@ private bool disableUploadValidation = false;
// Do we configure to disable the download validation routine
private bool disableDownloadValidation = false;
// Do we perform a local cleanup of files that are 'extra' on the local file system, when using --download-only
private bool cleanupLocalFiles = false;
private bool isItemFolder(const ref JSONValue item)
{
return ("folder" in item) != null;
@ -616,6 +619,13 @@ final class SyncEngine
log.vdebug("Setting nationalCloudDeployment = true");
}
// set cleanupLocalFiles to true
void setCleanupLocalFiles()
{
cleanupLocalFiles = true;
log.vdebug("Setting cleanupLocalFiles = true");
}
// return the OneDrive Account Type
auto getAccountType()
{
@ -3444,7 +3454,7 @@ final class SyncEngine
}
}
// scan the given directory for new items - for use with --monitor
// scan the given directory for new items - for use with --monitor or --cleanup-local-files
void scanForDifferencesFilesystemScan(const(string) path)
{
// To improve logging output for this function, what is the 'logical path' we are scanning for file & folder differences?
@ -3461,10 +3471,15 @@ final class SyncEngine
if (isDir(path)) {
// if this path is a directory, output this message.
// if a file, potentially leads to confusion as to what the client is actually doing
log.vlog("Uploading new items of ", logPath);
if (!cleanupLocalFiles) {
// if --cleanup-local-files was set, we will not be uploading data
log.vlog("Uploading new items of ", logPath);
}
}
// Filesystem walk to find new files not uploaded
// Filesystem walk to find extra files that reside locally.
// If --cleanup-local-files is not used, these will be uploaded (normal operation)
// If --download-only --cleanup-local-files is being used, extra files found locally will be deleted from the local filesystem
uploadNewItems(path);
}
@ -4324,14 +4339,45 @@ final class SyncEngine
// Was the path found in the database?
if (!pathFoundInDB) {
// Path not found in database when searching all drive id's
uploadCreateDir(path);
if (!cleanupLocalFiles) {
// --download-only --cleanup-local-files not used
uploadCreateDir(path);
} else {
// we need to clean up this directory
log.log("Removing local directory as --download-only & --cleanup-local-files configured");
// Remove any children of this path if they still exist
// Resolve 'Directory not empty' error when deleting local files
foreach (DirEntry child; dirEntries(path, SpanMode.depth, false)) {
// what sort of child is this?
if (isDir(child.name)) {
log.log("Removing local directory: ", child.name);
} else {
log.log("Removing local file: ", child.name);
}
// are we in a --dry-run scenario?
if (!dryRun) {
// No --dry-run ... process local delete
attrIsDir(child.linkAttributes) ? rmdir(child.name) : remove(child.name);
}
}
// Remove the path now that it is empty of children
log.log("Removing local directory: ", path);
// are we in a --dry-run scenario?
if (!dryRun) {
// No --dry-run ... process local delete
rmdirRecurse(path);
}
}
}
// recursively traverse children
// the above operation takes time and the directory might have
// disappeared in the meantime
if (!exists(path)) {
log.vlog("Directory disappeared during upload: ", path);
if (!cleanupLocalFiles) {
// --download-only --cleanup-local-files not used
log.vlog("Directory disappeared during upload: ", path);
}
return;
}
@ -4365,23 +4411,36 @@ final class SyncEngine
// Was the file found in the database?
if (!fileFoundInDB) {
// File not found in database when searching all drive id's, upload as new file
uploadNewFile(path);
// Did the upload fail?
if (!uploadFailed) {
// Upload did not fail
// Issue #763 - Delete local files after sync handling
// are we in an --upload-only & --remove-source-files scenario?
if ((uploadOnly) && (localDeleteAfterUpload)) {
// Log that we are deleting a local item
log.log("Removing local file as --upload-only & --remove-source-files configured");
// are we in a --dry-run scenario?
if (!dryRun) {
// No --dry-run ... process local file delete
// File not found in database when searching all drive id's
// Do we upload the file or clean up the file?
if (!cleanupLocalFiles) {
// --download-only --cleanup-local-files not used
uploadNewFile(path);
// Did the upload fail?
if (!uploadFailed) {
// Upload did not fail
// Issue #763 - Delete local files after sync handling
// are we in an --upload-only & --remove-source-files scenario?
if ((uploadOnly) && (localDeleteAfterUpload)) {
// Log that we are deleting a local item
log.log("Removing local file as --upload-only & --remove-source-files configured");
// are we in a --dry-run scenario?
log.vdebug("Removing local file: ", path);
safeRemove(path);
if (!dryRun) {
// No --dry-run ... process local file delete
safeRemove(path);
}
}
}
} else {
// we need to clean up this file
log.log("Removing local file as --download-only & --cleanup-local-files configured");
// are we in a --dry-run scenario?
log.log("Removing local file: ", path);
if (!dryRun) {
// No --dry-run ... process local file delete
safeRemove(path);
}
}
}
} else {