deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-17 21:26:41 +02:00
Code Issues Projects Releases Wiki Activity

Add option to cleanup local files regardless of sync state when using --download-only (#2113)

Browse source 
* Add option to cleanup local files regardless of sync state when using --download-only
This commit is contained in:
abraunegg 2022-08-31 06:41:52 +10:00 committed by GitHub
parent 5288f94ac4
commit 738be2d150
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 236 additions and 134 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
config
View file

@ -53,3 +53,4 @@
# space_reservation = "50"
# display_running_config = "false"
# read_only_auth_scope = "false"
# cleanup_local_files = "false"

3
docs/USAGE.md
View file

@ -505,6 +505,7 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con
# space_reservation = "50"
# display_running_config = "false"
# read_only_auth_scope = "false"
# cleanup_local_files = "false"
```
### 'config' file configuration examples:
@ -1180,6 +1181,8 @@ Options:
Check for the presence of .nosync in each directory. If found, skip directory from sync.
--classify-as-big-delete
Number of children in a path that is locally removed which will be classified as a 'big data delete'
--cleanup-local-files
Cleanup additional local files when using --download-only. This will remove local data.
--confdir ARG
Set the directory used to store the configuration files
--create-directory ARG

5
onedrive.1.in
View file

@ -41,6 +41,11 @@ Number of children in a path that is locally removed which will be classified as
.br
Configuration file key: \fBclassify_as_big_delete\fP (default: \fB1000\fP)
.TP
\fB\-\-cleanup\-local\-files\fP
Cleanup additional local files when using \-\-download-only. This will remove local data.
.br
Configuration file key: \fBcleanup_local_files\fP (default: \fBfalse\fP)
.TP
\fB\-\-confdir\fP ARG
Set the directory used to store the configuration files
.TP

5
src/config.d
View file

@ -138,6 +138,8 @@ final class Config
boolValues["display_running_config"] = false;
// Configure read-only authentication scope
boolValues["read_only_auth_scope"] = false;
// Flag to cleanup local files when using --download-only
boolValues["cleanup_local_files"] = false;
// DEVELOPER OPTIONS
// display_memory = true | false
@ -357,6 +359,9 @@ final class Config
"classify-as-big-delete",
"Number of children in a path that is locally removed which will be classified as a 'big data delete'",
&longValues["classify_as_big_delete"],
"cleanup-local-files",
"Cleanup additional local files when using --download-only. This will remove local data.",
&boolValues["cleanup_local_files"],
"create-directory",
"Create a directory on OneDrive - no sync will be performed.",
&stringValues["create_directory"],

41
src/main.d
View file

@ -54,6 +54,7 @@ int main(string[] args)
bool performSyncOK = false;
bool displayMemoryUsage = false;
bool displaySyncOptions = false;
bool cleanupLocalFilesGlobal = false;
// hash file permission values
string hashPermissionValue = "600";
@ -698,6 +699,7 @@ int main(string[] args)
writeln("Config option 'check_nomount' = ", cfg.getValueBool("check_nomount"));
writeln("Config option 'resync' = ", cfg.getValueBool("resync"));
writeln("Config option 'resync_auth' = ", cfg.getValueBool("resync_auth"));
writeln("Config option 'cleanup_local_files' = ", cfg.getValueBool("cleanup_local_files"));
// data integrity
writeln("Config option 'classify_as_big_delete' = ", cfg.getValueLong("classify_as_big_delete"));
@ -1155,6 +1157,16 @@ int main(string[] args)
sync.setBypassDataPreservation();
}
// Do we configure to clean up local files if using --download-only ?
if ((cfg.getValueBool("download_only")) && (cfg.getValueBool("cleanup_local_files"))) {
// --download-only and --cleanup-local-files were passed in
log.log("WARNING: Application has been configured to cleanup local files that are not present online.");
log.log("WARNING: Local data loss MAY occur in this scenario if you are expecting data to remain archived locally.");
sync.setCleanupLocalFiles();
// Set the global flag as we will use this as thhe item to be passed into the sync function below
cleanupLocalFilesGlobal = true;
}
// Are we configured to use a National Cloud Deployment
if (cfg.getValueString("azure_ad_endpoint") != "") {
// value is configured, is it a valid value?
@ -1299,7 +1311,7 @@ int main(string[] args)
// perform a --synchronize sync
// fullScanRequired = false, for final true-up
// but if we have sync_list configured, use syncListConfigured which = true
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, false, syncListConfigured, displaySyncOptions, cfg.getValueBool("monitor"), m);
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, false, syncListConfigured, displaySyncOptions, cfg.getValueBool("monitor"), m, cleanupLocalFilesGlobal);
// Write WAL and SHM data to file for this sync
log.vdebug("Merge contents of WAL and SHM files into main database file");
@ -1380,7 +1392,7 @@ int main(string[] args)
// monitor initialisation failed
log.error("ERROR: ", e.msg);
oneDrive.shutdown();
exit(-1);
return EXIT_FAILURE;
}
}
@ -1527,7 +1539,7 @@ int main(string[] args)
// log file only if enabled so we know when a sync started when not using --verbose
log.fileOnly(startMessage);
}
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired, syncListConfiguredFullScanOverride, displaySyncOptions, cfg.getValueBool("monitor"), m);
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), (logMonitorCounter == logInterval ? MONITOR_LOG_QUIET : MONITOR_LOG_SILENT), fullScanRequired, syncListConfiguredFullScanOverride, displaySyncOptions, cfg.getValueBool("monitor"), m, cleanupLocalFilesGlobal);
if (!cfg.getValueBool("download_only")) {
// discard all events that may have been generated by the sync that have not already been handled
try {
@ -1646,7 +1658,7 @@ bool initSyncEngine(SyncEngine sync)
}
// try to synchronize the folder three times
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired, bool syncListConfiguredFullScanOverride, bool displaySyncOptions, bool monitorEnabled, Monitor m)
void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, bool localFirst, bool uploadOnly, long logLevel, bool fullScanRequired, bool syncListConfiguredFullScanOverride, bool displaySyncOptions, bool monitorEnabled, Monitor m, bool cleanupLocalFiles)
{
int count;
string remotePath = "/";
@ -1701,7 +1713,15 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// OneDrive First
if (logLevel < MONITOR_LOG_QUIET) log.log("Syncing changes from selected OneDrive path ...");
sync.applyDifferencesSingleDirectory(remotePath);
// is this a download only request?
// Is this a --download-only --cleanup-local-files request?
// If yes, scan for local changes - but --cleanup-local-files is being used, a further flag will trigger local file deletes rather than attempt to upload files to OneDrive
if (cleanupLocalFiles) {
// --download-only and --cleanup-local-files were passed in
log.log("Searching local filesystem for extra files and folders which need to be removed");
sync.scanForDifferencesFilesystemScan(localPath);
} else {
// is this a --download-only request?
if (!downloadOnly) {
// process local changes
sync.scanForDifferences(localPath);
@ -1710,6 +1730,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
}
}
}
}
} else {
// no single directory sync
if (uploadOnly){
@ -1747,7 +1768,14 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
}
sync.applyDifferences(false);
// is this a download only request?
// Is this a --download-only --cleanup-local-files request?
// If yes, scan for local changes - but --cleanup-local-files is being used, a further flag will trigger local file deletes rather than attempt to upload files to OneDrive
if (cleanupLocalFiles) {
// --download-only and --cleanup-local-files were passed in
log.log("Searching local filesystem for extra files and folders which need to be removed");
sync.scanForDifferencesFilesystemScan(localPath);
} else {
// is this a --download-only request?
if (!downloadOnly) {
// process local changes walking the entire path checking for changes
// in monitor mode all local changes are captured via inotify
@ -1871,6 +1899,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
}
}
}
}
// sync is complete
logOutputMessage = "################################################ SYNC COMPLETE ###############################################";

67
src/sync.d
View file

@ -30,6 +30,9 @@ private bool disableUploadValidation = false;
// Do we configure to disable the download validation routine
private bool disableDownloadValidation = false;
// Do we perform a local cleanup of files that are 'extra' on the local file system, when using --download-only
private bool cleanupLocalFiles = false;
private bool isItemFolder(const ref JSONValue item)
{
return ("folder" in item) != null;
@ -616,6 +619,13 @@ final class SyncEngine
log.vdebug("Setting nationalCloudDeployment = true");
}
// set cleanupLocalFiles to true
void setCleanupLocalFiles()
{
cleanupLocalFiles = true;
log.vdebug("Setting cleanupLocalFiles = true");
}
// return the OneDrive Account Type
auto getAccountType()
{
@ -3444,7 +3454,7 @@ final class SyncEngine
}
}
// scan the given directory for new items - for use with --monitor
// scan the given directory for new items - for use with --monitor or --cleanup-local-files
void scanForDifferencesFilesystemScan(const(string) path)
{
// To improve logging output for this function, what is the 'logical path' we are scanning for file & folder differences?
@ -3461,10 +3471,15 @@ final class SyncEngine
if (isDir(path)) {
// if this path is a directory, output this message.
// if a file, potentially leads to confusion as to what the client is actually doing
if (!cleanupLocalFiles) {
// if --cleanup-local-files was set, we will not be uploading data
log.vlog("Uploading new items of ", logPath);
}
}
// Filesystem walk to find new files not uploaded
// Filesystem walk to find extra files that reside locally.
// If --cleanup-local-files is not used, these will be uploaded (normal operation)
// If --download-only --cleanup-local-files is being used, extra files found locally will be deleted from the local filesystem
uploadNewItems(path);
}
@ -4324,14 +4339,45 @@ final class SyncEngine
// Was the path found in the database?
if (!pathFoundInDB) {
// Path not found in database when searching all drive id's
if (!cleanupLocalFiles) {
// --download-only --cleanup-local-files not used
uploadCreateDir(path);
} else {
// we need to clean up this directory
log.log("Removing local directory as --download-only & --cleanup-local-files configured");
// Remove any children of this path if they still exist
// Resolve 'Directory not empty' error when deleting local files
foreach (DirEntry child; dirEntries(path, SpanMode.depth, false)) {
// what sort of child is this?
if (isDir(child.name)) {
log.log("Removing local directory: ", child.name);
} else {
log.log("Removing local file: ", child.name);
}
// are we in a --dry-run scenario?
if (!dryRun) {
// No --dry-run ... process local delete
attrIsDir(child.linkAttributes) ? rmdir(child.name) : remove(child.name);
}
}
// Remove the path now that it is empty of children
log.log("Removing local directory: ", path);
// are we in a --dry-run scenario?
if (!dryRun) {
// No --dry-run ... process local delete
rmdirRecurse(path);
}
}
}
// recursively traverse children
// the above operation takes time and the directory might have
// disappeared in the meantime
if (!exists(path)) {
if (!cleanupLocalFiles) {
// --download-only --cleanup-local-files not used
log.vlog("Directory disappeared during upload: ", path);
}
return;
}
@ -4365,7 +4411,10 @@ final class SyncEngine
// Was the file found in the database?
if (!fileFoundInDB) {
// File not found in database when searching all drive id's, upload as new file
// File not found in database when searching all drive id's
// Do we upload the file or clean up the file?
if (!cleanupLocalFiles) {
// --download-only --cleanup-local-files not used
uploadNewFile(path);
// Did the upload fail?
if (!uploadFailed) {
@ -4376,13 +4425,23 @@ final class SyncEngine
// Log that we are deleting a local item
log.log("Removing local file as --upload-only & --remove-source-files configured");
// are we in a --dry-run scenario?
log.vdebug("Removing local file: ", path);
if (!dryRun) {
// No --dry-run ... process local file delete
log.vdebug("Removing local file: ", path);
safeRemove(path);
}
}
}
} else {
// we need to clean up this file
log.log("Removing local file as --download-only & --cleanup-local-files configured");
// are we in a --dry-run scenario?
log.log("Removing local file: ", path);
if (!dryRun) {
// No --dry-run ... process local file delete
safeRemove(path);
}
}
}
} else {
// path is not a valid file