mirror of
https://github.com/abraunegg/onedrive
synced 2024-05-30 12:32:41 +02:00
Implement warning on big deletes to safeguard data on OneDrive (#621)
* When testing changes to onedrive client configuration, the new configuration might be invalid (see #458 for example) and you remove all your data on OneDrive accidentally. This new feature attempts to protect your data on OneDrive when performing large deletes, so that a large delete is detected and asks for confirmation before actually processing this request. This feature does not impact `--monitor` mode of operation, only standalone mode of operation.
This commit is contained in:
parent
2217fc2b97
commit
5c2ad041bd
1
config
1
config
|
@ -30,4 +30,5 @@
|
||||||
# monitor_log_frequency = "5"
|
# monitor_log_frequency = "5"
|
||||||
# monitor_fullscan_frequency = "10"
|
# monitor_fullscan_frequency = "10"
|
||||||
# sync_root_files = "false"
|
# sync_root_files = "false"
|
||||||
|
# classify_as_big_delete = "1000"
|
||||||
# user_agent = ""
|
# user_agent = ""
|
||||||
|
|
|
@ -62,6 +62,8 @@ final class Config
|
||||||
// Number of n sync runs before performing a full local scan of sync_dir
|
// Number of n sync runs before performing a full local scan of sync_dir
|
||||||
// By default 10 which means every ~7.5 minutes a full disk scan of sync_dir will occur
|
// By default 10 which means every ~7.5 minutes a full disk scan of sync_dir will occur
|
||||||
longValues["monitor_fullscan_frequency"] = 10;
|
longValues["monitor_fullscan_frequency"] = 10;
|
||||||
|
// Number of children in a path that is locally removed which will be classified as a 'big data delete'
|
||||||
|
longValues["classify_as_big_delete"] = 1000;
|
||||||
|
|
||||||
// Determine the users home directory.
|
// Determine the users home directory.
|
||||||
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
|
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
|
||||||
|
@ -163,6 +165,7 @@ final class Config
|
||||||
boolValues["logout"] = false;
|
boolValues["logout"] = false;
|
||||||
boolValues["monitor"] = false;
|
boolValues["monitor"] = false;
|
||||||
boolValues["synchronize"] = false;
|
boolValues["synchronize"] = false;
|
||||||
|
boolValues["force"] = false;
|
||||||
|
|
||||||
// Application Startup option validation
|
// Application Startup option validation
|
||||||
try {
|
try {
|
||||||
|
@ -182,6 +185,9 @@ final class Config
|
||||||
"check-for-nosync",
|
"check-for-nosync",
|
||||||
"Check for the presence of .nosync in each directory. If found, skip directory from sync.",
|
"Check for the presence of .nosync in each directory. If found, skip directory from sync.",
|
||||||
&boolValues["check_nosync"],
|
&boolValues["check_nosync"],
|
||||||
|
"classify-as-big-delete",
|
||||||
|
"Number of children in a path that is locally removed which will be classified as a 'big data delete'",
|
||||||
|
&longValues["classify_as_big_delete"],
|
||||||
"create-directory",
|
"create-directory",
|
||||||
"Create a directory on OneDrive - no sync will be performed.",
|
"Create a directory on OneDrive - no sync will be performed.",
|
||||||
&stringValues["create_directory"],
|
&stringValues["create_directory"],
|
||||||
|
@ -218,6 +224,9 @@ final class Config
|
||||||
"force-http-2",
|
"force-http-2",
|
||||||
"Force the use of HTTP/2 for all operations where applicable",
|
"Force the use of HTTP/2 for all operations where applicable",
|
||||||
&boolValues["force_http_2"],
|
&boolValues["force_http_2"],
|
||||||
|
"force",
|
||||||
|
"Force the deletion of data when a 'big delete' is detected",
|
||||||
|
&boolValues["force"],
|
||||||
"get-file-link",
|
"get-file-link",
|
||||||
"Display the file link of a synced file",
|
"Display the file link of a synced file",
|
||||||
&stringValues["get_file_link"],
|
&stringValues["get_file_link"],
|
||||||
|
|
42
src/main.d
42
src/main.d
|
@ -355,37 +355,43 @@ int main(string[] args)
|
||||||
string userConfigFilePath = cfg.configDirName ~ "/config";
|
string userConfigFilePath = cfg.configDirName ~ "/config";
|
||||||
string userSyncList = cfg.configDirName ~ "/sync_list";
|
string userSyncList = cfg.configDirName ~ "/sync_list";
|
||||||
// Display application version
|
// Display application version
|
||||||
std.stdio.write("onedrive version = ", import("version"));
|
writeln("onedrive version = ", strip(import("version")));
|
||||||
|
|
||||||
// Display all of the pertinent configuration options
|
// Display all of the pertinent configuration options
|
||||||
writeln("Config path = ", cfg.configDirName);
|
writeln("Config path = ", cfg.configDirName);
|
||||||
|
|
||||||
// Does a config file exist or are we using application defaults
|
// Does a config file exist or are we using application defaults
|
||||||
if (exists(userConfigFilePath)){
|
if (exists(userConfigFilePath)){
|
||||||
writeln("Config file found in config path = true");
|
writeln("Config file found in config path = true");
|
||||||
} else {
|
} else {
|
||||||
writeln("Config file found in config path = false");
|
writeln("Config file found in config path = false");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config Options
|
// Config Options
|
||||||
writeln("Config option 'check_nosync' = ", cfg.getValueBool("check_nosync"));
|
|
||||||
writeln("Config option 'sync_dir' = ", syncDir);
|
|
||||||
writeln("Config option 'skip_dir' = ", cfg.getValueString("skip_dir"));
|
writeln("Config option 'check_nosync' = ", cfg.getValueBool("check_nosync"));
|
||||||
writeln("Config option 'skip_file' = ", cfg.getValueString("skip_file"));
|
writeln("Config option 'sync_dir' = ", syncDir);
|
||||||
writeln("Config option 'skip_dotfiles' = ", cfg.getValueBool("skip_dotfiles"));
|
writeln("Config option 'skip_dir' = ", cfg.getValueString("skip_dir"));
|
||||||
writeln("Config option 'skip_symlinks' = ", cfg.getValueBool("skip_symlinks"));
|
writeln("Config option 'skip_file' = ", cfg.getValueString("skip_file"));
|
||||||
writeln("Config option 'monitor_interval' = ", cfg.getValueLong("monitor_interval"));
|
writeln("Config option 'skip_dotfiles' = ", cfg.getValueBool("skip_dotfiles"));
|
||||||
writeln("Config option 'min_notify_changes' = ", cfg.getValueLong("min_notify_changes"));
|
writeln("Config option 'skip_symlinks' = ", cfg.getValueBool("skip_symlinks"));
|
||||||
writeln("Config option 'log_dir' = ", cfg.getValueString("log_dir"));
|
writeln("Config option 'monitor_interval' = ", cfg.getValueLong("monitor_interval"));
|
||||||
|
writeln("Config option 'min_notify_changes' = ", cfg.getValueLong("min_notify_changes"));
|
||||||
|
writeln("Config option 'log_dir' = ", cfg.getValueString("log_dir"));
|
||||||
|
writeln("Config option 'classify_as_big_delete' = ", cfg.getValueLong("classify_as_big_delete"));
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// Is config option drive_id configured?
|
// Is config option drive_id configured?
|
||||||
if (cfg.getValueString("drive_id") != ""){
|
if (cfg.getValueString("drive_id") != ""){
|
||||||
writeln("Config option 'drive_id' = ", cfg.getValueString("drive_id"));
|
writeln("Config option 'drive_id' = ", cfg.getValueString("drive_id"));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Is sync_list configured?
|
// Is sync_list configured?
|
||||||
if (exists(userSyncList)){
|
if (exists(userSyncList)){
|
||||||
writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files"));
|
writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files"));
|
||||||
writeln("Selective sync configured = true");
|
writeln("Selective sync configured = true");
|
||||||
writeln("sync_list contents:");
|
writeln("sync_list contents:");
|
||||||
// Output the sync_list contents
|
// Output the sync_list contents
|
||||||
auto syncListFile = File(userSyncList);
|
auto syncListFile = File(userSyncList);
|
||||||
|
@ -395,8 +401,8 @@ int main(string[] args)
|
||||||
writeln(line);
|
writeln(line);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files"));
|
writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files"));
|
||||||
writeln("Selective sync configured = false");
|
writeln("Selective sync configured = false");
|
||||||
}
|
}
|
||||||
|
|
||||||
// exit
|
// exit
|
||||||
|
|
49
src/sync.d
49
src/sync.d
|
@ -3093,6 +3093,34 @@ final class SyncEngine
|
||||||
private void uploadDeleteItem(Item item, string path)
|
private void uploadDeleteItem(Item item, string path)
|
||||||
{
|
{
|
||||||
log.log("Deleting item from OneDrive: ", path);
|
log.log("Deleting item from OneDrive: ", path);
|
||||||
|
bool flagAsBigDelete = false;
|
||||||
|
|
||||||
|
// query the database - how many objects will this remove?
|
||||||
|
long itemsToDelete = 0;
|
||||||
|
auto children = itemdb.selectChildren(item.driveId, item.id);
|
||||||
|
itemsToDelete = count(children);
|
||||||
|
foreach (Item child; children) {
|
||||||
|
if (child.type != ItemType.file) {
|
||||||
|
// recursively count the children of this child
|
||||||
|
itemsToDelete = itemsToDelete + countChildren(child.driveId, child.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Are we running in monitor mode? A local delete of a file will issue a inotify event, which will trigger the local & remote data immediately
|
||||||
|
if (!cfg.getValueBool("monitor")) {
|
||||||
|
// not running in monitor mode
|
||||||
|
if (itemsToDelete > cfg.getValueLong("classify_as_big_delete")) {
|
||||||
|
// A big delete detected
|
||||||
|
flagAsBigDelete = true;
|
||||||
|
if (!cfg.getValueBool("force")) {
|
||||||
|
log.error("ERROR: An attempt to remove a large volume of data from OneDrive has been detected. Exiting client to preserve data on OneDrive");
|
||||||
|
log.error("ERROR: To delete delete a large volume of data use --force or increase the config value 'classify_as_big_delete' to a larger value");
|
||||||
|
// Must exit here to preserve data on OneDrive
|
||||||
|
exit(-1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (!dryRun) {
|
if (!dryRun) {
|
||||||
// we are not in a --dry-run situation, process deletion to OneDrive
|
// we are not in a --dry-run situation, process deletion to OneDrive
|
||||||
if ((item.driveId == "") && (item.id == "") && (item.eTag == "")){
|
if ((item.driveId == "") && (item.id == "") && (item.eTag == "")){
|
||||||
|
@ -3105,16 +3133,15 @@ final class SyncEngine
|
||||||
item.id = onedrivePathDetails["id"].str; // This item's ID. Should give something like 12345ABCDE1234A1!101
|
item.id = onedrivePathDetails["id"].str; // This item's ID. Should give something like 12345ABCDE1234A1!101
|
||||||
item.eTag = onedrivePathDetails["eTag"].str; // Should be something like aNjM2NjJFRUVGQjY2NjJFMSE5MzUuMA
|
item.eTag = onedrivePathDetails["eTag"].str; // Should be something like aNjM2NjJFRUVGQjY2NjJFMSE5MzUuMA
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// do the delete
|
||||||
try {
|
try {
|
||||||
onedrive.deleteById(item.driveId, item.id, item.eTag);
|
onedrive.deleteById(item.driveId, item.id, item.eTag);
|
||||||
} catch (OneDriveException e) {
|
} catch (OneDriveException e) {
|
||||||
if (e.httpStatusCode == 404) {
|
if (e.httpStatusCode == 404) {
|
||||||
// item.id, item.eTag could not be found on driveId
|
// item.id, item.eTag could not be found on driveId
|
||||||
log.vlog("OneDrive reported: The resource could not be found.");
|
log.vlog("OneDrive reported: The resource could not be found.");
|
||||||
}
|
} else {
|
||||||
|
|
||||||
else {
|
|
||||||
// Not a 404 response .. is this a 403 response due to OneDrive Business Retention Policy being enabled?
|
// Not a 404 response .. is this a 403 response due to OneDrive Business Retention Policy being enabled?
|
||||||
if ((e.httpStatusCode == 403) && (accountType != "personal")) {
|
if ((e.httpStatusCode == 403) && (accountType != "personal")) {
|
||||||
auto errorArray = splitLines(e.msg);
|
auto errorArray = splitLines(e.msg);
|
||||||
|
@ -3144,6 +3171,20 @@ final class SyncEngine
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private long countChildren(string driveId, string id){
|
||||||
|
// count children
|
||||||
|
long childrenCount = 0;
|
||||||
|
auto children = itemdb.selectChildren(driveId, id);
|
||||||
|
childrenCount = count(children);
|
||||||
|
foreach (Item child; children) {
|
||||||
|
if (child.type != ItemType.file) {
|
||||||
|
// recursively count the children of this child
|
||||||
|
childrenCount = childrenCount + countChildren(child.driveId, child.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return childrenCount;
|
||||||
|
}
|
||||||
|
|
||||||
// update the item's last modified time
|
// update the item's last modified time
|
||||||
private void uploadLastModifiedTime(const(char)[] driveId, const(char)[] id, const(char)[] eTag, SysTime mtime)
|
private void uploadLastModifiedTime(const(char)[] driveId, const(char)[] id, const(char)[] eTag, SysTime mtime)
|
||||||
|
|
Loading…
Reference in a new issue