From 73194b01698a860c742cc6f1664a8148325a6d80 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Mon, 12 May 2025 06:29:41 +1000 Subject: [PATCH] Implement FR #3234: Add configurable upload delay to support Obsidian (#3262) * As of now, Obsidian on Linux does not provide a built-in way to disable atomic saves or switch to a backup-copy method via configuration. Obsidian uses Electron and relies on the default save behavior of its underlying libraries and editor components (like CodeMirror), which typically perform atomic writes for every keystroke. This FR implements a delay in uploading changes to Microsoft OneDrive that is user configurable to better handle how Obsidian works * Add specific note regarding compatibility with Obsidian and the options that this PR introduces * Enforce a minimum | maximum for 'inotify_delay' * Update documentation for feature * Update 'config' file with 'delay_inotify_processing' option * Add Ubuntu 24.10 version of curl (8.9.1) to known bad curl versions * Add Ubuntu version name for Ubuntu 24.10 --- config | 3 ++ docs/application-config-options.md | 33 ++++++++++++++++++++++ docs/usage.md | 45 +++++++++++++++++++++++++++++- src/config.d | 19 ++++++++++++- src/main.d | 8 ++++++ src/util.d | 3 +- 6 files changed, 108 insertions(+), 3 deletions(-) diff --git a/config b/config index 007b6ba2..ae90d024 100644 --- a/config +++ b/config @@ -40,6 +40,9 @@ ## This setting controls whether the curl library is configured to output additional data to assist with diagnosing HTTPS issues and problems. #debug_https = "false" +## This setting controls whether 'inotify' events should be delayed or not. +#delay_inotify_processing = "false" + ## This option determines whether the client will conduct integrity validation on files downloaded from Microsoft OneDrive. #disable_download_validation = "false" diff --git a/docs/application-config-options.md b/docs/application-config-options.md index 675a7bb0..36094198 100644 --- a/docs/application-config-options.md +++ b/docs/application-config-options.md @@ -17,6 +17,7 @@ Before reading this document, please ensure you are running application version - [create_new_file_version](#create_new_file_version) - [data_timeout](#data_timeout) - [debug_https](#debug_https) + - [delay_inotify_processing](#delay_inotify_processing) - [disable_download_validation](#disable_download_validation) - [disable_notifications](#disable_notifications) - [disable_permission_set](#disable_permission_set) @@ -30,6 +31,7 @@ Before reading this document, please ensure you are running application version - [enable_logging](#enable_logging) - [force_http_11](#force_http_11) - [force_session_upload](#force_session_upload) + - [inotify_delay](#inotify_delay) - [ip_protocol_version](#ip_protocol_version) - [local_first](#local_first) - [log_dir](#log_dir) @@ -253,6 +255,20 @@ _**CLI Option Use:**_ `--debug-https` > [!WARNING] > Whilst this option can be used at any time, it is advisable that you only use this option when advised as this will output your `Authorization: bearer` - which is your authentication token to Microsoft OneDrive. + +### delay_inotify_processing +_**Description:**_ This setting controls whether 'inotify' events should be delayed or not. This option should only ever be enabled when attempting to reduce the impact of editors like Obsidian which constantly write change to disk in an atomic fashion. + +_**Value Type:**_ Boolean + +_**Default Value:**_ False + +_**Config Example:**_ `delay_inotify_processing = "false"` or `delay_inotify_processing = "true"` + +> [!NOTE] +> If you enable this option you *must* also enable 'force_session_upload' to ensure that your data uploads are done in a manner that editors, like Obsidian expect. + + ### disable_download_validation _**Description:**_ This option determines whether the client will conduct integrity validation on files downloaded from Microsoft OneDrive. Sometimes, when downloading files, particularly from SharePoint, there is a discrepancy between the file size reported by the OneDrive API and the byte count received from the SharePoint HTTP Server for the same file. Enable this option to disable the integrity checks performed by this client. @@ -417,6 +433,23 @@ _**Config Example:**_ `force_session_upload = "false"` or `force_session_upload _**CLI Option Use:**_ *None - this is a config file option only* + +### inotify_delay +_**Description:**_ This option specifies the number of seconds 'inotify' events are paused before they are processed by this client. This value is used to overcome aggressive write applications such as Obsidian which write each keystroke in an atomic manner to the local disk. Due to this atomic write, each 'save' causes the existing file to be deleted and replaced with a new file, which this client sees as multiple constant 'inotify' events. + +_**Value Type:**_ Integer + +_**Default Value:**_ 5 + +_**Maximum Value:**_ 15 + +_**Config Example:**_ `inotify_delay = "10"` + +_**CLI Option Use:**_ *None - this is a config file option only* + +> [!NOTE] +> This option is only used if 'delay_inotify_processing' is enabled, otherwise this option is ignored. + ### ip_protocol_version _**Description:**_ This setting controls the application IP protocol that should be used when communicating with Microsoft OneDrive. The default is to use IPv4 and IPv6 networks for communicating to Microsoft OneDrive. diff --git a/docs/usage.md b/docs/usage.md index 9e93c331..de9f96a3 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -10,6 +10,7 @@ Before reading this document, please ensure you are running application version - [Guidelines for Local File and Folder Naming in the Synchronisation Directory](#guidelines-for-local-file-and-folder-naming-in-the-synchronisation-directory) - [Support for Microsoft Azure Information Protected Files](#support-for-microsoft-azure-information-protected-files) - [Compatibility with Editors and Applications Using Atomic Save Operations](#compatibility-with-editors-and-applications-using-atomic-save-operations) + - [Compatibility with Obsidian](#compatibility-with-obsidian) - [Compatibility with curl](#compatibility-with-curl) - [First Steps](#first-steps) - [Authorise the Application with Your Microsoft OneDrive Account](#authorise-the-application-with-your-microsoft-onedrive-account) @@ -201,6 +202,48 @@ The `onedrive` client is fully compatible with applications that use atomic save This behaviour is by design and ensures consistency and data integrity between your local filesystem and the OneDrive cloud. +### Compatibility with Obsidian +Obsidian on Linux does not provide a built-in way to disable atomic saves or switch to a backup-copy method via configuration. The application is built on Electron and relies on the default save behaviour of its underlying libraries and editor components (such as CodeMirror), which typically perform *atomic writes* using the following process: + +1. A temporary file is created containing the updated content. +2. That temporary file is flushed to disk. +3. The temporary file is atomically renamed to replace the original file. + +This behaviour is intended to improve data integrity and crash resilience, but it results in high disk I/O — particularly in Obsidian, where auto-save is triggered nearly every keystroke. + +> [!IMPORTANT] +> Obsidian provides no mechanism to change how this save behaviour operates. This is a serious design limitation and should be treated as a bug in the application. The excessive and unnecessary write operations can significantly reduce the lifespan of SSDs over time due to increased wear, leading to broader consequences for system reliability. + +#### How This Affects the OneDrive Client + +Because Obsidian is constantly writing files, running the OneDrive Client for Linux in `--monitor` mode causes the client to continually receive inotify events. This leads to constant re-uploading of files, regardless of whether meaningful content has changed. + +The consequences of this are: + +1. Continuous upload attempts to Microsoft OneDrive. +2. Potential for repeated overwrites of online data. +3. Excessive API usage, which may result in Microsoft throttling your access — subsequently affecting the client’s ability to synchronise files reliably. + +#### Recommendation + +If you use Obsidian, it is *strongly* recommended that you enable the following two configuration options in your OneDrive client `config` file: +``` +force_session_upload = "true" +delay_inotify_processing = "true" +``` +These settings introduce a delay in processing local file change events, allowing the OneDrive client to batch or debounce Obsidian's frequent writes. By default, this delay is 5 seconds. + +To adjust this delay, you can add the following configuration option: +``` +inotify_delay = "10" +``` +This example sets the delay to 10 seconds. + +> [!CAUTION] +> Increasing `inotify_delay` too aggressively may have unintended side effects. All file system events are queued and processed in order, so setting a very high delay could result in large backlogs or undesirable data synchronisation outcomes — particularly in cases of rapid file changes or deletions. +> +> Adjust this setting with extreme caution and test thoroughly to ensure it does not impact your workflow or data integrity. + ### Compatibility with curl If your system uses curl < 7.47.0, curl will default to HTTP/1.1 for HTTPS operations, and the client will follow suit, using HTTP/1.1. @@ -236,7 +279,7 @@ If you explicitly want to use HTTP/1.1, you can do so by using the `--force-http | 7.88.1 | Debian 12 (Bookworm) | 6,7,8,9,10,11,12,13 | | 8.2.1 | Alpine Linux 3.14 | 7,8,9,10,11,12,13 | | 8.5.0 | Alpine Linux 3.15, Ubuntu 24.04 LTS (Noble Numbat) | 8,9,10,11,12,13 | -| 8.9.1 | Ubuntu 24.10 | 11,12,13 | +| 8.9.1 | Ubuntu 24.10 (Oracular Oriole) | 11,12,13 | | 8.10.0 | Alpine Linux 3.17 | 13 | > [!IMPORTANT] diff --git a/src/config.d b/src/config.d index f0957449..e4bf96a3 100644 --- a/src/config.d +++ b/src/config.d @@ -133,8 +133,11 @@ class ApplicationConfig { bool fullScanTrueUpRequired = false; bool suppressLoggingOutput = false; - // Number of concurrent threads when downloading and uploading data + // Default number of concurrent threads when downloading and uploading data ulong defaultConcurrentThreads = 8; + + // Default number of seconds inotify actions will be delayed by + ulong defaultInotifyDelay = 5; // All application run-time paths are formulated from this as a set of defaults // - What is the home path of the actual 'user' that is running the application @@ -376,6 +379,13 @@ class ApplicationConfig { // and Microsoft OneDrive should be respecting this timestamp as the timestamp to use|set when storing that file online boolValues["force_session_upload"] = false; + // Obsidian Editor has been written in such a way that it is constantly writing each and every keystroke to a file. + // Not only is this really bad application behaviour, for this client, this means the application is constantly writing to disk, thus attempting to upload file changes. + // Unfortunately Obsidian on Linux does not provide a built-in way to disable atomic saves or switch to a backup-copy method via configuration. + // This flag tells the 'onedrive' inotify monitor to 'sleep' for this period of time, so that constant system writes are not creating instant data uploads + boolValues["delay_inotify_processing"] = false; + longValues["inotify_delay"] = defaultInotifyDelay; // default of 5 seconds + // Webhook Feature Options boolValues["webhook_enabled"] = false; stringValues["webhook_public_url"] = ""; @@ -978,6 +988,13 @@ class ApplicationConfig { tempValue = defaultConcurrentThreads; } setValueLong("threads", tempValue); + } else if (key == "inotify_delay") { + ulong tempValue = thisConfigValue; + if ((tempValue < 5)||(tempValue > 15)) { + addLogEntry("Invalid value for key in config file - using default value: " ~ key); + tempValue = defaultInotifyDelay; + } + setValueLong("inotify_delay", tempValue); } } else { addLogEntry("Unknown key in config file: " ~ key); diff --git a/src/main.d b/src/main.d index 3c69ba79..79638c3a 100644 --- a/src/main.d +++ b/src/main.d @@ -1205,6 +1205,14 @@ int main(string[] cliArgs) { if(filesystemMonitor.initialised) { // If local monitor is on and is waiting (previous event was not from webhook) + + // Obsidian Editor has been written in such a way that it is constantly writing each and every keystroke to a file. + // Not only is this really bad application behaviour, for this client, this means the application is constantly writing to disk, thus attempting to upload file changes. + // Unfortunately Obsidian on Linux does not provide a built-in way to disable atomic saves or switch to a backup-copy method via configuration. + if (appConfig.getValueBool("delay_inotify_processing")) { + Thread.sleep(dur!("seconds")(to!int(appConfig.getValueLong("inotify_delay")))); + } + // start the worker and wait for event if (!notificationReceived) { filesystemMonitor.send(true); diff --git a/src/util.d b/src/util.d index 358a9678..bfb1605c 100644 --- a/src/util.d +++ b/src/util.d @@ -1605,7 +1605,8 @@ bool isBadCurlVersion(string curlVersion) { "7.81.0", // Ubuntu 22.x "7.88.1", // Debian 12 "8.2.1", // Ubuntu 23.10 - "8.5.0", // Ubuntu 24.x + "8.5.0", // Ubuntu 24.04 + "8.9.1", // Ubuntu 24.10 "8.10.0" // Various - HTTP/2 bug which was fixed in 8.10.1 ];