Implement #2414 to allow HTTP session timeout(s) tuning via config (#2425)

* Implement #2414 to allow HTTP session timeout(s) tuning via config (taken from v2.5.x)

config

@@ -44,7 +44,6 @@
 # sync_dir_permissions = "700"
 # sync_file_permissions = "600"
 # rate_limit = "131072"
-# operation_timeout = "3600"
 # webhook_enabled = "false"
 # webhook_public_url = ""
 # webhook_listening_host = ""
@@ -55,3 +54,8 @@
 # display_running_config = "false"
 # read_only_auth_scope = "false"
 # cleanup_local_files = "false"
+# operation_timeout = "3600"
+# dns_timeout = "60"
+# connect_timeout = "10"
+# data_timeout = "600"
+# ip_protocol_version = "0"

docs/USAGE.md

@@ -34,6 +34,7 @@ Before reading this document, please ensure you are running application version
     + [monitor_log_frequency](#monitor_log_frequency)
     + [min_notify_changes](#min_notify_changes)
     + [operation_timeout](#operation_timeout)
+    + [ip_protocol_version](#ip_protocol_version)
   * [Configuring the client for 'single tenant application' use](#configuring-the-client-for-single-tenant-application-use)
   * [Configuring the client to use older 'skilion' application identifier](#configuring-the-client-to-use-older-skilion-application-identifier)
 - [Frequently Asked Configuration Questions](#frequently-asked-configuration-questions)
@@ -518,7 +519,6 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con
 # sync_dir_permissions = "700"
 # sync_file_permissions = "600"
 # rate_limit = "131072"
-# operation_timeout = "3600"
 # webhook_enabled = "false"
 # webhook_public_url = ""
 # webhook_listening_host = ""
@@ -529,6 +529,11 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con
 # display_running_config = "false"
 # read_only_auth_scope = "false"
 # cleanup_local_files = "false"
+# operation_timeout = "3600"
+# dns_timeout = "60"
+# connect_timeout = "10"
+# data_timeout = "600"
+# ip_protocol_version = "0"
 ```
 
 ### 'config' file configuration examples:
@@ -741,6 +746,23 @@ Example:
 operation_timeout = "3600"
 ```
 
+#### ip_protocol_version
+By default, the application will use IPv4 and IPv6 to resolve and communicate with Microsoft OneDrive. In some Linux distributions (most notably Ubuntu and those distributions based on Ubuntu) this will cause problems due to how DNS resolution is being performed.
+
+To configure the application to use a specific IP version, configure the following in your config file:
+```text
+# operation_timeout = "3600"
+# dns_timeout = "60"
+# connect_timeout = "10"
+# data_timeout = "600"
+ip_protocol_version = "1"
+
+```
+**Note:**
+* A value of 0 will mean the client will use IPv4 and IPv6. This is the default.
+* A value of 1 will mean the client will use IPv4 only.
+* A value of 2 will mean the client will use IPv6 only.
+
 #### Configuring the client for 'single tenant application' use
 In some instances when using OneDrive Business Accounts, depending on the Azure organisational configuration, it will be necessary to configure the client as a 'single tenant application'.
 To configure this, after creating the application on your Azure tenant, update the 'config' file with the tenant name (not the GUID) and the newly created Application ID, then this will be used for the authentication process.

src/config.d

@@ -43,6 +43,35 @@ final class Config
 	// Default file permission mode
 	public long defaultFilePermissionMode = 600;
 	public int configuredFilePermissionMode;
+	
+	// Bring in v2.5.0 config items
+	
+	// HTTP Struct items, used for configuring HTTP()
+	// Curl Timeout Handling
+	// libcurl dns_cache_timeout timeout
+	immutable int defaultDnsTimeout = 60;
+	// Connect timeout for HTTP|HTTPS connections
+	immutable int defaultConnectTimeout = 10;
+	// With the following settings we force
+	// - if there is no data flow for 10min, abort
+	// - if the download time for one item exceeds 1h, abort
+	//
+	// Timeout for activity on connection
+	//  this translates into Curl's CURLOPT_LOW_SPEED_TIME
+	//  which says:
+	//   It contains the time in number seconds that the
+	//   transfer speed should be below the CURLOPT_LOW_SPEED_LIMIT
+	//   for the library to consider it too slow and abort.
+	immutable int defaultDataTimeout = 600;
+	// Maximum time any operation is allowed to take
+	// This includes dns resolution, connecting, data transfer, etc.
+	immutable int defaultOperationTimeout = 3600;
+	// Specify how many redirects should be allowed
+	immutable int defaultMaxRedirects = 5;
+	// Specify what IP protocol version should be used when communicating with OneDrive
+	immutable int defaultIpProtocol = 0; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
+	
+	
 
 	this(string confdirOption)
 	{
@@ -122,9 +151,6 @@ final class Config
 		longValues["sync_file_permissions"] = defaultFilePermissionMode;
 		// Configure download / upload rate limits
 		longValues["rate_limit"] = 0;
-		// maximum time an operation is allowed to take
-		// This includes dns resolution, connecting, data transfer, etc.
-		longValues["operation_timeout"] = 3600;
 		// To ensure we do not fill up the load disk, how much disk space should be reserved by default
 		longValues["space_reservation"] = 50 * 2^^20; // 50 MB as Bytes
 		// Webhook options
@@ -161,6 +187,19 @@ final class Config
 		// - Enabling this option will add function processing times to the console output
 		// - This then enables tracking of where the application is spending most amount of time when processing data when users have questions re performance
 		boolValues["display_processing_time"] = false;
+		
+		// HTTPS & CURL Operation Settings
+		// - Maximum time an operation is allowed to take
+		//   This includes dns resolution, connecting, data transfer, etc.
+		longValues["operation_timeout"] = defaultOperationTimeout;
+		// libcurl dns_cache_timeout timeout
+		longValues["dns_timeout"] = defaultDnsTimeout;
+		// Timeout for HTTPS connections
+		longValues["connect_timeout"] = defaultConnectTimeout;
+		// Timeout for activity on a HTTPS connection
+		longValues["data_timeout"] = defaultDataTimeout;
+		// What IP protocol version should be used when communicating with OneDrive
+		longValues["ip_protocol_version"] = defaultIpProtocol; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
 				
 		// EXPAND USERS HOME DIRECTORY
 		// Determine the users home directory.
@@ -451,9 +490,6 @@ final class Config
 				"no-remote-delete",
 					"Do not delete local file 'deletes' from OneDrive when using --upload-only",
 					&boolValues["no_remote_delete"],
-				"operation-timeout",
-					"Maximum amount of time (in seconds) an operation is allowed to take",
-					&longValues["operation_timeout"],
 				"print-token",
 					"Print the access token, useful for debugging",
 					&boolValues["print_token"],

src/main.d

@@ -761,7 +761,10 @@ int main(string[] args)
 		writeln("Config option 'debug_https'                  = ", cfg.getValueBool("debug_https"));
 		writeln("Config option 'rate_limit'                   = ", cfg.getValueLong("rate_limit"));
 		writeln("Config option 'operation_timeout'            = ", cfg.getValueLong("operation_timeout"));
-		
+		writeln("Config option 'dns_timeout'                  = ", cfg.getValueLong("dns_timeout"));
+		writeln("Config option 'connect_timeout'              = ", cfg.getValueLong("connect_timeout"));
+		writeln("Config option 'data_timeout'                 = ", cfg.getValueLong("data_timeout"));
+		writeln("Config option 'ip_protocol_version'          = ", cfg.getValueLong("ip_protocol_version"));
 		
 		// Is sync_list configured ?
 		writeln("Config option 'sync_root_files'              = ", cfg.getValueBool("sync_root_files"));
@@ -853,7 +856,7 @@ int main(string[] args)
 	
 	// Test if OneDrive service can be reached, exit if it cant be reached
 	log.vdebug("Testing network to ensure network connectivity to Microsoft OneDrive Service");
-	online = testNetwork();
+	online = testNetwork(cfg);
 	if (!online) {
 		// Cant initialise the API as we are not online
 		if (!cfg.getValueBool("monitor")) {
@@ -885,7 +888,7 @@ int main(string[] args)
 					Thread.sleep(dur!"seconds"(maxBackoffInterval));
 				}
 				// perform the re-rty
-				online = testNetwork();
+				online = testNetwork(cfg);
 				if (online) {
 					// We are now online
 					log.log("Internet connectivity to Microsoft OneDrive service has been restored");

src/onedrive.d

@@ -214,9 +214,9 @@ final class OneDriveApi
 		http = HTTP();
 		// Curl Timeout Handling
 		// libcurl dns_cache_timeout timeout
-		http.dnsTimeout = (dur!"seconds"(60));
+		http.dnsTimeout = (dur!"seconds"(cfg.getValueLong("dns_timeout")));
 		// Timeout for HTTPS connections
-		http.connectTimeout = (dur!"seconds"(10));
+		http.connectTimeout = (dur!"seconds"(cfg.getValueLong("connect_timeout")));
 		// with the following settings we force
 		// - if there is no data flow for 10min, abort
 		// - if the download time for one item exceeds 1h, abort
@@ -227,17 +227,27 @@ final class OneDriveApi
 		//   It contains the time in number seconds that the
 		//   transfer speed should be below the CURLOPT_LOW_SPEED_LIMIT
 		//   for the library to consider it too slow and abort.
-		http.dataTimeout = (dur!"seconds"(600));
+		http.dataTimeout = (dur!"seconds"(cfg.getValueLong("data_timeout")));
 		// maximum time an operation is allowed to take
 		// This includes dns resolution, connecting, data transfer, etc.
 		http.operationTimeout = (dur!"seconds"(cfg.getValueLong("operation_timeout")));
+		// What IP protocol version should be used when using Curl - IPv4 & IPv6, IPv4 or IPv6
+		http.handle.set(CurlOption.ipresolve,cfg.getValueLong("ip_protocol_version")); // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
 		// Specify how many redirects should be allowed
-		http.maxRedirects(5);
+		http.maxRedirects(cfg.defaultMaxRedirects);
 
 		// Do we enable curl debugging?
 		if (cfg.getValueBool("debug_https")) {
 			http.verbose = true;
 			.debugResponse = true;
+			
+			// Output what options we are using so that in the debug log this can be tracked
+			log.vdebug("http.dnsTimeout = ", cfg.getValueLong("dns_timeout"));
+			log.vdebug("http.connectTimeout = ", cfg.getValueLong("connect_timeout"));
+			log.vdebug("http.dataTimeout = ", cfg.getValueLong("data_timeout"));
+			log.vdebug("http.operationTimeout = ", cfg.getValueLong("operation_timeout"));
+			log.vdebug("http.CurlOption.ipresolve = ", cfg.getValueLong("ip_protocol_version"));
+			log.vdebug("http.maxRedirects = ", cfg.defaultMaxRedirects);
 		}
 
 		// Update clientId if application_id is set in config file

src/util.d

@@ -15,7 +15,9 @@ import std.json;
 import std.traits;
 import qxor;
 import core.stdc.stdlib;
-static import log;
+
+import log;
+import config;
 
 shared string deviceName;
 
@@ -114,15 +116,23 @@ Regex!char wild2regex(const(char)[] pattern)
 }
 
 // returns true if the network connection is available
-bool testNetwork()
+bool testNetwork(Config cfg)
 {
 	// Use low level HTTP struct
 	auto http = HTTP();
 	http.url = "https://login.microsoftonline.com";
 	// DNS lookup timeout
-	http.dnsTimeout = (dur!"seconds"(5));
+	http.dnsTimeout = (dur!"seconds"(cfg.getValueLong("dns_timeout")));
 	// Timeout for connecting
-	http.connectTimeout = (dur!"seconds"(5));
+	http.connectTimeout = (dur!"seconds"(cfg.getValueLong("connect_timeout")));
+	// Data Timeout for HTTPS connections
+	http.dataTimeout = (dur!"seconds"(cfg.getValueLong("data_timeout")));
+	// maximum time any operation is allowed to take
+	// This includes dns resolution, connecting, data transfer, etc.
+	http.operationTimeout = (dur!"seconds"(cfg.getValueLong("operation_timeout")));	
+	// What IP protocol version should be used when using Curl - IPv4 & IPv6, IPv4 or IPv6
+	http.handle.set(CurlOption.ipresolve,cfg.getValueLong("ip_protocol_version")); // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
+	
 	// HTTP connection test method
 	http.method = HTTP.Method.head;
 	// Attempt to contact the Microsoft Online Service