mirror of
https://github.com/abraunegg/onedrive
synced 2024-05-19 14:16:36 +02:00
* Implement #2414 to allow HTTP session timeout(s) tuning via config (taken from v2.5.x)
This commit is contained in:
parent
06420c9a0a
commit
dfd1d1aa4a
6
config
6
config
|
@ -44,7 +44,6 @@
|
||||||
# sync_dir_permissions = "700"
|
# sync_dir_permissions = "700"
|
||||||
# sync_file_permissions = "600"
|
# sync_file_permissions = "600"
|
||||||
# rate_limit = "131072"
|
# rate_limit = "131072"
|
||||||
# operation_timeout = "3600"
|
|
||||||
# webhook_enabled = "false"
|
# webhook_enabled = "false"
|
||||||
# webhook_public_url = ""
|
# webhook_public_url = ""
|
||||||
# webhook_listening_host = ""
|
# webhook_listening_host = ""
|
||||||
|
@ -55,3 +54,8 @@
|
||||||
# display_running_config = "false"
|
# display_running_config = "false"
|
||||||
# read_only_auth_scope = "false"
|
# read_only_auth_scope = "false"
|
||||||
# cleanup_local_files = "false"
|
# cleanup_local_files = "false"
|
||||||
|
# operation_timeout = "3600"
|
||||||
|
# dns_timeout = "60"
|
||||||
|
# connect_timeout = "10"
|
||||||
|
# data_timeout = "600"
|
||||||
|
# ip_protocol_version = "0"
|
||||||
|
|
|
@ -34,6 +34,7 @@ Before reading this document, please ensure you are running application version
|
||||||
+ [monitor_log_frequency](#monitor_log_frequency)
|
+ [monitor_log_frequency](#monitor_log_frequency)
|
||||||
+ [min_notify_changes](#min_notify_changes)
|
+ [min_notify_changes](#min_notify_changes)
|
||||||
+ [operation_timeout](#operation_timeout)
|
+ [operation_timeout](#operation_timeout)
|
||||||
|
+ [ip_protocol_version](#ip_protocol_version)
|
||||||
* [Configuring the client for 'single tenant application' use](#configuring-the-client-for-single-tenant-application-use)
|
* [Configuring the client for 'single tenant application' use](#configuring-the-client-for-single-tenant-application-use)
|
||||||
* [Configuring the client to use older 'skilion' application identifier](#configuring-the-client-to-use-older-skilion-application-identifier)
|
* [Configuring the client to use older 'skilion' application identifier](#configuring-the-client-to-use-older-skilion-application-identifier)
|
||||||
- [Frequently Asked Configuration Questions](#frequently-asked-configuration-questions)
|
- [Frequently Asked Configuration Questions](#frequently-asked-configuration-questions)
|
||||||
|
@ -518,7 +519,6 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con
|
||||||
# sync_dir_permissions = "700"
|
# sync_dir_permissions = "700"
|
||||||
# sync_file_permissions = "600"
|
# sync_file_permissions = "600"
|
||||||
# rate_limit = "131072"
|
# rate_limit = "131072"
|
||||||
# operation_timeout = "3600"
|
|
||||||
# webhook_enabled = "false"
|
# webhook_enabled = "false"
|
||||||
# webhook_public_url = ""
|
# webhook_public_url = ""
|
||||||
# webhook_listening_host = ""
|
# webhook_listening_host = ""
|
||||||
|
@ -529,6 +529,11 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con
|
||||||
# display_running_config = "false"
|
# display_running_config = "false"
|
||||||
# read_only_auth_scope = "false"
|
# read_only_auth_scope = "false"
|
||||||
# cleanup_local_files = "false"
|
# cleanup_local_files = "false"
|
||||||
|
# operation_timeout = "3600"
|
||||||
|
# dns_timeout = "60"
|
||||||
|
# connect_timeout = "10"
|
||||||
|
# data_timeout = "600"
|
||||||
|
# ip_protocol_version = "0"
|
||||||
```
|
```
|
||||||
|
|
||||||
### 'config' file configuration examples:
|
### 'config' file configuration examples:
|
||||||
|
@ -741,6 +746,23 @@ Example:
|
||||||
operation_timeout = "3600"
|
operation_timeout = "3600"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### ip_protocol_version
|
||||||
|
By default, the application will use IPv4 and IPv6 to resolve and communicate with Microsoft OneDrive. In some Linux distributions (most notably Ubuntu and those distributions based on Ubuntu) this will cause problems due to how DNS resolution is being performed.
|
||||||
|
|
||||||
|
To configure the application to use a specific IP version, configure the following in your config file:
|
||||||
|
```text
|
||||||
|
# operation_timeout = "3600"
|
||||||
|
# dns_timeout = "60"
|
||||||
|
# connect_timeout = "10"
|
||||||
|
# data_timeout = "600"
|
||||||
|
ip_protocol_version = "1"
|
||||||
|
|
||||||
|
```
|
||||||
|
**Note:**
|
||||||
|
* A value of 0 will mean the client will use IPv4 and IPv6. This is the default.
|
||||||
|
* A value of 1 will mean the client will use IPv4 only.
|
||||||
|
* A value of 2 will mean the client will use IPv6 only.
|
||||||
|
|
||||||
#### Configuring the client for 'single tenant application' use
|
#### Configuring the client for 'single tenant application' use
|
||||||
In some instances when using OneDrive Business Accounts, depending on the Azure organisational configuration, it will be necessary to configure the client as a 'single tenant application'.
|
In some instances when using OneDrive Business Accounts, depending on the Azure organisational configuration, it will be necessary to configure the client as a 'single tenant application'.
|
||||||
To configure this, after creating the application on your Azure tenant, update the 'config' file with the tenant name (not the GUID) and the newly created Application ID, then this will be used for the authentication process.
|
To configure this, after creating the application on your Azure tenant, update the 'config' file with the tenant name (not the GUID) and the newly created Application ID, then this will be used for the authentication process.
|
||||||
|
|
48
src/config.d
48
src/config.d
|
@ -43,6 +43,35 @@ final class Config
|
||||||
// Default file permission mode
|
// Default file permission mode
|
||||||
public long defaultFilePermissionMode = 600;
|
public long defaultFilePermissionMode = 600;
|
||||||
public int configuredFilePermissionMode;
|
public int configuredFilePermissionMode;
|
||||||
|
|
||||||
|
// Bring in v2.5.0 config items
|
||||||
|
|
||||||
|
// HTTP Struct items, used for configuring HTTP()
|
||||||
|
// Curl Timeout Handling
|
||||||
|
// libcurl dns_cache_timeout timeout
|
||||||
|
immutable int defaultDnsTimeout = 60;
|
||||||
|
// Connect timeout for HTTP|HTTPS connections
|
||||||
|
immutable int defaultConnectTimeout = 10;
|
||||||
|
// With the following settings we force
|
||||||
|
// - if there is no data flow for 10min, abort
|
||||||
|
// - if the download time for one item exceeds 1h, abort
|
||||||
|
//
|
||||||
|
// Timeout for activity on connection
|
||||||
|
// this translates into Curl's CURLOPT_LOW_SPEED_TIME
|
||||||
|
// which says:
|
||||||
|
// It contains the time in number seconds that the
|
||||||
|
// transfer speed should be below the CURLOPT_LOW_SPEED_LIMIT
|
||||||
|
// for the library to consider it too slow and abort.
|
||||||
|
immutable int defaultDataTimeout = 600;
|
||||||
|
// Maximum time any operation is allowed to take
|
||||||
|
// This includes dns resolution, connecting, data transfer, etc.
|
||||||
|
immutable int defaultOperationTimeout = 3600;
|
||||||
|
// Specify how many redirects should be allowed
|
||||||
|
immutable int defaultMaxRedirects = 5;
|
||||||
|
// Specify what IP protocol version should be used when communicating with OneDrive
|
||||||
|
immutable int defaultIpProtocol = 0; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
this(string confdirOption)
|
this(string confdirOption)
|
||||||
{
|
{
|
||||||
|
@ -122,9 +151,6 @@ final class Config
|
||||||
longValues["sync_file_permissions"] = defaultFilePermissionMode;
|
longValues["sync_file_permissions"] = defaultFilePermissionMode;
|
||||||
// Configure download / upload rate limits
|
// Configure download / upload rate limits
|
||||||
longValues["rate_limit"] = 0;
|
longValues["rate_limit"] = 0;
|
||||||
// maximum time an operation is allowed to take
|
|
||||||
// This includes dns resolution, connecting, data transfer, etc.
|
|
||||||
longValues["operation_timeout"] = 3600;
|
|
||||||
// To ensure we do not fill up the load disk, how much disk space should be reserved by default
|
// To ensure we do not fill up the load disk, how much disk space should be reserved by default
|
||||||
longValues["space_reservation"] = 50 * 2^^20; // 50 MB as Bytes
|
longValues["space_reservation"] = 50 * 2^^20; // 50 MB as Bytes
|
||||||
// Webhook options
|
// Webhook options
|
||||||
|
@ -161,6 +187,19 @@ final class Config
|
||||||
// - Enabling this option will add function processing times to the console output
|
// - Enabling this option will add function processing times to the console output
|
||||||
// - This then enables tracking of where the application is spending most amount of time when processing data when users have questions re performance
|
// - This then enables tracking of where the application is spending most amount of time when processing data when users have questions re performance
|
||||||
boolValues["display_processing_time"] = false;
|
boolValues["display_processing_time"] = false;
|
||||||
|
|
||||||
|
// HTTPS & CURL Operation Settings
|
||||||
|
// - Maximum time an operation is allowed to take
|
||||||
|
// This includes dns resolution, connecting, data transfer, etc.
|
||||||
|
longValues["operation_timeout"] = defaultOperationTimeout;
|
||||||
|
// libcurl dns_cache_timeout timeout
|
||||||
|
longValues["dns_timeout"] = defaultDnsTimeout;
|
||||||
|
// Timeout for HTTPS connections
|
||||||
|
longValues["connect_timeout"] = defaultConnectTimeout;
|
||||||
|
// Timeout for activity on a HTTPS connection
|
||||||
|
longValues["data_timeout"] = defaultDataTimeout;
|
||||||
|
// What IP protocol version should be used when communicating with OneDrive
|
||||||
|
longValues["ip_protocol_version"] = defaultIpProtocol; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
|
||||||
|
|
||||||
// EXPAND USERS HOME DIRECTORY
|
// EXPAND USERS HOME DIRECTORY
|
||||||
// Determine the users home directory.
|
// Determine the users home directory.
|
||||||
|
@ -451,9 +490,6 @@ final class Config
|
||||||
"no-remote-delete",
|
"no-remote-delete",
|
||||||
"Do not delete local file 'deletes' from OneDrive when using --upload-only",
|
"Do not delete local file 'deletes' from OneDrive when using --upload-only",
|
||||||
&boolValues["no_remote_delete"],
|
&boolValues["no_remote_delete"],
|
||||||
"operation-timeout",
|
|
||||||
"Maximum amount of time (in seconds) an operation is allowed to take",
|
|
||||||
&longValues["operation_timeout"],
|
|
||||||
"print-token",
|
"print-token",
|
||||||
"Print the access token, useful for debugging",
|
"Print the access token, useful for debugging",
|
||||||
&boolValues["print_token"],
|
&boolValues["print_token"],
|
||||||
|
|
|
@ -761,7 +761,10 @@ int main(string[] args)
|
||||||
writeln("Config option 'debug_https' = ", cfg.getValueBool("debug_https"));
|
writeln("Config option 'debug_https' = ", cfg.getValueBool("debug_https"));
|
||||||
writeln("Config option 'rate_limit' = ", cfg.getValueLong("rate_limit"));
|
writeln("Config option 'rate_limit' = ", cfg.getValueLong("rate_limit"));
|
||||||
writeln("Config option 'operation_timeout' = ", cfg.getValueLong("operation_timeout"));
|
writeln("Config option 'operation_timeout' = ", cfg.getValueLong("operation_timeout"));
|
||||||
|
writeln("Config option 'dns_timeout' = ", cfg.getValueLong("dns_timeout"));
|
||||||
|
writeln("Config option 'connect_timeout' = ", cfg.getValueLong("connect_timeout"));
|
||||||
|
writeln("Config option 'data_timeout' = ", cfg.getValueLong("data_timeout"));
|
||||||
|
writeln("Config option 'ip_protocol_version' = ", cfg.getValueLong("ip_protocol_version"));
|
||||||
|
|
||||||
// Is sync_list configured ?
|
// Is sync_list configured ?
|
||||||
writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files"));
|
writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files"));
|
||||||
|
@ -853,7 +856,7 @@ int main(string[] args)
|
||||||
|
|
||||||
// Test if OneDrive service can be reached, exit if it cant be reached
|
// Test if OneDrive service can be reached, exit if it cant be reached
|
||||||
log.vdebug("Testing network to ensure network connectivity to Microsoft OneDrive Service");
|
log.vdebug("Testing network to ensure network connectivity to Microsoft OneDrive Service");
|
||||||
online = testNetwork();
|
online = testNetwork(cfg);
|
||||||
if (!online) {
|
if (!online) {
|
||||||
// Cant initialise the API as we are not online
|
// Cant initialise the API as we are not online
|
||||||
if (!cfg.getValueBool("monitor")) {
|
if (!cfg.getValueBool("monitor")) {
|
||||||
|
@ -885,7 +888,7 @@ int main(string[] args)
|
||||||
Thread.sleep(dur!"seconds"(maxBackoffInterval));
|
Thread.sleep(dur!"seconds"(maxBackoffInterval));
|
||||||
}
|
}
|
||||||
// perform the re-rty
|
// perform the re-rty
|
||||||
online = testNetwork();
|
online = testNetwork(cfg);
|
||||||
if (online) {
|
if (online) {
|
||||||
// We are now online
|
// We are now online
|
||||||
log.log("Internet connectivity to Microsoft OneDrive service has been restored");
|
log.log("Internet connectivity to Microsoft OneDrive service has been restored");
|
||||||
|
|
|
@ -214,9 +214,9 @@ final class OneDriveApi
|
||||||
http = HTTP();
|
http = HTTP();
|
||||||
// Curl Timeout Handling
|
// Curl Timeout Handling
|
||||||
// libcurl dns_cache_timeout timeout
|
// libcurl dns_cache_timeout timeout
|
||||||
http.dnsTimeout = (dur!"seconds"(60));
|
http.dnsTimeout = (dur!"seconds"(cfg.getValueLong("dns_timeout")));
|
||||||
// Timeout for HTTPS connections
|
// Timeout for HTTPS connections
|
||||||
http.connectTimeout = (dur!"seconds"(10));
|
http.connectTimeout = (dur!"seconds"(cfg.getValueLong("connect_timeout")));
|
||||||
// with the following settings we force
|
// with the following settings we force
|
||||||
// - if there is no data flow for 10min, abort
|
// - if there is no data flow for 10min, abort
|
||||||
// - if the download time for one item exceeds 1h, abort
|
// - if the download time for one item exceeds 1h, abort
|
||||||
|
@ -227,17 +227,27 @@ final class OneDriveApi
|
||||||
// It contains the time in number seconds that the
|
// It contains the time in number seconds that the
|
||||||
// transfer speed should be below the CURLOPT_LOW_SPEED_LIMIT
|
// transfer speed should be below the CURLOPT_LOW_SPEED_LIMIT
|
||||||
// for the library to consider it too slow and abort.
|
// for the library to consider it too slow and abort.
|
||||||
http.dataTimeout = (dur!"seconds"(600));
|
http.dataTimeout = (dur!"seconds"(cfg.getValueLong("data_timeout")));
|
||||||
// maximum time an operation is allowed to take
|
// maximum time an operation is allowed to take
|
||||||
// This includes dns resolution, connecting, data transfer, etc.
|
// This includes dns resolution, connecting, data transfer, etc.
|
||||||
http.operationTimeout = (dur!"seconds"(cfg.getValueLong("operation_timeout")));
|
http.operationTimeout = (dur!"seconds"(cfg.getValueLong("operation_timeout")));
|
||||||
|
// What IP protocol version should be used when using Curl - IPv4 & IPv6, IPv4 or IPv6
|
||||||
|
http.handle.set(CurlOption.ipresolve,cfg.getValueLong("ip_protocol_version")); // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
|
||||||
// Specify how many redirects should be allowed
|
// Specify how many redirects should be allowed
|
||||||
http.maxRedirects(5);
|
http.maxRedirects(cfg.defaultMaxRedirects);
|
||||||
|
|
||||||
// Do we enable curl debugging?
|
// Do we enable curl debugging?
|
||||||
if (cfg.getValueBool("debug_https")) {
|
if (cfg.getValueBool("debug_https")) {
|
||||||
http.verbose = true;
|
http.verbose = true;
|
||||||
.debugResponse = true;
|
.debugResponse = true;
|
||||||
|
|
||||||
|
// Output what options we are using so that in the debug log this can be tracked
|
||||||
|
log.vdebug("http.dnsTimeout = ", cfg.getValueLong("dns_timeout"));
|
||||||
|
log.vdebug("http.connectTimeout = ", cfg.getValueLong("connect_timeout"));
|
||||||
|
log.vdebug("http.dataTimeout = ", cfg.getValueLong("data_timeout"));
|
||||||
|
log.vdebug("http.operationTimeout = ", cfg.getValueLong("operation_timeout"));
|
||||||
|
log.vdebug("http.CurlOption.ipresolve = ", cfg.getValueLong("ip_protocol_version"));
|
||||||
|
log.vdebug("http.maxRedirects = ", cfg.defaultMaxRedirects);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update clientId if application_id is set in config file
|
// Update clientId if application_id is set in config file
|
||||||
|
|
18
src/util.d
18
src/util.d
|
@ -15,7 +15,9 @@ import std.json;
|
||||||
import std.traits;
|
import std.traits;
|
||||||
import qxor;
|
import qxor;
|
||||||
import core.stdc.stdlib;
|
import core.stdc.stdlib;
|
||||||
static import log;
|
|
||||||
|
import log;
|
||||||
|
import config;
|
||||||
|
|
||||||
shared string deviceName;
|
shared string deviceName;
|
||||||
|
|
||||||
|
@ -114,15 +116,23 @@ Regex!char wild2regex(const(char)[] pattern)
|
||||||
}
|
}
|
||||||
|
|
||||||
// returns true if the network connection is available
|
// returns true if the network connection is available
|
||||||
bool testNetwork()
|
bool testNetwork(Config cfg)
|
||||||
{
|
{
|
||||||
// Use low level HTTP struct
|
// Use low level HTTP struct
|
||||||
auto http = HTTP();
|
auto http = HTTP();
|
||||||
http.url = "https://login.microsoftonline.com";
|
http.url = "https://login.microsoftonline.com";
|
||||||
// DNS lookup timeout
|
// DNS lookup timeout
|
||||||
http.dnsTimeout = (dur!"seconds"(5));
|
http.dnsTimeout = (dur!"seconds"(cfg.getValueLong("dns_timeout")));
|
||||||
// Timeout for connecting
|
// Timeout for connecting
|
||||||
http.connectTimeout = (dur!"seconds"(5));
|
http.connectTimeout = (dur!"seconds"(cfg.getValueLong("connect_timeout")));
|
||||||
|
// Data Timeout for HTTPS connections
|
||||||
|
http.dataTimeout = (dur!"seconds"(cfg.getValueLong("data_timeout")));
|
||||||
|
// maximum time any operation is allowed to take
|
||||||
|
// This includes dns resolution, connecting, data transfer, etc.
|
||||||
|
http.operationTimeout = (dur!"seconds"(cfg.getValueLong("operation_timeout")));
|
||||||
|
// What IP protocol version should be used when using Curl - IPv4 & IPv6, IPv4 or IPv6
|
||||||
|
http.handle.set(CurlOption.ipresolve,cfg.getValueLong("ip_protocol_version")); // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
|
||||||
|
|
||||||
// HTTP connection test method
|
// HTTP connection test method
|
||||||
http.method = HTTP.Method.head;
|
http.method = HTTP.Method.head;
|
||||||
// Attempt to contact the Microsoft Online Service
|
// Attempt to contact the Microsoft Online Service
|
||||||
|
|
Loading…
Reference in a new issue