deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-07 16:26:40 +02:00
Code Issues Projects Releases Wiki Activity

Add Real-Time syncing of remote updates via webhooks (#1626)

Browse source 
* Add Real-Time syncing of remote updates via webhooks

Co-authored-by: abraunegg <alex.braunegg@gmail.com>
This commit is contained in:
Yuan Liu 2021-11-22 12:06:13 -08:00 committed by GitHub
parent 540c2369f3
commit bfeeae91de
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 11050 additions and 291 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
Makefile.in
View file

@ -78,7 +78,8 @@ SOURCES = \
src/sync.d \ src/sync.d \
src/upload.d \ src/upload.d \
src/util.d \ src/util.d \
src/progress.d src/progress.d \
src/arsd/cgi.d
ifeq ($(NOTIFICATIONS),yes) ifeq ($(NOTIFICATIONS),yes)
SOURCES += src/notifications/notify.d src/notifications/dnotify.d SOURCES += src/notifications/notify.d src/notifications/dnotify.d
@ -123,7 +124,7 @@ else
ifeq ($(RHEL_VERSION),6) ifeq ($(RHEL_VERSION),6)
install -D contrib/init.d/onedrive.init $(DESTDIR)/etc/init.d/onedrive install -D contrib/init.d/onedrive.init $(DESTDIR)/etc/init.d/onedrive
install -D contrib/init.d/onedrive_service.sh $(DESTDIR)$(bindir)/onedrive_service.sh install -D contrib/init.d/onedrive_service.sh $(DESTDIR)$(bindir)/onedrive_service.sh
endif endif
endif endif
ifeq ($(COMPLETIONS),yes) ifeq ($(COMPLETIONS),yes)
$(INSTALL) -D -m 644 contrib/completions/complete.zsh $(DESTDIR)$(ZSH_COMPLETION_DIR)/_onedrive $(INSTALL) -D -m 644 contrib/completions/complete.zsh $(DESTDIR)$(ZSH_COMPLETION_DIR)/_onedrive

4
README.md
View file

@ -13,7 +13,8 @@ This client is a 'fork' of the [skilion](https://github.com/skilion/onedrive) cl
## Features ## Features
* State caching * State caching
* Real-Time file monitoring with Inotify * Real-Time local file monitoring with inotify
* Real-Time syncing of remote updates via webhooks
* File upload / download validation to ensure data integrity * File upload / download validation to ensure data integrity
* Resumable uploads * Resumable uploads
* Support OneDrive for Business (part of Office 365) * Support OneDrive for Business (part of Office 365)
@ -27,7 +28,6 @@ This client is a 'fork' of the [skilion](https://github.com/skilion/onedrive) cl
* Supports rate limiting of traffic * Supports rate limiting of traffic
## What's missing ## What's missing
* While local changes are uploaded right away, remote changes are delayed until next automated sync cycle when using --monitor
* Ability to encrypt/decrypt files on-the-fly when uploading/downloading files from OneDrive * Ability to encrypt/decrypt files on-the-fly when uploading/downloading files from OneDrive
* Support for Windows 'On-Demand' functionality so file is only downloaded when accessed locally * Support for Windows 'On-Demand' functionality so file is only downloaded when accessed locally
* A GUI for configuration management * A GUI for configuration management

8
config
View file

@ -41,4 +41,10 @@
# sync_business_shared_folders = "false" # sync_business_shared_folders = "false"
# sync_dir_permissions = "700" # sync_dir_permissions = "700"
# sync_file_permissions = "600" # sync_file_permissions = "600"
# rate_limit = "131072" # rate_limit = "131072"
# webhook_enabled = "false"
# webhook_public_url = ""
# webhook_listening_host = ""
# webhook_listening_port = "8888"
# webhook_expiration_interval = "86400"
# webhook_renewal_interval = "43200"

83
docs/USAGE.md
View file

@ -36,6 +36,10 @@
* [Shared folders (OneDrive Business or Office 365)](#shared-folders-onedrive-business-or-office-365) * [Shared folders (OneDrive Business or Office 365)](#shared-folders-onedrive-business-or-office-365)
* [SharePoint / Office 365 Shared Libraries](#sharepoint--office-365-shared-libraries) * [SharePoint / Office 365 Shared Libraries](#sharepoint--office-365-shared-libraries)
- [Running 'onedrive' in 'monitor' mode](#running-onedrive-in-monitor-mode) - [Running 'onedrive' in 'monitor' mode](#running-onedrive-in-monitor-mode)
* [Use webhook to subscribe to remote updates in 'monitor' mode](#use-webhook-to-subscribe-to-remote-updates-in-monitor-mode)
* [More webhook configuration options](#more-webhook-configuration-options)
+ [webhook_listening_host and webhook_listening_port](#webhook_listening_host-and-webhook_listening_port)
+ [webhook_expiration_interval and webhook_renewal_interval](#webhook_expiration_interval-and-webhook_renewal_interval)
- [Running 'onedrive' as a system service](#running-onedrive-as-a-system-service) - [Running 'onedrive' as a system service](#running-onedrive-as-a-system-service)
* [OneDrive service running as root user via init.d](#onedrive-service-running-as-root-user-via-initd) * [OneDrive service running as root user via init.d](#onedrive-service-running-as-root-user-via-initd)
* [OneDrive service running as root user via systemd (Arch, Ubuntu, Debian, OpenSuSE, Fedora)](#onedrive-service-running-as-root-user-via-systemd-arch-ubuntu-debian-opensuse-fedora) * [OneDrive service running as root user via systemd (Arch, Ubuntu, Debian, OpenSuSE, Fedora)](#onedrive-service-running-as-root-user-via-systemd-arch-ubuntu-debian-opensuse-fedora)
@ -84,19 +88,19 @@ After installing the application you must authorize the application with your On
You will be asked to open a specific URL by using your web browser where you will have to login into your Microsoft Account and give the application the permission to access your files. After giving permission to the application, you will be redirected to a blank page. Copy the URI of the blank page into the application. You will be asked to open a specific URL by using your web browser where you will have to login into your Microsoft Account and give the application the permission to access your files. After giving permission to the application, you will be redirected to a blank page. Copy the URI of the blank page into the application.
```text ```text
[user@hostname ~]$ onedrive [user@hostname ~]$ onedrive
Authorize this app visiting: Authorize this app visiting:
https://..... https://.....
Enter the response uri: Enter the response uri:
``` ```
**Example:** **Example:**
``` ```
[user@hostname ~]$ onedrive [user@hostname ~]$ onedrive
Authorize this app visiting: Authorize this app visiting:
https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=22c49a0d-d21c-4792-aed1-8f163c982546&scope=Files.ReadWrite%20Files.ReadWrite.all%20Sites.ReadWrite.All%20offline_access&response_type=code&redirect_uri=https://login.microsoftonline.com/common/oauth2/nativeclient https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=22c49a0d-d21c-4792-aed1-8f163c982546&scope=Files.ReadWrite%20Files.ReadWrite.all%20Sites.ReadWrite.All%20offline_access&response_type=code&redirect_uri=https://login.microsoftonline.com/common/oauth2/nativeclient
@ -120,7 +124,7 @@ Config path = /home/alex/.config/onedrive
Config file found in config path = false Config file found in config path = false
Config option 'check_nosync' = false Config option 'check_nosync' = false
Config option 'sync_dir' = /home/alex/OneDrive Config option 'sync_dir' = /home/alex/OneDrive
Config option 'skip_dir' = Config option 'skip_dir' =
Config option 'skip_file' = ~*|.~*|*.tmp Config option 'skip_file' = ~*|.~*|*.tmp
Config option 'skip_dotfiles' = false Config option 'skip_dotfiles' = false
Config option 'skip_symlinks' = false Config option 'skip_symlinks' = false
@ -188,7 +192,7 @@ Example: If the full path is `~/OneDrive/mydir`, the command would be `onedrive
### Performing a 'one-way' download sync ### Performing a 'one-way' download sync
In some cases it may be desirable to 'download only' from OneDrive. To do this use the following command: In some cases it may be desirable to 'download only' from OneDrive. To do this use the following command:
```text ```text
onedrive --synchronize --download-only onedrive --synchronize --download-only
``` ```
### Performing a 'one-way' upload sync ### Performing a 'one-way' upload sync
@ -353,6 +357,12 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con
# sync_file_permissions = "600" # sync_file_permissions = "600"
# rate_limit = "131072" # rate_limit = "131072"
# operation_timeout = "3600" # operation_timeout = "3600"
# webhook_enabled = "false"
# webhook_public_url = ""
# webhook_listening_host = ""
# webhook_listening_port = "8888"
# webhook_expiration_interval = "86400"
# webhook_renewal_interval = "43200"
``` ```
@ -401,7 +411,7 @@ sync_file_permissions = "600"
**Important:** Special permission bits (setuid, setgid, sticky bit) are not supported. Valid permission values are from `000` to `777` only. **Important:** Special permission bits (setuid, setgid, sticky bit) are not supported. Valid permission values are from `000` to `777` only.
#### skip_dir #### skip_dir
Example: Example:
```text ```text
# When changing a config option below, remove the '#' from the start of the line # When changing a config option below, remove the '#' from the start of the line
# For explanations of all config options below see docs/USAGE.md or the man page. # For explanations of all config options below see docs/USAGE.md or the man page.
@ -528,7 +538,7 @@ Each line of the file represents a relative path from your `sync_dir`. All files
Here is an example of `sync_list`: Here is an example of `sync_list`:
```text ```text
# sync_list supports comments # sync_list supports comments
# #
# The ordering of entries is highly recommended - exclusions before inclusions # The ordering of entries is highly recommended - exclusions before inclusions
# #
# Exclude temp folders under Documents # Exclude temp folders under Documents
@ -562,7 +572,7 @@ The following are supported for pattern matching and exclusion rules:
To simplify 'exclusions' and 'inclusions', the following is also possible: To simplify 'exclusions' and 'inclusions', the following is also possible:
```text ```text
# sync_list supports comments # sync_list supports comments
# #
# The ordering of entries is highly recommended - exclusions before inclusions # The ordering of entries is highly recommended - exclusions before inclusions
# #
# Exclude temp folders under Documents # Exclude temp folders under Documents
@ -583,7 +593,7 @@ sync_root_files = "true"
This will tell the application to sync any file that it finds in your 'sync_dir' root by default. This will tell the application to sync any file that it finds in your 'sync_dir' root by default.
### Configuring the client for 'single tenant application' use ### Configuring the client for 'single tenant application' use
In some instances when using OneDrive Business Accounts, depending on the Azure organisational configuration, it will be necessary to configure the client as a 'single tenant application'. In some instances when using OneDrive Business Accounts, depending on the Azure organisational configuration, it will be necessary to configure the client as a 'single tenant application'.
To configure this, after creating the application on your Azure tenant, update the 'config' file with the tenant name (not the GUID) and the newly created Application ID, then this will be used for the authentication process. To configure this, after creating the application on your Azure tenant, update the 'config' file with the tenant name (not the GUID) and the newly created Application ID, then this will be used for the authentication process.
```text ```text
# skip_dir_strict_match = "false" # skip_dir_strict_match = "false"
@ -682,6 +692,59 @@ sudo sysctl fs.inotify.max_user_watches=<new_value>
To make these changes permanent, refer to your OS reference documentation. To make these changes permanent, refer to your OS reference documentation.
### Use webhook to subscribe to remote updates in 'monitor' mode
A webhook can be optionally enabled in the monitor mode to allow the onedrive process to subscribe to remote updates. Remote changes can be synced to your local file system as soon as possible, without waiting for the next sync cycle.
To enable this feature, you need to configure the following options in the config file:
```
webhook_enabled = "true"
webhook_public_url = "<public-facing url to reach your webhook>"
```
Setting `webhook_enabled` to `true` enables the webhook in 'monitor' mode. The onedrive process will listen for incoming updates at a configurable endpoint, which defaults to `0.0.0.0:8888`. The `webhook_public_url` must be set to an public-facing url for Microsoft to send updates to your webhook. If your host is directly exposed to the Internet, the `webhook_public_url` can be set to `http://<your_host>:8888/` to match the default endpoint. However, the recommended approach is to configure a reverse proxy like nginx.
For example, below is a nginx config snippet to proxy traffic into the webhook:
```
http {
server {
listen 80;
location /webhooks/onedrive {
proxy_pass http://127.0.0.1:8888;
}
}
}
```
With nginx running, you can configure `webhook_public_url` to `http://<your_host>/webhooks/onedrive`.
### More webhook configuration options
Below options can be optionally configured. The default is usually good enough.
#### webhook_listening_host and webhook_listening_port
Set `webhook_listening_host` and `webhook_listening_port` to change the webhook listening endpoint. If `webhook_listening_host` is left empty, which is the default, the webhook will bind to `0.0.0.0`. The default `webhook_listening_port` is `8888`.
```
webhook_listening_host = ""
webhook_listening_port = "8888"
```
#### webhook_expiration_interval and webhook_renewal_interval
Set `webhook_expiration_interval` and `webhook_renewal_interval` to change the frequency of subscription renewal. By default, the webhook asks Microsoft to keep subscriptions alive for 24 hours, and it renews subscriptions when it is less than 12 hours before their expiration.
```
# Default expiration interval is 24 hours
webhook_expiration_interval = "86400"
# Default renewal interval is 12 hours
webhook_renewal_interval = "43200"
```
## Running 'onedrive' as a system service ## Running 'onedrive' as a system service
There are a few ways to use onedrive as a service There are a few ways to use onedrive as a service
* via init.d * via init.d
@ -869,7 +932,7 @@ for extra details.
* Configuring the client for use in dual-boot (Windows / Linux) situations * Configuring the client for use in dual-boot (Windows / Linux) situations
* Configuring the client for use when 'sync_dir' is a mounted directory * Configuring the client for use when 'sync_dir' is a mounted directory
* Upload data from the local ~/OneDrive folder to a specific location on OneDrive * Upload data from the local ~/OneDrive folder to a specific location on OneDrive
Refer to [./advanced-usage.md](advanced-usage.md) for configuration assistance. Refer to [./advanced-usage.md](advanced-usage.md) for configuration assistance.
### Access OneDrive service through a proxy ### Access OneDrive service through a proxy

8
src/arsd/README.md Normal file
View file

@ -0,0 +1,8 @@
The files in this directory have been obtained form the following places:
cgi.d
https://github.com/adamdruppe/arsd/blob/a870179988b8881b04126856105f0fad2cc0018d/cgi.d
License: Boost Software License - Version 1.0
Copyright 2008-2021, Adam D. Ruppe
see https://github.com/adamdruppe/arsd/blob/a870179988b8881b04126856105f0fad2cc0018d/LICENSE

10481
src/arsd/cgi.d Normal file
View file

File diff suppressed because it is too large Load diff

60
src/config.d
View file

@ -43,7 +43,7 @@ final class Config
// Default file permission mode // Default file permission mode
public long defaultFilePermissionMode = 600; public long defaultFilePermissionMode = 600;
public int configuredFilePermissionMode; public int configuredFilePermissionMode;
this(string confdirOption) this(string confdirOption)
{ {
// default configuration - entries in config file ~/.config/onedrive/config // default configuration - entries in config file ~/.config/onedrive/config
@ -118,8 +118,16 @@ final class Config
longValues["sync_file_permissions"] = defaultFilePermissionMode; longValues["sync_file_permissions"] = defaultFilePermissionMode;
// Configure download / upload rate limits // Configure download / upload rate limits
longValues["rate_limit"] = 0; longValues["rate_limit"] = 0;
// DEVELOPER OPTIONS // Webhook options
boolValues["webhook_enabled"] = false;
stringValues["webhook_public_url"] = "";
stringValues["webhook_listening_host"] = "";
longValues["webhook_listening_port"] = 8888;
longValues["webhook_expiration_interval"] = 3600 * 24;
longValues["webhook_renewal_interval"] = 3600 * 12;
// DEVELOPER OPTIONS
// display_memory = true | false // display_memory = true | false
// - It may be desirable to display the memory usage of the application to assist with diagnosing memory issues with the application // - It may be desirable to display the memory usage of the application to assist with diagnosing memory issues with the application
// - This is especially beneficial when debugging or performing memory tests with Valgrind // - This is especially beneficial when debugging or performing memory tests with Valgrind
@ -136,7 +144,7 @@ final class Config
// This includes dns resolution, connecting, data transfer, etc. // This includes dns resolution, connecting, data transfer, etc.
longValues["operation_timeout"] = 3600; longValues["operation_timeout"] = 3600;
// Determine the users home directory. // Determine the users home directory.
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts // Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
// Check for HOME environment variable // Check for HOME environment variable
if (environment.get("HOME") != ""){ if (environment.get("HOME") != ""){
@ -156,10 +164,10 @@ final class Config
homePath = "~"; homePath = "~";
} }
} }
// Output homePath calculation // Output homePath calculation
log.vdebug("homePath: ", homePath); log.vdebug("homePath: ", homePath);
// Determine the correct configuration directory to use // Determine the correct configuration directory to use
string configDirBase; string configDirBase;
string systemConfigDirBase; string systemConfigDirBase;
@ -185,7 +193,7 @@ final class Config
// Also set up a path to pre-shipped shared configs (which can be overridden by supplying a config file in userspace) // Also set up a path to pre-shipped shared configs (which can be overridden by supplying a config file in userspace)
systemConfigDirBase = "/etc"; systemConfigDirBase = "/etc";
} }
// Output configDirBase calculation // Output configDirBase calculation
log.vdebug("configDirBase: ", configDirBase); log.vdebug("configDirBase: ", configDirBase);
// Set the default application configuration directory // Set the default application configuration directory
@ -195,7 +203,7 @@ final class Config
// systemConfigDirBase contains the correct path so we do not need to check for presence of '~' // systemConfigDirBase contains the correct path so we do not need to check for presence of '~'
systemConfigDirName = systemConfigDirBase ~ "/onedrive"; systemConfigDirName = systemConfigDirBase ~ "/onedrive";
} }
// Config directory options all determined // Config directory options all determined
if (!exists(configDirName)) { if (!exists(configDirName)) {
// create the directory // create the directory
@ -203,11 +211,11 @@ final class Config
// Configure the applicable permissions for the folder // Configure the applicable permissions for the folder
configDirName.setAttributes(returnRequiredDirectoryPermisions()); configDirName.setAttributes(returnRequiredDirectoryPermisions());
} }
// configDirName has a trailing / // configDirName has a trailing /
log.vlog("Using 'user' Config Dir: ", configDirName); log.vlog("Using 'user' Config Dir: ", configDirName);
log.vlog("Using 'system' Config Dir: ", systemConfigDirName); log.vlog("Using 'system' Config Dir: ", systemConfigDirName);
// Update application set variables based on configDirName // Update application set variables based on configDirName
refreshTokenFilePath = buildNormalizedPath(configDirName ~ "/refresh_token"); refreshTokenFilePath = buildNormalizedPath(configDirName ~ "/refresh_token");
deltaLinkFilePath = buildNormalizedPath(configDirName ~ "/delta_link"); deltaLinkFilePath = buildNormalizedPath(configDirName ~ "/delta_link");
@ -218,7 +226,7 @@ final class Config
syncListFilePath = buildNormalizedPath(configDirName ~ "/sync_list"); syncListFilePath = buildNormalizedPath(configDirName ~ "/sync_list");
systemConfigFilePath = buildNormalizedPath(systemConfigDirName ~ "/config"); systemConfigFilePath = buildNormalizedPath(systemConfigDirName ~ "/config");
businessSharedFolderFilePath = buildNormalizedPath(configDirName ~ "/business_shared_folders"); businessSharedFolderFilePath = buildNormalizedPath(configDirName ~ "/business_shared_folders");
// Debug Output for application set variables based on configDirName // Debug Output for application set variables based on configDirName
log.vdebug("refreshTokenFilePath = ", refreshTokenFilePath); log.vdebug("refreshTokenFilePath = ", refreshTokenFilePath);
log.vdebug("deltaLinkFilePath = ", deltaLinkFilePath); log.vdebug("deltaLinkFilePath = ", deltaLinkFilePath);
@ -289,7 +297,7 @@ final class Config
boolValues["synchronize"] = false; boolValues["synchronize"] = false;
boolValues["force"] = false; boolValues["force"] = false;
boolValues["list_business_shared_folders"] = false; boolValues["list_business_shared_folders"] = false;
// Application Startup option validation // Application Startup option validation
try { try {
string tmpStr; string tmpStr;
@ -297,7 +305,7 @@ final class Config
long tmpVerb; long tmpVerb;
// duplicated from main.d to get full help output! // duplicated from main.d to get full help output!
auto opt = getopt( auto opt = getopt(
args, args,
std.getopt.config.bundling, std.getopt.config.bundling,
std.getopt.config.caseSensitive, std.getopt.config.caseSensitive,
@ -305,7 +313,7 @@ final class Config
"Perform authentication not via interactive dialog but via files read/writes to these files.", "Perform authentication not via interactive dialog but via files read/writes to these files.",
&stringValues["auth_files"], &stringValues["auth_files"],
"check-for-nomount", "check-for-nomount",
"Check for the presence of .nosync in the syncdir root. If found, do not perform sync.", "Check for the presence of .nosync in the syncdir root. If found, do not perform sync.",
&boolValues["check_nomount"], &boolValues["check_nomount"],
"check-for-nosync", "check-for-nosync",
"Check for the presence of .nosync in each directory. If found, skip directory from sync.", "Check for the presence of .nosync in each directory. If found, skip directory from sync.",
@ -319,8 +327,8 @@ final class Config
"create-share-link", "create-share-link",
"Create a shareable link for an existing file on OneDrive", "Create a shareable link for an existing file on OneDrive",
&stringValues["create_share_link"], &stringValues["create_share_link"],
"debug-https", "debug-https",
"Debug OneDrive HTTPS communication.", "Debug OneDrive HTTPS communication.",
&boolValues["debug_https"], &boolValues["debug_https"],
"destination-directory", "destination-directory",
"Destination directory for renamed or move on OneDrive - no sync will be performed.", "Destination directory for renamed or move on OneDrive - no sync will be performed.",
@ -519,7 +527,7 @@ final class Config
// configure function variables // configure function variables
auto file = File(filename, "r"); auto file = File(filename, "r");
string lineBuffer; string lineBuffer;
// configure scopes // configure scopes
// - failure // - failure
scope(failure) { scope(failure) {
@ -538,7 +546,7 @@ final class Config
file.close(); file.close();
} }
} }
// read file line by line // read file line by line
auto range = file.byLine(); auto range = file.byLine();
foreach (line; range) { foreach (line; range) {
@ -585,7 +593,7 @@ final class Config
setValueString("skip_dir", configFileSkipDir); setValueString("skip_dir", configFileSkipDir);
} }
} }
// Azure AD Configuration // Azure AD Configuration
if (key == "azure_ad_endpoint") { if (key == "azure_ad_endpoint") {
string azureConfigValue = c.front.dup; string azureConfigValue = c.front.dup;
@ -605,7 +613,7 @@ final class Config
case "CN": case "CN":
log.log("Using config option for Azure AD China operated by 21Vianet"); log.log("Using config option for Azure AD China operated by 21Vianet");
break; break;
// Default - all other entries // Default - all other entries
default: default:
log.log("Unknown Azure AD Endpoint - using Global Azure AD Endpoints"); log.log("Unknown Azure AD Endpoint - using Global Azure AD Endpoints");
} }
@ -628,10 +636,10 @@ final class Config
} }
return true; return true;
} }
void configureRequiredDirectoryPermisions() { void configureRequiredDirectoryPermisions() {
// return the directory permission mode required // return the directory permission mode required
// - return octal!defaultDirectoryPermissionMode; ... cant be used .. which is odd // - return octal!defaultDirectoryPermissionMode; ... cant be used .. which is odd
// Error: variable defaultDirectoryPermissionMode cannot be read at compile time // Error: variable defaultDirectoryPermissionMode cannot be read at compile time
if (getValueLong("sync_dir_permissions") != defaultDirectoryPermissionMode) { if (getValueLong("sync_dir_permissions") != defaultDirectoryPermissionMode) {
// return user configured permissions as octal integer // return user configured permissions as octal integer
@ -645,10 +653,10 @@ final class Config
configuredDirectoryPermissionMode = to!int(convertedValue); configuredDirectoryPermissionMode = to!int(convertedValue);
} }
} }
void configureRequiredFilePermisions() { void configureRequiredFilePermisions() {
// return the file permission mode required // return the file permission mode required
// - return octal!defaultFilePermissionMode; ... cant be used .. which is odd // - return octal!defaultFilePermissionMode; ... cant be used .. which is odd
// Error: variable defaultFilePermissionMode cannot be read at compile time // Error: variable defaultFilePermissionMode cannot be read at compile time
if (getValueLong("sync_file_permissions") != defaultFilePermissionMode) { if (getValueLong("sync_file_permissions") != defaultFilePermissionMode) {
// return user configured permissions as octal integer // return user configured permissions as octal integer
@ -662,7 +670,7 @@ final class Config
configuredFilePermissionMode = to!int(convertedValue); configuredFilePermissionMode = to!int(convertedValue);
} }
} }
int returnRequiredDirectoryPermisions() { int returnRequiredDirectoryPermisions() {
// read the configuredDirectoryPermissionMode and return // read the configuredDirectoryPermissionMode and return
if (configuredDirectoryPermissionMode == 0) { if (configuredDirectoryPermissionMode == 0) {
@ -672,7 +680,7 @@ final class Config
} }
return configuredDirectoryPermissionMode; return configuredDirectoryPermissionMode;
} }
int returnRequiredFilePermisions() { int returnRequiredFilePermisions() {
// read the configuredFilePermissionMode and return // read the configuredFilePermissionMode and return
if (configuredFilePermissionMode == 0) { if (configuredFilePermissionMode == 0) {

291
src/main.d
View file

@ -5,11 +5,13 @@ import config, itemdb, monitor, onedrive, selective, sync, util;
import std.net.curl: CurlException; import std.net.curl: CurlException;
import core.stdc.signal; import core.stdc.signal;
import std.traits; import std.traits;
import std.concurrency: receiveTimeout;
static import log; static import log;
OneDriveApi oneDrive; OneDriveApi oneDrive;
ItemDatabase itemDb; ItemDatabase itemDb;
bool onedriveInitialised = false;
const int EXIT_UNAUTHORIZED = 3; const int EXIT_UNAUTHORIZED = 3;
enum MONITOR_LOG_SILENT = 2; enum MONITOR_LOG_SILENT = 2;
@ -20,7 +22,7 @@ int main(string[] args)
{ {
// Disable buffering on stdout // Disable buffering on stdout
stdout.setvbuf(0, _IONBF); stdout.setvbuf(0, _IONBF);
// main function variables // main function variables
string confdirOption; string confdirOption;
string configFilePath; string configFilePath;
@ -48,10 +50,9 @@ int main(string[] args)
bool skipDirDifferent = false; bool skipDirDifferent = false;
bool online = false; bool online = false;
bool performSyncOK = false; bool performSyncOK = false;
bool onedriveInitialised = false;
bool displayMemoryUsage = false; bool displayMemoryUsage = false;
bool displaySyncOptions = false; bool displaySyncOptions = false;
// Define scopes // Define scopes
scope(exit) { scope(exit) {
// Display memory details // Display memory details
@ -79,7 +80,7 @@ int main(string[] args)
log.displayMemoryUsagePostGC(); log.displayMemoryUsagePostGC();
} }
} }
scope(failure) { scope(failure) {
// Display memory details // Display memory details
if (displayMemoryUsage) { if (displayMemoryUsage) {
@ -139,7 +140,7 @@ int main(string[] args)
log.error("Try 'onedrive -h' for more information"); log.error("Try 'onedrive -h' for more information");
return EXIT_FAILURE; return EXIT_FAILURE;
} }
// load configuration file if available // load configuration file if available
auto cfg = new config.Config(confdirOption); auto cfg = new config.Config(confdirOption);
if (!cfg.initialize()) { if (!cfg.initialize()) {
@ -147,22 +148,22 @@ int main(string[] args)
// Error message already printed // Error message already printed
return EXIT_FAILURE; return EXIT_FAILURE;
} }
// set memory display // set memory display
displayMemoryUsage = cfg.getValueBool("display_memory"); displayMemoryUsage = cfg.getValueBool("display_memory");
// set display sync options // set display sync options
displaySyncOptions = cfg.getValueBool("display_sync_options"); displaySyncOptions = cfg.getValueBool("display_sync_options");
// update configuration from command line args // update configuration from command line args
cfg.update_from_args(args); cfg.update_from_args(args);
// Initialise normalised file paths // Initialise normalised file paths
configFilePath = buildNormalizedPath(cfg.configDirName ~ "/config"); configFilePath = buildNormalizedPath(cfg.configDirName ~ "/config");
syncListFilePath = buildNormalizedPath(cfg.configDirName ~ "/sync_list"); syncListFilePath = buildNormalizedPath(cfg.configDirName ~ "/sync_list");
databaseFilePath = buildNormalizedPath(cfg.configDirName ~ "/items.db"); databaseFilePath = buildNormalizedPath(cfg.configDirName ~ "/items.db");
businessSharedFolderFilePath = buildNormalizedPath(cfg.configDirName ~ "/business_shared_folders"); businessSharedFolderFilePath = buildNormalizedPath(cfg.configDirName ~ "/business_shared_folders");
// Has any of our configuration that would require a --resync been changed? // Has any of our configuration that would require a --resync been changed?
// 1. sync_list file modification // 1. sync_list file modification
// 2. config file modification - but only if sync_dir, skip_dir, skip_file or drive_id was modified // 2. config file modification - but only if sync_dir, skip_dir, skip_file or drive_id was modified
@ -171,25 +172,25 @@ int main(string[] args)
syncListHashFile = buildNormalizedPath(cfg.configDirName ~ "/.sync_list.hash"); syncListHashFile = buildNormalizedPath(cfg.configDirName ~ "/.sync_list.hash");
configBackupFile = buildNormalizedPath(cfg.configDirName ~ "/.config.backup"); configBackupFile = buildNormalizedPath(cfg.configDirName ~ "/.config.backup");
businessSharedFoldersHashFile = buildNormalizedPath(cfg.configDirName ~ "/.business_shared_folders.hash"); businessSharedFoldersHashFile = buildNormalizedPath(cfg.configDirName ~ "/.business_shared_folders.hash");
// Does a config file exist with a valid hash file // Does a config file exist with a valid hash file
if ((exists(configFilePath)) && (!exists(configHashFile))) { if ((exists(configFilePath)) && (!exists(configHashFile))) {
// Hash of config file needs to be created // Hash of config file needs to be created
std.file.write(configHashFile, computeQuickXorHash(configFilePath)); std.file.write(configHashFile, computeQuickXorHash(configFilePath));
} }
// Does a sync_list file exist with a valid hash file // Does a sync_list file exist with a valid hash file
if ((exists(syncListFilePath)) && (!exists(syncListHashFile))) { if ((exists(syncListFilePath)) && (!exists(syncListHashFile))) {
// Hash of sync_list file needs to be created // Hash of sync_list file needs to be created
std.file.write(syncListHashFile, computeQuickXorHash(syncListFilePath)); std.file.write(syncListHashFile, computeQuickXorHash(syncListFilePath));
} }
// check if business_shared_folders & business_shared_folders hash exists // check if business_shared_folders & business_shared_folders hash exists
if ((exists(businessSharedFolderFilePath)) && (!exists(businessSharedFoldersHashFile))) { if ((exists(businessSharedFolderFilePath)) && (!exists(businessSharedFoldersHashFile))) {
// Hash of business_shared_folders file needs to be created // Hash of business_shared_folders file needs to be created
std.file.write(businessSharedFoldersHashFile, computeQuickXorHash(businessSharedFolderFilePath)); std.file.write(businessSharedFoldersHashFile, computeQuickXorHash(businessSharedFolderFilePath));
} }
// If hash files exist, but config files do not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different // If hash files exist, but config files do not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different
if ((!exists(configFilePath)) && (exists(configHashFile))) { if ((!exists(configFilePath)) && (exists(configHashFile))) {
// if --resync safe remove config.hash and config.backup // if --resync safe remove config.hash and config.backup
@ -198,18 +199,18 @@ int main(string[] args)
safeRemove(configBackupFile); safeRemove(configBackupFile);
} }
} }
// If sync_list hash file exists, but sync_list file does not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different // If sync_list hash file exists, but sync_list file does not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different
if ((!exists(syncListFilePath)) && (exists(syncListHashFile))) { if ((!exists(syncListFilePath)) && (exists(syncListHashFile))) {
// if --resync safe remove sync_list.hash // if --resync safe remove sync_list.hash
if (cfg.getValueBool("resync")) safeRemove(syncListHashFile); if (cfg.getValueBool("resync")) safeRemove(syncListHashFile);
} }
if ((!exists(businessSharedFolderFilePath)) && (exists(businessSharedFoldersHashFile))) { if ((!exists(businessSharedFolderFilePath)) && (exists(businessSharedFoldersHashFile))) {
// if --resync safe remove business_shared_folders.hash // if --resync safe remove business_shared_folders.hash
if (cfg.getValueBool("resync")) safeRemove(businessSharedFoldersHashFile); if (cfg.getValueBool("resync")) safeRemove(businessSharedFoldersHashFile);
} }
// Read config hashes if they exist // Read config hashes if they exist
if (exists(configFilePath)) currentConfigHash = computeQuickXorHash(configFilePath); if (exists(configFilePath)) currentConfigHash = computeQuickXorHash(configFilePath);
if (exists(syncListFilePath)) currentSyncListHash = computeQuickXorHash(syncListFilePath); if (exists(syncListFilePath)) currentSyncListHash = computeQuickXorHash(syncListFilePath);
@ -217,21 +218,21 @@ int main(string[] args)
if (exists(configHashFile)) previousConfigHash = readText(configHashFile); if (exists(configHashFile)) previousConfigHash = readText(configHashFile);
if (exists(syncListHashFile)) previousSyncListHash = readText(syncListHashFile); if (exists(syncListHashFile)) previousSyncListHash = readText(syncListHashFile);
if (exists(businessSharedFoldersHashFile)) previousBusinessSharedFoldersHash = readText(businessSharedFoldersHashFile); if (exists(businessSharedFoldersHashFile)) previousBusinessSharedFoldersHash = readText(businessSharedFoldersHashFile);
// Was sync_list file updated? // Was sync_list file updated?
if (currentSyncListHash != previousSyncListHash) { if (currentSyncListHash != previousSyncListHash) {
// Debugging output to assist what changed // Debugging output to assist what changed
log.vdebug("sync_list file has been updated, --resync needed"); log.vdebug("sync_list file has been updated, --resync needed");
syncListDifferent = true; syncListDifferent = true;
} }
// Was business_shared_folders updated? // Was business_shared_folders updated?
if (currentBusinessSharedFoldersHash != previousBusinessSharedFoldersHash) { if (currentBusinessSharedFoldersHash != previousBusinessSharedFoldersHash) {
// Debugging output to assist what changed // Debugging output to assist what changed
log.vdebug("business_shared_folders file has been updated, --resync needed"); log.vdebug("business_shared_folders file has been updated, --resync needed");
businessSharedFoldersDifferent = true; businessSharedFoldersDifferent = true;
} }
// Was config file updated between last execution ang this execution? // Was config file updated between last execution ang this execution?
if (currentConfigHash != previousConfigHash) { if (currentConfigHash != previousConfigHash) {
// config file was updated, however we only want to trigger a --resync requirement if sync_dir, skip_dir, skip_file or drive_id was modified // config file was updated, however we only want to trigger a --resync requirement if sync_dir, skip_dir, skip_file or drive_id was modified
@ -266,7 +267,7 @@ int main(string[] args)
log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed"); log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed");
configOptionsDifferent = true; configOptionsDifferent = true;
} }
if ((key == "skip_file") && (c.front.dup != cfg.getValueString("skip_file"))){ if ((key == "skip_file") && (c.front.dup != cfg.getValueString("skip_file"))){
log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed"); log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed");
configOptionsDifferent = true; configOptionsDifferent = true;
@ -291,7 +292,7 @@ int main(string[] args)
// no backup to check // no backup to check
log.vdebug("WARNING: no backup config file was found, unable to validate if any changes made"); log.vdebug("WARNING: no backup config file was found, unable to validate if any changes made");
} }
// If there was a backup, any modified values we need to worry about would been detected // If there was a backup, any modified values we need to worry about would been detected
if (!cfg.getValueBool("display_config")) { if (!cfg.getValueBool("display_config")) {
// we are not testing the configuration // we are not testing the configuration
@ -309,13 +310,13 @@ int main(string[] args)
} }
} }
} }
// Is there a backup of the config file if the config file exists? // Is there a backup of the config file if the config file exists?
if ((exists(configFilePath)) && (!exists(configBackupFile))) { if ((exists(configFilePath)) && (!exists(configBackupFile))) {
// create backup copy of current config file // create backup copy of current config file
std.file.copy(configFilePath, configBackupFile); std.file.copy(configFilePath, configBackupFile);
} }
// config file set options can be changed via CLI input, specifically these will impact sync and --resync will be needed: // config file set options can be changed via CLI input, specifically these will impact sync and --resync will be needed:
// --syncdir ARG // --syncdir ARG
// --skip-file ARG // --skip-file ARG
@ -331,7 +332,7 @@ int main(string[] args)
syncDirDifferent = true; syncDirDifferent = true;
} }
} }
// was the skip_file updated by CLI? // was the skip_file updated by CLI?
if (cfg.configFileSkipFile != "") { if (cfg.configFileSkipFile != "") {
// skip_file was set in config file // skip_file was set in config file
@ -340,8 +341,8 @@ int main(string[] args)
log.vdebug("skip_file: CLI override of config file option, --resync needed"); log.vdebug("skip_file: CLI override of config file option, --resync needed");
skipFileDifferent = true; skipFileDifferent = true;
} }
} }
// was the skip_dir updated by CLI? // was the skip_dir updated by CLI?
if (cfg.configFileSkipDir != "") { if (cfg.configFileSkipDir != "") {
// skip_dir was set in config file // skip_dir was set in config file
@ -352,7 +353,7 @@ int main(string[] args)
} }
} }
} }
// Has anything triggered a --resync requirement? // Has anything triggered a --resync requirement?
if (configOptionsDifferent || syncListDifferent || syncDirDifferent || skipFileDifferent || skipDirDifferent || businessSharedFoldersDifferent) { if (configOptionsDifferent || syncListDifferent || syncDirDifferent || skipFileDifferent || skipDirDifferent || businessSharedFoldersDifferent) {
// --resync needed, is the user just testing configuration changes? // --resync needed, is the user just testing configuration changes?
@ -388,7 +389,7 @@ int main(string[] args)
} }
} }
} }
// dry-run notification and database setup // dry-run notification and database setup
if (cfg.getValueBool("dry_run")) { if (cfg.getValueBool("dry_run")) {
log.log("DRY-RUN Configured. Output below shows what 'would' have occurred."); log.log("DRY-RUN Configured. Output below shows what 'would' have occurred.");
@ -398,18 +399,18 @@ int main(string[] args)
if (exists(cfg.databaseFilePathDryRun)) { if (exists(cfg.databaseFilePathDryRun)) {
// remove the existing file // remove the existing file
log.vdebug("Removing items-dryrun.sqlite3 as it still exists for some reason"); log.vdebug("Removing items-dryrun.sqlite3 as it still exists for some reason");
safeRemove(cfg.databaseFilePathDryRun); safeRemove(cfg.databaseFilePathDryRun);
} }
// silent cleanup of shm and wal files if they exist // silent cleanup of shm and wal files if they exist
if (exists(dryRunShmFile)) { if (exists(dryRunShmFile)) {
// remove items-dryrun.sqlite3-shm // remove items-dryrun.sqlite3-shm
safeRemove(dryRunShmFile); safeRemove(dryRunShmFile);
} }
if (exists(dryRunWalFile)) { if (exists(dryRunWalFile)) {
// remove items-dryrun.sqlite3-wal // remove items-dryrun.sqlite3-wal
safeRemove(dryRunWalFile); safeRemove(dryRunWalFile);
} }
// Make a copy of the original items.sqlite3 for use as the dry run copy if it exists // Make a copy of the original items.sqlite3 for use as the dry run copy if it exists
if (exists(cfg.databaseFilePath)) { if (exists(cfg.databaseFilePath)) {
// in a --dry-run --resync scenario, we should not copy the existing database file // in a --dry-run --resync scenario, we should not copy the existing database file
@ -423,7 +424,7 @@ int main(string[] args)
} }
} }
} }
// sync_dir environment handling to handle ~ expansion properly // sync_dir environment handling to handle ~ expansion properly
bool shellEnvSet = false; bool shellEnvSet = false;
if ((environment.get("SHELL") == "") && (environment.get("USER") == "")){ if ((environment.get("SHELL") == "") && (environment.get("USER") == "")){
@ -450,10 +451,10 @@ int main(string[] args)
syncDir = cfg.getValueString("sync_dir"); syncDir = cfg.getValueString("sync_dir");
} }
} }
// vdebug syncDir as set and calculated // vdebug syncDir as set and calculated
log.vdebug("syncDir: ", syncDir); log.vdebug("syncDir: ", syncDir);
// Configure the logging directory if different from application default // Configure the logging directory if different from application default
// log_dir environment handling to handle ~ expansion properly // log_dir environment handling to handle ~ expansion properly
string logDir = cfg.getValueString("log_dir"); string logDir = cfg.getValueString("log_dir");
@ -478,7 +479,7 @@ int main(string[] args)
// update log_dir with normalised path, with '~' expanded correctly // update log_dir with normalised path, with '~' expanded correctly
cfg.setValueString("log_dir", logDir); cfg.setValueString("log_dir", logDir);
} }
// Configure logging only if enabled // Configure logging only if enabled
if (cfg.getValueBool("enable_logging")){ if (cfg.getValueBool("enable_logging")){
// Initialise using the configured logging directory // Initialise using the configured logging directory
@ -488,7 +489,7 @@ int main(string[] args)
// Configure whether notifications are used // Configure whether notifications are used
log.setNotifications(cfg.getValueBool("monitor") && !cfg.getValueBool("disable_notifications")); log.setNotifications(cfg.getValueBool("monitor") && !cfg.getValueBool("disable_notifications"));
// Application upgrades - skilion version etc // Application upgrades - skilion version etc
if (exists(databaseFilePath)) { if (exists(databaseFilePath)) {
if (!cfg.getValueBool("dry_run")) { if (!cfg.getValueBool("dry_run")) {
@ -497,7 +498,7 @@ int main(string[] args)
log.logAndNotify("Database schema changed, resync needed"); log.logAndNotify("Database schema changed, resync needed");
cfg.setValueBool("resync", true); cfg.setValueBool("resync", true);
} }
// Handle --logout as separate item, do not 'resync' on a --logout / reauth // Handle --logout as separate item, do not 'resync' on a --logout / reauth
if (cfg.getValueBool("logout")) { if (cfg.getValueBool("logout")) {
log.vdebug("--logout requested"); log.vdebug("--logout requested");
@ -508,7 +509,7 @@ int main(string[] args)
// Exit // Exit
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }
// Handle --resync to remove local files // Handle --resync to remove local files
if (cfg.getValueBool("resync")) { if (cfg.getValueBool("resync")) {
if (cfg.getValueBool("resync")) log.vdebug("--resync requested"); if (cfg.getValueBool("resync")) log.vdebug("--resync requested");
@ -519,7 +520,7 @@ int main(string[] args)
safeRemove(cfg.uploadStateFilePath); safeRemove(cfg.uploadStateFilePath);
} }
} }
// Display current application configuration, no application initialisation // Display current application configuration, no application initialisation
if (cfg.getValueBool("display_config")){ if (cfg.getValueBool("display_config")){
// Display application version // Display application version
@ -528,7 +529,7 @@ int main(string[] args)
writeln("Config path = ", cfg.configDirName); writeln("Config path = ", cfg.configDirName);
// Does a config file exist or are we using application defaults // Does a config file exist or are we using application defaults
writeln("Config file found in config path = ", exists(configFilePath)); writeln("Config file found in config path = ", exists(configFilePath));
// Config Options // Config Options
writeln("Config option 'check_nosync' = ", cfg.getValueBool("check_nosync")); writeln("Config option 'check_nosync' = ", cfg.getValueBool("check_nosync"));
writeln("Config option 'sync_dir' = ", syncDir); writeln("Config option 'sync_dir' = ", syncDir);
@ -543,12 +544,12 @@ int main(string[] args)
writeln("Config option 'upload_only' = ", cfg.getValueBool("upload_only")); writeln("Config option 'upload_only' = ", cfg.getValueBool("upload_only"));
writeln("Config option 'no_remote_delete' = ", cfg.getValueBool("no_remote_delete")); writeln("Config option 'no_remote_delete' = ", cfg.getValueBool("no_remote_delete"));
writeln("Config option 'remove_source_files' = ", cfg.getValueBool("remove_source_files")); writeln("Config option 'remove_source_files' = ", cfg.getValueBool("remove_source_files"));
// Is config option drive_id configured? // Is config option drive_id configured?
if (cfg.getValueString("drive_id") != ""){ if (cfg.getValueString("drive_id") != ""){
writeln("Config option 'drive_id' = ", cfg.getValueString("drive_id")); writeln("Config option 'drive_id' = ", cfg.getValueString("drive_id"));
} }
// Is sync_list configured? // Is sync_list configured?
if (exists(syncListFilePath)){ if (exists(syncListFilePath)){
writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files")); writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files"));
@ -565,7 +566,7 @@ int main(string[] args)
writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files")); writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files"));
writeln("Selective sync 'sync_list' configured = false"); writeln("Selective sync 'sync_list' configured = false");
} }
// Is business_shared_folders configured // Is business_shared_folders configured
if (exists(businessSharedFolderFilePath)){ if (exists(businessSharedFolderFilePath)){
writeln("Business Shared Folders configured = true"); writeln("Business Shared Folders configured = true");
@ -580,11 +581,11 @@ int main(string[] args)
} else { } else {
writeln("Business Shared Folders configured = false"); writeln("Business Shared Folders configured = false");
} }
// Exit // Exit
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }
// Test if OneDrive service can be reached, exit if it cant be reached // Test if OneDrive service can be reached, exit if it cant be reached
log.vdebug("Testing network to ensure network connectivity to Microsoft OneDrive Service"); log.vdebug("Testing network to ensure network connectivity to Microsoft OneDrive Service");
online = testNetwork(); online = testNetwork();
@ -597,7 +598,7 @@ int main(string[] args)
} else { } else {
// Running as --monitor // Running as --monitor
log.error("Unable to reach Microsoft OneDrive API service at this point in time, re-trying network tests\n"); log.error("Unable to reach Microsoft OneDrive API service at this point in time, re-trying network tests\n");
// re-try network connection to OneDrive // re-try network connection to OneDrive
// https://github.com/abraunegg/onedrive/issues/1184 // https://github.com/abraunegg/onedrive/issues/1184
// Back off & retry with incremental delay // Back off & retry with incremental delay
@ -605,13 +606,13 @@ int main(string[] args)
int retryAttempts = 1; int retryAttempts = 1;
int backoffInterval = 1; int backoffInterval = 1;
int maxBackoffInterval = 3600; int maxBackoffInterval = 3600;
bool retrySuccess = false; bool retrySuccess = false;
while (!retrySuccess){ while (!retrySuccess){
// retry to access OneDrive API // retry to access OneDrive API
backoffInterval++; backoffInterval++;
int thisBackOffInterval = retryAttempts*backoffInterval; int thisBackOffInterval = retryAttempts*backoffInterval;
log.vdebug(" Retry Attempt: ", retryAttempts); log.vdebug(" Retry Attempt: ", retryAttempts);
if (thisBackOffInterval <= maxBackoffInterval) { if (thisBackOffInterval <= maxBackoffInterval) {
log.vdebug(" Retry In (seconds): ", thisBackOffInterval); log.vdebug(" Retry In (seconds): ", thisBackOffInterval);
Thread.sleep(dur!"seconds"(thisBackOffInterval)); Thread.sleep(dur!"seconds"(thisBackOffInterval));
@ -631,7 +632,7 @@ int main(string[] args)
// we have attempted to re-connect X number of times // we have attempted to re-connect X number of times
// false set this to true to break out of while loop // false set this to true to break out of while loop
retrySuccess = true; retrySuccess = true;
} }
} }
// Increment & loop around // Increment & loop around
retryAttempts++; retryAttempts++;
@ -643,7 +644,7 @@ int main(string[] args)
} }
} }
} }
// Initialize OneDrive, check for authorization // Initialize OneDrive, check for authorization
if (online) { if (online) {
// we can only initialise if we are online // we can only initialise if we are online
@ -652,24 +653,24 @@ int main(string[] args)
onedriveInitialised = oneDrive.init(); onedriveInitialised = oneDrive.init();
oneDrive.printAccessToken = cfg.getValueBool("print_token"); oneDrive.printAccessToken = cfg.getValueBool("print_token");
} }
if (!onedriveInitialised) { if (!onedriveInitialised) {
log.error("Could not initialize the OneDrive API"); log.error("Could not initialize the OneDrive API");
// Use exit scopes to shutdown API // Use exit scopes to shutdown API
return EXIT_UNAUTHORIZED; return EXIT_UNAUTHORIZED;
} }
// if --synchronize or --monitor not passed in, configure the flag to display help & exit // if --synchronize or --monitor not passed in, configure the flag to display help & exit
if (cfg.getValueBool("synchronize") || cfg.getValueBool("monitor")) { if (cfg.getValueBool("synchronize") || cfg.getValueBool("monitor")) {
performSyncOK = true; performSyncOK = true;
} }
// create-directory, remove-directory, source-directory, destination-directory // create-directory, remove-directory, source-directory, destination-directory
// these are activities that dont perform a sync, so to not generate an error message for these items either // these are activities that dont perform a sync, so to not generate an error message for these items either
if (((cfg.getValueString("create_directory") != "") || (cfg.getValueString("remove_directory") != "")) || ((cfg.getValueString("source_directory") != "") && (cfg.getValueString("destination_directory") != "")) || (cfg.getValueString("get_file_link") != "") || (cfg.getValueString("create_share_link") != "") || (cfg.getValueString("get_o365_drive_id") != "") || cfg.getValueBool("display_sync_status") || cfg.getValueBool("list_business_shared_folders")) { if (((cfg.getValueString("create_directory") != "") || (cfg.getValueString("remove_directory") != "")) || ((cfg.getValueString("source_directory") != "") && (cfg.getValueString("destination_directory") != "")) || (cfg.getValueString("get_file_link") != "") || (cfg.getValueString("create_share_link") != "") || (cfg.getValueString("get_o365_drive_id") != "") || cfg.getValueBool("display_sync_status") || cfg.getValueBool("list_business_shared_folders")) {
performSyncOK = true; performSyncOK = true;
} }
// Were acceptable sync operations provided? Was --synchronize or --monitor passed in // Were acceptable sync operations provided? Was --synchronize or --monitor passed in
if (!performSyncOK) { if (!performSyncOK) {
// was the application just authorised? // was the application just authorised?
@ -694,7 +695,7 @@ int main(string[] args)
return EXIT_FAILURE; return EXIT_FAILURE;
} }
} }
// if --synchronize && --monitor passed in, exit & display help as these conflict with each other // if --synchronize && --monitor passed in, exit & display help as these conflict with each other
if (cfg.getValueBool("synchronize") && cfg.getValueBool("monitor")) { if (cfg.getValueBool("synchronize") && cfg.getValueBool("monitor")) {
writeln("\nERROR: --synchronize and --monitor cannot be used together\n"); writeln("\nERROR: --synchronize and --monitor cannot be used together\n");
@ -702,7 +703,7 @@ int main(string[] args)
// Use exit scopes to shutdown API // Use exit scopes to shutdown API
return EXIT_FAILURE; return EXIT_FAILURE;
} }
// Initialize the item database // Initialize the item database
log.vlog("Opening the item database ..."); log.vlog("Opening the item database ...");
if (!cfg.getValueBool("dry_run")) { if (!cfg.getValueBool("dry_run")) {
@ -714,7 +715,7 @@ int main(string[] args)
log.vdebug("Using database file: ", asNormalizedPath(cfg.databaseFilePathDryRun)); log.vdebug("Using database file: ", asNormalizedPath(cfg.databaseFilePathDryRun));
itemDb = new ItemDatabase(cfg.databaseFilePathDryRun); itemDb = new ItemDatabase(cfg.databaseFilePathDryRun);
} }
// What are the permission that have been set for the application? // What are the permission that have been set for the application?
// These are relevant for: // These are relevant for:
// - The ~/OneDrive parent folder or 'sync_dir' configured item // - The ~/OneDrive parent folder or 'sync_dir' configured item
@ -731,7 +732,7 @@ int main(string[] args)
log.vdebug("Configuring default new file permissions as: ", cfg.getValueLong("sync_file_permissions")); log.vdebug("Configuring default new file permissions as: ", cfg.getValueLong("sync_file_permissions"));
cfg.configureRequiredFilePermisions(); cfg.configureRequiredFilePermisions();
} }
// configure the sync direcory based on syncDir config option // configure the sync direcory based on syncDir config option
log.vlog("All operations will be performed in: ", syncDir); log.vlog("All operations will be performed in: ", syncDir);
if (!exists(syncDir)) { if (!exists(syncDir)) {
@ -749,13 +750,13 @@ int main(string[] args)
return EXIT_FAILURE; return EXIT_FAILURE;
} }
} }
// Change the working directory to the 'sync_dir' configured item // Change the working directory to the 'sync_dir' configured item
chdir(syncDir); chdir(syncDir);
// Configure selective sync by parsing and getting a regex for skip_file config component // Configure selective sync by parsing and getting a regex for skip_file config component
auto selectiveSync = new SelectiveSync(); auto selectiveSync = new SelectiveSync();
// load sync_list if it exists // load sync_list if it exists
if (exists(syncListFilePath)){ if (exists(syncListFilePath)){
log.vdebug("Loading user configured sync_list file ..."); log.vdebug("Loading user configured sync_list file ...");
@ -774,7 +775,7 @@ int main(string[] args)
} }
} }
selectiveSync.load(syncListFilePath); selectiveSync.load(syncListFilePath);
// load business_shared_folders if it exists // load business_shared_folders if it exists
if (exists(businessSharedFolderFilePath)){ if (exists(businessSharedFolderFilePath)){
log.vdebug("Loading user configured business_shared_folders file ..."); log.vdebug("Loading user configured business_shared_folders file ...");
@ -787,27 +788,27 @@ int main(string[] args)
} }
} }
selectiveSync.loadSharedFolders(businessSharedFolderFilePath); selectiveSync.loadSharedFolders(businessSharedFolderFilePath);
// Configure skip_dir, skip_file, skip-dir-strict-match & skip_dotfiles from config entries // Configure skip_dir, skip_file, skip-dir-strict-match & skip_dotfiles from config entries
// Handle skip_dir configuration in config file // Handle skip_dir configuration in config file
log.vdebug("Configuring skip_dir ..."); log.vdebug("Configuring skip_dir ...");
log.vdebug("skip_dir: ", cfg.getValueString("skip_dir")); log.vdebug("skip_dir: ", cfg.getValueString("skip_dir"));
selectiveSync.setDirMask(cfg.getValueString("skip_dir")); selectiveSync.setDirMask(cfg.getValueString("skip_dir"));
// Was --skip-dir-strict-match configured? // Was --skip-dir-strict-match configured?
log.vdebug("Configuring skip_dir_strict_match ..."); log.vdebug("Configuring skip_dir_strict_match ...");
log.vdebug("skip_dir_strict_match: ", cfg.getValueBool("skip_dir_strict_match")); log.vdebug("skip_dir_strict_match: ", cfg.getValueBool("skip_dir_strict_match"));
if (cfg.getValueBool("skip_dir_strict_match")) { if (cfg.getValueBool("skip_dir_strict_match")) {
selectiveSync.setSkipDirStrictMatch(); selectiveSync.setSkipDirStrictMatch();
} }
// Was --skip-dot-files configured? // Was --skip-dot-files configured?
log.vdebug("Configuring skip_dotfiles ..."); log.vdebug("Configuring skip_dotfiles ...");
log.vdebug("skip_dotfiles: ", cfg.getValueBool("skip_dotfiles")); log.vdebug("skip_dotfiles: ", cfg.getValueBool("skip_dotfiles"));
if (cfg.getValueBool("skip_dotfiles")) { if (cfg.getValueBool("skip_dotfiles")) {
selectiveSync.setSkipDotfiles(); selectiveSync.setSkipDotfiles();
} }
// Handle skip_file configuration in config file // Handle skip_file configuration in config file
log.vdebug("Configuring skip_file ..."); log.vdebug("Configuring skip_file ...");
// Validate skip_file to ensure that this does not contain an invalid configuration // Validate skip_file to ensure that this does not contain an invalid configuration
@ -822,7 +823,7 @@ int main(string[] args)
// All skip_file entries are valid // All skip_file entries are valid
log.vdebug("skip_file: ", cfg.getValueString("skip_file")); log.vdebug("skip_file: ", cfg.getValueString("skip_file"));
selectiveSync.setFileMask(cfg.getValueString("skip_file")); selectiveSync.setFileMask(cfg.getValueString("skip_file"));
// Initialize the sync engine // Initialize the sync engine
auto sync = new SyncEngine(cfg, oneDrive, itemDb, selectiveSync); auto sync = new SyncEngine(cfg, oneDrive, itemDb, selectiveSync);
try { try {
@ -847,7 +848,7 @@ int main(string[] args)
if (syncListConfigured) { if (syncListConfigured) {
sync.setSyncListConfigured(); sync.setSyncListConfigured();
} }
// Do we need to configure specific --upload-only options? // Do we need to configure specific --upload-only options?
if (cfg.getValueBool("upload_only")) { if (cfg.getValueBool("upload_only")) {
// --upload-only was passed in or configured // --upload-only was passed in or configured
@ -866,17 +867,17 @@ int main(string[] args)
sync.setLocalDeleteAfterUpload(); sync.setLocalDeleteAfterUpload();
} }
} }
// Do we configure to disable the upload validation routine // Do we configure to disable the upload validation routine
if (cfg.getValueBool("disable_upload_validation")) sync.setDisableUploadValidation(); if (cfg.getValueBool("disable_upload_validation")) sync.setDisableUploadValidation();
// Has the user enabled to bypass data preservation of renaming local files when there is a conflict? // Has the user enabled to bypass data preservation of renaming local files when there is a conflict?
if (cfg.getValueBool("bypass_data_preservation")) { if (cfg.getValueBool("bypass_data_preservation")) {
log.log("WARNING: Application has been configured to bypass local data preservation in the event of file conflict."); log.log("WARNING: Application has been configured to bypass local data preservation in the event of file conflict.");
log.log("WARNING: Local data loss MAY occur in this scenario."); log.log("WARNING: Local data loss MAY occur in this scenario.");
sync.setBypassDataPreservation(); sync.setBypassDataPreservation();
} }
// Are we configured to use a National Cloud Deployment // Are we configured to use a National Cloud Deployment
if (cfg.getValueString("azure_ad_endpoint") != "") { if (cfg.getValueString("azure_ad_endpoint") != "") {
// value is configured, is it a valid value? // value is configured, is it a valid value?
@ -885,7 +886,7 @@ int main(string[] args)
sync.setNationalCloudDeployment(); sync.setNationalCloudDeployment();
} }
} }
// Do we need to validate the syncDir to check for the presence of a '.nosync' file // Do we need to validate the syncDir to check for the presence of a '.nosync' file
if (cfg.getValueBool("check_nomount")) { if (cfg.getValueBool("check_nomount")) {
// we were asked to check the mounts // we were asked to check the mounts
@ -895,53 +896,53 @@ int main(string[] args)
return EXIT_FAILURE; return EXIT_FAILURE;
} }
} }
// Do we need to create or remove a directory? // Do we need to create or remove a directory?
if ((cfg.getValueString("create_directory") != "") || (cfg.getValueString("remove_directory") != "")) { if ((cfg.getValueString("create_directory") != "") || (cfg.getValueString("remove_directory") != "")) {
if (cfg.getValueString("create_directory") != "") { if (cfg.getValueString("create_directory") != "") {
// create a directory on OneDrive // create a directory on OneDrive
sync.createDirectoryNoSync(cfg.getValueString("create_directory")); sync.createDirectoryNoSync(cfg.getValueString("create_directory"));
} }
if (cfg.getValueString("remove_directory") != "") { if (cfg.getValueString("remove_directory") != "") {
// remove a directory on OneDrive // remove a directory on OneDrive
sync.deleteDirectoryNoSync(cfg.getValueString("remove_directory")); sync.deleteDirectoryNoSync(cfg.getValueString("remove_directory"));
} }
} }
// Are we renaming or moving a directory? // Are we renaming or moving a directory?
if ((cfg.getValueString("source_directory") != "") && (cfg.getValueString("destination_directory") != "")) { if ((cfg.getValueString("source_directory") != "") && (cfg.getValueString("destination_directory") != "")) {
// We are renaming or moving a directory // We are renaming or moving a directory
sync.renameDirectoryNoSync(cfg.getValueString("source_directory"), cfg.getValueString("destination_directory")); sync.renameDirectoryNoSync(cfg.getValueString("source_directory"), cfg.getValueString("destination_directory"));
} }
// Are we obtaining the Office 365 Drive ID for a given Office 365 SharePoint Shared Library? // Are we obtaining the Office 365 Drive ID for a given Office 365 SharePoint Shared Library?
if (cfg.getValueString("get_o365_drive_id") != "") { if (cfg.getValueString("get_o365_drive_id") != "") {
sync.querySiteCollectionForDriveID(cfg.getValueString("get_o365_drive_id")); sync.querySiteCollectionForDriveID(cfg.getValueString("get_o365_drive_id"));
// Exit application // Exit application
// Use exit scopes to shutdown API // Use exit scopes to shutdown API
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }
// Are we createing an anonymous read-only shareable link for an existing file on OneDrive? // Are we createing an anonymous read-only shareable link for an existing file on OneDrive?
if (cfg.getValueString("create_share_link") != "") { if (cfg.getValueString("create_share_link") != "") {
// Query OneDrive for the file, and if valid, create a shareable link for the file // Query OneDrive for the file, and if valid, create a shareable link for the file
sync.createShareableLinkForFile(cfg.getValueString("create_share_link")); sync.createShareableLinkForFile(cfg.getValueString("create_share_link"));
// Exit application // Exit application
// Use exit scopes to shutdown API // Use exit scopes to shutdown API
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }
// Are we obtaining the URL path for a synced file? // Are we obtaining the URL path for a synced file?
if (cfg.getValueString("get_file_link") != "") { if (cfg.getValueString("get_file_link") != "") {
// Query OneDrive for the file link // Query OneDrive for the file link
sync.queryOneDriveForFileURL(cfg.getValueString("get_file_link"), syncDir); sync.queryOneDriveForFileURL(cfg.getValueString("get_file_link"), syncDir);
// Exit application // Exit application
// Use exit scopes to shutdown API // Use exit scopes to shutdown API
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }
// Are we listing OneDrive Business Shared Folders // Are we listing OneDrive Business Shared Folders
if (cfg.getValueBool("list_business_shared_folders")) { if (cfg.getValueBool("list_business_shared_folders")) {
// Is this a business account type? // Is this a business account type?
@ -951,11 +952,11 @@ int main(string[] args)
} else { } else {
log.error("ERROR: Unsupported account type for listing OneDrive Business Shared Folders"); log.error("ERROR: Unsupported account type for listing OneDrive Business Shared Folders");
} }
// Exit application // Exit application
// Use exit scopes to shutdown API // Use exit scopes to shutdown API
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }
// Are we going to sync OneDrive Business Shared Folders // Are we going to sync OneDrive Business Shared Folders
if (cfg.getValueBool("sync_business_shared_folders")) { if (cfg.getValueBool("sync_business_shared_folders")) {
// Is this a business account type? // Is this a business account type?
@ -966,7 +967,7 @@ int main(string[] args)
log.error("ERROR: Unsupported account type for syncing OneDrive Business Shared Folders"); log.error("ERROR: Unsupported account type for syncing OneDrive Business Shared Folders");
} }
} }
// Are we displaying the sync status of the client? // Are we displaying the sync status of the client?
if (cfg.getValueBool("display_sync_status")) { if (cfg.getValueBool("display_sync_status")) {
string remotePath = "/"; string remotePath = "/";
@ -977,7 +978,7 @@ int main(string[] args)
} }
sync.queryDriveForChanges(remotePath); sync.queryDriveForChanges(remotePath);
} }
// Are we performing a sync, or monitor operation? // Are we performing a sync, or monitor operation?
if ((cfg.getValueBool("synchronize")) || (cfg.getValueBool("monitor"))) { if ((cfg.getValueBool("synchronize")) || (cfg.getValueBool("monitor"))) {
// Initialise the monitor class, so that we can do more granular inotify handling when performing the actual sync // Initialise the monitor class, so that we can do more granular inotify handling when performing the actual sync
@ -1004,17 +1005,17 @@ int main(string[] args)
// fullScanRequired = false, for final true-up // fullScanRequired = false, for final true-up
// but if we have sync_list configured, use syncListConfigured which = true // but if we have sync_list configured, use syncListConfigured which = true
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, false, syncListConfigured, displaySyncOptions, cfg.getValueBool("monitor"), m); performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, false, syncListConfigured, displaySyncOptions, cfg.getValueBool("monitor"), m);
// Write WAL and SHM data to file for this sync // Write WAL and SHM data to file for this sync
log.vdebug("Merge contents of WAL and SHM files into main database file"); log.vdebug("Merge contents of WAL and SHM files into main database file");
itemDb.performVacuum(); itemDb.performVacuum();
} }
} }
if (cfg.getValueBool("monitor")) { if (cfg.getValueBool("monitor")) {
log.logAndNotify("Initializing monitor ..."); log.logAndNotify("Initializing monitor ...");
log.log("OneDrive monitor interval (seconds): ", cfg.getValueLong("monitor_interval")); log.log("OneDrive monitor interval (seconds): ", cfg.getValueLong("monitor_interval"));
m.onDirCreated = delegate(string path) { m.onDirCreated = delegate(string path) {
// Handle .folder creation if skip_dotfiles is enabled // Handle .folder creation if skip_dotfiles is enabled
if ((cfg.getValueBool("skip_dotfiles")) && (selectiveSync.isDotFile(path))) { if ((cfg.getValueBool("skip_dotfiles")) && (selectiveSync.isDotFile(path))) {
@ -1103,7 +1104,8 @@ int main(string[] args)
// sync list is configured // sync list is configured
syncListConfiguredFullScanOverride = true; syncListConfiguredFullScanOverride = true;
} }
immutable bool webhookEnabled = cfg.getValueBool("webhook_enabled");
while (performMonitor) { while (performMonitor) {
if (!cfg.getValueBool("download_only")) { if (!cfg.getValueBool("download_only")) {
try { try {
@ -1113,7 +1115,35 @@ int main(string[] args)
log.error("ERROR: The following inotify error was generated: ", e.msg); log.error("ERROR: The following inotify error was generated: ", e.msg);
} }
} }
// Check for notifications pushed from Microsoft to the webhook
bool notificationReceived = false;
if (webhookEnabled) {
// Create a subscription on the first run, or renew the subscription
// on subsequent runs when it is about to expire.
oneDrive.createOrRenewSubscription();
// Process incoming notifications if any.
// Empirical evidence shows that Microsoft often sends multiple
// notifications for one single change, so we need a loop to exhaust
// all signals that were queued up by the webhook. The notifications
// do not contain any actual changes, and we will always rely do the
// delta endpoint to sync to latest. Therefore, only one sync run is
// good enough to catch up for multiple notifications.
for (int signalCount = 0;; signalCount++) {
const auto signalExists = receiveTimeout(dur!"seconds"(-1), (ulong _) => {});
if (signalExists) {
notificationReceived = true;
} else {
if (notificationReceived) {
log.log("Received ", signalCount," refresh signals from the webhook");
}
break;
}
}
}
auto currTime = MonoTime.currTime(); auto currTime = MonoTime.currTime();
// has monitor_interval elapsed or are we at application startup / monitor startup? // has monitor_interval elapsed or are we at application startup / monitor startup?
// in a --resync scenario, if we have not 're-populated' the database, valid changes will get skipped: // in a --resync scenario, if we have not 're-populated' the database, valid changes will get skipped:
@ -1123,7 +1153,7 @@ int main(string[] args)
// Moving random_files/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby to target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby // Moving random_files/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby to target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
// Skipping uploading this new file as parent path is not in the database: target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby // Skipping uploading this new file as parent path is not in the database: target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
// 'target' should be in the DB, it should also exist online, but because of --resync, it does not exist in the database thus parent check fails // 'target' should be in the DB, it should also exist online, but because of --resync, it does not exist in the database thus parent check fails
if ((currTime - lastCheckTime > checkInterval) || (monitorLoopFullCount == 0)) { if (notificationReceived || (currTime - lastCheckTime > checkInterval) || (monitorLoopFullCount == 0)) {
// monitor sync loop // monitor sync loop
logOutputMessage = "################################################## NEW LOOP ##################################################"; logOutputMessage = "################################################## NEW LOOP ##################################################";
if (displaySyncOptions) { if (displaySyncOptions) {
@ -1137,7 +1167,7 @@ int main(string[] args)
if (displayMemoryUsage) { if (displayMemoryUsage) {
log.displayMemoryUsagePreGC(); log.displayMemoryUsagePreGC();
} }
// log monitor output suppression // log monitor output suppression
logMonitorCounter += 1; logMonitorCounter += 1;
if (logMonitorCounter > logInterval) { if (logMonitorCounter > logInterval) {
@ -1159,7 +1189,7 @@ int main(string[] args)
fullScanRequired = false; fullScanRequired = false;
} }
} }
if (displaySyncOptions) { if (displaySyncOptions) {
// sync option handling per sync loop // sync option handling per sync loop
log.log("fullScanCounter = ", fullScanCounter); log.log("fullScanCounter = ", fullScanCounter);
@ -1219,14 +1249,14 @@ int main(string[] args)
if (displayMemoryUsage) { if (displayMemoryUsage) {
log.displayMemoryUsagePostGC(); log.displayMemoryUsagePostGC();
} }
// Write WAL and SHM data to file for this loop // Write WAL and SHM data to file for this loop
log.vdebug("Merge contents of WAL and SHM files into main database file"); log.vdebug("Merge contents of WAL and SHM files into main database file");
itemDb.performVacuum(); itemDb.performVacuum();
// monitor loop complete // monitor loop complete
logOutputMessage = "################################################ LOOP COMPLETE ###############################################"; logOutputMessage = "################################################ LOOP COMPLETE ###############################################";
// Handle display options // Handle display options
if (displaySyncOptions) { if (displaySyncOptions) {
log.log(logOutputMessage); log.log(logOutputMessage);
@ -1255,20 +1285,20 @@ int main(string[] args)
// remove the file // remove the file
log.vdebug("Removing items-dryrun.sqlite3 as dry run operations complete"); log.vdebug("Removing items-dryrun.sqlite3 as dry run operations complete");
// remove items-dryrun.sqlite3 // remove items-dryrun.sqlite3
safeRemove(cfg.databaseFilePathDryRun); safeRemove(cfg.databaseFilePathDryRun);
} }
// silent cleanup of shm and wal files if they exist // silent cleanup of shm and wal files if they exist
if (exists(dryRunShmFile)) { if (exists(dryRunShmFile)) {
// remove items-dryrun.sqlite3-shm // remove items-dryrun.sqlite3-shm
safeRemove(dryRunShmFile); safeRemove(dryRunShmFile);
} }
if (exists(dryRunWalFile)) { if (exists(dryRunWalFile)) {
// remove items-dryrun.sqlite3-wal // remove items-dryrun.sqlite3-wal
safeRemove(dryRunWalFile); safeRemove(dryRunWalFile);
} }
} }
// Exit application // Exit application
// Use exit scopes to shutdown API // Use exit scopes to shutdown API
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }
@ -1298,14 +1328,14 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
string remotePath = "/"; string remotePath = "/";
string localPath = "."; string localPath = ".";
string logOutputMessage; string logOutputMessage;
// performSync API scan triggers // performSync API scan triggers
log.vdebug("performSync API scan triggers"); log.vdebug("performSync API scan triggers");
log.vdebug("-----------------------------"); log.vdebug("-----------------------------");
log.vdebug("fullScanRequired = ", fullScanRequired); log.vdebug("fullScanRequired = ", fullScanRequired);
log.vdebug("syncListConfiguredFullScanOverride = ", syncListConfiguredFullScanOverride); log.vdebug("syncListConfiguredFullScanOverride = ", syncListConfiguredFullScanOverride);
log.vdebug("-----------------------------"); log.vdebug("-----------------------------");
// Are we doing a single directory sync? // Are we doing a single directory sync?
if (singleDirectory != ""){ if (singleDirectory != ""){
// Need two different path strings here // Need two different path strings here
@ -1314,12 +1344,12 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// Set flag for singleDirectoryScope for change handling // Set flag for singleDirectoryScope for change handling
sync.setSingleDirectoryScope(); sync.setSingleDirectoryScope();
} }
// Due to Microsoft Sharepoint 'enrichment' of files, we try to download the Microsoft modified file automatically // Due to Microsoft Sharepoint 'enrichment' of files, we try to download the Microsoft modified file automatically
// Set flag if we are in upload only state to handle this differently // Set flag if we are in upload only state to handle this differently
// See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details // See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details
if (uploadOnly) sync.setUploadOnly(); if (uploadOnly) sync.setUploadOnly();
do { do {
try { try {
// starting a sync // starting a sync
@ -1380,7 +1410,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
} else { } else {
// sync from OneDrive first before uploading files to OneDrive // sync from OneDrive first before uploading files to OneDrive
if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ..."); if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ...");
// For the initial sync, always use the delta link so that we capture all the right delta changes including adds, moves & deletes // For the initial sync, always use the delta link so that we capture all the right delta changes including adds, moves & deletes
logOutputMessage = "Initial Scan: Call OneDrive Delta API for delta changes as compared to last successful sync."; logOutputMessage = "Initial Scan: Call OneDrive Delta API for delta changes as compared to last successful sync.";
syncCallLogOutput = "Calling sync.applyDifferences(false);"; syncCallLogOutput = "Calling sync.applyDifferences(false);";
@ -1392,8 +1422,8 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
log.vdebug(syncCallLogOutput); log.vdebug(syncCallLogOutput);
} }
sync.applyDifferences(false); sync.applyDifferences(false);
// is this a download only request? // is this a download only request?
if (!downloadOnly) { if (!downloadOnly) {
// process local changes walking the entire path checking for changes // process local changes walking the entire path checking for changes
// in monitor mode all local changes are captured via inotify // in monitor mode all local changes are captured via inotify
@ -1407,7 +1437,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
log.vdebug(logOutputMessage); log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput); log.vdebug(syncCallLogOutput);
} }
// What sort of local scan do we want to do? // What sort of local scan do we want to do?
// In --monitor mode, when performing the DB scan, a race condition occurs where by if a file or folder is moved during this process // In --monitor mode, when performing the DB scan, a race condition occurs where by if a file or folder is moved during this process
// the inotify event is discarded once performSync() is finished (see m.update(false) above), so these events need to be handled // the inotify event is discarded once performSync() is finished (see m.update(false) above), so these events need to be handled
@ -1428,20 +1458,20 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
// handle any inotify events that occured 'whilst' we were scanning the local filesystem // handle any inotify events that occured 'whilst' we were scanning the local filesystem
m.update(true); m.update(true);
} }
// At this point, all OneDrive changes / local changes should be uploaded and in sync // At this point, all OneDrive changes / local changes should be uploaded and in sync
// This MAY not be the case when using sync_list, thus a full walk of OneDrive ojects is required // This MAY not be the case when using sync_list, thus a full walk of OneDrive ojects is required
// --synchronize & no sync_list : fullScanRequired = false, syncListConfiguredFullScanOverride = false // --synchronize & no sync_list : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// --synchronize & sync_list in use : fullScanRequired = false, syncListConfiguredFullScanOverride = true // --synchronize & sync_list in use : fullScanRequired = false, syncListConfiguredFullScanOverride = true
// --monitor loops around 10 iterations. On the 1st loop, sets fullScanRequired = false, syncListConfiguredFullScanOverride = true if requried // --monitor loops around 10 iterations. On the 1st loop, sets fullScanRequired = false, syncListConfiguredFullScanOverride = true if requried
// --monitor & no sync_list (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = false // --monitor & no sync_list (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = false
// --monitor & no sync_list (loop #2 - #10) : fullScanRequired = false, syncListConfiguredFullScanOverride = false // --monitor & no sync_list (loop #2 - #10) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// --monitor & sync_list in use (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = true // --monitor & sync_list in use (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = true
// --monitor & sync_list in use (loop #2 - #10) : fullScanRequired = false, syncListConfiguredFullScanOverride = false // --monitor & sync_list in use (loop #2 - #10) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
// Do not perform a full walk of the OneDrive objects // Do not perform a full walk of the OneDrive objects
if ((!fullScanRequired) && (!syncListConfiguredFullScanOverride)){ if ((!fullScanRequired) && (!syncListConfiguredFullScanOverride)){
logOutputMessage = "Final True-Up: Do not perform a full walk of the OneDrive objects - not required"; logOutputMessage = "Final True-Up: Do not perform a full walk of the OneDrive objects - not required";
@ -1455,7 +1485,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
} }
sync.applyDifferences(false); sync.applyDifferences(false);
} }
// Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop // Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop
if ((!fullScanRequired) && (syncListConfiguredFullScanOverride)){ if ((!fullScanRequired) && (syncListConfiguredFullScanOverride)){
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop"; logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop";
@ -1469,7 +1499,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
} }
sync.applyDifferences(true); sync.applyDifferences(true);
} }
// Perform a full walk of OneDrive objects because a full scan was required // Perform a full walk of OneDrive objects because a full scan was required
if ((fullScanRequired) && (!syncListConfiguredFullScanOverride)){ if ((fullScanRequired) && (!syncListConfiguredFullScanOverride)){
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required"; logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required";
@ -1480,10 +1510,10 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
} else { } else {
log.vdebug(logOutputMessage); log.vdebug(logOutputMessage);
log.vdebug(syncCallLogOutput); log.vdebug(syncCallLogOutput);
} }
sync.applyDifferences(true); sync.applyDifferences(true);
} }
// Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop // Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop
if ((fullScanRequired) && (syncListConfiguredFullScanOverride)){ if ((fullScanRequired) && (syncListConfiguredFullScanOverride)){
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop"; logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop";
@ -1501,7 +1531,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
} }
} }
} }
// sync is complete // sync is complete
logOutputMessage = "################################################ SYNC COMPLETE ###############################################"; logOutputMessage = "################################################ SYNC COMPLETE ###############################################";
if (displaySyncOptions) { if (displaySyncOptions) {
@ -1509,13 +1539,13 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
} else { } else {
log.vdebug(logOutputMessage); log.vdebug(logOutputMessage);
} }
count = -1; count = -1;
} catch (Exception e) { } catch (Exception e) {
if (++count == 3) { if (++count == 3) {
log.log("Giving up on sync after three attempts: ", e.msg); log.log("Giving up on sync after three attempts: ", e.msg);
throw e; throw e;
} else } else
log.log("Retry sync count: ", count, ": ", e.msg); log.log("Retry sync count: ", count, ": ", e.msg);
} }
} while (count != -1); } while (count != -1);
@ -1532,14 +1562,19 @@ auto assumeNoGC(T) (T t) if (isFunctionPointer!T || isDelegate!T)
extern(C) nothrow @nogc @system void exitHandler(int value) { extern(C) nothrow @nogc @system void exitHandler(int value) {
try { try {
assumeNoGC ( () { assumeNoGC ( () {
log.log("Got termination signal, shutting down db connection"); log.log("Got termination signal, performing clean up");
// if initialised, shut down the HTTP instance
if (onedriveInitialised) {
log.log("Shutting down the HTTP instance");
oneDrive.shutdown();
}
// was itemDb initialised? // was itemDb initialised?
if (itemDb !is null) { if (itemDb !is null) {
// Make sure the .wal file is incorporated into the main db before we exit // Make sure the .wal file is incorporated into the main db before we exit
log.log("Shutting down db connection");
itemDb.performVacuum(); itemDb.performVacuum();
destroy(itemDb); destroy(itemDb);
} }
// Use exit scopes to shutdown OneDrive API
})(); })();
} catch(Exception e) {} } catch(Exception e) {}
exit(0); exit(0);

401
src/onedrive.d
View file

@ -1,14 +1,17 @@
import std.net.curl; import std.net.curl;
import etc.c.curl: CurlOption; import etc.c.curl: CurlOption;
import std.datetime, std.exception, std.file, std.json, std.path; import std.datetime, std.datetime.systime, std.exception, std.file, std.json, std.path;
import std.stdio, std.string, std.uni, std.uri, std.file; import std.stdio, std.string, std.uni, std.uri, std.file, std.uuid;
import std.array: split; import std.array: split;
import core.atomic : atomicOp;
import core.stdc.stdlib; import core.stdc.stdlib;
import core.thread, std.conv, std.math; import core.thread, std.conv, std.math;
import std.algorithm.searching; import std.algorithm.searching;
import std.concurrency;
import progress; import progress;
import config; import config;
import util; import util;
import arsd.cgi;
static import log; static import log;
shared bool debugResponse = false; shared bool debugResponse = false;
private bool dryRun = false; private bool dryRun = false;
@ -23,22 +26,22 @@ private immutable {
// Global & Defaults // Global & Defaults
string globalAuthEndpoint = "https://login.microsoftonline.com"; string globalAuthEndpoint = "https://login.microsoftonline.com";
string globalGraphEndpoint = "https://graph.microsoft.com"; string globalGraphEndpoint = "https://graph.microsoft.com";
// US Government L4 // US Government L4
string usl4AuthEndpoint = "https://login.microsoftonline.us"; string usl4AuthEndpoint = "https://login.microsoftonline.us";
string usl4GraphEndpoint = "https://graph.microsoft.us"; string usl4GraphEndpoint = "https://graph.microsoft.us";
// US Government L5 // US Government L5
string usl5AuthEndpoint = "https://login.microsoftonline.us"; string usl5AuthEndpoint = "https://login.microsoftonline.us";
string usl5GraphEndpoint = "https://dod-graph.microsoft.us"; string usl5GraphEndpoint = "https://dod-graph.microsoft.us";
// Germany // Germany
string deAuthEndpoint = "https://login.microsoftonline.de"; string deAuthEndpoint = "https://login.microsoftonline.de";
string deGraphEndpoint = "https://graph.microsoft.de"; string deGraphEndpoint = "https://graph.microsoft.de";
// China // China
string cnAuthEndpoint = "https://login.chinacloudapi.cn"; string cnAuthEndpoint = "https://login.chinacloudapi.cn";
string cnGraphEndpoint = "https://microsoftgraph.chinacloudapi.cn"; string cnGraphEndpoint = "https://microsoftgraph.chinacloudapi.cn";
} }
private { private {
@ -53,27 +56,30 @@ private {
// Default Drive ID // Default Drive ID
string driveId = ""; string driveId = "";
// API Query URL's, based on using defaults, but can be updated by config option 'azure_ad_endpoint' // API Query URL's, based on using defaults, but can be updated by config option 'azure_ad_endpoint'
// Authentication // Authentication
string authUrl = globalAuthEndpoint ~ "/common/oauth2/v2.0/authorize"; string authUrl = globalAuthEndpoint ~ "/common/oauth2/v2.0/authorize";
string redirectUrl = globalAuthEndpoint ~ "/common/oauth2/nativeclient"; string redirectUrl = globalAuthEndpoint ~ "/common/oauth2/nativeclient";
string tokenUrl = globalAuthEndpoint ~ "/common/oauth2/v2.0/token"; string tokenUrl = globalAuthEndpoint ~ "/common/oauth2/v2.0/token";
// Drive Queries // Drive Queries
string driveUrl = globalGraphEndpoint ~ "/v1.0/me/drive"; string driveUrl = globalGraphEndpoint ~ "/v1.0/me/drive";
string driveByIdUrl = globalGraphEndpoint ~ "/v1.0/drives/"; string driveByIdUrl = globalGraphEndpoint ~ "/v1.0/drives/";
// What is 'shared with me' Query // What is 'shared with me' Query
string sharedWithMeUrl = globalGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe"; string sharedWithMeUrl = globalGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
// Item Queries // Item Queries
string itemByIdUrl = globalGraphEndpoint ~ "/v1.0/me/drive/items/"; string itemByIdUrl = globalGraphEndpoint ~ "/v1.0/me/drive/items/";
string itemByPathUrl = globalGraphEndpoint ~ "/v1.0/me/drive/root:/"; string itemByPathUrl = globalGraphEndpoint ~ "/v1.0/me/drive/root:/";
// Office 365 / SharePoint Queries // Office 365 / SharePoint Queries
string siteSearchUrl = globalGraphEndpoint ~ "/v1.0/sites?search"; string siteSearchUrl = globalGraphEndpoint ~ "/v1.0/sites?search";
string siteDriveUrl = globalGraphEndpoint ~ "/v1.0/sites/"; string siteDriveUrl = globalGraphEndpoint ~ "/v1.0/sites/";
// Subscriptions
string subscriptionUrl = globalGraphEndpoint ~ "/v1.0/subscriptions";
} }
class OneDriveException: Exception class OneDriveException: Exception
@ -99,12 +105,64 @@ class OneDriveException: Exception
} }
} }
shared class OneDriveWebhook {
private string host;
private ushort port;
private uint count;
this(string host, ushort port) {
this.host = host;
this.port = port;
this.count = 0;
}
// The static serve() is necessary because spawn() does not like instance methods
static serve(shared OneDriveWebhook webhook, Tid parentTid) {
webhook.serve(parentTid);
}
private void serve(Tid parentTid) {
auto server = new RequestServer(host, port);
server.serveEmbeddedHttp!(cgi => handle(cgi, parentTid))();
}
private void handle(Cgi cgi, Tid parentTid) {
if (.debugResponse) {
log.log("Webhook request: ", cgi.requestMethod, " ", cgi.requestUri);
if (!cgi.postBody.empty) {
log.log("Webhook post body: ", cgi.postBody);
}
}
cgi.setResponseContentType("text/plain");
if ("validationToken" in cgi.get) {
// For validation requests, respond with the validation token passed in the query string
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/webhook-receiver-validation-request
cgi.write(cgi.get["validationToken"]);
log.log("Webhook: handled validation request");
} else {
// Notifications don't include any information about the changes that triggered them.
// Put a refresh signal in the queue and let the main monitor loop process it.
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/using-webhooks
count.atomicOp!"+="(1);
send(parentTid, to!ulong(count));
cgi.write("OK");
log.log("Webhook: sent refresh signal #", count);
}
}
}
final class OneDriveApi final class OneDriveApi
{ {
private Config cfg; private Config cfg;
private string refreshToken, accessToken; private string refreshToken, accessToken, subscriptionId;
private SysTime accessTokenExpiration; private SysTime accessTokenExpiration;
private HTTP http; private HTTP http;
private shared OneDriveWebhook webhook;
private SysTime subscriptionExpiration;
private Duration subscriptionExpirationInterval, subscriptionRenewalInterval;
private string notificationUrl;
// if true, every new access token is printed // if true, every new access token is printed
bool printAccessToken; bool printAccessToken;
@ -132,16 +190,16 @@ final class OneDriveApi
// maximum time an operation is allowed to take // maximum time an operation is allowed to take
// This includes dns resolution, connecting, data transfer, etc. // This includes dns resolution, connecting, data transfer, etc.
http.operationTimeout = (dur!"seconds"(cfg.getValueLong("operation_timeout"))); http.operationTimeout = (dur!"seconds"(cfg.getValueLong("operation_timeout")));
// Specify how many redirects should be allowed // Specify how many redirects should be allowed
http.maxRedirects(5); http.maxRedirects(5);
// Do we enable curl debugging? // Do we enable curl debugging?
if (cfg.getValueBool("debug_https")) { if (cfg.getValueBool("debug_https")) {
http.verbose = true; http.verbose = true;
.debugResponse = true; .debugResponse = true;
} }
// Update clientId if application_id is set in config file // Update clientId if application_id is set in config file
if (cfg.getValueString("application_id") != "") { if (cfg.getValueString("application_id") != "") {
// an application_id is set in config file // an application_id is set in config file
@ -149,7 +207,7 @@ final class OneDriveApi
clientId = cfg.getValueString("application_id"); clientId = cfg.getValueString("application_id");
companyName = "custom_application"; companyName = "custom_application";
} }
// Configure tenant id value, if 'azure_tenant_id' is configured, // Configure tenant id value, if 'azure_tenant_id' is configured,
// otherwise use the "common" multiplexer // otherwise use the "common" multiplexer
string tenantId = "common"; string tenantId = "common";
@ -157,7 +215,7 @@ final class OneDriveApi
// Use the value entered by the user // Use the value entered by the user
tenantId = cfg.getValueString("azure_tenant_id"); tenantId = cfg.getValueString("azure_tenant_id");
} }
// Configure Azure AD endpoints if 'azure_ad_endpoint' is configured // Configure Azure AD endpoints if 'azure_ad_endpoint' is configured
string azureConfigValue = cfg.getValueString("azure_ad_endpoint"); string azureConfigValue = cfg.getValueString("azure_ad_endpoint");
switch(azureConfigValue) { switch(azureConfigValue) {
@ -185,10 +243,10 @@ final class OneDriveApi
// custom application_id // custom application_id
redirectUrl = usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; redirectUrl = usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
} }
// Drive Queries // Drive Queries
driveUrl = usl4GraphEndpoint ~ "/v1.0/me/drive"; driveUrl = usl4GraphEndpoint ~ "/v1.0/me/drive";
driveByIdUrl = usl4GraphEndpoint ~ "/v1.0/drives/"; driveByIdUrl = usl4GraphEndpoint ~ "/v1.0/drives/";
// Item Queries // Item Queries
itemByIdUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/items/"; itemByIdUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/items/";
itemByPathUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/root:/"; itemByPathUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/root:/";
@ -197,6 +255,8 @@ final class OneDriveApi
siteDriveUrl = usl4GraphEndpoint ~ "/v1.0/sites/"; siteDriveUrl = usl4GraphEndpoint ~ "/v1.0/sites/";
// Shared With Me // Shared With Me
sharedWithMeUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe"; sharedWithMeUrl = usl4GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
// Subscriptions
subscriptionUrl = usl4GraphEndpoint ~ "/v1.0/subscriptions";
break; break;
case "USL5": case "USL5":
log.log("Configuring Azure AD for US Government Endpoints (DOD)"); log.log("Configuring Azure AD for US Government Endpoints (DOD)");
@ -211,10 +271,10 @@ final class OneDriveApi
// custom application_id // custom application_id
redirectUrl = usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; redirectUrl = usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
} }
// Drive Queries // Drive Queries
driveUrl = usl5GraphEndpoint ~ "/v1.0/me/drive"; driveUrl = usl5GraphEndpoint ~ "/v1.0/me/drive";
driveByIdUrl = usl5GraphEndpoint ~ "/v1.0/drives/"; driveByIdUrl = usl5GraphEndpoint ~ "/v1.0/drives/";
// Item Queries // Item Queries
itemByIdUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/items/"; itemByIdUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/items/";
itemByPathUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/root:/"; itemByPathUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/root:/";
@ -223,6 +283,8 @@ final class OneDriveApi
siteDriveUrl = usl5GraphEndpoint ~ "/v1.0/sites/"; siteDriveUrl = usl5GraphEndpoint ~ "/v1.0/sites/";
// Shared With Me // Shared With Me
sharedWithMeUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe"; sharedWithMeUrl = usl5GraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
// Subscriptions
subscriptionUrl = usl5GraphEndpoint ~ "/v1.0/subscriptions";
break; break;
case "DE": case "DE":
log.log("Configuring Azure AD Germany"); log.log("Configuring Azure AD Germany");
@ -237,10 +299,10 @@ final class OneDriveApi
// custom application_id // custom application_id
redirectUrl = deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; redirectUrl = deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
} }
// Drive Queries // Drive Queries
driveUrl = deGraphEndpoint ~ "/v1.0/me/drive"; driveUrl = deGraphEndpoint ~ "/v1.0/me/drive";
driveByIdUrl = deGraphEndpoint ~ "/v1.0/drives/"; driveByIdUrl = deGraphEndpoint ~ "/v1.0/drives/";
// Item Queries // Item Queries
itemByIdUrl = deGraphEndpoint ~ "/v1.0/me/drive/items/"; itemByIdUrl = deGraphEndpoint ~ "/v1.0/me/drive/items/";
itemByPathUrl = deGraphEndpoint ~ "/v1.0/me/drive/root:/"; itemByPathUrl = deGraphEndpoint ~ "/v1.0/me/drive/root:/";
@ -249,6 +311,8 @@ final class OneDriveApi
siteDriveUrl = deGraphEndpoint ~ "/v1.0/sites/"; siteDriveUrl = deGraphEndpoint ~ "/v1.0/sites/";
// Shared With Me // Shared With Me
sharedWithMeUrl = deGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe"; sharedWithMeUrl = deGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
// Subscriptions
subscriptionUrl = deGraphEndpoint ~ "/v1.0/subscriptions";
break; break;
case "CN": case "CN":
log.log("Configuring AD China operated by 21Vianet"); log.log("Configuring AD China operated by 21Vianet");
@ -263,10 +327,10 @@ final class OneDriveApi
// custom application_id // custom application_id
redirectUrl = cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient"; redirectUrl = cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
} }
// Drive Queries // Drive Queries
driveUrl = cnGraphEndpoint ~ "/v1.0/me/drive"; driveUrl = cnGraphEndpoint ~ "/v1.0/me/drive";
driveByIdUrl = cnGraphEndpoint ~ "/v1.0/drives/"; driveByIdUrl = cnGraphEndpoint ~ "/v1.0/drives/";
// Item Queries // Item Queries
itemByIdUrl = cnGraphEndpoint ~ "/v1.0/me/drive/items/"; itemByIdUrl = cnGraphEndpoint ~ "/v1.0/me/drive/items/";
itemByPathUrl = cnGraphEndpoint ~ "/v1.0/me/drive/root:/"; itemByPathUrl = cnGraphEndpoint ~ "/v1.0/me/drive/root:/";
@ -275,33 +339,35 @@ final class OneDriveApi
siteDriveUrl = cnGraphEndpoint ~ "/v1.0/sites/"; siteDriveUrl = cnGraphEndpoint ~ "/v1.0/sites/";
// Shared With Me // Shared With Me
sharedWithMeUrl = cnGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe"; sharedWithMeUrl = cnGraphEndpoint ~ "/v1.0/me/drive/sharedWithMe";
// Subscriptions
subscriptionUrl = cnGraphEndpoint ~ "/v1.0/subscriptions";
break; break;
// Default - all other entries // Default - all other entries
default: default:
log.log("Unknown Azure AD Endpoint request - using Global Azure AD Endpoints"); log.log("Unknown Azure AD Endpoint request - using Global Azure AD Endpoints");
} }
// Debug output of configured URL's // Debug output of configured URL's
// Authentication // Authentication
log.vdebug("Configured authUrl: ", authUrl); log.vdebug("Configured authUrl: ", authUrl);
log.vdebug("Configured redirectUrl: ", redirectUrl); log.vdebug("Configured redirectUrl: ", redirectUrl);
log.vdebug("Configured tokenUrl: ", tokenUrl); log.vdebug("Configured tokenUrl: ", tokenUrl);
// Drive Queries // Drive Queries
log.vdebug("Configured driveUrl: ", driveUrl); log.vdebug("Configured driveUrl: ", driveUrl);
log.vdebug("Configured driveByIdUrl: ", driveByIdUrl); log.vdebug("Configured driveByIdUrl: ", driveByIdUrl);
// Shared With Me // Shared With Me
log.vdebug("Configured sharedWithMeUrl: ", sharedWithMeUrl); log.vdebug("Configured sharedWithMeUrl: ", sharedWithMeUrl);
// Item Queries // Item Queries
log.vdebug("Configured itemByIdUrl: ", itemByIdUrl); log.vdebug("Configured itemByIdUrl: ", itemByIdUrl);
log.vdebug("Configured itemByPathUrl: ", itemByPathUrl); log.vdebug("Configured itemByPathUrl: ", itemByPathUrl);
// SharePoint Queries // SharePoint Queries
log.vdebug("Configured siteSearchUrl: ", siteSearchUrl); log.vdebug("Configured siteSearchUrl: ", siteSearchUrl);
log.vdebug("Configured siteDriveUrl: ", siteDriveUrl); log.vdebug("Configured siteDriveUrl: ", siteDriveUrl);
// Configure the User Agent string // Configure the User Agent string
if (cfg.getValueString("user_agent") == "") { if (cfg.getValueString("user_agent") == "") {
// Application User Agent string defaults // Application User Agent string defaults
@ -315,7 +381,7 @@ final class OneDriveApi
// Use the value entered by the user // Use the value entered by the user
http.setUserAgent = cfg.getValueString("user_agent"); http.setUserAgent = cfg.getValueString("user_agent");
} }
// What version of HTTP protocol do we use? // What version of HTTP protocol do we use?
// Curl >= 7.62.0 defaults to http2 for a significant number of operations // Curl >= 7.62.0 defaults to http2 for a significant number of operations
if (cfg.getValueBool("force_http_2")) { if (cfg.getValueBool("force_http_2")) {
@ -328,28 +394,28 @@ final class OneDriveApi
// Downgrade to HTTP 1.1 - yes version = 2 is HTTP 1.1 // Downgrade to HTTP 1.1 - yes version = 2 is HTTP 1.1
http.handle.set(CurlOption.http_version,2); http.handle.set(CurlOption.http_version,2);
} }
// Configure upload / download rate limits if configured // Configure upload / download rate limits if configured
long userRateLimit = cfg.getValueLong("rate_limit"); long userRateLimit = cfg.getValueLong("rate_limit");
// 131072 = 128 KB/s - minimum for basic application operations to prevent timeouts // 131072 = 128 KB/s - minimum for basic application operations to prevent timeouts
// A 0 value means rate is unlimited, and is the curl default // A 0 value means rate is unlimited, and is the curl default
if (userRateLimit > 0) { if (userRateLimit > 0) {
// User configured rate limit // User configured rate limit
writeln("User Configured Rate Limit: ", userRateLimit); writeln("User Configured Rate Limit: ", userRateLimit);
// If user provided rate limit is < 131072, flag that this is too low, setting to the minimum of 131072 // If user provided rate limit is < 131072, flag that this is too low, setting to the minimum of 131072
if (userRateLimit < 131072) { if (userRateLimit < 131072) {
// user provided limit too low // user provided limit too low
log.log("WARNING: User configured rate limit too low for normal application processing and preventing application timeouts. Overriding to default minimum of 131072 (128KB/s)"); log.log("WARNING: User configured rate limit too low for normal application processing and preventing application timeouts. Overriding to default minimum of 131072 (128KB/s)");
userRateLimit = 131072; userRateLimit = 131072;
} }
// set rate limit // set rate limit
http.handle.set(CurlOption.max_send_speed_large,userRateLimit); http.handle.set(CurlOption.max_send_speed_large,userRateLimit);
http.handle.set(CurlOption.max_recv_speed_large,userRateLimit); http.handle.set(CurlOption.max_recv_speed_large,userRateLimit);
} }
// Do we set the dryRun handlers? // Do we set the dryRun handlers?
if (cfg.getValueBool("dry_run")) { if (cfg.getValueBool("dry_run")) {
.dryRun = true; .dryRun = true;
@ -357,11 +423,19 @@ final class OneDriveApi
.simulateNoRefreshTokenFile = true; .simulateNoRefreshTokenFile = true;
} }
} }
subscriptionExpiration = Clock.currTime(UTC());
subscriptionExpirationInterval = dur!"seconds"(cfg.getValueLong("webhook_expiration_interval"));
subscriptionRenewalInterval = dur!"seconds"(cfg.getValueLong("webhook_renewal_interval"));
notificationUrl = cfg.getValueString("webhook_public_url");
} }
// Shutdown OneDrive HTTP construct // Shutdown OneDrive HTTP construct
void shutdown() void shutdown()
{ {
// delete subscription if there exists any
deleteSubscription();
// reset any values to defaults, freeing any set objects // reset any values to defaults, freeing any set objects
http.clearRequestHeaders(); http.clearRequestHeaders();
http.onSend = null; http.onSend = null;
@ -380,7 +454,7 @@ final class OneDriveApi
log.vdebug("clientId = ", clientId); log.vdebug("clientId = ", clientId);
log.vdebug("companyName = ", companyName); log.vdebug("companyName = ", companyName);
log.vdebug("appTitle = ", appTitle); log.vdebug("appTitle = ", appTitle);
try { try {
driveId = cfg.getValueString("drive_id"); driveId = cfg.getValueString("drive_id");
if (driveId.length) { if (driveId.length) {
@ -389,7 +463,7 @@ final class OneDriveApi
itemByPathUrl = driveUrl ~ "/root:/"; itemByPathUrl = driveUrl ~ "/root:/";
} }
} catch (Exception e) {} } catch (Exception e) {}
if (!.dryRun) { if (!.dryRun) {
// original code // original code
try { try {
@ -450,7 +524,7 @@ final class OneDriveApi
while (!exists(responseUrl)) { while (!exists(responseUrl)) {
Thread.sleep(dur!("msecs")(100)); Thread.sleep(dur!("msecs")(100));
} }
// read response from OneDrive // read response from OneDrive
try { try {
response = cast(char[]) read(responseUrl); response = cast(char[]) read(responseUrl);
@ -458,8 +532,8 @@ final class OneDriveApi
// exception generated // exception generated
displayOneDriveErrorMessage(e.msg, getFunctionName!({})); displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
return false; return false;
} }
// try to remove old files // try to remove old files
try { try {
std.file.remove(authUrl); std.file.remove(authUrl);
@ -485,7 +559,7 @@ final class OneDriveApi
// Return the current value of retryAfterValue if it has been set to something other than 0 // Return the current value of retryAfterValue if it has been set to something other than 0
return .retryAfterValue; return .retryAfterValue;
} }
void resetRetryAfterValue() void resetRetryAfterValue()
{ {
// Reset the current value of retryAfterValue to 0 after it has been used // Reset the current value of retryAfterValue to 0 after it has been used
@ -509,7 +583,7 @@ final class OneDriveApi
url = driveUrl ~ "/root"; url = driveUrl ~ "/root";
return get(url); return get(url);
} }
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getDriveIdRoot(const(char)[] driveId) JSONValue getDriveIdRoot(const(char)[] driveId)
{ {
@ -525,7 +599,7 @@ final class OneDriveApi
checkAccessTokenExpired(); checkAccessTokenExpired();
return get(sharedWithMeUrl); return get(sharedWithMeUrl);
} }
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get
JSONValue getDriveQuota(const(char)[] driveId) JSONValue getDriveQuota(const(char)[] driveId)
{ {
@ -535,7 +609,7 @@ final class OneDriveApi
url ~= "?select=quota"; url ~= "?select=quota";
return get(url); return get(url);
} }
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta
JSONValue viewChangesByItemId(const(char)[] driveId, const(char)[] id, const(char)[] deltaLink) JSONValue viewChangesByItemId(const(char)[] driveId, const(char)[] id, const(char)[] deltaLink)
{ {
@ -550,7 +624,7 @@ final class OneDriveApi
} }
return get(url); return get(url);
} }
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta
JSONValue viewChangesByDriveId(const(char)[] driveId, const(char)[] deltaLink) JSONValue viewChangesByDriveId(const(char)[] driveId, const(char)[] deltaLink)
{ {
@ -562,7 +636,7 @@ final class OneDriveApi
} }
return get(url); return get(url);
} }
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_list_children // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_list_children
JSONValue listChildren(const(char)[] driveId, const(char)[] id, const(char)[] nextLink) JSONValue listChildren(const(char)[] driveId, const(char)[] id, const(char)[] nextLink)
{ {
@ -590,13 +664,13 @@ final class OneDriveApi
} catch (FileException e) { } catch (FileException e) {
// display the error message // display the error message
displayFileSystemErrorMessage(e.msg, getFunctionName!({})); displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
} }
} }
} }
// Create the required local directory // Create the required local directory
string newPath = dirName(saveToPath); string newPath = dirName(saveToPath);
// Does the path exist locally? // Does the path exist locally?
if (!exists(newPath)) { if (!exists(newPath)) {
try { try {
@ -610,7 +684,7 @@ final class OneDriveApi
displayFileSystemErrorMessage(e.msg, getFunctionName!({})); displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
} }
} }
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content?AVOverride=1"; const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content?AVOverride=1";
// Download file // Download file
download(url, saveToPath, fileSize); download(url, saveToPath, fileSize);
@ -667,7 +741,7 @@ final class OneDriveApi
{ {
checkAccessTokenExpired(); checkAccessTokenExpired();
const(char)[] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ "/children"; const(char)[] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ "/children";
http.addRequestHeader("Content-Type", "application/json"); http.addRequestHeader("Content-Type", "application/json");
return post(url, item.toString()); return post(url, item.toString());
} }
@ -681,7 +755,7 @@ final class OneDriveApi
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
return get(url); return get(url);
} }
// Return the details of the specified id // Return the details of the specified id
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getPathDetailsById(const(char)[] driveId, const(char)[] id) JSONValue getPathDetailsById(const(char)[] driveId, const(char)[] id)
@ -692,7 +766,7 @@ final class OneDriveApi
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
return get(url); return get(url);
} }
// Return the requested details of the specified path on the specified drive id and path // Return the requested details of the specified path on the specified drive id and path
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get?view=odsp-graph-online // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get?view=odsp-graph-online
JSONValue getPathDetailsByDriveId(const(char)[] driveId, const(string) path) JSONValue getPathDetailsByDriveId(const(char)[] driveId, const(string) path)
@ -705,7 +779,7 @@ final class OneDriveApi
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
return get(url); return get(url);
} }
// Return the requested details of the specified path on the specified drive id and item id // Return the requested details of the specified path on the specified drive id and item id
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get?view=odsp-graph-online // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get?view=odsp-graph-online
JSONValue getPathDetailsByDriveIdAndItemId(const(char)[] driveId, const(char)[] itemId) JSONValue getPathDetailsByDriveIdAndItemId(const(char)[] driveId, const(char)[] itemId)
@ -718,7 +792,7 @@ final class OneDriveApi
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
return get(url); return get(url);
} }
// Return the requested details of the specified id // Return the requested details of the specified id
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getFileDetails(const(char)[] driveId, const(char)[] id) JSONValue getFileDetails(const(char)[] driveId, const(char)[] id)
@ -729,7 +803,7 @@ final class OneDriveApi
url ~= "?select=size,malware,file,webUrl"; url ~= "?select=size,malware,file,webUrl";
return get(url); return get(url);
} }
// Create an anonymous read-only shareable link for an existing file on OneDrive // Create an anonymous read-only shareable link for an existing file on OneDrive
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createlink // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createlink
JSONValue createShareableLink(const(char)[] driveId, const(char)[] id, JSONValue accessScope) JSONValue createShareableLink(const(char)[] driveId, const(char)[] id, JSONValue accessScope)
@ -737,10 +811,10 @@ final class OneDriveApi
checkAccessTokenExpired(); checkAccessTokenExpired();
const(char)[] url; const(char)[] url;
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/createLink"; url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/createLink";
http.addRequestHeader("Content-Type", "application/json"); http.addRequestHeader("Content-Type", "application/json");
return post(url, accessScope.toString()); return post(url, accessScope.toString());
} }
// https://dev.onedrive.com/items/move.htm // https://dev.onedrive.com/items/move.htm
JSONValue moveByPath(const(char)[] sourcePath, JSONValue moveData) JSONValue moveByPath(const(char)[] sourcePath, JSONValue moveData)
{ {
@ -750,7 +824,7 @@ final class OneDriveApi
http.addRequestHeader("Content-Type", "application/json"); http.addRequestHeader("Content-Type", "application/json");
return move(url, moveData.toString()); return move(url, moveData.toString());
} }
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
JSONValue createUploadSession(const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null, JSONValue item = null) JSONValue createUploadSession(const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null, JSONValue item = null)
{ {
@ -770,7 +844,7 @@ final class OneDriveApi
file.seek(offset); file.seek(offset);
string contentRange = "bytes " ~ to!string(offset) ~ "-" ~ to!string(offset + offsetSize - 1) ~ "/" ~ to!string(fileSize); string contentRange = "bytes " ~ to!string(offset) ~ "-" ~ to!string(offset + offsetSize - 1) ~ "/" ~ to!string(fileSize);
log.vdebugNewLine("contentRange: ", contentRange); log.vdebugNewLine("contentRange: ", contentRange);
// function scopes // function scopes
scope(exit) { scope(exit) {
http.clearRequestHeaders(); http.clearRequestHeaders();
@ -785,7 +859,7 @@ final class OneDriveApi
file.close(); file.close();
} }
} }
http.method = HTTP.Method.put; http.method = HTTP.Method.put;
http.url = uploadUrl; http.url = uploadUrl;
http.addRequestHeader("Content-Range", contentRange); http.addRequestHeader("Content-Range", contentRange);
@ -818,7 +892,7 @@ final class OneDriveApi
} }
return get(url); return get(url);
} }
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_list?view=odsp-graph-online // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_list?view=odsp-graph-online
JSONValue o365SiteDrives(string site_id){ JSONValue o365SiteDrives(string site_id){
checkAccessTokenExpired(); checkAccessTokenExpired();
@ -827,6 +901,89 @@ final class OneDriveApi
return get(url); return get(url);
} }
// Create a new subscription or renew the existing subscription
void createOrRenewSubscription() {
checkAccessTokenExpired();
// Kick off the webhook server first
if (webhook is null) {
webhook = new shared OneDriveWebhook(
cfg.getValueString("webhook_listening_host"),
to!ushort(cfg.getValueLong("webhook_listening_port"))
);
spawn(&OneDriveWebhook.serve, webhook, thisTid);
}
if (!hasValidSubscription()) {
createSubscription();
} else if (isSubscriptionUpForRenewal()) {
try {
renewSubscription();
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
log.log("The subscription is not found on the server. Recreating subscription ...");
createSubscription();
}
}
}
}
private bool hasValidSubscription() {
return !subscriptionId.empty && subscriptionExpiration > Clock.currTime(UTC());
}
private bool isSubscriptionUpForRenewal() {
return subscriptionExpiration < Clock.currTime(UTC()) + subscriptionRenewalInterval;
}
private void createSubscription() {
log.log("Initializing subscription for updates ...");
auto expirationDateTime = Clock.currTime(UTC()) + subscriptionExpirationInterval;
const(char)[] url;
url = subscriptionUrl;
const JSONValue request = [
"changeType": "updated",
"notificationUrl": notificationUrl,
"resource": "/me/drive/root",
"expirationDateTime": expirationDateTime.toISOExtString(),
"clientState": randomUUID().toString()
];
http.addRequestHeader("Content-Type", "application/json");
JSONValue response = post(url, request.toString());
// Save important subscription metadata including id and expiration
subscriptionId = response["id"].str;
subscriptionExpiration = SysTime.fromISOExtString(response["expirationDateTime"].str);
}
private void renewSubscription() {
log.log("Renewing subscription for updates ...");
auto expirationDateTime = Clock.currTime(UTC()) + subscriptionExpirationInterval;
const(char)[] url;
url = subscriptionUrl ~ "/" ~ subscriptionId;
const JSONValue request = [
"expirationDateTime": expirationDateTime.toISOExtString()
];
http.addRequestHeader("Content-Type", "application/json");
JSONValue response = patch(url, request.toString());
// Update subscription expiration from the response
subscriptionExpiration = SysTime.fromISOExtString(response["expirationDateTime"].str);
}
private void deleteSubscription() {
if (!hasValidSubscription()) {
return;
}
const(char)[] url;
url = subscriptionUrl ~ "/" ~ subscriptionId;
del(url);
log.log("Deleted subscription");
}
private void redeemToken(const(char)[] authCode) private void redeemToken(const(char)[] authCode)
{ {
const(char)[] postData = const(char)[] postData =
@ -850,14 +1007,14 @@ final class OneDriveApi
private void acquireToken(const(char)[] postData) private void acquireToken(const(char)[] postData)
{ {
JSONValue response; JSONValue response;
try { try {
response = post(tokenUrl, postData); response = post(tokenUrl, postData);
} catch (OneDriveException e) { } catch (OneDriveException e) {
// an error was generated // an error was generated
displayOneDriveErrorMessage(e.msg, getFunctionName!({})); displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
} }
if (response.type() == JSONType.object) { if (response.type() == JSONType.object) {
if ("access_token" in response){ if ("access_token" in response){
accessToken = "bearer " ~ response["access_token"].str(); accessToken = "bearer " ~ response["access_token"].str();
@ -939,7 +1096,7 @@ final class OneDriveApi
long thresholdFileSize = 4 * 2^^20; // 4 MiB long thresholdFileSize = 4 * 2^^20; // 4 MiB
// open file as write in binary mode // open file as write in binary mode
auto file = File(filename, "wb"); auto file = File(filename, "wb");
// function scopes // function scopes
scope(exit) { scope(exit) {
http.clearRequestHeaders(); http.clearRequestHeaders();
@ -959,16 +1116,16 @@ final class OneDriveApi
file.close(); file.close();
} }
} }
http.method = HTTP.Method.get; http.method = HTTP.Method.get;
http.url = url; http.url = url;
addAccessTokenHeader(); addAccessTokenHeader();
http.onReceive = (ubyte[] data) { http.onReceive = (ubyte[] data) {
file.rawWrite(data); file.rawWrite(data);
return data.length; return data.length;
}; };
if (fileSize >= thresholdFileSize){ if (fileSize >= thresholdFileSize){
// Download Progress Bar // Download Progress Bar
size_t iteration = 20; size_t iteration = 20;
@ -1006,7 +1163,7 @@ final class OneDriveApi
} }
return 0; return 0;
}; };
// Perform download & display progress bar // Perform download & display progress bar
try { try {
// try and catch any curl error // try and catch any curl error
@ -1031,7 +1188,7 @@ final class OneDriveApi
displayOneDriveErrorMessage(e.msg, getFunctionName!({})); displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
} }
} }
// Check the HTTP response code, which, if a 429, will also check response headers // Check the HTTP response code, which, if a 429, will also check response headers
checkHttpCode(); checkHttpCode();
} }
@ -1069,13 +1226,13 @@ final class OneDriveApi
checkHttpCode(); checkHttpCode();
return response; return response;
} }
private JSONValue upload(string filepath, string url) private JSONValue upload(string filepath, string url)
{ {
checkAccessTokenExpired(); checkAccessTokenExpired();
// open file as read-only in binary mode // open file as read-only in binary mode
auto file = File(filepath, "rb"); auto file = File(filepath, "rb");
// function scopes // function scopes
scope(exit) { scope(exit) {
http.clearRequestHeaders(); http.clearRequestHeaders();
@ -1090,7 +1247,7 @@ final class OneDriveApi
file.close(); file.close();
} }
} }
http.method = HTTP.Method.put; http.method = HTTP.Method.put;
http.url = url; http.url = url;
addAccessTokenHeader(); addAccessTokenHeader();
@ -1139,7 +1296,7 @@ final class OneDriveApi
} }
return data.length; return data.length;
}; };
try { try {
http.perform(); http.perform();
// Check the HTTP Response headers - needed for correct 429 handling // Check the HTTP Response headers - needed for correct 429 handling
@ -1151,7 +1308,7 @@ final class OneDriveApi
auto errorArray = splitLines(e.msg); auto errorArray = splitLines(e.msg);
string errorMessage = errorArray[0]; string errorMessage = errorArray[0];
string defaultTimeoutErrorMessage = " Error Message: There was a timeout in accessing the Microsoft OneDrive service - Internet connectivity issue?"; string defaultTimeoutErrorMessage = " Error Message: There was a timeout in accessing the Microsoft OneDrive service - Internet connectivity issue?";
if (canFind(errorMessage, "Couldn't connect to server on handle") || canFind(errorMessage, "Couldn't resolve host name on handle") || canFind(errorMessage, "Timeout was reached on handle")) { if (canFind(errorMessage, "Couldn't connect to server on handle") || canFind(errorMessage, "Couldn't resolve host name on handle") || canFind(errorMessage, "Timeout was reached on handle")) {
// This is a curl timeout // This is a curl timeout
log.error(defaultTimeoutErrorMessage); log.error(defaultTimeoutErrorMessage);
@ -1166,7 +1323,7 @@ final class OneDriveApi
while (!retrySuccess){ while (!retrySuccess){
backoffInterval++; backoffInterval++;
int thisBackOffInterval = retryAttempts*backoffInterval; int thisBackOffInterval = retryAttempts*backoffInterval;
log.vdebug(" Retry Attempt: ", retryAttempts); log.vdebug(" Retry Attempt: ", retryAttempts);
if (thisBackOffInterval <= maxBackoffInterval) { if (thisBackOffInterval <= maxBackoffInterval) {
log.vdebug(" Retry In (seconds): ", thisBackOffInterval); log.vdebug(" Retry In (seconds): ", thisBackOffInterval);
Thread.sleep(dur!"seconds"(thisBackOffInterval)); Thread.sleep(dur!"seconds"(thisBackOffInterval));
@ -1206,7 +1363,7 @@ final class OneDriveApi
// return an empty JSON for handling // return an empty JSON for handling
return json; return json;
} }
try { try {
json = content.parseJSON(); json = content.parseJSON();
} catch (JSONException e) { } catch (JSONException e) {
@ -1215,7 +1372,7 @@ final class OneDriveApi
} }
return json; return json;
} }
private void checkHTTPResponseHeaders() private void checkHTTPResponseHeaders()
{ {
// Get the HTTP Response headers - needed for correct 429 handling // Get the HTTP Response headers - needed for correct 429 handling
@ -1223,7 +1380,7 @@ final class OneDriveApi
if (.debugResponse){ if (.debugResponse){
log.vdebug("http.perform() => HTTP Response Headers: ", responseHeaders); log.vdebug("http.perform() => HTTP Response Headers: ", responseHeaders);
} }
// is retry-after in the response headers // is retry-after in the response headers
if ("retry-after" in http.responseHeaders) { if ("retry-after" in http.responseHeaders) {
// Set the retry-after value // Set the retry-after value
@ -1237,18 +1394,18 @@ final class OneDriveApi
{ {
// https://dev.onedrive.com/misc/errors.htm // https://dev.onedrive.com/misc/errors.htm
// https://developer.overdrive.com/docs/reference-guide // https://developer.overdrive.com/docs/reference-guide
/* /*
HTTP/1.1 Response handling HTTP/1.1 Response handling
Errors in the OneDrive API are returned using standard HTTP status codes, as well as a JSON error response object. The following HTTP status codes should be expected. Errors in the OneDrive API are returned using standard HTTP status codes, as well as a JSON error response object. The following HTTP status codes should be expected.
Status code Status message Description Status code Status message Description
100 Continue Continue 100 Continue Continue
200 OK Request was handled OK 200 OK Request was handled OK
201 Created This means you've made a successful POST to checkout, lock in a format, or place a hold 201 Created This means you've made a successful POST to checkout, lock in a format, or place a hold
204 No Content This means you've made a successful DELETE to remove a hold or return a title 204 No Content This means you've made a successful DELETE to remove a hold or return a title
400 Bad Request Cannot process the request because it is malformed or incorrect. 400 Bad Request Cannot process the request because it is malformed or incorrect.
401 Unauthorized Required authentication information is either missing or not valid for the resource. 401 Unauthorized Required authentication information is either missing or not valid for the resource.
403 Forbidden Access is denied to the requested resource. The user might not have enough permission. 403 Forbidden Access is denied to the requested resource. The user might not have enough permission.
@ -1265,20 +1422,20 @@ final class OneDriveApi
416 Requested Range Not Satisfiable The specified byte range is invalid or unavailable. 416 Requested Range Not Satisfiable The specified byte range is invalid or unavailable.
422 Unprocessable Entity Cannot process the request because it is semantically incorrect. 422 Unprocessable Entity Cannot process the request because it is semantically incorrect.
429 Too Many Requests Client application has been throttled and should not attempt to repeat the request until an amount of time has elapsed. 429 Too Many Requests Client application has been throttled and should not attempt to repeat the request until an amount of time has elapsed.
500 Internal Server Error There was an internal server error while processing the request. 500 Internal Server Error There was an internal server error while processing the request.
501 Not Implemented The requested feature isn’t implemented. 501 Not Implemented The requested feature isn’t implemented.
502 Bad Gateway The service was unreachable 502 Bad Gateway The service was unreachable
503 Service Unavailable The service is temporarily unavailable. You may repeat the request after a delay. There may be a Retry-After header. 503 Service Unavailable The service is temporarily unavailable. You may repeat the request after a delay. There may be a Retry-After header.
507 Insufficient Storage The maximum storage quota has been reached. 507 Insufficient Storage The maximum storage quota has been reached.
509 Bandwidth Limit Exceeded Your app has been throttled for exceeding the maximum bandwidth cap. Your app can retry the request again after more time has elapsed. 509 Bandwidth Limit Exceeded Your app has been throttled for exceeding the maximum bandwidth cap. Your app can retry the request again after more time has elapsed.
HTTP/2 Response handling HTTP/2 Response handling
0 OK 0 OK
*/ */
switch(http.statusLine.code) switch(http.statusLine.code)
{ {
// 0 - OK ... HTTP2 version of 200 OK // 0 - OK ... HTTP2 version of 200 OK
@ -1289,7 +1446,7 @@ final class OneDriveApi
break; break;
// 200 - OK // 200 - OK
case 200: case 200:
// No Log .. // No Log ..
break; break;
// 201 - Created OK // 201 - Created OK
// 202 - Accepted // 202 - Accepted
@ -1298,53 +1455,53 @@ final class OneDriveApi
// No actions, but log if verbose logging // No actions, but log if verbose logging
//log.vlog("OneDrive Response: '", http.statusLine.code, " - ", http.statusLine.reason, "'"); //log.vlog("OneDrive Response: '", http.statusLine.code, " - ", http.statusLine.reason, "'");
break; break;
// 302 - resource found and available at another location, redirect // 302 - resource found and available at another location, redirect
case 302: case 302:
break; break;
// 400 - Bad Request // 400 - Bad Request
case 400: case 400:
// Bad Request .. how should we act? // Bad Request .. how should we act?
log.vlog("OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error"); log.vlog("OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error");
break; break;
// 403 - Forbidden // 403 - Forbidden
case 403: case 403:
// OneDrive responded that the user is forbidden // OneDrive responded that the user is forbidden
log.vlog("OneDrive returned a 'HTTP 403 - Forbidden' - gracefully handling error"); log.vlog("OneDrive returned a 'HTTP 403 - Forbidden' - gracefully handling error");
break; break;
// 404 - Item not found // 404 - Item not found
case 404: case 404:
// Item was not found - do not throw an exception // Item was not found - do not throw an exception
log.vlog("OneDrive returned a 'HTTP 404 - Item not found' - gracefully handling error"); log.vlog("OneDrive returned a 'HTTP 404 - Item not found' - gracefully handling error");
break; break;
// 408 - Request Timeout // 408 - Request Timeout
case 408: case 408:
// Request to connect to OneDrive service timed out // Request to connect to OneDrive service timed out
log.vlog("Request Timeout - gracefully handling error"); log.vlog("Request Timeout - gracefully handling error");
throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?"); throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?");
// 409 - Conflict // 409 - Conflict
case 409: case 409:
// Conflict handling .. how should we act? This only really gets triggered if we are using --local-first & we remove items.db as the DB thinks the file is not uploaded but it is // Conflict handling .. how should we act? This only really gets triggered if we are using --local-first & we remove items.db as the DB thinks the file is not uploaded but it is
log.vlog("OneDrive returned a 'HTTP 409 - Conflict' - gracefully handling error"); log.vlog("OneDrive returned a 'HTTP 409 - Conflict' - gracefully handling error");
break; break;
// 412 - Precondition Failed // 412 - Precondition Failed
case 412: case 412:
// A precondition provided in the request (such as an if-match header) does not match the resource's current state. // A precondition provided in the request (such as an if-match header) does not match the resource's current state.
log.vlog("OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error"); log.vlog("OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error");
break; break;
// 415 - Unsupported Media Type // 415 - Unsupported Media Type
case 415: case 415:
// Unsupported Media Type ... sometimes triggered on image files, especially PNG // Unsupported Media Type ... sometimes triggered on image files, especially PNG
log.vlog("OneDrive returned a 'HTTP 415 - Unsupported Media Type' - gracefully handling error"); log.vlog("OneDrive returned a 'HTTP 415 - Unsupported Media Type' - gracefully handling error");
break; break;
// 429 - Too Many Requests // 429 - Too Many Requests
case 429: case 429:
// Too many requests in a certain time window // Too many requests in a certain time window
@ -1353,7 +1510,7 @@ final class OneDriveApi
// https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online // https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
log.vlog("OneDrive returned a 'HTTP 429 - Too Many Requests' - gracefully handling error"); log.vlog("OneDrive returned a 'HTTP 429 - Too Many Requests' - gracefully handling error");
throw new OneDriveException(http.statusLine.code, http.statusLine.reason); throw new OneDriveException(http.statusLine.code, http.statusLine.reason);
// Server side (OneDrive) Errors // Server side (OneDrive) Errors
// 500 - Internal Server Error // 500 - Internal Server Error
// 502 - Bad Gateway // 502 - Bad Gateway
@ -1363,17 +1520,17 @@ final class OneDriveApi
// No actions // No actions
log.vlog("OneDrive returned a 'HTTP 500 Internal Server Error' - gracefully handling error"); log.vlog("OneDrive returned a 'HTTP 500 Internal Server Error' - gracefully handling error");
break; break;
case 502: case 502:
// No actions // No actions
log.vlog("OneDrive returned a 'HTTP 502 Bad Gateway Error' - gracefully handling error"); log.vlog("OneDrive returned a 'HTTP 502 Bad Gateway Error' - gracefully handling error");
break; break;
case 503: case 503:
// No actions // No actions
log.vlog("OneDrive returned a 'HTTP 503 Service Unavailable Error' - gracefully handling error"); log.vlog("OneDrive returned a 'HTTP 503 Service Unavailable Error' - gracefully handling error");
break; break;
case 504: case 504:
// No actions // No actions
log.vlog("OneDrive returned a 'HTTP 504 Gateway Timeout Error' - gracefully handling error"); log.vlog("OneDrive returned a 'HTTP 504 Gateway Timeout Error' - gracefully handling error");
@ -1381,7 +1538,7 @@ final class OneDriveApi
// "else" // "else"
default: default:
throw new OneDriveException(http.statusLine.code, http.statusLine.reason); throw new OneDriveException(http.statusLine.code, http.statusLine.reason);
} }
} }
@ -1397,7 +1554,7 @@ final class OneDriveApi
break; break;
// 200 - OK // 200 - OK
case 200: case 200:
// No Log .. // No Log ..
break; break;
// 201 - Created OK // 201 - Created OK
// 202 - Accepted // 202 - Accepted
@ -1406,29 +1563,29 @@ final class OneDriveApi
// No actions, but log if verbose logging // No actions, but log if verbose logging
//log.vlog("OneDrive Response: '", http.statusLine.code, " - ", http.statusLine.reason, "'"); //log.vlog("OneDrive Response: '", http.statusLine.code, " - ", http.statusLine.reason, "'");
break; break;
// 302 - resource found and available at another location, redirect // 302 - resource found and available at another location, redirect
case 302: case 302:
break; break;
// 400 - Bad Request // 400 - Bad Request
case 400: case 400:
// Bad Request .. how should we act? // Bad Request .. how should we act?
// make sure this is thrown so that it is caught // make sure this is thrown so that it is caught
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response); throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
// 403 - Forbidden // 403 - Forbidden
case 403: case 403:
// OneDrive responded that the user is forbidden // OneDrive responded that the user is forbidden
log.vlog("OneDrive returned a 'HTTP 403 - Forbidden' - gracefully handling error"); log.vlog("OneDrive returned a 'HTTP 403 - Forbidden' - gracefully handling error");
// Throw this as a specific exception so this is caught when performing sync.o365SiteSearch // Throw this as a specific exception so this is caught when performing sync.o365SiteSearch
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response); throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
// 412 - Precondition Failed // 412 - Precondition Failed
case 412: case 412:
// Throw this as a specific exception so this is caught when performing sync.uploadLastModifiedTime // Throw this as a specific exception so this is caught when performing sync.uploadLastModifiedTime
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response); throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
// Server side (OneDrive) Errors // Server side (OneDrive) Errors
// 500 - Internal Server Error // 500 - Internal Server Error
// 502 - Bad Gateway // 502 - Bad Gateway
@ -1437,19 +1594,19 @@ final class OneDriveApi
case 500: case 500:
// Throw this as a specific exception so this is caught // Throw this as a specific exception so this is caught
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response); throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
case 502: case 502:
// Throw this as a specific exception so this is caught // Throw this as a specific exception so this is caught
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response); throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
case 503: case 503:
// Throw this as a specific exception so this is caught // Throw this as a specific exception so this is caught
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response); throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
case 504: case 504:
// Throw this as a specific exception so this is caught // Throw this as a specific exception so this is caught
throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response); throw new OneDriveException(http.statusLine.code, http.statusLine.reason, response);
// Default - all other errors that are not a 2xx or a 302 // Default - all other errors that are not a 2xx or a 302
default: default:
if (http.statusLine.code / 100 != 2 && http.statusLine.code != 302) { if (http.statusLine.code / 100 != 2 && http.statusLine.code != 302) {