Add Real-Time syncing of remote updates via webhooks (#1718)
* Add Real-Time syncing of remote updates via webhooks Co-authored-by: abraunegg <alex.braunegg@gmail.com>
This commit is contained in:
parent
2f47beab60
commit
1e2827ad1c
|
@ -78,7 +78,8 @@ SOURCES = \
|
|||
src/sync.d \
|
||||
src/upload.d \
|
||||
src/util.d \
|
||||
src/progress.d
|
||||
src/progress.d \
|
||||
src/arsd/cgi.d
|
||||
|
||||
ifeq ($(NOTIFICATIONS),yes)
|
||||
SOURCES += src/notifications/notify.d src/notifications/dnotify.d
|
||||
|
@ -123,7 +124,7 @@ else
|
|||
ifeq ($(RHEL_VERSION),6)
|
||||
install -D contrib/init.d/onedrive.init $(DESTDIR)/etc/init.d/onedrive
|
||||
install -D contrib/init.d/onedrive_service.sh $(DESTDIR)$(bindir)/onedrive_service.sh
|
||||
endif
|
||||
endif
|
||||
endif
|
||||
ifeq ($(COMPLETIONS),yes)
|
||||
$(INSTALL) -D -m 644 contrib/completions/complete.zsh $(DESTDIR)$(ZSH_COMPLETION_DIR)/_onedrive
|
||||
|
|
|
@ -13,7 +13,8 @@ This client is a 'fork' of the [skilion](https://github.com/skilion/onedrive) cl
|
|||
|
||||
## Features
|
||||
* State caching
|
||||
* Real-Time file monitoring with Inotify
|
||||
* Real-Time local file monitoring with inotify
|
||||
* Real-Time syncing of remote updates via webhooks
|
||||
* File upload / download validation to ensure data integrity
|
||||
* Resumable uploads
|
||||
* Support OneDrive for Business (part of Office 365)
|
||||
|
@ -27,7 +28,6 @@ This client is a 'fork' of the [skilion](https://github.com/skilion/onedrive) cl
|
|||
* Supports rate limiting of traffic
|
||||
|
||||
## What's missing
|
||||
* While local changes are uploaded right away, remote changes are delayed until next automated sync cycle when using --monitor
|
||||
* Ability to encrypt/decrypt files on-the-fly when uploading/downloading files from OneDrive
|
||||
* Support for Windows 'On-Demand' functionality so file is only downloaded when accessed locally
|
||||
* A GUI for configuration management
|
||||
|
|
8
config
8
config
|
@ -41,4 +41,10 @@
|
|||
# sync_business_shared_folders = "false"
|
||||
# sync_dir_permissions = "700"
|
||||
# sync_file_permissions = "600"
|
||||
# rate_limit = "131072"
|
||||
# rate_limit = "131072"
|
||||
# webhook_enabled = "false"
|
||||
# webhook_public_url = ""
|
||||
# webhook_listening_host = ""
|
||||
# webhook_listening_port = "8888"
|
||||
# webhook_expiration_interval = "86400"
|
||||
# webhook_renewal_interval = "43200"
|
|
@ -36,6 +36,10 @@
|
|||
* [Shared folders (OneDrive Business or Office 365)](#shared-folders-onedrive-business-or-office-365)
|
||||
* [SharePoint / Office 365 Shared Libraries](#sharepoint--office-365-shared-libraries)
|
||||
- [Running 'onedrive' in 'monitor' mode](#running-onedrive-in-monitor-mode)
|
||||
* [Use webhook to subscribe to remote updates in 'monitor' mode](#use-webhook-to-subscribe-to-remote-updates-in-monitor-mode)
|
||||
* [More webhook configuration options](#more-webhook-configuration-options)
|
||||
+ [webhook_listening_host and webhook_listening_port](#webhook_listening_host-and-webhook_listening_port)
|
||||
+ [webhook_expiration_interval and webhook_renewal_interval](#webhook_expiration_interval-and-webhook_renewal_interval)
|
||||
- [Running 'onedrive' as a system service](#running-onedrive-as-a-system-service)
|
||||
* [OneDrive service running as root user via init.d](#onedrive-service-running-as-root-user-via-initd)
|
||||
* [OneDrive service running as root user via systemd (Arch, Ubuntu, Debian, OpenSuSE, Fedora)](#onedrive-service-running-as-root-user-via-systemd-arch-ubuntu-debian-opensuse-fedora)
|
||||
|
@ -84,19 +88,19 @@ After installing the application you must authorize the application with your On
|
|||
|
||||
You will be asked to open a specific URL by using your web browser where you will have to login into your Microsoft Account and give the application the permission to access your files. After giving permission to the application, you will be redirected to a blank page. Copy the URI of the blank page into the application.
|
||||
```text
|
||||
[user@hostname ~]$ onedrive
|
||||
[user@hostname ~]$ onedrive
|
||||
|
||||
Authorize this app visiting:
|
||||
|
||||
https://.....
|
||||
|
||||
Enter the response uri:
|
||||
Enter the response uri:
|
||||
|
||||
```
|
||||
|
||||
**Example:**
|
||||
```
|
||||
[user@hostname ~]$ onedrive
|
||||
[user@hostname ~]$ onedrive
|
||||
Authorize this app visiting:
|
||||
|
||||
https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=22c49a0d-d21c-4792-aed1-8f163c982546&scope=Files.ReadWrite%20Files.ReadWrite.all%20Sites.ReadWrite.All%20offline_access&response_type=code&redirect_uri=https://login.microsoftonline.com/common/oauth2/nativeclient
|
||||
|
@ -120,7 +124,7 @@ Config path = /home/alex/.config/onedrive
|
|||
Config file found in config path = false
|
||||
Config option 'check_nosync' = false
|
||||
Config option 'sync_dir' = /home/alex/OneDrive
|
||||
Config option 'skip_dir' =
|
||||
Config option 'skip_dir' =
|
||||
Config option 'skip_file' = ~*|.~*|*.tmp
|
||||
Config option 'skip_dotfiles' = false
|
||||
Config option 'skip_symlinks' = false
|
||||
|
@ -188,7 +192,7 @@ Example: If the full path is `~/OneDrive/mydir`, the command would be `onedrive
|
|||
### Performing a 'one-way' download sync
|
||||
In some cases it may be desirable to 'download only' from OneDrive. To do this use the following command:
|
||||
```text
|
||||
onedrive --synchronize --download-only
|
||||
onedrive --synchronize --download-only
|
||||
```
|
||||
|
||||
### Performing a 'one-way' upload sync
|
||||
|
@ -353,6 +357,12 @@ See the [config](https://raw.githubusercontent.com/abraunegg/onedrive/master/con
|
|||
# sync_file_permissions = "600"
|
||||
# rate_limit = "131072"
|
||||
# operation_timeout = "3600"
|
||||
# webhook_enabled = "false"
|
||||
# webhook_public_url = ""
|
||||
# webhook_listening_host = ""
|
||||
# webhook_listening_port = "8888"
|
||||
# webhook_expiration_interval = "86400"
|
||||
# webhook_renewal_interval = "43200"
|
||||
```
|
||||
|
||||
|
||||
|
@ -401,7 +411,7 @@ sync_file_permissions = "600"
|
|||
**Important:** Special permission bits (setuid, setgid, sticky bit) are not supported. Valid permission values are from `000` to `777` only.
|
||||
|
||||
#### skip_dir
|
||||
Example:
|
||||
Example:
|
||||
```text
|
||||
# When changing a config option below, remove the '#' from the start of the line
|
||||
# For explanations of all config options below see docs/USAGE.md or the man page.
|
||||
|
@ -528,7 +538,7 @@ Each line of the file represents a relative path from your `sync_dir`. All files
|
|||
Here is an example of `sync_list`:
|
||||
```text
|
||||
# sync_list supports comments
|
||||
#
|
||||
#
|
||||
# The ordering of entries is highly recommended - exclusions before inclusions
|
||||
#
|
||||
# Exclude temp folders under Documents
|
||||
|
@ -562,7 +572,7 @@ The following are supported for pattern matching and exclusion rules:
|
|||
To simplify 'exclusions' and 'inclusions', the following is also possible:
|
||||
```text
|
||||
# sync_list supports comments
|
||||
#
|
||||
#
|
||||
# The ordering of entries is highly recommended - exclusions before inclusions
|
||||
#
|
||||
# Exclude temp folders under Documents
|
||||
|
@ -583,7 +593,7 @@ sync_root_files = "true"
|
|||
This will tell the application to sync any file that it finds in your 'sync_dir' root by default.
|
||||
|
||||
### Configuring the client for 'single tenant application' use
|
||||
In some instances when using OneDrive Business Accounts, depending on the Azure organisational configuration, it will be necessary to configure the client as a 'single tenant application'.
|
||||
In some instances when using OneDrive Business Accounts, depending on the Azure organisational configuration, it will be necessary to configure the client as a 'single tenant application'.
|
||||
To configure this, after creating the application on your Azure tenant, update the 'config' file with the tenant name (not the GUID) and the newly created Application ID, then this will be used for the authentication process.
|
||||
```text
|
||||
# skip_dir_strict_match = "false"
|
||||
|
@ -682,6 +692,59 @@ sudo sysctl fs.inotify.max_user_watches=<new_value>
|
|||
|
||||
To make these changes permanent, refer to your OS reference documentation.
|
||||
|
||||
### Use webhook to subscribe to remote updates in 'monitor' mode
|
||||
|
||||
A webhook can be optionally enabled in the monitor mode to allow the onedrive process to subscribe to remote updates. Remote changes can be synced to your local file system as soon as possible, without waiting for the next sync cycle.
|
||||
|
||||
To enable this feature, you need to configure the following options in the config file:
|
||||
|
||||
```
|
||||
webhook_enabled = "true"
|
||||
webhook_public_url = "<public-facing url to reach your webhook>"
|
||||
```
|
||||
|
||||
Setting `webhook_enabled` to `true` enables the webhook in 'monitor' mode. The onedrive process will listen for incoming updates at a configurable endpoint, which defaults to `0.0.0.0:8888`. The `webhook_public_url` must be set to an public-facing url for Microsoft to send updates to your webhook. If your host is directly exposed to the Internet, the `webhook_public_url` can be set to `http://<your_host>:8888/` to match the default endpoint. However, the recommended approach is to configure a reverse proxy like nginx.
|
||||
|
||||
For example, below is a nginx config snippet to proxy traffic into the webhook:
|
||||
|
||||
```
|
||||
http {
|
||||
server {
|
||||
listen 80;
|
||||
location /webhooks/onedrive {
|
||||
proxy_pass http://127.0.0.1:8888;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
With nginx running, you can configure `webhook_public_url` to `http://<your_host>/webhooks/onedrive`.
|
||||
|
||||
### More webhook configuration options
|
||||
|
||||
Below options can be optionally configured. The default is usually good enough.
|
||||
|
||||
#### webhook_listening_host and webhook_listening_port
|
||||
|
||||
Set `webhook_listening_host` and `webhook_listening_port` to change the webhook listening endpoint. If `webhook_listening_host` is left empty, which is the default, the webhook will bind to `0.0.0.0`. The default `webhook_listening_port` is `8888`.
|
||||
|
||||
```
|
||||
webhook_listening_host = ""
|
||||
webhook_listening_port = "8888"
|
||||
```
|
||||
|
||||
#### webhook_expiration_interval and webhook_renewal_interval
|
||||
|
||||
Set `webhook_expiration_interval` and `webhook_renewal_interval` to change the frequency of subscription renewal. By default, the webhook asks Microsoft to keep subscriptions alive for 24 hours, and it renews subscriptions when it is less than 12 hours before their expiration.
|
||||
|
||||
```
|
||||
# Default expiration interval is 24 hours
|
||||
webhook_expiration_interval = "86400"
|
||||
|
||||
# Default renewal interval is 12 hours
|
||||
webhook_renewal_interval = "43200"
|
||||
```
|
||||
|
||||
## Running 'onedrive' as a system service
|
||||
There are a few ways to use onedrive as a service
|
||||
* via init.d
|
||||
|
@ -869,7 +932,7 @@ for extra details.
|
|||
* Configuring the client for use in dual-boot (Windows / Linux) situations
|
||||
* Configuring the client for use when 'sync_dir' is a mounted directory
|
||||
* Upload data from the local ~/OneDrive folder to a specific location on OneDrive
|
||||
|
||||
|
||||
Refer to [./advanced-usage.md](advanced-usage.md) for configuration assistance.
|
||||
|
||||
### Access OneDrive service through a proxy
|
||||
|
|
|
@ -0,0 +1,8 @@
|
|||
The files in this directory have been obtained form the following places:
|
||||
|
||||
cgi.d
|
||||
https://github.com/adamdruppe/arsd/blob/a870179988b8881b04126856105f0fad2cc0018d/cgi.d
|
||||
License: Boost Software License - Version 1.0
|
||||
|
||||
Copyright 2008-2021, Adam D. Ruppe
|
||||
see https://github.com/adamdruppe/arsd/blob/a870179988b8881b04126856105f0fad2cc0018d/LICENSE
|
File diff suppressed because it is too large
Load Diff
54
src/config.d
54
src/config.d
|
@ -43,7 +43,7 @@ final class Config
|
|||
// Default file permission mode
|
||||
public long defaultFilePermissionMode = 600;
|
||||
public int configuredFilePermissionMode;
|
||||
|
||||
|
||||
this(string confdirOption)
|
||||
{
|
||||
// default configuration - entries in config file ~/.config/onedrive/config
|
||||
|
@ -123,6 +123,14 @@ final class Config
|
|||
// This includes dns resolution, connecting, data transfer, etc.
|
||||
longValues["operation_timeout"] = 3600;
|
||||
|
||||
// Webhook options
|
||||
boolValues["webhook_enabled"] = false;
|
||||
stringValues["webhook_public_url"] = "";
|
||||
stringValues["webhook_listening_host"] = "";
|
||||
longValues["webhook_listening_port"] = 8888;
|
||||
longValues["webhook_expiration_interval"] = 3600 * 24;
|
||||
longValues["webhook_renewal_interval"] = 3600 * 12;
|
||||
|
||||
// DEVELOPER OPTIONS
|
||||
// display_memory = true | false
|
||||
// - It may be desirable to display the memory usage of the application to assist with diagnosing memory issues with the application
|
||||
|
@ -156,10 +164,10 @@ final class Config
|
|||
homePath = "~";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Output homePath calculation
|
||||
log.vdebug("homePath: ", homePath);
|
||||
|
||||
|
||||
// Determine the correct configuration directory to use
|
||||
string configDirBase;
|
||||
string systemConfigDirBase;
|
||||
|
@ -185,7 +193,7 @@ final class Config
|
|||
// Also set up a path to pre-shipped shared configs (which can be overridden by supplying a config file in userspace)
|
||||
systemConfigDirBase = "/etc";
|
||||
}
|
||||
|
||||
|
||||
// Output configDirBase calculation
|
||||
log.vdebug("configDirBase: ", configDirBase);
|
||||
// Set the default application configuration directory
|
||||
|
@ -195,7 +203,7 @@ final class Config
|
|||
// systemConfigDirBase contains the correct path so we do not need to check for presence of '~'
|
||||
systemConfigDirName = systemConfigDirBase ~ "/onedrive";
|
||||
}
|
||||
|
||||
|
||||
// Config directory options all determined
|
||||
if (!exists(configDirName)) {
|
||||
// create the directory
|
||||
|
@ -203,11 +211,11 @@ final class Config
|
|||
// Configure the applicable permissions for the folder
|
||||
configDirName.setAttributes(returnRequiredDirectoryPermisions());
|
||||
}
|
||||
|
||||
|
||||
// configDirName has a trailing /
|
||||
log.vlog("Using 'user' Config Dir: ", configDirName);
|
||||
log.vlog("Using 'system' Config Dir: ", systemConfigDirName);
|
||||
|
||||
|
||||
// Update application set variables based on configDirName
|
||||
refreshTokenFilePath = buildNormalizedPath(configDirName ~ "/refresh_token");
|
||||
deltaLinkFilePath = buildNormalizedPath(configDirName ~ "/delta_link");
|
||||
|
@ -218,7 +226,7 @@ final class Config
|
|||
syncListFilePath = buildNormalizedPath(configDirName ~ "/sync_list");
|
||||
systemConfigFilePath = buildNormalizedPath(systemConfigDirName ~ "/config");
|
||||
businessSharedFolderFilePath = buildNormalizedPath(configDirName ~ "/business_shared_folders");
|
||||
|
||||
|
||||
// Debug Output for application set variables based on configDirName
|
||||
log.vdebug("refreshTokenFilePath = ", refreshTokenFilePath);
|
||||
log.vdebug("deltaLinkFilePath = ", deltaLinkFilePath);
|
||||
|
@ -290,7 +298,7 @@ final class Config
|
|||
boolValues["synchronize"] = false;
|
||||
boolValues["force"] = false;
|
||||
boolValues["list_business_shared_folders"] = false;
|
||||
|
||||
|
||||
// Application Startup option validation
|
||||
try {
|
||||
string tmpStr;
|
||||
|
@ -298,7 +306,7 @@ final class Config
|
|||
long tmpVerb;
|
||||
// duplicated from main.d to get full help output!
|
||||
auto opt = getopt(
|
||||
|
||||
|
||||
args,
|
||||
std.getopt.config.bundling,
|
||||
std.getopt.config.caseSensitive,
|
||||
|
@ -309,7 +317,7 @@ final class Config
|
|||
"Perform authentication not via interactive dialog but via providing the reponse url directly.",
|
||||
&stringValues["auth_response"],
|
||||
"check-for-nomount",
|
||||
"Check for the presence of .nosync in the syncdir root. If found, do not perform sync.",
|
||||
"Check for the presence of .nosync in the syncdir root. If found, do not perform sync.",
|
||||
&boolValues["check_nomount"],
|
||||
"check-for-nosync",
|
||||
"Check for the presence of .nosync in each directory. If found, skip directory from sync.",
|
||||
|
@ -323,8 +331,8 @@ final class Config
|
|||
"create-share-link",
|
||||
"Create a shareable link for an existing file on OneDrive",
|
||||
&stringValues["create_share_link"],
|
||||
"debug-https",
|
||||
"Debug OneDrive HTTPS communication.",
|
||||
"debug-https",
|
||||
"Debug OneDrive HTTPS communication.",
|
||||
&boolValues["debug_https"],
|
||||
"destination-directory",
|
||||
"Destination directory for renamed or move on OneDrive - no sync will be performed.",
|
||||
|
@ -526,7 +534,7 @@ final class Config
|
|||
// configure function variables
|
||||
auto file = File(filename, "r");
|
||||
string lineBuffer;
|
||||
|
||||
|
||||
// configure scopes
|
||||
// - failure
|
||||
scope(failure) {
|
||||
|
@ -545,7 +553,7 @@ final class Config
|
|||
file.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// read file line by line
|
||||
auto range = file.byLine();
|
||||
foreach (line; range) {
|
||||
|
@ -592,7 +600,7 @@ final class Config
|
|||
setValueString("skip_dir", configFileSkipDir);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Azure AD Configuration
|
||||
if (key == "azure_ad_endpoint") {
|
||||
string azureConfigValue = c.front.dup;
|
||||
|
@ -612,7 +620,7 @@ final class Config
|
|||
case "CN":
|
||||
log.log("Using config option for Azure AD China operated by 21Vianet");
|
||||
break;
|
||||
// Default - all other entries
|
||||
// Default - all other entries
|
||||
default:
|
||||
log.log("Unknown Azure AD Endpoint - using Global Azure AD Endpoints");
|
||||
}
|
||||
|
@ -635,10 +643,10 @@ final class Config
|
|||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void configureRequiredDirectoryPermisions() {
|
||||
// return the directory permission mode required
|
||||
// - return octal!defaultDirectoryPermissionMode; ... cant be used .. which is odd
|
||||
// - return octal!defaultDirectoryPermissionMode; ... cant be used .. which is odd
|
||||
// Error: variable defaultDirectoryPermissionMode cannot be read at compile time
|
||||
if (getValueLong("sync_dir_permissions") != defaultDirectoryPermissionMode) {
|
||||
// return user configured permissions as octal integer
|
||||
|
@ -652,10 +660,10 @@ final class Config
|
|||
configuredDirectoryPermissionMode = to!int(convertedValue);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void configureRequiredFilePermisions() {
|
||||
// return the file permission mode required
|
||||
// - return octal!defaultFilePermissionMode; ... cant be used .. which is odd
|
||||
// - return octal!defaultFilePermissionMode; ... cant be used .. which is odd
|
||||
// Error: variable defaultFilePermissionMode cannot be read at compile time
|
||||
if (getValueLong("sync_file_permissions") != defaultFilePermissionMode) {
|
||||
// return user configured permissions as octal integer
|
||||
|
@ -669,7 +677,7 @@ final class Config
|
|||
configuredFilePermissionMode = to!int(convertedValue);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int returnRequiredDirectoryPermisions() {
|
||||
// read the configuredDirectoryPermissionMode and return
|
||||
if (configuredDirectoryPermissionMode == 0) {
|
||||
|
@ -679,7 +687,7 @@ final class Config
|
|||
}
|
||||
return configuredDirectoryPermissionMode;
|
||||
}
|
||||
|
||||
|
||||
int returnRequiredFilePermisions() {
|
||||
// read the configuredFilePermissionMode and return
|
||||
if (configuredFilePermissionMode == 0) {
|
||||
|
|
293
src/main.d
293
src/main.d
|
@ -5,11 +5,13 @@ import config, itemdb, monitor, onedrive, selective, sync, util;
|
|||
import std.net.curl: CurlException;
|
||||
import core.stdc.signal;
|
||||
import std.traits;
|
||||
import std.concurrency: receiveTimeout;
|
||||
static import log;
|
||||
|
||||
OneDriveApi oneDrive;
|
||||
ItemDatabase itemDb;
|
||||
|
||||
bool onedriveInitialised = false;
|
||||
const int EXIT_UNAUTHORIZED = 3;
|
||||
|
||||
enum MONITOR_LOG_SILENT = 2;
|
||||
|
@ -20,7 +22,7 @@ int main(string[] args)
|
|||
{
|
||||
// Disable buffering on stdout
|
||||
stdout.setvbuf(0, _IONBF);
|
||||
|
||||
|
||||
// main function variables
|
||||
string confdirOption;
|
||||
string configFilePath;
|
||||
|
@ -48,10 +50,9 @@ int main(string[] args)
|
|||
bool skipDirDifferent = false;
|
||||
bool online = false;
|
||||
bool performSyncOK = false;
|
||||
bool onedriveInitialised = false;
|
||||
bool displayMemoryUsage = false;
|
||||
bool displaySyncOptions = false;
|
||||
|
||||
|
||||
// Define scopes
|
||||
scope(exit) {
|
||||
// Display memory details
|
||||
|
@ -79,7 +80,7 @@ int main(string[] args)
|
|||
log.displayMemoryUsagePostGC();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
scope(failure) {
|
||||
// Display memory details
|
||||
if (displayMemoryUsage) {
|
||||
|
@ -139,7 +140,7 @@ int main(string[] args)
|
|||
log.error("Try 'onedrive -h' for more information");
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
|
||||
// load configuration file if available
|
||||
auto cfg = new config.Config(confdirOption);
|
||||
if (!cfg.initialize()) {
|
||||
|
@ -147,22 +148,22 @@ int main(string[] args)
|
|||
// Error message already printed
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
|
||||
// set memory display
|
||||
displayMemoryUsage = cfg.getValueBool("display_memory");
|
||||
|
||||
|
||||
// set display sync options
|
||||
displaySyncOptions = cfg.getValueBool("display_sync_options");
|
||||
|
||||
|
||||
// update configuration from command line args
|
||||
cfg.update_from_args(args);
|
||||
|
||||
|
||||
// Initialise normalised file paths
|
||||
configFilePath = buildNormalizedPath(cfg.configDirName ~ "/config");
|
||||
syncListFilePath = buildNormalizedPath(cfg.configDirName ~ "/sync_list");
|
||||
databaseFilePath = buildNormalizedPath(cfg.configDirName ~ "/items.db");
|
||||
businessSharedFolderFilePath = buildNormalizedPath(cfg.configDirName ~ "/business_shared_folders");
|
||||
|
||||
|
||||
// Has any of our configuration that would require a --resync been changed?
|
||||
// 1. sync_list file modification
|
||||
// 2. config file modification - but only if sync_dir, skip_dir, skip_file or drive_id was modified
|
||||
|
@ -171,25 +172,25 @@ int main(string[] args)
|
|||
syncListHashFile = buildNormalizedPath(cfg.configDirName ~ "/.sync_list.hash");
|
||||
configBackupFile = buildNormalizedPath(cfg.configDirName ~ "/.config.backup");
|
||||
businessSharedFoldersHashFile = buildNormalizedPath(cfg.configDirName ~ "/.business_shared_folders.hash");
|
||||
|
||||
|
||||
// Does a config file exist with a valid hash file
|
||||
if ((exists(configFilePath)) && (!exists(configHashFile))) {
|
||||
// Hash of config file needs to be created
|
||||
std.file.write(configHashFile, computeQuickXorHash(configFilePath));
|
||||
}
|
||||
|
||||
|
||||
// Does a sync_list file exist with a valid hash file
|
||||
if ((exists(syncListFilePath)) && (!exists(syncListHashFile))) {
|
||||
// Hash of sync_list file needs to be created
|
||||
std.file.write(syncListHashFile, computeQuickXorHash(syncListFilePath));
|
||||
}
|
||||
|
||||
|
||||
// check if business_shared_folders & business_shared_folders hash exists
|
||||
if ((exists(businessSharedFolderFilePath)) && (!exists(businessSharedFoldersHashFile))) {
|
||||
// Hash of business_shared_folders file needs to be created
|
||||
std.file.write(businessSharedFoldersHashFile, computeQuickXorHash(businessSharedFolderFilePath));
|
||||
}
|
||||
|
||||
|
||||
// If hash files exist, but config files do not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different
|
||||
if ((!exists(configFilePath)) && (exists(configHashFile))) {
|
||||
// if --resync safe remove config.hash and config.backup
|
||||
|
@ -198,18 +199,18 @@ int main(string[] args)
|
|||
safeRemove(configBackupFile);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// If sync_list hash file exists, but sync_list file does not ... remove the hash, but only if --resync was issued as now the application will use 'defaults' which 'may' be different
|
||||
if ((!exists(syncListFilePath)) && (exists(syncListHashFile))) {
|
||||
// if --resync safe remove sync_list.hash
|
||||
if (cfg.getValueBool("resync")) safeRemove(syncListHashFile);
|
||||
}
|
||||
|
||||
|
||||
if ((!exists(businessSharedFolderFilePath)) && (exists(businessSharedFoldersHashFile))) {
|
||||
// if --resync safe remove business_shared_folders.hash
|
||||
if (cfg.getValueBool("resync")) safeRemove(businessSharedFoldersHashFile);
|
||||
}
|
||||
|
||||
|
||||
// Read config hashes if they exist
|
||||
if (exists(configFilePath)) currentConfigHash = computeQuickXorHash(configFilePath);
|
||||
if (exists(syncListFilePath)) currentSyncListHash = computeQuickXorHash(syncListFilePath);
|
||||
|
@ -217,21 +218,21 @@ int main(string[] args)
|
|||
if (exists(configHashFile)) previousConfigHash = readText(configHashFile);
|
||||
if (exists(syncListHashFile)) previousSyncListHash = readText(syncListHashFile);
|
||||
if (exists(businessSharedFoldersHashFile)) previousBusinessSharedFoldersHash = readText(businessSharedFoldersHashFile);
|
||||
|
||||
|
||||
// Was sync_list file updated?
|
||||
if (currentSyncListHash != previousSyncListHash) {
|
||||
// Debugging output to assist what changed
|
||||
log.vdebug("sync_list file has been updated, --resync needed");
|
||||
syncListDifferent = true;
|
||||
}
|
||||
|
||||
|
||||
// Was business_shared_folders updated?
|
||||
if (currentBusinessSharedFoldersHash != previousBusinessSharedFoldersHash) {
|
||||
// Debugging output to assist what changed
|
||||
log.vdebug("business_shared_folders file has been updated, --resync needed");
|
||||
businessSharedFoldersDifferent = true;
|
||||
}
|
||||
|
||||
|
||||
// Was config file updated between last execution ang this execution?
|
||||
if (currentConfigHash != previousConfigHash) {
|
||||
// config file was updated, however we only want to trigger a --resync requirement if sync_dir, skip_dir, skip_file or drive_id was modified
|
||||
|
@ -266,7 +267,7 @@ int main(string[] args)
|
|||
log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed");
|
||||
configOptionsDifferent = true;
|
||||
}
|
||||
|
||||
|
||||
if ((key == "skip_file") && (c.front.dup != cfg.getValueString("skip_file"))){
|
||||
log.vdebug(key, " was modified since the last time the application was successfully run, --resync needed");
|
||||
configOptionsDifferent = true;
|
||||
|
@ -291,7 +292,7 @@ int main(string[] args)
|
|||
// no backup to check
|
||||
log.vdebug("WARNING: no backup config file was found, unable to validate if any changes made");
|
||||
}
|
||||
|
||||
|
||||
// If there was a backup, any modified values we need to worry about would been detected
|
||||
if (!cfg.getValueBool("display_config")) {
|
||||
// we are not testing the configuration
|
||||
|
@ -309,13 +310,13 @@ int main(string[] args)
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Is there a backup of the config file if the config file exists?
|
||||
if ((exists(configFilePath)) && (!exists(configBackupFile))) {
|
||||
// create backup copy of current config file
|
||||
std.file.copy(configFilePath, configBackupFile);
|
||||
}
|
||||
|
||||
|
||||
// config file set options can be changed via CLI input, specifically these will impact sync and --resync will be needed:
|
||||
// --syncdir ARG
|
||||
// --skip-file ARG
|
||||
|
@ -331,7 +332,7 @@ int main(string[] args)
|
|||
syncDirDifferent = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// was the skip_file updated by CLI?
|
||||
if (cfg.configFileSkipFile != "") {
|
||||
// skip_file was set in config file
|
||||
|
@ -340,8 +341,8 @@ int main(string[] args)
|
|||
log.vdebug("skip_file: CLI override of config file option, --resync needed");
|
||||
skipFileDifferent = true;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// was the skip_dir updated by CLI?
|
||||
if (cfg.configFileSkipDir != "") {
|
||||
// skip_dir was set in config file
|
||||
|
@ -352,7 +353,7 @@ int main(string[] args)
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Has anything triggered a --resync requirement?
|
||||
if (configOptionsDifferent || syncListDifferent || syncDirDifferent || skipFileDifferent || skipDirDifferent || businessSharedFoldersDifferent) {
|
||||
// --resync needed, is the user just testing configuration changes?
|
||||
|
@ -388,7 +389,7 @@ int main(string[] args)
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// dry-run notification and database setup
|
||||
if (cfg.getValueBool("dry_run")) {
|
||||
log.log("DRY-RUN Configured. Output below shows what 'would' have occurred.");
|
||||
|
@ -398,18 +399,18 @@ int main(string[] args)
|
|||
if (exists(cfg.databaseFilePathDryRun)) {
|
||||
// remove the existing file
|
||||
log.vdebug("Removing items-dryrun.sqlite3 as it still exists for some reason");
|
||||
safeRemove(cfg.databaseFilePathDryRun);
|
||||
safeRemove(cfg.databaseFilePathDryRun);
|
||||
}
|
||||
// silent cleanup of shm and wal files if they exist
|
||||
if (exists(dryRunShmFile)) {
|
||||
// remove items-dryrun.sqlite3-shm
|
||||
safeRemove(dryRunShmFile);
|
||||
safeRemove(dryRunShmFile);
|
||||
}
|
||||
if (exists(dryRunWalFile)) {
|
||||
// remove items-dryrun.sqlite3-wal
|
||||
safeRemove(dryRunWalFile);
|
||||
safeRemove(dryRunWalFile);
|
||||
}
|
||||
|
||||
|
||||
// Make a copy of the original items.sqlite3 for use as the dry run copy if it exists
|
||||
if (exists(cfg.databaseFilePath)) {
|
||||
// in a --dry-run --resync scenario, we should not copy the existing database file
|
||||
|
@ -423,7 +424,7 @@ int main(string[] args)
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// sync_dir environment handling to handle ~ expansion properly
|
||||
bool shellEnvSet = false;
|
||||
if ((environment.get("SHELL") == "") && (environment.get("USER") == "")){
|
||||
|
@ -450,10 +451,10 @@ int main(string[] args)
|
|||
syncDir = cfg.getValueString("sync_dir");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// vdebug syncDir as set and calculated
|
||||
log.vdebug("syncDir: ", syncDir);
|
||||
|
||||
|
||||
// Configure the logging directory if different from application default
|
||||
// log_dir environment handling to handle ~ expansion properly
|
||||
string logDir = cfg.getValueString("log_dir");
|
||||
|
@ -478,7 +479,7 @@ int main(string[] args)
|
|||
// update log_dir with normalised path, with '~' expanded correctly
|
||||
cfg.setValueString("log_dir", logDir);
|
||||
}
|
||||
|
||||
|
||||
// Configure logging only if enabled
|
||||
if (cfg.getValueBool("enable_logging")){
|
||||
// Initialise using the configured logging directory
|
||||
|
@ -488,7 +489,7 @@ int main(string[] args)
|
|||
|
||||
// Configure whether notifications are used
|
||||
log.setNotifications(cfg.getValueBool("monitor") && !cfg.getValueBool("disable_notifications"));
|
||||
|
||||
|
||||
// Application upgrades - skilion version etc
|
||||
if (exists(databaseFilePath)) {
|
||||
if (!cfg.getValueBool("dry_run")) {
|
||||
|
@ -497,7 +498,7 @@ int main(string[] args)
|
|||
log.logAndNotify("Database schema changed, resync needed");
|
||||
cfg.setValueBool("resync", true);
|
||||
}
|
||||
|
||||
|
||||
// Handle --logout as separate item, do not 'resync' on a --logout / reauth
|
||||
if (cfg.getValueBool("logout")) {
|
||||
log.vdebug("--logout requested");
|
||||
|
@ -508,7 +509,7 @@ int main(string[] args)
|
|||
// Exit
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
// Handle --resync to remove local files
|
||||
if (cfg.getValueBool("resync")) {
|
||||
if (cfg.getValueBool("resync")) log.vdebug("--resync requested");
|
||||
|
@ -519,7 +520,7 @@ int main(string[] args)
|
|||
safeRemove(cfg.uploadStateFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Display current application configuration, no application initialisation
|
||||
if (cfg.getValueBool("display_config")){
|
||||
// Display application version
|
||||
|
@ -528,7 +529,7 @@ int main(string[] args)
|
|||
writeln("Config path = ", cfg.configDirName);
|
||||
// Does a config file exist or are we using application defaults
|
||||
writeln("Config file found in config path = ", exists(configFilePath));
|
||||
|
||||
|
||||
// Config Options
|
||||
writeln("Config option 'check_nosync' = ", cfg.getValueBool("check_nosync"));
|
||||
writeln("Config option 'sync_dir' = ", syncDir);
|
||||
|
@ -543,12 +544,12 @@ int main(string[] args)
|
|||
writeln("Config option 'upload_only' = ", cfg.getValueBool("upload_only"));
|
||||
writeln("Config option 'no_remote_delete' = ", cfg.getValueBool("no_remote_delete"));
|
||||
writeln("Config option 'remove_source_files' = ", cfg.getValueBool("remove_source_files"));
|
||||
|
||||
|
||||
// Is config option drive_id configured?
|
||||
if (cfg.getValueString("drive_id") != ""){
|
||||
writeln("Config option 'drive_id' = ", cfg.getValueString("drive_id"));
|
||||
}
|
||||
|
||||
|
||||
// Is sync_list configured?
|
||||
if (exists(syncListFilePath)){
|
||||
writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files"));
|
||||
|
@ -565,7 +566,7 @@ int main(string[] args)
|
|||
writeln("Config option 'sync_root_files' = ", cfg.getValueBool("sync_root_files"));
|
||||
writeln("Selective sync 'sync_list' configured = false");
|
||||
}
|
||||
|
||||
|
||||
// Is business_shared_folders configured
|
||||
if (exists(businessSharedFolderFilePath)){
|
||||
writeln("Business Shared Folders configured = true");
|
||||
|
@ -580,11 +581,11 @@ int main(string[] args)
|
|||
} else {
|
||||
writeln("Business Shared Folders configured = false");
|
||||
}
|
||||
|
||||
|
||||
// Exit
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
// Test if OneDrive service can be reached, exit if it cant be reached
|
||||
log.vdebug("Testing network to ensure network connectivity to Microsoft OneDrive Service");
|
||||
online = testNetwork();
|
||||
|
@ -597,7 +598,7 @@ int main(string[] args)
|
|||
} else {
|
||||
// Running as --monitor
|
||||
log.error("Unable to reach Microsoft OneDrive API service at this point in time, re-trying network tests\n");
|
||||
|
||||
|
||||
// re-try network connection to OneDrive
|
||||
// https://github.com/abraunegg/onedrive/issues/1184
|
||||
// Back off & retry with incremental delay
|
||||
|
@ -605,13 +606,13 @@ int main(string[] args)
|
|||
int retryAttempts = 1;
|
||||
int backoffInterval = 1;
|
||||
int maxBackoffInterval = 3600;
|
||||
|
||||
|
||||
bool retrySuccess = false;
|
||||
while (!retrySuccess){
|
||||
// retry to access OneDrive API
|
||||
backoffInterval++;
|
||||
int thisBackOffInterval = retryAttempts*backoffInterval;
|
||||
log.vdebug(" Retry Attempt: ", retryAttempts);
|
||||
log.vdebug(" Retry Attempt: ", retryAttempts);
|
||||
if (thisBackOffInterval <= maxBackoffInterval) {
|
||||
log.vdebug(" Retry In (seconds): ", thisBackOffInterval);
|
||||
Thread.sleep(dur!"seconds"(thisBackOffInterval));
|
||||
|
@ -631,7 +632,7 @@ int main(string[] args)
|
|||
// we have attempted to re-connect X number of times
|
||||
// false set this to true to break out of while loop
|
||||
retrySuccess = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Increment & loop around
|
||||
retryAttempts++;
|
||||
|
@ -643,7 +644,7 @@ int main(string[] args)
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Initialize OneDrive, check for authorization
|
||||
if (online) {
|
||||
// we can only initialise if we are online
|
||||
|
@ -652,24 +653,24 @@ int main(string[] args)
|
|||
onedriveInitialised = oneDrive.init();
|
||||
oneDrive.printAccessToken = cfg.getValueBool("print_token");
|
||||
}
|
||||
|
||||
|
||||
if (!onedriveInitialised) {
|
||||
log.error("Could not initialize the OneDrive API");
|
||||
// Use exit scopes to shutdown API
|
||||
return EXIT_UNAUTHORIZED;
|
||||
}
|
||||
|
||||
|
||||
// if --synchronize or --monitor not passed in, configure the flag to display help & exit
|
||||
if (cfg.getValueBool("synchronize") || cfg.getValueBool("monitor")) {
|
||||
performSyncOK = true;
|
||||
}
|
||||
|
||||
|
||||
// create-directory, remove-directory, source-directory, destination-directory
|
||||
// these are activities that dont perform a sync, so to not generate an error message for these items either
|
||||
if (((cfg.getValueString("create_directory") != "") || (cfg.getValueString("remove_directory") != "")) || ((cfg.getValueString("source_directory") != "") && (cfg.getValueString("destination_directory") != "")) || (cfg.getValueString("get_file_link") != "") || (cfg.getValueString("create_share_link") != "") || (cfg.getValueString("get_o365_drive_id") != "") || cfg.getValueBool("display_sync_status") || cfg.getValueBool("list_business_shared_folders")) {
|
||||
performSyncOK = true;
|
||||
}
|
||||
|
||||
|
||||
// Were acceptable sync operations provided? Was --synchronize or --monitor passed in
|
||||
if (!performSyncOK) {
|
||||
// was the application just authorised?
|
||||
|
@ -694,7 +695,7 @@ int main(string[] args)
|
|||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// if --synchronize && --monitor passed in, exit & display help as these conflict with each other
|
||||
if (cfg.getValueBool("synchronize") && cfg.getValueBool("monitor")) {
|
||||
writeln("\nERROR: --synchronize and --monitor cannot be used together\n");
|
||||
|
@ -702,7 +703,7 @@ int main(string[] args)
|
|||
// Use exit scopes to shutdown API
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
|
||||
// Initialize the item database
|
||||
log.vlog("Opening the item database ...");
|
||||
if (!cfg.getValueBool("dry_run")) {
|
||||
|
@ -714,7 +715,7 @@ int main(string[] args)
|
|||
log.vdebug("Using database file: ", asNormalizedPath(cfg.databaseFilePathDryRun));
|
||||
itemDb = new ItemDatabase(cfg.databaseFilePathDryRun);
|
||||
}
|
||||
|
||||
|
||||
// What are the permission that have been set for the application?
|
||||
// These are relevant for:
|
||||
// - The ~/OneDrive parent folder or 'sync_dir' configured item
|
||||
|
@ -731,7 +732,7 @@ int main(string[] args)
|
|||
log.vdebug("Configuring default new file permissions as: ", cfg.getValueLong("sync_file_permissions"));
|
||||
cfg.configureRequiredFilePermisions();
|
||||
}
|
||||
|
||||
|
||||
// configure the sync direcory based on syncDir config option
|
||||
log.vlog("All operations will be performed in: ", syncDir);
|
||||
if (!exists(syncDir)) {
|
||||
|
@ -749,13 +750,13 @@ int main(string[] args)
|
|||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Change the working directory to the 'sync_dir' configured item
|
||||
chdir(syncDir);
|
||||
|
||||
|
||||
// Configure selective sync by parsing and getting a regex for skip_file config component
|
||||
auto selectiveSync = new SelectiveSync();
|
||||
|
||||
|
||||
// load sync_list if it exists
|
||||
if (exists(syncListFilePath)){
|
||||
log.vdebug("Loading user configured sync_list file ...");
|
||||
|
@ -774,7 +775,7 @@ int main(string[] args)
|
|||
}
|
||||
}
|
||||
selectiveSync.load(syncListFilePath);
|
||||
|
||||
|
||||
// load business_shared_folders if it exists
|
||||
if (exists(businessSharedFolderFilePath)){
|
||||
log.vdebug("Loading user configured business_shared_folders file ...");
|
||||
|
@ -787,27 +788,27 @@ int main(string[] args)
|
|||
}
|
||||
}
|
||||
selectiveSync.loadSharedFolders(businessSharedFolderFilePath);
|
||||
|
||||
|
||||
// Configure skip_dir, skip_file, skip-dir-strict-match & skip_dotfiles from config entries
|
||||
// Handle skip_dir configuration in config file
|
||||
log.vdebug("Configuring skip_dir ...");
|
||||
log.vdebug("skip_dir: ", cfg.getValueString("skip_dir"));
|
||||
selectiveSync.setDirMask(cfg.getValueString("skip_dir"));
|
||||
|
||||
|
||||
// Was --skip-dir-strict-match configured?
|
||||
log.vdebug("Configuring skip_dir_strict_match ...");
|
||||
log.vdebug("skip_dir_strict_match: ", cfg.getValueBool("skip_dir_strict_match"));
|
||||
if (cfg.getValueBool("skip_dir_strict_match")) {
|
||||
selectiveSync.setSkipDirStrictMatch();
|
||||
}
|
||||
|
||||
|
||||
// Was --skip-dot-files configured?
|
||||
log.vdebug("Configuring skip_dotfiles ...");
|
||||
log.vdebug("skip_dotfiles: ", cfg.getValueBool("skip_dotfiles"));
|
||||
if (cfg.getValueBool("skip_dotfiles")) {
|
||||
selectiveSync.setSkipDotfiles();
|
||||
}
|
||||
|
||||
|
||||
// Handle skip_file configuration in config file
|
||||
log.vdebug("Configuring skip_file ...");
|
||||
// Validate skip_file to ensure that this does not contain an invalid configuration
|
||||
|
@ -822,7 +823,7 @@ int main(string[] args)
|
|||
// All skip_file entries are valid
|
||||
log.vdebug("skip_file: ", cfg.getValueString("skip_file"));
|
||||
selectiveSync.setFileMask(cfg.getValueString("skip_file"));
|
||||
|
||||
|
||||
// Initialize the sync engine
|
||||
auto sync = new SyncEngine(cfg, oneDrive, itemDb, selectiveSync);
|
||||
try {
|
||||
|
@ -847,7 +848,7 @@ int main(string[] args)
|
|||
if (syncListConfigured) {
|
||||
sync.setSyncListConfigured();
|
||||
}
|
||||
|
||||
|
||||
// Do we need to configure specific --upload-only options?
|
||||
if (cfg.getValueBool("upload_only")) {
|
||||
// --upload-only was passed in or configured
|
||||
|
@ -866,12 +867,12 @@ int main(string[] args)
|
|||
sync.setLocalDeleteAfterUpload();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Do we configure to disable the upload validation routine
|
||||
if (cfg.getValueBool("disable_upload_validation")) sync.setDisableUploadValidation();
|
||||
|
||||
|
||||
// Do we configure to disable the download validation routine
|
||||
if (cfg.getValueBool("disable_download_validation")) sync.setDisableDownloadValidation();
|
||||
if (cfg.getValueBool("disable_download_validation")) sync.setDisableDownloadValidation();
|
||||
|
||||
// Has the user enabled to bypass data preservation of renaming local files when there is a conflict?
|
||||
if (cfg.getValueBool("bypass_data_preservation")) {
|
||||
|
@ -879,7 +880,7 @@ int main(string[] args)
|
|||
log.log("WARNING: Local data loss MAY occur in this scenario.");
|
||||
sync.setBypassDataPreservation();
|
||||
}
|
||||
|
||||
|
||||
// Are we configured to use a National Cloud Deployment
|
||||
if (cfg.getValueString("azure_ad_endpoint") != "") {
|
||||
// value is configured, is it a valid value?
|
||||
|
@ -888,7 +889,7 @@ int main(string[] args)
|
|||
sync.setNationalCloudDeployment();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Do we need to validate the syncDir to check for the presence of a '.nosync' file
|
||||
if (cfg.getValueBool("check_nomount")) {
|
||||
// we were asked to check the mounts
|
||||
|
@ -898,53 +899,53 @@ int main(string[] args)
|
|||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Do we need to create or remove a directory?
|
||||
if ((cfg.getValueString("create_directory") != "") || (cfg.getValueString("remove_directory") != "")) {
|
||||
|
||||
|
||||
if (cfg.getValueString("create_directory") != "") {
|
||||
// create a directory on OneDrive
|
||||
sync.createDirectoryNoSync(cfg.getValueString("create_directory"));
|
||||
}
|
||||
|
||||
|
||||
if (cfg.getValueString("remove_directory") != "") {
|
||||
// remove a directory on OneDrive
|
||||
sync.deleteDirectoryNoSync(cfg.getValueString("remove_directory"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Are we renaming or moving a directory?
|
||||
if ((cfg.getValueString("source_directory") != "") && (cfg.getValueString("destination_directory") != "")) {
|
||||
// We are renaming or moving a directory
|
||||
sync.renameDirectoryNoSync(cfg.getValueString("source_directory"), cfg.getValueString("destination_directory"));
|
||||
}
|
||||
|
||||
|
||||
// Are we obtaining the Office 365 Drive ID for a given Office 365 SharePoint Shared Library?
|
||||
if (cfg.getValueString("get_o365_drive_id") != "") {
|
||||
sync.querySiteCollectionForDriveID(cfg.getValueString("get_o365_drive_id"));
|
||||
// Exit application
|
||||
// Exit application
|
||||
// Use exit scopes to shutdown API
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
// Are we createing an anonymous read-only shareable link for an existing file on OneDrive?
|
||||
if (cfg.getValueString("create_share_link") != "") {
|
||||
// Query OneDrive for the file, and if valid, create a shareable link for the file
|
||||
sync.createShareableLinkForFile(cfg.getValueString("create_share_link"));
|
||||
// Exit application
|
||||
// Exit application
|
||||
// Use exit scopes to shutdown API
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
// Are we obtaining the URL path for a synced file?
|
||||
if (cfg.getValueString("get_file_link") != "") {
|
||||
// Query OneDrive for the file link
|
||||
sync.queryOneDriveForFileURL(cfg.getValueString("get_file_link"), syncDir);
|
||||
// Exit application
|
||||
// Exit application
|
||||
// Use exit scopes to shutdown API
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
// Are we listing OneDrive Business Shared Folders
|
||||
if (cfg.getValueBool("list_business_shared_folders")) {
|
||||
// Is this a business account type?
|
||||
|
@ -954,11 +955,11 @@ int main(string[] args)
|
|||
} else {
|
||||
log.error("ERROR: Unsupported account type for listing OneDrive Business Shared Folders");
|
||||
}
|
||||
// Exit application
|
||||
// Exit application
|
||||
// Use exit scopes to shutdown API
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
// Are we going to sync OneDrive Business Shared Folders
|
||||
if (cfg.getValueBool("sync_business_shared_folders")) {
|
||||
// Is this a business account type?
|
||||
|
@ -969,7 +970,7 @@ int main(string[] args)
|
|||
log.error("ERROR: Unsupported account type for syncing OneDrive Business Shared Folders");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Are we displaying the sync status of the client?
|
||||
if (cfg.getValueBool("display_sync_status")) {
|
||||
string remotePath = "/";
|
||||
|
@ -980,7 +981,7 @@ int main(string[] args)
|
|||
}
|
||||
sync.queryDriveForChanges(remotePath);
|
||||
}
|
||||
|
||||
|
||||
// Are we performing a sync, or monitor operation?
|
||||
if ((cfg.getValueBool("synchronize")) || (cfg.getValueBool("monitor"))) {
|
||||
// Initialise the monitor class, so that we can do more granular inotify handling when performing the actual sync
|
||||
|
@ -1007,17 +1008,17 @@ int main(string[] args)
|
|||
// fullScanRequired = false, for final true-up
|
||||
// but if we have sync_list configured, use syncListConfigured which = true
|
||||
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, false, syncListConfigured, displaySyncOptions, cfg.getValueBool("monitor"), m);
|
||||
|
||||
|
||||
// Write WAL and SHM data to file for this sync
|
||||
log.vdebug("Merge contents of WAL and SHM files into main database file");
|
||||
itemDb.performVacuum();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (cfg.getValueBool("monitor")) {
|
||||
log.logAndNotify("Initializing monitor ...");
|
||||
log.log("OneDrive monitor interval (seconds): ", cfg.getValueLong("monitor_interval"));
|
||||
|
||||
|
||||
m.onDirCreated = delegate(string path) {
|
||||
// Handle .folder creation if skip_dotfiles is enabled
|
||||
if ((cfg.getValueBool("skip_dotfiles")) && (selectiveSync.isDotFile(path))) {
|
||||
|
@ -1106,7 +1107,8 @@ int main(string[] args)
|
|||
// sync list is configured
|
||||
syncListConfiguredFullScanOverride = true;
|
||||
}
|
||||
|
||||
immutable bool webhookEnabled = cfg.getValueBool("webhook_enabled");
|
||||
|
||||
while (performMonitor) {
|
||||
if (!cfg.getValueBool("download_only")) {
|
||||
try {
|
||||
|
@ -1116,7 +1118,35 @@ int main(string[] args)
|
|||
log.error("ERROR: The following inotify error was generated: ", e.msg);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Check for notifications pushed from Microsoft to the webhook
|
||||
bool notificationReceived = false;
|
||||
if (webhookEnabled) {
|
||||
// Create a subscription on the first run, or renew the subscription
|
||||
// on subsequent runs when it is about to expire.
|
||||
oneDrive.createOrRenewSubscription();
|
||||
|
||||
// Process incoming notifications if any.
|
||||
|
||||
// Empirical evidence shows that Microsoft often sends multiple
|
||||
// notifications for one single change, so we need a loop to exhaust
|
||||
// all signals that were queued up by the webhook. The notifications
|
||||
// do not contain any actual changes, and we will always rely do the
|
||||
// delta endpoint to sync to latest. Therefore, only one sync run is
|
||||
// good enough to catch up for multiple notifications.
|
||||
for (int signalCount = 0;; signalCount++) {
|
||||
const auto signalExists = receiveTimeout(dur!"seconds"(-1), (ulong _) {});
|
||||
if (signalExists) {
|
||||
notificationReceived = true;
|
||||
} else {
|
||||
if (notificationReceived) {
|
||||
log.log("Received ", signalCount," refresh signals from the webhook");
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto currTime = MonoTime.currTime();
|
||||
// has monitor_interval elapsed or are we at application startup / monitor startup?
|
||||
// in a --resync scenario, if we have not 're-populated' the database, valid changes will get skipped:
|
||||
|
@ -1126,7 +1156,7 @@ int main(string[] args)
|
|||
// Moving random_files/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby to target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
|
||||
// Skipping uploading this new file as parent path is not in the database: target/2eVPInOMTFNXzRXeNMEoJch5OR9XpGby
|
||||
// 'target' should be in the DB, it should also exist online, but because of --resync, it does not exist in the database thus parent check fails
|
||||
if ((currTime - lastCheckTime > checkInterval) || (monitorLoopFullCount == 0)) {
|
||||
if (notificationReceived || (currTime - lastCheckTime > checkInterval) || (monitorLoopFullCount == 0)) {
|
||||
// monitor sync loop
|
||||
logOutputMessage = "################################################## NEW LOOP ##################################################";
|
||||
if (displaySyncOptions) {
|
||||
|
@ -1140,7 +1170,7 @@ int main(string[] args)
|
|||
if (displayMemoryUsage) {
|
||||
log.displayMemoryUsagePreGC();
|
||||
}
|
||||
|
||||
|
||||
// log monitor output suppression
|
||||
logMonitorCounter += 1;
|
||||
if (logMonitorCounter > logInterval) {
|
||||
|
@ -1162,7 +1192,7 @@ int main(string[] args)
|
|||
fullScanRequired = false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (displaySyncOptions) {
|
||||
// sync option handling per sync loop
|
||||
log.log("fullScanCounter = ", fullScanCounter);
|
||||
|
@ -1222,14 +1252,14 @@ int main(string[] args)
|
|||
if (displayMemoryUsage) {
|
||||
log.displayMemoryUsagePostGC();
|
||||
}
|
||||
|
||||
|
||||
// Write WAL and SHM data to file for this loop
|
||||
log.vdebug("Merge contents of WAL and SHM files into main database file");
|
||||
itemDb.performVacuum();
|
||||
|
||||
|
||||
// monitor loop complete
|
||||
logOutputMessage = "################################################ LOOP COMPLETE ###############################################";
|
||||
|
||||
|
||||
// Handle display options
|
||||
if (displaySyncOptions) {
|
||||
log.log(logOutputMessage);
|
||||
|
@ -1258,20 +1288,20 @@ int main(string[] args)
|
|||
// remove the file
|
||||
log.vdebug("Removing items-dryrun.sqlite3 as dry run operations complete");
|
||||
// remove items-dryrun.sqlite3
|
||||
safeRemove(cfg.databaseFilePathDryRun);
|
||||
safeRemove(cfg.databaseFilePathDryRun);
|
||||
}
|
||||
// silent cleanup of shm and wal files if they exist
|
||||
if (exists(dryRunShmFile)) {
|
||||
// remove items-dryrun.sqlite3-shm
|
||||
safeRemove(dryRunShmFile);
|
||||
safeRemove(dryRunShmFile);
|
||||
}
|
||||
if (exists(dryRunWalFile)) {
|
||||
// remove items-dryrun.sqlite3-wal
|
||||
safeRemove(dryRunWalFile);
|
||||
safeRemove(dryRunWalFile);
|
||||
}
|
||||
}
|
||||
|
||||
// Exit application
|
||||
|
||||
// Exit application
|
||||
// Use exit scopes to shutdown API
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
@ -1301,14 +1331,14 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
string remotePath = "/";
|
||||
string localPath = ".";
|
||||
string logOutputMessage;
|
||||
|
||||
|
||||
// performSync API scan triggers
|
||||
log.vdebug("performSync API scan triggers");
|
||||
log.vdebug("-----------------------------");
|
||||
log.vdebug("fullScanRequired = ", fullScanRequired);
|
||||
log.vdebug("syncListConfiguredFullScanOverride = ", syncListConfiguredFullScanOverride);
|
||||
log.vdebug("-----------------------------");
|
||||
|
||||
|
||||
// Are we doing a single directory sync?
|
||||
if (singleDirectory != ""){
|
||||
// Need two different path strings here
|
||||
|
@ -1317,12 +1347,12 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
// Set flag for singleDirectoryScope for change handling
|
||||
sync.setSingleDirectoryScope();
|
||||
}
|
||||
|
||||
|
||||
// Due to Microsoft Sharepoint 'enrichment' of files, we try to download the Microsoft modified file automatically
|
||||
// Set flag if we are in upload only state to handle this differently
|
||||
// See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details
|
||||
// See: https://github.com/OneDrive/onedrive-api-docs/issues/935 for further details
|
||||
if (uploadOnly) sync.setUploadOnly();
|
||||
|
||||
|
||||
do {
|
||||
try {
|
||||
// starting a sync
|
||||
|
@ -1383,7 +1413,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
} else {
|
||||
// sync from OneDrive first before uploading files to OneDrive
|
||||
if (logLevel < MONITOR_LOG_SILENT) log.log("Syncing changes from OneDrive ...");
|
||||
|
||||
|
||||
// For the initial sync, always use the delta link so that we capture all the right delta changes including adds, moves & deletes
|
||||
logOutputMessage = "Initial Scan: Call OneDrive Delta API for delta changes as compared to last successful sync.";
|
||||
syncCallLogOutput = "Calling sync.applyDifferences(false);";
|
||||
|
@ -1395,8 +1425,8 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
log.vdebug(syncCallLogOutput);
|
||||
}
|
||||
sync.applyDifferences(false);
|
||||
|
||||
// is this a download only request?
|
||||
|
||||
// is this a download only request?
|
||||
if (!downloadOnly) {
|
||||
// process local changes walking the entire path checking for changes
|
||||
// in monitor mode all local changes are captured via inotify
|
||||
|
@ -1410,7 +1440,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
log.vdebug(logOutputMessage);
|
||||
log.vdebug(syncCallLogOutput);
|
||||
}
|
||||
|
||||
|
||||
// What sort of local scan do we want to do?
|
||||
// In --monitor mode, when performing the DB scan, a race condition occurs where by if a file or folder is moved during this process
|
||||
// the inotify event is discarded once performSync() is finished (see m.update(false) above), so these events need to be handled
|
||||
|
@ -1431,20 +1461,20 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
// handle any inotify events that occured 'whilst' we were scanning the local filesystem
|
||||
m.update(true);
|
||||
}
|
||||
|
||||
|
||||
// At this point, all OneDrive changes / local changes should be uploaded and in sync
|
||||
// This MAY not be the case when using sync_list, thus a full walk of OneDrive ojects is required
|
||||
|
||||
|
||||
// --synchronize & no sync_list : fullScanRequired = false, syncListConfiguredFullScanOverride = false
|
||||
// --synchronize & sync_list in use : fullScanRequired = false, syncListConfiguredFullScanOverride = true
|
||||
|
||||
|
||||
// --monitor loops around 10 iterations. On the 1st loop, sets fullScanRequired = false, syncListConfiguredFullScanOverride = true if requried
|
||||
|
||||
|
||||
// --monitor & no sync_list (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = false
|
||||
// --monitor & no sync_list (loop #2 - #10) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
|
||||
// --monitor & sync_list in use (loop #1) : fullScanRequired = true, syncListConfiguredFullScanOverride = true
|
||||
// --monitor & sync_list in use (loop #2 - #10) : fullScanRequired = false, syncListConfiguredFullScanOverride = false
|
||||
|
||||
|
||||
// Do not perform a full walk of the OneDrive objects
|
||||
if ((!fullScanRequired) && (!syncListConfiguredFullScanOverride)){
|
||||
logOutputMessage = "Final True-Up: Do not perform a full walk of the OneDrive objects - not required";
|
||||
|
@ -1458,7 +1488,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
}
|
||||
sync.applyDifferences(false);
|
||||
}
|
||||
|
||||
|
||||
// Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop
|
||||
if ((!fullScanRequired) && (syncListConfiguredFullScanOverride)){
|
||||
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because sync_list is in use / or trigger was set in --monitor loop";
|
||||
|
@ -1472,7 +1502,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
}
|
||||
sync.applyDifferences(true);
|
||||
}
|
||||
|
||||
|
||||
// Perform a full walk of OneDrive objects because a full scan was required
|
||||
if ((fullScanRequired) && (!syncListConfiguredFullScanOverride)){
|
||||
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required";
|
||||
|
@ -1483,10 +1513,10 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
} else {
|
||||
log.vdebug(logOutputMessage);
|
||||
log.vdebug(syncCallLogOutput);
|
||||
}
|
||||
}
|
||||
sync.applyDifferences(true);
|
||||
}
|
||||
|
||||
|
||||
// Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop
|
||||
if ((fullScanRequired) && (syncListConfiguredFullScanOverride)){
|
||||
logOutputMessage = "Final True-Up: Perform a full walk of OneDrive objects because a full scan was required and sync_list is in use and trigger was set in --monitor loop";
|
||||
|
@ -1504,7 +1534,7 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// sync is complete
|
||||
logOutputMessage = "################################################ SYNC COMPLETE ###############################################";
|
||||
if (displaySyncOptions) {
|
||||
|
@ -1512,13 +1542,13 @@ void performSync(SyncEngine sync, string singleDirectory, bool downloadOnly, boo
|
|||
} else {
|
||||
log.vdebug(logOutputMessage);
|
||||
}
|
||||
|
||||
|
||||
count = -1;
|
||||
} catch (Exception e) {
|
||||
if (++count == 3) {
|
||||
log.log("Giving up on sync after three attempts: ", e.msg);
|
||||
throw e;
|
||||
} else
|
||||
} else
|
||||
log.log("Retry sync count: ", count, ": ", e.msg);
|
||||
}
|
||||
} while (count != -1);
|
||||
|
@ -1535,14 +1565,19 @@ auto assumeNoGC(T) (T t) if (isFunctionPointer!T || isDelegate!T)
|
|||
extern(C) nothrow @nogc @system void exitHandler(int value) {
|
||||
try {
|
||||
assumeNoGC ( () {
|
||||
log.log("Got termination signal, shutting down db connection");
|
||||
log.log("Got termination signal, performing clean up");
|
||||
// if initialised, shut down the HTTP instance
|
||||
if (onedriveInitialised) {
|
||||
log.log("Shutting down the HTTP instance");
|
||||
oneDrive.shutdown();
|
||||
}
|
||||
// was itemDb initialised?
|
||||
if (itemDb !is null) {
|
||||
// Make sure the .wal file is incorporated into the main db before we exit
|
||||
log.log("Shutting down db connection");
|
||||
itemDb.performVacuum();
|
||||
destroy(itemDb);
|
||||
}
|
||||
// Use exit scopes to shutdown OneDrive API
|
||||
})();
|
||||
} catch(Exception e) {}
|
||||
exit(0);
|
||||
|
|
442
src/onedrive.d
442
src/onedrive.d
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue