mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-10 01:42:32 +02:00
Merge 'master' to PR
* Merge 'master' to PR
This commit is contained in:
abraunegg
commit
50137b5f3b
5
.github/ISSUE_TEMPLATE/bug_report.md
vendored
5
.github/ISSUE_TEMPLATE/bug_report.md
vendored
|
|
@ -15,10 +15,13 @@ A clear and concise description of what the bug is.
|
|||
* OneDrive Account Type
|
||||
* Did you build from source or install from a package?
|
||||
* If you installed from source, what is your DMD or LDC compiler version: `dmd --version` or `ldmd2 --version`
|
||||
* OneDrive Application Version: Output of `onedrive --version`
|
||||
* OneDrive Application Configuration: Output of `onedrive --display-config`
|
||||
* Provide the version of curl you are using: Output of `curl --version`
|
||||
* Is your configured 'sync_dir' a local directory or a network mount point?
|
||||
* Provide all the mountpoints in your system: Output of: `mount`
|
||||
* If *not* local, provide all the mountpoints in your system: Output of: `mount`
|
||||
* What partition format type does your configured 'sync_dir' reside on? Output of: `lsblk -f`
|
||||
* Explain your entire configuration setup - is the OneDrive folder shared with any other system, shared with any other platform at the same time, is the OneDrive account you use shared across multiple systems / platforms / Operating Systems and in use at the same time
|
||||
|
||||
**Note:** Please generate a full debug log whilst reproducing the issue as per [https://github.com/abraunegg/onedrive/wiki/Generate-debug-log-for-support](https://github.com/abraunegg/onedrive/wiki/Generate-debug-log-for-support) and email to support@mynas.com.au
|
||||
|
||||
|
|
|
|||
42
CHANGELOG.md
42
CHANGELOG.md
|
|
@ -1,7 +1,47 @@
|
|||
# Changelog
|
||||
|
||||
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
||||
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## 2.4.8 - 2020-11-30
|
||||
### Fixed
|
||||
* Fix to use config set option for 'remove_source_files' and 'skip_dir_strict_match' rather than ignore if set
|
||||
* Fix download failure and crash due to incorrect local filesystem permissions when using mounted external devices
|
||||
* Fix to not change permissions on pre-existing local directories
|
||||
* Fix logging output when authentication authorisation fails to not say authorisation was successful
|
||||
* Fix to check application_id before setting redirect URL when using specific Azure endpoints
|
||||
* Fix application crash in --monitor mode due to 'Failed to stat file' when setgid is used on a directory and data cannot be read
|
||||
|
||||
### Added
|
||||
* Added advanced-usage.md to document advaced client usage such as multi account configurations and Windows dual-boot
|
||||
|
||||
### Updated
|
||||
* Updated --verbose logging output for config options when set
|
||||
* Updated documentation (man page, USAGE.md, Office365.md, BusinessSharedFolders.md)
|
||||
|
||||
## 2.4.7 - 2020-11-09
|
||||
### Fixed
|
||||
* Fix debugging output for /delta changes available queries
|
||||
* Fix logging output for modification comparison source data
|
||||
* Fix Business Shared Folder handling to process only Shared Folders, not individually shared files
|
||||
* Fix cleanup dryrun shm and wal files if they exist
|
||||
* Fix --list-shared-folders to only show folders
|
||||
* Fix to check for the presence of .nosync when processing DB entries
|
||||
* Fix skip_dir matching when using --resync
|
||||
* Fix uploading data to shared business folders when using --upload-only
|
||||
* Fix to merge contents of SQLite WAL file into main database file on sync completion
|
||||
* Fix to check if localModifiedTime is >= than item.mtime to avoid re-upload for equal modified time
|
||||
* Fix to correctly set config directory permissions at first start
|
||||
|
||||
### Added
|
||||
* Added environment variable to allow easy HTTPS debug in docker
|
||||
* Added environment variable to allow download-only mode in Docker
|
||||
* Implement Feature: Allow config to specify a tenant id for non-multi-tenant applications
|
||||
* Implement Feature: Adding support for authentication with single tenant custom applications
|
||||
* Implement Feature: Configure specific File and Folder Permissions
|
||||
|
||||
### Updated
|
||||
* Updated documentation (readme.md, install.md, usage.md, bug_report.md)
|
||||
|
||||
## 2.4.6 - 2020-10-04
|
||||
### Fixed
|
||||
* Fix flagging of remaining free space when value is being restricted
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ endif
|
|||
system_unit_files = contrib/systemd/onedrive@.service
|
||||
user_unit_files = contrib/systemd/onedrive.service
|
||||
|
||||
DOCFILES = README.md config LICENSE CHANGELOG.md docs/Docker.md docs/INSTALL.md docs/Office365.md docs/USAGE.md docs/BusinessSharedFolders.md
|
||||
DOCFILES = README.md config LICENSE CHANGELOG.md docs/Docker.md docs/INSTALL.md docs/Office365.md docs/USAGE.md docs/BusinessSharedFolders.md docs/advanced-usage.md
|
||||
|
||||
ifneq ("$(wildcard /etc/redhat-release)","")
|
||||
RHEL = $(shell cat /etc/redhat-release | grep -E "(Red Hat Enterprise Linux Server|CentOS)" | wc -l)
|
||||
|
|
|
|||
2
config
2
config
|
|
@ -40,3 +40,5 @@
|
|||
# azure_ad_endpoint = ""
|
||||
# azure_tenant_id = "common"
|
||||
# sync_business_shared_folders = "false"
|
||||
# sync_dir_permissions = "700"
|
||||
# sync_file_permissions = "600"
|
||||
20
configure
vendored
20
configure
vendored
|
|
@ -1,6 +1,6 @@
|
|||
#! /bin/sh
|
||||
# Guess values for system-dependent variables and create Makefiles.
|
||||
# Generated by GNU Autoconf 2.69 for onedrive v2.4.7-dev.
|
||||
# Generated by GNU Autoconf 2.69 for onedrive v2.4.8.
|
||||
#
|
||||
# Report bugs to <https://github.com/abraunegg/onedrive>.
|
||||
#
|
||||
|
|
@ -579,8 +579,8 @@ MAKEFLAGS=
|
|||
# Identity of this package.
|
||||
PACKAGE_NAME='onedrive'
|
||||
PACKAGE_TARNAME='onedrive'
|
||||
PACKAGE_VERSION='v2.4.7-dev'
|
||||
PACKAGE_STRING='onedrive v2.4.7-dev'
|
||||
PACKAGE_VERSION='v2.4.8'
|
||||
PACKAGE_STRING='onedrive v2.4.8'
|
||||
PACKAGE_BUGREPORT='https://github.com/abraunegg/onedrive'
|
||||
PACKAGE_URL=''
|
||||
|
||||
|
|
@ -1219,7 +1219,7 @@ if test "$ac_init_help" = "long"; then
|
|||
# Omit some internal or obsolete options to make the list less imposing.
|
||||
# This message is too long to be a string in the A/UX 3.1 sh.
|
||||
cat <<_ACEOF
|
||||
\`configure' configures onedrive v2.4.7-dev to adapt to many kinds of systems.
|
||||
\`configure' configures onedrive v2.4.8 to adapt to many kinds of systems.
|
||||
|
||||
Usage: $0 [OPTION]... [VAR=VALUE]...
|
||||
|
||||
|
|
@ -1280,7 +1280,7 @@ fi
|
|||
|
||||
if test -n "$ac_init_help"; then
|
||||
case $ac_init_help in
|
||||
short | recursive ) echo "Configuration of onedrive v2.4.7-dev:";;
|
||||
short | recursive ) echo "Configuration of onedrive v2.4.8:";;
|
||||
esac
|
||||
cat <<\_ACEOF
|
||||
|
||||
|
|
@ -1393,7 +1393,7 @@ fi
|
|||
test -n "$ac_init_help" && exit $ac_status
|
||||
if $ac_init_version; then
|
||||
cat <<\_ACEOF
|
||||
onedrive configure v2.4.7-dev
|
||||
onedrive configure v2.4.8
|
||||
generated by GNU Autoconf 2.69
|
||||
|
||||
Copyright (C) 2012 Free Software Foundation, Inc.
|
||||
|
|
@ -1410,7 +1410,7 @@ cat >config.log <<_ACEOF
|
|||
This file contains any messages produced by compilers while
|
||||
running configure, to aid debugging if configure makes a mistake.
|
||||
|
||||
It was created by onedrive $as_me v2.4.7-dev, which was
|
||||
It was created by onedrive $as_me v2.4.8, which was
|
||||
generated by GNU Autoconf 2.69. Invocation command line was
|
||||
|
||||
$ $0 $@
|
||||
|
|
@ -2162,7 +2162,7 @@ fi
|
|||
|
||||
|
||||
|
||||
PACKAGE_DATE="October 2020"
|
||||
PACKAGE_DATE="November 2020"
|
||||
|
||||
|
||||
|
||||
|
|
@ -3159,7 +3159,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
|
|||
# report actual input values of CONFIG_FILES etc. instead of their
|
||||
# values after options handling.
|
||||
ac_log="
|
||||
This file was extended by onedrive $as_me v2.4.7-dev, which was
|
||||
This file was extended by onedrive $as_me v2.4.8, which was
|
||||
generated by GNU Autoconf 2.69. Invocation command line was
|
||||
|
||||
CONFIG_FILES = $CONFIG_FILES
|
||||
|
|
@ -3212,7 +3212,7 @@ _ACEOF
|
|||
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
|
||||
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
|
||||
ac_cs_version="\\
|
||||
onedrive config.status v2.4.7-dev
|
||||
onedrive config.status v2.4.8
|
||||
configured by $0, generated by GNU Autoconf 2.69,
|
||||
with options \\"\$ac_cs_config\\"
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ dnl - commit the changed files (configure.ac, configure)
|
|||
dnl - tag the release
|
||||
|
||||
AC_PREREQ([2.69])
|
||||
AC_INIT([onedrive],[v2.4.7-dev], [https://github.com/abraunegg/onedrive], [onedrive])
|
||||
AC_INIT([onedrive],[v2.4.8], [https://github.com/abraunegg/onedrive], [onedrive])
|
||||
AC_CONFIG_SRCDIR([src/main.d])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -40,12 +40,24 @@ if [ "${ONEDRIVE_DEBUG:=0}" == "1" ]; then
|
|||
ARGS=(--verbose --verbose ${ARGS[@]})
|
||||
fi
|
||||
|
||||
# Tell client to perform HTTPS debug output, based on an environment variable
|
||||
if [ "${ONEDRIVE_DEBUG_HTTPS:=0}" == "1" ]; then
|
||||
echo "# We are performing HTTPS debug output"
|
||||
ARGS=(--debug-https ${ARGS[@]})
|
||||
fi
|
||||
|
||||
# Tell client to perform a resync based on environment variable
|
||||
if [ "${ONEDRIVE_RESYNC:=0}" == "1" ]; then
|
||||
echo "# We are performing a --resync"
|
||||
ARGS=(--resync ${ARGS[@]})
|
||||
fi
|
||||
|
||||
# Tell client to sync in download-only mode based on environment variable
|
||||
if [ "${ONEDRIVE_DOWNLOADONLY:=0}" == "1" ]; then
|
||||
echo "# We are synchronizing in download-only mode"
|
||||
ARGS=(--download-only ${ARGS[@]})
|
||||
fi
|
||||
|
||||
if [ ${#} -gt 0 ]; then
|
||||
ARGS=("${@}")
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
%endif
|
||||
|
||||
Name: onedrive
|
||||
Version: 2.4.6
|
||||
Version: 2.4.8
|
||||
Release: 1%{?dist}
|
||||
Summary: Microsoft OneDrive Client
|
||||
Group: System Environment/Network
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@
|
|||
Syncing OneDrive Business Shared Folders requires additional configuration for your 'onedrive' client:
|
||||
1. List available shared folders to determine which folder you wish to sync & to validate that you have access to that folder
|
||||
2. Create a new file called 'business_shared_folders' in your config directory which contains a list of the shared folders you wish to sync
|
||||
3. Perform a sync
|
||||
3. Test the configuration using '--dry-run'
|
||||
4. Sync the OneDrive Business Shared folders as required
|
||||
|
||||
## Listing available OneDrive Business Shared Folders
|
||||
List the available OneDrive Business Shared folders with the following command:
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ docker volume inspect onedrive_conf
|
|||
|
||||
Or you can map your own config folder to the config volume. Make sure to copy all files from the docker volume into your mapped folder first.
|
||||
|
||||
The detailed document for the config can be found here: [additional-configuration](https://github.com/abraunegg/onedrive#additional-configuration)
|
||||
The detailed document for the config can be found here: [Configuration](https://github.com/abraunegg/onedrive/blob/master/docs/USAGE.md#configuration)
|
||||
|
||||
### 7. Sync multiple accounts
|
||||
|
||||
|
|
@ -152,7 +152,9 @@ docker run $firstRun --restart unless-stopped --name onedrive -v onedrive_conf:/
|
|||
| <B>ONEDRIVE_GID</B> | GroupID (GID) to run as | 1000 |
|
||||
| <B>ONEDRIVE_VERBOSE</B> | Controls "--verbose" switch on onedrive sync. Default is 0 | 1 |
|
||||
| <B>ONEDRIVE_DEBUG</B> | Controls "--verbose --verbose" switch on onedrive sync. Default is 0 | 1 |
|
||||
| <B>ONEDRIVE_DEBUG_HTTPS</B> | Controls "--debug-https" switch on onedrive sync. Default is 0 | 1 |
|
||||
| <B>ONEDRIVE_RESYNC</B> | Controls "--resync" switch on onedrive sync. Default is 0 | 1 |
|
||||
| <B>ONEDRIVE_DOWNLOADONLY</B> | Controls "--download-only" switch on onedrive sync. Default is 0 | 1 |
|
||||
|
||||
### Usage Examples
|
||||
**Verbose Output:**
|
||||
|
|
|
|||
|
|
@ -6,8 +6,9 @@ This project has been packaged for the following Linux distributions:
|
|||
* Arch Linux, available from AUR as [onedrive-abraunegg](https://aur.archlinux.org/packages/onedrive-abraunegg/)
|
||||
* Debian, available from the package repository as [onedrive](https://packages.debian.org/sid/net/onedrive)
|
||||
* Fedora, available via package repositories as [onedrive](https://koji.fedoraproject.org/koji/packageinfo?packageID=26044)
|
||||
* Gentoo, available via portage overlay as [onedrive](https://gpo.zugaina.org/net-misc/onedrive)
|
||||
* NixOS, use package `onedrive` either by adding it to `configuration.nix` or by using the command `nix-env -iA <channel name>.onedrive`. This does not install a service. To install a service, use unstable channel (will stabilize in 20.09) and add `services.onedrive.enable=true` in `configuration.nix`. You can also add a custom package using the `services.onedrive.package` option (recommended since package lags upstream). Enabling the service installs a default package too (based on the channel). You can also add multiple onedrive accounts trivially, see [documentation](https://github.com/NixOS/nixpkgs/pull/77734#issuecomment-575874225)`.
|
||||
* openSUSE, available for Tumbleweed as [onedrive](https://software.opensuse.org/package/onedrive)
|
||||
* openSUSE, available for Tumbleweed, Leap 15.2, Leap 15.1 as [onedrive](https://software.opensuse.org/package/onedrive)
|
||||
* Slackware, available from the slackbuilds.org repository as [onedrive](https://slackbuilds.org/repository/14.2/network/onedrive/)
|
||||
* Solus, available from the package repository as [onedrive](https://dev.getsol.us/search/query/FB7PIf1jG9Z9/#R)
|
||||
* Ubuntu, available as a package from the following PPA [onedrive](https://launchpad.net/~yann1ck/+archive/ubuntu/onedrive)
|
||||
|
|
@ -233,6 +234,7 @@ sudo pacman -S libnotify
|
|||
```
|
||||
|
||||
### Dependencies: Raspbian (ARMHF)
|
||||
**Note:** Build environment must have at least 1GB of memory & 1GB swap space. Check with `swapon`.
|
||||
```text
|
||||
sudo apt-get install libcurl4-openssl-dev
|
||||
sudo apt-get install libsqlite3-dev
|
||||
|
|
@ -322,8 +324,8 @@ as far as possible automatically, but can be overridden by passing
|
|||
`--with-fish-completion-dir=<DIR>` to `configure`.
|
||||
|
||||
### Building using a different compiler (for example [LDC](https://wiki.dlang.org/LDC))
|
||||
#### ARMHF Architecture
|
||||
**Note:** Build environment must have at least 1GB of memory & 1GB swap space. Check with `swapon -s`
|
||||
#### ARMHF Architecture (Raspbian etc)
|
||||
**Note:** Build environment must have at least 1GB of memory & 1GB swap space. Check with `swapon`.
|
||||
```text
|
||||
git clone https://github.com/abraunegg/onedrive.git
|
||||
cd onedrive
|
||||
|
|
@ -333,7 +335,7 @@ sudo make install
|
|||
```
|
||||
|
||||
#### ARM64 Architecture
|
||||
**Note:** Build environment must have at least 1GB of memory & 1GB swap space. Check with `swapon -s`
|
||||
**Note:** Build environment must have at least 1GB of memory & 1GB swap space. Check with `swapon`
|
||||
```text
|
||||
git clone https://github.com/abraunegg/onedrive.git
|
||||
cd onedrive
|
||||
|
|
|
|||
|
|
@ -1,12 +1,24 @@
|
|||
# Show how to access a Sharepoint group drive in Office 365 business or education
|
||||
## Obtaining the Sharepoint Site Details
|
||||
# How to configure OneDrive SharePoint Shared Library sync
|
||||
Syncing a OneDrive SharePoint library requires additional configuration for your 'onedrive' client:
|
||||
1. Login to OneDrive and under 'Shared Libraries' obtain the shared library name
|
||||
2. Query that shared library name using the client to obtain the required configuration details
|
||||
3. Configure the client's config file with the required 'drive_id'
|
||||
4. Test the configuration using '--dry-run'
|
||||
5. Sync the SharePoint Library as required
|
||||
|
||||
## Listing available OneDrive SharePoint Libraries
|
||||
1. Login to the OneDrive web interface and determine which shared library you wish to configure the client for:
|
||||
|
||||
|
||||
## Query that shared library name using the client to obtain the required configuration details
|
||||
2. Run the following command using the 'onedrive' client
|
||||
```text
|
||||
onedrive --get-O365-drive-id '<your library name>'
|
||||
```
|
||||
3. This will return the following:
|
||||
```text
|
||||
Configuration file successfully loaded
|
||||
Configuring Global Azure AD Endpoints
|
||||
Initializing the Synchronization Engine ...
|
||||
Office 365 Library Name Query: <your library name>
|
||||
SiteName: <your library name>
|
||||
|
|
@ -14,10 +26,21 @@ drive_id: b!6H_y8B...xU5
|
|||
URL: <your site URL>
|
||||
```
|
||||
|
||||
## Configuring the onedrive client
|
||||
Once you have obtained the 'drive_id' above, add to your 'onedrive' configuration file (`~/.config/onedrive/config`) the following:
|
||||
## Configure the client's config file with the required 'drive_id'
|
||||
4. Once you have obtained the 'drive_id' above, add to your 'onedrive' configuration file (`~/.config/onedrive/config`) the following:
|
||||
```text
|
||||
drive_id = "insert the drive id from above here"
|
||||
drive_id = "insert the drive_id value from above here"
|
||||
```
|
||||
The OneDrive client will now be configured to sync this SharePoint shared library to your local system.
|
||||
|
||||
The OneDrive client will now sync this SharePoint shared library to your local system.
|
||||
**Note:** After changing `drive_id`, you must perform a full re-synchronization by adding `--resync` to your existing command line.
|
||||
|
||||
## Test the configuration using '--dry-run'
|
||||
5. Test your new configuration using the `--dry-run` option to validate the the new configuration
|
||||
|
||||
## Sync the SharePoint Library as required
|
||||
6. Sync the SharePoint Library to your system with either `--synchronize` or `--monitor` operations
|
||||
|
||||
|
||||
# How to configure multiple OneDrive SharePoint Shared Library sync
|
||||
Refer to [./advanced-usage.md](advanced-usage.md) for configuration assistance.
|
||||
|
|
|
|||
|
|
@ -300,6 +300,8 @@ The default configuration file is listed below:
|
|||
# azure_ad_endpoint = ""
|
||||
# azure_tenant_id = "common"
|
||||
# sync_business_shared_folders = "false"
|
||||
# sync_dir_permissions = "700"
|
||||
# sync_file_permissions = "600"
|
||||
```
|
||||
|
||||
|
||||
|
|
@ -327,6 +329,26 @@ The issue here is around how the client stores the sync_dir path in the database
|
|||
|
||||
**Important Note:** If your `sync_dir` is pointing to a network mount point (a network share via NFS, Windows Network Share, Samba Network Share) these types of network mount points do not support 'inotify', thus tracking real-time changes via inotify of local files is not possible. Local filesystem changes will be replicated between the local filesystem and OneDrive based on the `monitor_interval` value. This is not something (inotify support for NFS, Samba) that this client can fix.
|
||||
|
||||
#### sync_dir directory and file permissions
|
||||
The following are directory and file default permissions for any new directory or file that is created:
|
||||
* Directories: 700 - This provides the following permissions: `drwx------`
|
||||
* Files: 600 - This provides the following permissions: `-rw-------`
|
||||
|
||||
To change the default permissions, update the following 2 configuration options with the required permissions. Utilise [Unix Permissions Calculator](http://permissions-calculator.org/) to assist in determining the required permissions.
|
||||
|
||||
```text
|
||||
# When changing a config option below, remove the '#' from the start of the line
|
||||
# For explanations of all config options below see docs/USAGE.md or the man page.
|
||||
#
|
||||
...
|
||||
# sync_business_shared_folders = "false"
|
||||
sync_dir_permissions = "700"
|
||||
sync_file_permissions = "600"
|
||||
|
||||
```
|
||||
|
||||
**Important:** Special permission bits (setuid, setgid, sticky bit) are not supported. Valid permission values are from `000` to `777` only.
|
||||
|
||||
#### skip_dir
|
||||
Example:
|
||||
```text
|
||||
|
|
@ -474,6 +496,33 @@ The following are supported for pattern matching and exclusion rules:
|
|||
|
||||
**Note:** after changing the sync_list, you must perform a full re-synchronization by adding `--resync` to your existing command line - for example: `onedrive --synchronize --resync`
|
||||
|
||||
### Configuring the client for 'single tenant application' use
|
||||
In some instances when using OneDrive Business Accounts, depending on the Azure organisational configuration, it will be necessary to configure the client as a 'single tenant application'.
|
||||
To configure this, after creating the application on your Azure tenant, update the 'config' file with the tenant name (not the GUID) and the newly created Application ID, then this will be used for the authentication process.
|
||||
```text
|
||||
# skip_dir_strict_match = "false"
|
||||
application_id = "your.application.id.guid"
|
||||
# resync = "false"
|
||||
# bypass_data_preservation = "false"
|
||||
# azure_ad_endpoint = "xxxxxx"
|
||||
azure_tenant_id = "your.azure.tenant.name"
|
||||
# sync_business_shared_folders = "false"
|
||||
```
|
||||
|
||||
### Configuring the client to use older 'skilion' application identifier
|
||||
In some instances it may be desirable to utilise the older 'skilion' application identifier to avoid authorising a new application ID within Microsoft Azure environments.
|
||||
To configure this, update the 'config' file with the old Application ID, then this will be used for the authentication process.
|
||||
```text
|
||||
# skip_dir_strict_match = "false"
|
||||
application_id = "22c49a0d-d21c-4792-aed1-8f163c982546"
|
||||
# resync = "false"
|
||||
# bypass_data_preservation = "false"
|
||||
```
|
||||
**Note:** The application will now use the older 'skilion' client identifier, however this may increase your chances of getting a OneDrive 429 error.
|
||||
|
||||
**Note:** After changing the 'application_id' you will need to restart any 'onedrive' process you have running, and potentially issue a `--logout` to re-auth the client with this updated application ID.
|
||||
|
||||
|
||||
### How to 'skip' directories from syncing?
|
||||
There are several mechanisms available to 'skip' a directory from the sync process:
|
||||
* Utilise 'skip_dir'
|
||||
|
|
@ -596,7 +645,7 @@ journalctl --unit=onedrive@<username> -f
|
|||
In some cases you may wish to receive GUI notifications when using the client when logged in as a non-root user. In this case, follow the directions below:
|
||||
|
||||
1. Login via graphical UI as user you wish to enable the service for
|
||||
2. Disable any `onedive@` service files for your username - eg:
|
||||
2. Disable any `onedrive@` service files for your username - eg:
|
||||
```text
|
||||
sudo systemctl stop onedrive@alex.service
|
||||
sudo systemctl disable onedrive@alex.service
|
||||
|
|
@ -616,42 +665,7 @@ journalctl --user-unit=onedrive -f
|
|||
|
||||
## Additional Configuration
|
||||
### Using multiple OneDrive accounts
|
||||
You can run multiple instances of the application by specifying a different config directory in order to handle multiple OneDrive accounts. For example, if you have a work and a personal account, you can run the onedrive command using the --confdir parameter. Here is an example:
|
||||
|
||||
```text
|
||||
onedrive --synchronize --verbose --confdir="~/.config/onedrivePersonal" &
|
||||
onedrive --synchronize --verbose --confdir="~/.config/onedriveWork" &
|
||||
```
|
||||
or
|
||||
```text
|
||||
onedrive --monitor --verbose --confdir="~/.config/onedrivePersonal" &
|
||||
onedrive --monitor --verbose --confdir="~/.config/onedriveWork" &
|
||||
```
|
||||
|
||||
* `--synchronize` does a one-time sync
|
||||
* `--monitor` keeps the application running and monitoring for changes both local and remote
|
||||
* `&` puts the application in background and leaves the terminal interactive
|
||||
|
||||
**Important:** For each configuration, change the 'sync_dir' to a new value, unique for each specific configuration. Leaving this at the default of `sync_dir = "~/OneDrive"` will cause all data from both accounts to be synced to the same folder, then to each other.
|
||||
|
||||
### Automatic syncing of both OneDrive accounts
|
||||
In order to automatically start syncing your OneDrive accounts, you will need to create a service file for each account. From the applicable 'user systemd folder':
|
||||
* RHEL / CentOS: `/usr/lib/systemd/system`
|
||||
* Others: `/usr/lib/systemd/user`
|
||||
|
||||
```text
|
||||
cp onedrive.service onedrive-work.service
|
||||
```
|
||||
And edit the line beginning with `ExecStart` so that the confdir mirrors the one you used above:
|
||||
```text
|
||||
ExecStart=/usr/local/bin/onedrive --monitor --confdir="/path/to/config/dir"
|
||||
```
|
||||
Then you can safely run these commands:
|
||||
```text
|
||||
systemctl --user enable onedrive-work
|
||||
systemctl --user start onedrive-work
|
||||
```
|
||||
Repeat these steps for each OneDrive account that you wish to use.
|
||||
Refer to [./advanced-usage.md](advanced-usage.md) for configuration assistance.
|
||||
|
||||
### Access OneDrive service through a proxy
|
||||
If you have a requirement to run the client through a proxy, there are a couple of ways to achieve this:
|
||||
|
|
@ -690,7 +704,6 @@ sudo restorecon -R -v /path/to/onedriveSyncFolder
|
|||
```
|
||||
|
||||
## All available commands
|
||||
|
||||
Output of `onedrive --help`
|
||||
```text
|
||||
OneDrive - a client for OneDrive Cloud Services
|
||||
|
|
|
|||
139
docs/advanced-usage.md
Normal file
139
docs/advanced-usage.md
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
# Advanced Configuration of the OneDrive Free Client
|
||||
This document covers the following scenarios:
|
||||
* Configuring the client to use mutlitple OneDrive accounts / configurations
|
||||
* Configuring the client for use in dual-boot (Windows / Linux) situations
|
||||
|
||||
## Configuring the client to use mutlitple OneDrive accounts / configurations
|
||||
Essentially, each OneDrive account or SharePoint Shared Library which you require to be synced needs to have it's own and unique configuration, local sync directory and service files. To do this, the following steps are needed:
|
||||
1. Create a unique configuration folder for each onedrive client configuration that you need
|
||||
2. Copy to this folder a copy of the default configuration file
|
||||
3. Update the default configuration file as required, changing the required minimum config options and any additional options as needed to support your multi-account configuration
|
||||
4. Authenticate the client using the new configuration directory
|
||||
5. Test the configuration using '--display-config' and '--dry-run'
|
||||
6. Sync the OneDrive account data as required using `--synchronize` or `--monitor`
|
||||
7. Configure a unique systemd service file for this account configuration
|
||||
|
||||
### 1. Create a unique configuration folder for each onedrive client configuration that you need
|
||||
Make the configuration folder as required for this new configuration, for example:
|
||||
```text
|
||||
mkdir ~/.config/my-new-config
|
||||
```
|
||||
|
||||
### 2. Copy to this folder a copy of the default configuration file
|
||||
Copy to this folder a copy of the default configuration file by downloading this file from GitHub and saving this file in the directory created above:
|
||||
```text
|
||||
wget https://raw.githubusercontent.com/abraunegg/onedrive/master/config -O ~/.config/my-new-config/config
|
||||
```
|
||||
|
||||
### 3. Update the default configuration file
|
||||
The following config options *must* be updated to ensure that individual account data is not cross populated with other OneDrive accounts or other configurations:
|
||||
* sync_dir
|
||||
|
||||
Other options that may require to be updated, depending on the OneDrive account that is being configured:
|
||||
* drive_id
|
||||
* application_id
|
||||
* sync_business_shared_folders
|
||||
* skip_dir
|
||||
* skip_file
|
||||
* Creation of a 'sync_list' file if required
|
||||
* Creation of a 'business_shared_folders' file if required
|
||||
|
||||
### 4. Authenticate the client
|
||||
Authenticate the client using the specific configuration file:
|
||||
```text
|
||||
onedrive --confdir="~/.config/my-new-config"
|
||||
```
|
||||
You will be asked to open a specific URL by using your web browser where you will have to login into your Microsoft Account and give the application the permission to access your files. After giving permission to the application, you will be redirected to a blank page. Copy the URI of the blank page into the application.
|
||||
```text
|
||||
[user@hostname ~]$ onedrive --confdir="~/.config/my-new-config"
|
||||
Configuration file successfully loaded
|
||||
Configuring Global Azure AD Endpoints
|
||||
Authorize this app visiting:
|
||||
|
||||
https://.....
|
||||
|
||||
Enter the response uri:
|
||||
|
||||
```
|
||||
|
||||
### 5. Display and Test the configuration
|
||||
Test the configuration using '--display-config' and '--dry-run'. By doing so, this allows you to test any configuration that you have currently made, enabling you to fix this configuration before using the configuration.
|
||||
|
||||
#### Display the configuration
|
||||
```text
|
||||
onedrive --confdir="~/.config/my-new-config --display-config"
|
||||
```
|
||||
|
||||
#### Test the configuration by performing a dry-run
|
||||
```text
|
||||
onedrive --confdir="~/.config/my-new-config" --synchronize --verbose --dry-run
|
||||
```
|
||||
|
||||
If both of these operate as per your expectation, the configuration of this client setup is complete and validated. If not, amend your configuration as required.
|
||||
|
||||
### 6. Sync the OneDrive account data as required
|
||||
Sync the data for the new account configuration as required:
|
||||
```text
|
||||
onedrive --confdir="~/.config/my-new-config" --synchronize --verbose
|
||||
```
|
||||
or
|
||||
```text
|
||||
onedrive --confdir="~/.config/my-new-config" --monitor --verbose
|
||||
```
|
||||
|
||||
* `--synchronize` does a one-time sync
|
||||
* `--monitor` keeps the application running and monitoring for changes both local and remote
|
||||
|
||||
### 7. Automatic syncing of new OneDrive configuration
|
||||
In order to automatically start syncing your OneDrive accounts, you will need to create a service file for each account. From the applicable 'systemd folder' where the applicable systemd service file exists:
|
||||
* RHEL / CentOS: `/usr/lib/systemd/system`
|
||||
* Others: `/usr/lib/systemd/user` and `/lib/systemd/system`
|
||||
|
||||
**Note:** The `onedrive.service` runs the service as the 'root' user, whereas the `onedrive@.service` runs the service as your user account.
|
||||
|
||||
Copy the required service file to a new name:
|
||||
```text
|
||||
cp onedrive.service onedrive-my-new-config.service
|
||||
```
|
||||
or
|
||||
```text
|
||||
cp onedrive@.service onedrive-my-new-config@.service
|
||||
```
|
||||
|
||||
Edit the line beginning with `ExecStart` so that the confdir mirrors the one you used above:
|
||||
```text
|
||||
ExecStart=/usr/local/bin/onedrive --monitor --confdir="/full/path/to/config/dir"
|
||||
```
|
||||
|
||||
Example:
|
||||
```text
|
||||
ExecStart=/usr/local/bin/onedrive --monitor --confdir="/home/myusername/.config/my-new-config"
|
||||
```
|
||||
|
||||
Then you can safely run these commands:
|
||||
```text
|
||||
systemctl --user enable onedrive-my-new-config
|
||||
systemctl --user start onedrive-my-new-config
|
||||
```
|
||||
or
|
||||
```text
|
||||
systemctl --user enable onedrive-my-new-config@myusername.service
|
||||
systemctl --user start onedrive-my-new-config@myusername.service
|
||||
```
|
||||
|
||||
Repeat these steps for each OneDrive new account that you wish to use.
|
||||
|
||||
## Configuring the client for use in dual-boot (Windows / Linux) situations
|
||||
When dual booting Windows and Linux, depending on the Windows OneDrive account configuration, the 'Files On-Demand' option may be enabled when running OneDrive within your Windows environment.
|
||||
|
||||
When this option is enabled in Windows, if you are sharing this location between your Windows and Linux systems, all files will be a 0 byte link, and cannot be used under Linux.
|
||||
|
||||
To fix the problem of windows turning all files (that should be kept offline) into links, you have to uncheck a specific option in the onedrive settings window. The option in question is `Save space and download files as you use them`.
|
||||
|
||||
To find this setting, open the onedrive pop-up window from the taskbar, click "Help & Settings" > "Settings". This opens a new window. Go to the tab "Settings" and look for the section "Files On-Demand".
|
||||
|
||||
After unchecking the option and clicking "OK", the Windows OneDrive client should restart itself and start actually downloading your files so they will truely be available on your disk when offline. These files will then be fully accessible under Linux and the Linux OneDrive client.
|
||||
|
||||
| OneDrive Personal | Onedrive Business<br>SharePoint |
|
||||
|---|---|
|
||||
|  |  |
|
||||
BIN
docs/images/SharedLibraries.jpg
Normal file
BIN
docs/images/SharedLibraries.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
BIN
docs/images/business-files-on-demand.png
Normal file
BIN
docs/images/business-files-on-demand.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
BIN
docs/images/personal-files-on-demand.png
Normal file
BIN
docs/images/personal-files-on-demand.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
|
|
@ -241,13 +241,23 @@ State caching
|
|||
|
||||
Real-Time file monitoring with Inotify
|
||||
|
||||
File upload / download validation to ensure data integrity
|
||||
|
||||
Resumable uploads
|
||||
|
||||
Support OneDrive for Business (part of Office 365)
|
||||
|
||||
Shared folders (OneDrive Personal)
|
||||
Shared Folder support for OneDrive Personal and OneDrive Business accounts
|
||||
|
||||
SharePoint / Office 365 Group Drives (refer to README.Office365.md to configure)
|
||||
SharePoint / Office365 Shared Libraries
|
||||
|
||||
Desktop notifications via libnotify
|
||||
|
||||
Dry-run capability to test configuration changes
|
||||
|
||||
Prevent major OneDrive accidental data deletion after configuration change
|
||||
|
||||
Support for National cloud deployments (Microsoft Cloud for US Government, Microsoft Cloud Germany, Azure and Office 365 operated by 21Vianet in China)
|
||||
|
||||
|
||||
.SH CONFIGURATION
|
||||
|
|
@ -339,4 +349,8 @@ for a user via the \fBonedrive@<username>\fP service.
|
|||
|
||||
Further examples and documentation is available in
|
||||
\f[C]README.md\f[]
|
||||
\f[C]README.Office365.md\f[]
|
||||
\f[C]docs/USAGE.md\f[]
|
||||
\f[C]docs/advanced-usage.md\f[]
|
||||
\f[C]docs/BusinessSharedFolders.md\f[]
|
||||
\f[C]docs/Office365.md\f[]
|
||||
\f[C]docs/national-cloud-deployments.md\f[]
|
||||
|
|
|
|||
75
src/config.d
75
src/config.d
|
|
@ -36,11 +36,17 @@ final class Config
|
|||
private long[string] longValues;
|
||||
// Compile time regex - this does not change
|
||||
public auto configRegex = ctRegex!(`^(\w+)\s*=\s*"(.*)"\s*$`);
|
||||
// Default directory permission mode
|
||||
public long defaultDirectoryPermissionMode = 700;
|
||||
public int configuredDirectoryPermissionMode;
|
||||
// Default file permission mode
|
||||
public long defaultFilePermissionMode = 600;
|
||||
public int configuredFilePermissionMode;
|
||||
|
||||
this(string confdirOption)
|
||||
{
|
||||
// default configuration - entries in config file ~/.config/onedrive/config
|
||||
// an entry here means it can be set via the config file if there is a coresponding read and set in update_from_args()
|
||||
// an entry here means it can be set via the config file if there is a coresponding entry, read from config and set via update_from_args()
|
||||
stringValues["sync_dir"] = defaultSyncDir;
|
||||
stringValues["skip_file"] = defaultSkipFile;
|
||||
stringValues["skip_dir"] = defaultSkipDir;
|
||||
|
|
@ -106,6 +112,10 @@ final class Config
|
|||
stringValues["azure_tenant_id"] = "common";
|
||||
// Allow enable / disable of the syncing of OneDrive Business Shared Folders via configuration file
|
||||
boolValues["sync_business_shared_folders"] = false;
|
||||
// Configure the default folder permission attributes for newly created folders
|
||||
longValues["sync_dir_permissions"] = defaultDirectoryPermissionMode;
|
||||
// Configure the default file permission attributes for newly created file
|
||||
longValues["sync_file_permissions"] = defaultFilePermissionMode;
|
||||
|
||||
// DEVELOPER OPTIONS
|
||||
// display_memory = true | false
|
||||
|
|
@ -181,7 +191,13 @@ final class Config
|
|||
}
|
||||
|
||||
// Config directory options all determined
|
||||
if (!exists(configDirName)) mkdirRecurse(configDirName);
|
||||
if (!exists(configDirName)) {
|
||||
// create the directory
|
||||
mkdirRecurse(configDirName);
|
||||
// Configure the applicable permissions for the folder
|
||||
configDirName.setAttributes(returnRequiredDirectoryPermisions());
|
||||
}
|
||||
|
||||
// configDirName has a trailing /
|
||||
log.vlog("Using 'user' Config Dir: ", configDirName);
|
||||
log.vlog("Using 'system' Config Dir: ", systemConfigDirName);
|
||||
|
|
@ -266,8 +282,6 @@ final class Config
|
|||
boolValues["monitor"] = false;
|
||||
boolValues["synchronize"] = false;
|
||||
boolValues["force"] = false;
|
||||
boolValues["remove_source_files"] = false;
|
||||
boolValues["skip_dir_strict_match"] = false;
|
||||
boolValues["list_business_shared_folders"] = false;
|
||||
|
||||
// Application Startup option validation
|
||||
|
|
@ -611,6 +625,59 @@ final class Config
|
|||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void configureRequiredDirectoryPermisions() {
|
||||
// return the directory permission mode required
|
||||
// - return octal!defaultDirectoryPermissionMode; ... cant be used .. which is odd
|
||||
// Error: variable defaultDirectoryPermissionMode cannot be read at compile time
|
||||
if (getValueLong("sync_dir_permissions") != defaultDirectoryPermissionMode) {
|
||||
// return user configured permissions as octal integer
|
||||
string valueToConvert = to!string(getValueLong("sync_dir_permissions"));
|
||||
auto convertedValue = parse!long(valueToConvert, 8);
|
||||
configuredDirectoryPermissionMode = to!int(convertedValue);
|
||||
} else {
|
||||
// return default as octal integer
|
||||
string valueToConvert = to!string(defaultDirectoryPermissionMode);
|
||||
auto convertedValue = parse!long(valueToConvert, 8);
|
||||
configuredDirectoryPermissionMode = to!int(convertedValue);
|
||||
}
|
||||
}
|
||||
|
||||
void configureRequiredFilePermisions() {
|
||||
// return the file permission mode required
|
||||
// - return octal!defaultFilePermissionMode; ... cant be used .. which is odd
|
||||
// Error: variable defaultFilePermissionMode cannot be read at compile time
|
||||
if (getValueLong("sync_file_permissions") != defaultFilePermissionMode) {
|
||||
// return user configured permissions as octal integer
|
||||
string valueToConvert = to!string(getValueLong("sync_file_permissions"));
|
||||
auto convertedValue = parse!long(valueToConvert, 8);
|
||||
configuredFilePermissionMode = to!int(convertedValue);
|
||||
} else {
|
||||
// return default as octal integer
|
||||
string valueToConvert = to!string(defaultFilePermissionMode);
|
||||
auto convertedValue = parse!long(valueToConvert, 8);
|
||||
configuredFilePermissionMode = to!int(convertedValue);
|
||||
}
|
||||
}
|
||||
|
||||
int returnRequiredDirectoryPermisions() {
|
||||
// read the configuredDirectoryPermissionMode and return
|
||||
if (configuredDirectoryPermissionMode == 0) {
|
||||
// the configured value is zero, this means that directories would get
|
||||
// values of d---------
|
||||
configureRequiredDirectoryPermisions();
|
||||
}
|
||||
return configuredDirectoryPermissionMode;
|
||||
}
|
||||
|
||||
int returnRequiredFilePermisions() {
|
||||
// read the configuredFilePermissionMode and return
|
||||
if (configuredFilePermissionMode == 0) {
|
||||
// the configured value is zero
|
||||
configureRequiredFilePermisions();
|
||||
}
|
||||
return configuredFilePermissionMode;
|
||||
}
|
||||
}
|
||||
|
||||
void outputLongHelp(Option[] opt)
|
||||
|
|
|
|||
|
|
@ -474,4 +474,11 @@ final class ItemDatabase
|
|||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
// Perform a vacuum on the database, commit WAL / SHM to file
|
||||
void performVacuum()
|
||||
{
|
||||
auto stmt = db.prepare("VACUUM;");
|
||||
stmt.exec();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
118
src/main.d
118
src/main.d
|
|
@ -62,10 +62,16 @@ int main(string[] args)
|
|||
if (onedriveInitialised) {
|
||||
oneDrive.shutdown();
|
||||
}
|
||||
// Make sure the .wal file is incorporated into the main db before we exit
|
||||
destroy(itemDb);
|
||||
// was itemDb initialised?
|
||||
if (itemDb !is null) {
|
||||
// Make sure the .wal file is incorporated into the main db before we exit
|
||||
itemDb.performVacuum();
|
||||
destroy(itemDb);
|
||||
}
|
||||
// free API instance
|
||||
oneDrive = null;
|
||||
if (oneDrive !is null) {
|
||||
destroy(oneDrive);
|
||||
}
|
||||
// Perform Garbage Cleanup
|
||||
GC.collect();
|
||||
// Display memory details
|
||||
|
|
@ -83,10 +89,16 @@ int main(string[] args)
|
|||
if (onedriveInitialised) {
|
||||
oneDrive.shutdown();
|
||||
}
|
||||
// Make sure the .wal file is incorporated into the main db before we exit
|
||||
destroy(itemDb);
|
||||
// was itemDb initialised?
|
||||
if (itemDb !is null) {
|
||||
// Make sure the .wal file is incorporated into the main db before we exit
|
||||
itemDb.performVacuum();
|
||||
destroy(itemDb);
|
||||
}
|
||||
// free API instance
|
||||
oneDrive = null;
|
||||
if (oneDrive !is null) {
|
||||
destroy(oneDrive);
|
||||
}
|
||||
// Perform Garbage Cleanup
|
||||
GC.collect();
|
||||
// Display memory details
|
||||
|
|
@ -380,12 +392,23 @@ int main(string[] args)
|
|||
// dry-run notification and database setup
|
||||
if (cfg.getValueBool("dry_run")) {
|
||||
log.log("DRY-RUN Configured. Output below shows what 'would' have occurred.");
|
||||
string dryRunShmFile = cfg.databaseFilePathDryRun ~ "-shm";
|
||||
string dryRunWalFile = cfg.databaseFilePathDryRun ~ "-wal";
|
||||
// If the dry run database exists, clean this up
|
||||
if (exists(cfg.databaseFilePathDryRun)) {
|
||||
// remove the existing file
|
||||
log.vdebug("Removing items-dryrun.sqlite3 as it still exists for some reason");
|
||||
safeRemove(cfg.databaseFilePathDryRun);
|
||||
}
|
||||
// silent cleanup of shm and wal files if they exist
|
||||
if (exists(dryRunShmFile)) {
|
||||
// remove items-dryrun.sqlite3-shm
|
||||
safeRemove(dryRunShmFile);
|
||||
}
|
||||
if (exists(dryRunWalFile)) {
|
||||
// remove items-dryrun.sqlite3-wal
|
||||
safeRemove(dryRunWalFile);
|
||||
}
|
||||
|
||||
// Make a copy of the original items.sqlite3 for use as the dry run copy if it exists
|
||||
if (exists(cfg.databaseFilePath)) {
|
||||
|
|
@ -486,6 +509,9 @@ int main(string[] args)
|
|||
writeln("Config option 'min_notify_changes' = ", cfg.getValueLong("min_notify_changes"));
|
||||
writeln("Config option 'log_dir' = ", cfg.getValueString("log_dir"));
|
||||
writeln("Config option 'classify_as_big_delete' = ", cfg.getValueLong("classify_as_big_delete"));
|
||||
writeln("Config option 'upload_only' = ", cfg.getValueBool("upload_only"));
|
||||
writeln("Config option 'no_remote_delete' = ", cfg.getValueBool("no_remote_delete"));
|
||||
writeln("Config option 'remove_source_files' = ", cfg.getValueBool("remove_source_files"));
|
||||
|
||||
// Is config option drive_id configured?
|
||||
if (cfg.getValueString("drive_id") != ""){
|
||||
|
|
@ -574,10 +600,17 @@ int main(string[] args)
|
|||
// was the application just authorised?
|
||||
if (cfg.applicationAuthorizeResponseUri) {
|
||||
// Application was just authorised
|
||||
log.log("\nApplication has been successfully authorised, however no additional command switches were provided.\n");
|
||||
log.log("Please use --help for further assistance in regards to running this application.\n");
|
||||
// Use exit scopes to shutdown API
|
||||
return EXIT_SUCCESS;
|
||||
if (exists(cfg.refreshTokenFilePath)) {
|
||||
// OneDrive refresh token exists
|
||||
log.log("\nApplication has been successfully authorised, however no additional command switches were provided.\n");
|
||||
log.log("Please use --help for further assistance in regards to running this application.\n");
|
||||
// Use exit scopes to shutdown API
|
||||
return EXIT_SUCCESS;
|
||||
} else {
|
||||
// we just authorised, but refresh_token does not exist .. probably an auth error
|
||||
log.log("\nApplication has not been successfully authorised. Please check your URI response entry and try again.\n");
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
} else {
|
||||
// Application was not just authorised
|
||||
log.log("\n--synchronize or --monitor switches missing from your command line input. Please add one (not both) of these switches to your command line or use --help for further assistance.\n");
|
||||
|
|
@ -607,6 +640,23 @@ int main(string[] args)
|
|||
itemDb = new ItemDatabase(cfg.databaseFilePathDryRun);
|
||||
}
|
||||
|
||||
// What are the permission that have been set for the application?
|
||||
// These are relevant for:
|
||||
// - The ~/OneDrive parent folder or 'sync_dir' configured item
|
||||
// - Any new folder created under ~/OneDrive or 'sync_dir'
|
||||
// - Any new file created under ~/OneDrive or 'sync_dir'
|
||||
// valid permissions are 000 -> 777 - anything else is invalid
|
||||
if ((cfg.getValueLong("sync_dir_permissions") < 0) || (cfg.getValueLong("sync_file_permissions") < 0) || (cfg.getValueLong("sync_dir_permissions") > 777) || (cfg.getValueLong("sync_file_permissions") > 777)) {
|
||||
log.error("ERROR: Invalid 'User|Group|Other' permissions set within config file. Please check.");
|
||||
return EXIT_FAILURE;
|
||||
} else {
|
||||
// debug log output what permissions are being set to
|
||||
log.vdebug("Configuring default new folder permissions as: ", cfg.getValueLong("sync_dir_permissions"));
|
||||
cfg.configureRequiredDirectoryPermisions();
|
||||
log.vdebug("Configuring default new file permissions as: ", cfg.getValueLong("sync_file_permissions"));
|
||||
cfg.configureRequiredFilePermisions();
|
||||
}
|
||||
|
||||
// configure the sync direcory based on syncDir config option
|
||||
log.vlog("All operations will be performed in: ", syncDir);
|
||||
if (!exists(syncDir)) {
|
||||
|
|
@ -614,6 +664,9 @@ int main(string[] args)
|
|||
try {
|
||||
// Attempt to create the sync dir we have been configured with
|
||||
mkdirRecurse(syncDir);
|
||||
// Configure the applicable permissions for the folder
|
||||
log.vdebug("Setting directory permissions for: ", syncDir);
|
||||
syncDir.setAttributes(cfg.returnRequiredDirectoryPermisions());
|
||||
} catch (std.file.FileException e) {
|
||||
// Creating the sync directory failed
|
||||
log.error("ERROR: Unable to create local OneDrive syncDir - ", e.msg);
|
||||
|
|
@ -621,6 +674,8 @@ int main(string[] args)
|
|||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
// Change the working directory to the 'sync_dir' configured item
|
||||
chdir(syncDir);
|
||||
|
||||
// Configure selective sync by parsing and getting a regex for skip_file config component
|
||||
|
|
@ -721,14 +776,18 @@ int main(string[] args)
|
|||
// Do we need to configure specific --upload-only options?
|
||||
if (cfg.getValueBool("upload_only")) {
|
||||
// --upload-only was passed in or configured
|
||||
log.vdebug("Configuring uploadOnly flag to TRUE as --upload-only passed in or configured");
|
||||
sync.setUploadOnly();
|
||||
// was --no-remote-delete passed in or configured
|
||||
if (cfg.getValueBool("no_remote_delete")) {
|
||||
// Configure the noRemoteDelete flag
|
||||
log.vdebug("Configuring noRemoteDelete flag to TRUE as --no-remote-delete passed in or configured");
|
||||
sync.setNoRemoteDelete();
|
||||
}
|
||||
// was --remove-source-files passed in or configured
|
||||
if (cfg.getValueBool("remove_source_files")) {
|
||||
// Configure the localDeleteAfterUpload flag
|
||||
log.vdebug("Configuring localDeleteAfterUpload flag to TRUE as --remove-source-files passed in or configured");
|
||||
sync.setLocalDeleteAfterUpload();
|
||||
}
|
||||
}
|
||||
|
|
@ -858,14 +917,22 @@ int main(string[] args)
|
|||
if (!exists(cfg.getValueString("single_directory"))) {
|
||||
// The requested path to use with --single-directory does not exist locally within the configured 'sync_dir'
|
||||
log.logAndNotify("WARNING: The requested path for --single-directory does not exist locally. Creating requested path within ", syncDir);
|
||||
// Make the required path locally
|
||||
mkdirRecurse(cfg.getValueString("single_directory"));
|
||||
// Make the required --single-directory path locally
|
||||
string singleDirectoryPath = cfg.getValueString("single_directory");
|
||||
mkdirRecurse(singleDirectoryPath);
|
||||
// Configure the applicable permissions for the folder
|
||||
log.vdebug("Setting directory permissions for: ", singleDirectoryPath);
|
||||
singleDirectoryPath.setAttributes(cfg.returnRequiredDirectoryPermisions());
|
||||
}
|
||||
}
|
||||
// perform a --synchronize sync
|
||||
// fullScanRequired = false, for final true-up
|
||||
// but if we have sync_list configured, use syncListConfigured which = true
|
||||
performSync(sync, cfg.getValueString("single_directory"), cfg.getValueBool("download_only"), cfg.getValueBool("local_first"), cfg.getValueBool("upload_only"), LOG_NORMAL, false, syncListConfigured, displaySyncOptions, cfg.getValueBool("monitor"), m);
|
||||
|
||||
// Write WAL and SHM data to file for this sync
|
||||
log.vdebug("Merge contents of WAL and SHM files into main database file");
|
||||
itemDb.performVacuum();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1077,8 +1144,15 @@ int main(string[] args)
|
|||
if (displayMemoryUsage) {
|
||||
log.displayMemoryUsagePostGC();
|
||||
}
|
||||
|
||||
// Write WAL and SHM data to file for this loop
|
||||
log.vdebug("Merge contents of WAL and SHM files into main database file");
|
||||
itemDb.performVacuum();
|
||||
|
||||
// monitor loop complete
|
||||
logOutputMessage = "################################################ LOOP COMPLETE ###############################################";
|
||||
|
||||
// Handle display options
|
||||
if (displaySyncOptions) {
|
||||
log.log(logOutputMessage);
|
||||
} else {
|
||||
|
|
@ -1100,11 +1174,23 @@ int main(string[] args)
|
|||
|
||||
// --dry-run temp database cleanup
|
||||
if (cfg.getValueBool("dry_run")) {
|
||||
string dryRunShmFile = cfg.databaseFilePathDryRun ~ "-shm";
|
||||
string dryRunWalFile = cfg.databaseFilePathDryRun ~ "-wal";
|
||||
if (exists(cfg.databaseFilePathDryRun)) {
|
||||
// remove the file
|
||||
log.vdebug("Removing items-dryrun.sqlite3 as dry run operations complete");
|
||||
// remove items-dryrun.sqlite3
|
||||
safeRemove(cfg.databaseFilePathDryRun);
|
||||
}
|
||||
// silent cleanup of shm and wal files if they exist
|
||||
if (exists(dryRunShmFile)) {
|
||||
// remove items-dryrun.sqlite3-shm
|
||||
safeRemove(dryRunShmFile);
|
||||
}
|
||||
if (exists(dryRunWalFile)) {
|
||||
// remove items-dryrun.sqlite3-wal
|
||||
safeRemove(dryRunWalFile);
|
||||
}
|
||||
}
|
||||
|
||||
// Exit application
|
||||
|
|
@ -1372,8 +1458,12 @@ extern(C) nothrow @nogc @system void exitHandler(int value) {
|
|||
try {
|
||||
assumeNoGC ( () {
|
||||
log.log("Got termination signal, shutting down db connection");
|
||||
// make sure the .wal file is incorporated into the main db
|
||||
destroy(itemDb);
|
||||
// was itemDb initialised?
|
||||
if (itemDb !is null) {
|
||||
// Make sure the .wal file is incorporated into the main db before we exit
|
||||
itemDb.performVacuum();
|
||||
destroy(itemDb);
|
||||
}
|
||||
// Use exit scopes to shutdown OneDrive API
|
||||
})();
|
||||
} catch(Exception e) {}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import core.sys.linux.sys.inotify;
|
||||
import core.stdc.errno;
|
||||
import core.sys.posix.poll, core.sys.posix.unistd;
|
||||
import std.exception, std.file, std.path, std.regex, std.stdio, std.string, std.algorithm.mutation;
|
||||
import std.exception, std.file, std.path, std.regex, std.stdio, std.string, std.algorithm;
|
||||
import core.stdc.stdlib;
|
||||
import config;
|
||||
import selective;
|
||||
import util;
|
||||
|
|
@ -132,17 +133,43 @@ final class Monitor
|
|||
}
|
||||
|
||||
// passed all potential exclusions
|
||||
// add inotify watch for this path / directory / file
|
||||
log.vdebug("Calling add() for this dirname: ", dirname);
|
||||
add(dirname);
|
||||
try {
|
||||
auto pathList = dirEntries(dirname, SpanMode.shallow, false);
|
||||
foreach(DirEntry entry; pathList) {
|
||||
if (entry.isDir) {
|
||||
addRecursive(entry.name);
|
||||
|
||||
// if this is a directory, recursivly add this path
|
||||
if (isDir(dirname)) {
|
||||
// try and get all the directory entities for this path
|
||||
try {
|
||||
auto pathList = dirEntries(dirname, SpanMode.shallow, false);
|
||||
foreach(DirEntry entry; pathList) {
|
||||
if (entry.isDir) {
|
||||
log.vdebug("Calling addRecursive() for this directory: ", entry.name);
|
||||
addRecursive(entry.name);
|
||||
}
|
||||
}
|
||||
// catch any error which is generated
|
||||
} catch (std.file.FileException e) {
|
||||
// Standard filesystem error
|
||||
displayFileSystemErrorMessage(e.msg);
|
||||
return;
|
||||
} catch (Exception e) {
|
||||
// Issue #1154 handling
|
||||
// Need to check for: Failed to stat file in error message
|
||||
if (canFind(e.msg, "Failed to stat file")) {
|
||||
// File system access issue
|
||||
log.error("ERROR: The local file system returned an error with the following message:");
|
||||
log.error(" Error Message: ", e.msg);
|
||||
log.error("ACCESS ERROR: Please check your UID and GID access to this file, as the permissions on this file is preventing this application to read it");
|
||||
log.error("\nFATAL: Exiting application to avoid deleting data due to local file system access issues\n");
|
||||
// Must exit here
|
||||
exit(-1);
|
||||
} else {
|
||||
// some other error
|
||||
displayFileSystemErrorMessage(e.msg);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (std.file.FileException e) {
|
||||
log.vdebug("ERROR: ", e.msg);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -173,6 +200,7 @@ final class Monitor
|
|||
|
||||
// Add path to inotify watch - required regardless if a '.folder' or 'folder'
|
||||
wdToDirName[wd] = buildNormalizedPath(pathname) ~ "/";
|
||||
log.vdebug("inotify_add_watch successfully added for: ", pathname);
|
||||
|
||||
// Do we log that we are monitoring this directory?
|
||||
if (isDir(pathname)) {
|
||||
|
|
@ -360,4 +388,12 @@ final class Monitor
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse and display error message received from the local file system
|
||||
private void displayFileSystemErrorMessage(string message)
|
||||
{
|
||||
log.error("ERROR: The local file system returned an error with the following message:");
|
||||
auto errorArray = splitLines(message);
|
||||
log.error(" Error Message: ", errorArray[0]);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
131
src/onedrive.d
131
src/onedrive.d
|
|
@ -15,6 +15,9 @@ private bool simulateNoRefreshTokenFile = false;
|
|||
private ulong retryAfterValue = 0;
|
||||
|
||||
private immutable {
|
||||
// Client ID / Application ID (abraunegg)
|
||||
string clientIdDefault = "d50ca740-c83f-4d1b-b616-12c519384f0c";
|
||||
|
||||
// Azure Active Directory & Graph Explorer Endpoints
|
||||
// Global & Defaults
|
||||
string globalAuthEndpoint = "https://login.microsoftonline.com";
|
||||
|
|
@ -38,8 +41,8 @@ private immutable {
|
|||
}
|
||||
|
||||
private {
|
||||
// Client ID / Application ID (abraunegg)
|
||||
string clientId = "d50ca740-c83f-4d1b-b616-12c519384f0c";
|
||||
// Client ID / Application ID
|
||||
string clientId = clientIdDefault;
|
||||
|
||||
// Default User Agent configuration
|
||||
string isvTag = "ISV";
|
||||
|
|
@ -149,6 +152,14 @@ final class OneDriveApi
|
|||
.debugResponse = true;
|
||||
}
|
||||
|
||||
// Update clientId if application_id is set in config file
|
||||
if (cfg.getValueString("application_id") != "") {
|
||||
// an application_id is set in config file
|
||||
log.vdebug("Setting custom application_id to: " , cfg.getValueString("application_id"));
|
||||
clientId = cfg.getValueString("application_id");
|
||||
companyName = "custom_application";
|
||||
}
|
||||
|
||||
// Configure tenant id value, if 'azure_tenant_id' is configured,
|
||||
// otherwise use the "common" multiplexer
|
||||
string tenantId = "common";
|
||||
|
|
@ -156,19 +167,35 @@ final class OneDriveApi
|
|||
// Use the value entered by the user
|
||||
tenantId = cfg.getValueString("azure_tenant_id");
|
||||
}
|
||||
|
||||
|
||||
// Configure Azure AD endpoints if 'azure_ad_endpoint' is configured
|
||||
string azureConfigValue = cfg.getValueString("azure_ad_endpoint");
|
||||
switch(azureConfigValue) {
|
||||
case "":
|
||||
log.log("Configuring Global Azure AD Endpoints");
|
||||
if (tenantId == "common") {
|
||||
log.log("Configuring Global Azure AD Endpoints");
|
||||
} else {
|
||||
log.log("Configuring Global Azure AD Endpoints - Single Tenant Application");
|
||||
}
|
||||
// Authentication
|
||||
authUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
|
||||
redirectUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
tokenUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
|
||||
break;
|
||||
case "USL4":
|
||||
log.log("Configuring Azure AD for US Government Endpoints");
|
||||
// Authentication
|
||||
authUrl = usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
|
||||
redirectUrl = usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
tokenUrl = usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
|
||||
if (clientId == clientIdDefault) {
|
||||
// application_id == default
|
||||
log.vdebug("USL4 AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint");
|
||||
redirectUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
} else {
|
||||
// custom application_id
|
||||
redirectUrl = usl4AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
}
|
||||
|
||||
// Drive Queries
|
||||
driveUrl = usl4GraphEndpoint ~ "/v1.0/me/drive";
|
||||
driveByIdUrl = usl4GraphEndpoint ~ "/v1.0/drives/";
|
||||
|
|
@ -186,8 +213,16 @@ final class OneDriveApi
|
|||
log.log("Configuring Azure AD for US Government Endpoints (DOD)");
|
||||
// Authentication
|
||||
authUrl = usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
|
||||
redirectUrl = usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
tokenUrl = usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
|
||||
if (clientId == clientIdDefault) {
|
||||
// application_id == default
|
||||
log.vdebug("USL5 AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint");
|
||||
redirectUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
} else {
|
||||
// custom application_id
|
||||
redirectUrl = usl5AuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
}
|
||||
|
||||
// Drive Queries
|
||||
driveUrl = usl5GraphEndpoint ~ "/v1.0/me/drive";
|
||||
driveByIdUrl = usl5GraphEndpoint ~ "/v1.0/drives/";
|
||||
|
|
@ -205,8 +240,16 @@ final class OneDriveApi
|
|||
log.log("Configuring Azure AD Germany");
|
||||
// Authentication
|
||||
authUrl = deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
|
||||
redirectUrl = deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
tokenUrl = deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
|
||||
if (clientId == clientIdDefault) {
|
||||
// application_id == default
|
||||
log.vdebug("DE AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint");
|
||||
redirectUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
} else {
|
||||
// custom application_id
|
||||
redirectUrl = deAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
}
|
||||
|
||||
// Drive Queries
|
||||
driveUrl = deGraphEndpoint ~ "/v1.0/me/drive";
|
||||
driveByIdUrl = deGraphEndpoint ~ "/v1.0/drives/";
|
||||
|
|
@ -224,8 +267,16 @@ final class OneDriveApi
|
|||
log.log("Configuring AD China operated by 21Vianet");
|
||||
// Authentication
|
||||
authUrl = cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/authorize";
|
||||
redirectUrl = cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
tokenUrl = cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/v2.0/token";
|
||||
if (clientId == clientIdDefault) {
|
||||
// application_id == default
|
||||
log.vdebug("CN AD Endpoint but default application_id, redirectUrl needs to be aligned to globalAuthEndpoint");
|
||||
redirectUrl = globalAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
} else {
|
||||
// custom application_id
|
||||
redirectUrl = cnAuthEndpoint ~ "/" ~ tenantId ~ "/oauth2/nativeclient";
|
||||
}
|
||||
|
||||
// Drive Queries
|
||||
driveUrl = cnGraphEndpoint ~ "/v1.0/me/drive";
|
||||
driveByIdUrl = cnGraphEndpoint ~ "/v1.0/drives/";
|
||||
|
|
@ -296,13 +347,6 @@ final class OneDriveApi
|
|||
|
||||
bool init()
|
||||
{
|
||||
// Update clientId if application_id is set in config file
|
||||
if (cfg.getValueString("application_id") != "") {
|
||||
// an application_id is set in config file
|
||||
clientId = cfg.getValueString("application_id");
|
||||
companyName = "custom_application";
|
||||
}
|
||||
|
||||
// detail what we are using for applicaion identification
|
||||
log.vdebug("clientId = ", clientId);
|
||||
log.vdebug("companyName = ", companyName);
|
||||
|
|
@ -556,11 +600,43 @@ final class OneDriveApi
|
|||
{
|
||||
checkAccessTokenExpired();
|
||||
scope(failure) {
|
||||
if (exists(saveToPath)) remove(saveToPath);
|
||||
if (exists(saveToPath)) {
|
||||
// try and remove the file, catch error
|
||||
try {
|
||||
remove(saveToPath);
|
||||
} catch (FileException e) {
|
||||
// display the error message
|
||||
displayFileSystemErrorMessage(e.msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
mkdirRecurse(dirName(saveToPath));
|
||||
|
||||
// Create the required local directory
|
||||
string newPath = dirName(saveToPath);
|
||||
|
||||
// Does the path exist locally?
|
||||
if (!exists(newPath)) {
|
||||
try {
|
||||
log.vdebug("Requested path does not exist, creating directory structure: ", newPath);
|
||||
mkdirRecurse(newPath);
|
||||
// Configure the applicable permissions for the folder
|
||||
log.vdebug("Setting directory permissions for: ", newPath);
|
||||
newPath.setAttributes(cfg.returnRequiredDirectoryPermisions());
|
||||
} catch (FileException e) {
|
||||
// display the error message
|
||||
displayFileSystemErrorMessage(e.msg);
|
||||
}
|
||||
}
|
||||
|
||||
const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content?AVOverride=1";
|
||||
// Download file
|
||||
download(url, saveToPath, fileSize);
|
||||
// Does path exist?
|
||||
if (exists(saveToPath)) {
|
||||
// File was downloaded sucessfully - configure the applicable permissions for the file
|
||||
log.vdebug("Setting file permissions for: ", saveToPath);
|
||||
saveToPath.setAttributes(cfg.returnRequiredFilePermisions());
|
||||
}
|
||||
}
|
||||
|
||||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
|
||||
|
|
@ -792,7 +868,16 @@ final class OneDriveApi
|
|||
refreshToken = response["refresh_token"].str();
|
||||
accessTokenExpiration = Clock.currTime() + dur!"seconds"(response["expires_in"].integer());
|
||||
if (!.dryRun) {
|
||||
std.file.write(cfg.refreshTokenFilePath, refreshToken);
|
||||
try {
|
||||
// try and update the refresh_token file
|
||||
log.vdebug("Updating refresh_token file with new token from OneDrive");
|
||||
std.file.write(cfg.refreshTokenFilePath, refreshToken);
|
||||
log.vdebug("Setting file permissions for: ", cfg.refreshTokenFilePath);
|
||||
cfg.refreshTokenFilePath.setAttributes(cfg.returnRequiredFilePermisions());
|
||||
} catch (FileException e) {
|
||||
// display the error message
|
||||
displayFileSystemErrorMessage(e.msg);
|
||||
}
|
||||
}
|
||||
if (printAccessToken) writeln("New access token: ", accessToken);
|
||||
} else {
|
||||
|
|
@ -803,7 +888,9 @@ final class OneDriveApi
|
|||
} else {
|
||||
// External tenant token handling
|
||||
if ("access_token" in response){
|
||||
log.vdebug("Updating access token with new token received from External OneDrive 3rd Party");
|
||||
externalAccessToken = "bearer " ~ response["access_token"].str();
|
||||
log.vdebug("Updating refresh_token with new token received from External OneDrive 3rd Party");
|
||||
externalRefreshToken = response["refresh_token"].str();
|
||||
externalAccessTokenExpiration = Clock.currTime() + dur!"seconds"(response["expires_in"].integer());
|
||||
if (printAccessToken) writeln("New external access token: ", externalAccessToken);
|
||||
|
|
@ -1399,6 +1486,14 @@ final class OneDriveApi
|
|||
log.error(" Error Reason: ", errorMessage["error_description"].str);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse and display error message received from the local file system
|
||||
private void displayFileSystemErrorMessage(string message)
|
||||
{
|
||||
log.error("ERROR: The local file system returned an error with the following message:");
|
||||
auto errorArray = splitLines(message);
|
||||
log.error(" Error Message: ", errorArray[0]);
|
||||
}
|
||||
}
|
||||
|
||||
unittest
|
||||
|
|
|
|||
|
|
@ -89,15 +89,30 @@ final class SelectiveSync
|
|||
|
||||
// Try full path match first
|
||||
if (!name.matchFirst(dirmask).empty) {
|
||||
log.vdebug("'!name.matchFirst(dirmask).empty' returned true = matched");
|
||||
return true;
|
||||
} else {
|
||||
// Do we check the base name as well?
|
||||
if (!skipDirStrictMatch) {
|
||||
// check just the basename in the path
|
||||
string parent = baseName(name);
|
||||
if(!parent.matchFirst(dirmask).empty) {
|
||||
return true;
|
||||
log.vdebug("No Strict Matching Enforced");
|
||||
|
||||
// Test the entire path working backwards from child
|
||||
string path = buildNormalizedPath(name);
|
||||
string checkPath;
|
||||
auto paths = pathSplitter(path);
|
||||
|
||||
foreach_reverse(directory; paths) {
|
||||
if (directory != "/") {
|
||||
// This will add a leading '/' but that needs to be stripped to check
|
||||
checkPath = "/" ~ directory ~ checkPath;
|
||||
if(!checkPath.strip('/').matchFirst(dirmask).empty) {
|
||||
log.vdebug("'!checkPath.matchFirst(dirmask).empty' returned true = matched");
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.vdebug("Strict Matching Enforced - No Match");
|
||||
}
|
||||
}
|
||||
// no match
|
||||
|
|
|
|||
601
src/sync.d
601
src/sync.d
|
|
@ -130,10 +130,14 @@ private Item makeItem(const ref JSONValue driveItem)
|
|||
// Resolve 'Key not found: fileSystemInfo' when then item is a remote item
|
||||
// https://github.com/abraunegg/onedrive/issues/11
|
||||
if (isItemRemote(driveItem)) {
|
||||
// remoteItem is a OneDrive object that exists on a 'different' OneDrive drive id, when compared to account default
|
||||
item.mtime = SysTime.fromISOExtString(driveItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str);
|
||||
} else {
|
||||
// item exists on account default drive id
|
||||
item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str);
|
||||
}
|
||||
// debug output of what the OneDrive item modified time is
|
||||
log.vdebug("lastModifiedDateTime (OneDrive item): ", item.mtime);
|
||||
}
|
||||
|
||||
if (isItemFile(driveItem)) {
|
||||
|
|
@ -603,133 +607,158 @@ final class SyncEngine
|
|||
// Log that an invalid JSON object was returned
|
||||
log.error("ERROR: onedrive.getTenantID call returned an invalid JSON Object");
|
||||
}
|
||||
|
||||
// query OneDrive Business Shared Folders shared with me
|
||||
|
||||
// Query OneDrive Business Shared Folders shared with me
|
||||
log.vlog("Attempting to sync OneDrive Business Shared Folders");
|
||||
JSONValue graphQuery = onedrive.getSharedWithMe();
|
||||
if (graphQuery.type() == JSONType.object) {
|
||||
string sharedFolderName;
|
||||
bool isExternalTenant = false;
|
||||
foreach (searchResult; graphQuery["value"].array) {
|
||||
sharedFolderName = searchResult["name"].str;
|
||||
// Compare this to values in business_shared_folders
|
||||
if(selectiveSync.isSharedFolderMatched(sharedFolderName)){
|
||||
// Folder name matches what we are looking for
|
||||
// Flags for matching
|
||||
bool itemInDatabase = false;
|
||||
bool itemLocalDirExists = false;
|
||||
bool itemPathIsLocal = false;
|
||||
isExternalTenant = false;
|
||||
|
||||
// "what if" there are 2 or more folders shared with me have the "same" name?
|
||||
// The folder name will be the same, but driveId will be different
|
||||
// This will then cause these 'shared folders' to cross populate data, which may not be desirable
|
||||
log.vdebug("Shared Folder Name: ", sharedFolderName);
|
||||
log.vdebug("Parent Drive Id: ", searchResult["remoteItem"]["parentReference"]["driveId"].str);
|
||||
log.vdebug("Shared Item Id: ", searchResult["remoteItem"]["id"].str);
|
||||
|
||||
// Is this OneDrive Shared Folder on an external tenant?
|
||||
if (searchResult["remoteItem"]["sharepointIds"]["tenantId"].str != myTenantID) {
|
||||
isExternalTenant = true;
|
||||
log.vdebug("This shared folder is shared from an external organisation as the tenant is different");
|
||||
// Have to configure the access to this tenant, which requires separate tokenUrl for that tenant
|
||||
onedrive.setExternalTenant(searchResult["remoteItem"]["sharepointIds"]["tenantId"].str);
|
||||
// Configure additional logging items for this array element
|
||||
string sharedByName;
|
||||
string sharedByEmail;
|
||||
// Extra details for verbose logging
|
||||
if ("sharedBy" in searchResult["remoteItem"]["shared"]) {
|
||||
if ("displayName" in searchResult["remoteItem"]["shared"]["sharedBy"]["user"]) {
|
||||
sharedByName = searchResult["remoteItem"]["shared"]["sharedBy"]["user"]["displayName"].str;
|
||||
}
|
||||
|
||||
// for each driveid in the existing driveIDsArray
|
||||
Item databaseItem;
|
||||
foreach (searchDriveId; driveIDsArray) {
|
||||
log.vdebug("Searching database for: ", searchDriveId, ", ", sharedFolderName);
|
||||
if (itemdb.selectByPath(sharedFolderName, searchDriveId, databaseItem)) {
|
||||
log.vdebug("Found shared folder name in database");
|
||||
itemInDatabase = true;
|
||||
log.vdebug("databaseItem: ", databaseItem);
|
||||
// Does the databaseItem.driveId == defaultDriveId?
|
||||
if (databaseItem.driveId == defaultDriveId) {
|
||||
itemPathIsLocal = true;
|
||||
}
|
||||
} else {
|
||||
log.vdebug("Shared folder name not found in database");
|
||||
// "what if" there is 'already' a local folder with this name
|
||||
// Check if in the database
|
||||
// If NOT in the database, but resides on disk, this could be a new local folder created after last sync but before this one
|
||||
// However we sync 'shared folders' before checking for local changes
|
||||
string localpath = expandTilde(cfg.getValueString("sync_dir")) ~ "/" ~ sharedFolderName;
|
||||
if (exists(localpath)) {
|
||||
// local path exists
|
||||
log.vdebug("Found shared folder name in local OneDrive sync_dir");
|
||||
itemLocalDirExists = true;
|
||||
if ("email" in searchResult["remoteItem"]["shared"]["sharedBy"]["user"]) {
|
||||
sharedByEmail = searchResult["remoteItem"]["shared"]["sharedBy"]["user"]["email"].str;
|
||||
}
|
||||
}
|
||||
|
||||
// is the shared item with us a 'folder' ?
|
||||
if (isItemFolder(searchResult)) {
|
||||
// item returned is a shared folder, not a shared file
|
||||
sharedFolderName = searchResult["name"].str;
|
||||
// Compare this to values in business_shared_folders
|
||||
if(selectiveSync.isSharedFolderMatched(sharedFolderName)){
|
||||
// Folder name matches what we are looking for
|
||||
// Flags for matching
|
||||
bool itemInDatabase = false;
|
||||
bool itemLocalDirExists = false;
|
||||
bool itemPathIsLocal = false;
|
||||
isExternalTenant = false;
|
||||
|
||||
// "what if" there are 2 or more folders shared with me have the "same" name?
|
||||
// The folder name will be the same, but driveId will be different
|
||||
// This will then cause these 'shared folders' to cross populate data, which may not be desirable
|
||||
log.vdebug("Shared Folder Name: ", sharedFolderName);
|
||||
log.vdebug("Parent Drive Id: ", searchResult["remoteItem"]["parentReference"]["driveId"].str);
|
||||
log.vdebug("Shared Item Id: ", searchResult["remoteItem"]["id"].str);
|
||||
|
||||
// Is this OneDrive Shared Folder on an external tenant?
|
||||
if (searchResult["remoteItem"]["sharepointIds"]["tenantId"].str != myTenantID) {
|
||||
isExternalTenant = true;
|
||||
log.vdebug("This shared folder is shared from an external organisation as the tenant is different");
|
||||
// Have to configure the access to this tenant, which requires separate tokenUrl for that tenant
|
||||
onedrive.setExternalTenant(searchResult["remoteItem"]["sharepointIds"]["tenantId"].str);
|
||||
}
|
||||
|
||||
// for each driveid in the existing driveIDsArray
|
||||
Item databaseItem;
|
||||
foreach (searchDriveId; driveIDsArray) {
|
||||
log.vdebug("searching database for: ", searchDriveId, " ", sharedFolderName);
|
||||
if (itemdb.selectByPath(sharedFolderName, searchDriveId, databaseItem)) {
|
||||
log.vdebug("Found shared folder name in database");
|
||||
itemInDatabase = true;
|
||||
log.vdebug("databaseItem: ", databaseItem);
|
||||
// Does the databaseItem.driveId == defaultDriveId?
|
||||
if (databaseItem.driveId == defaultDriveId) {
|
||||
itemPathIsLocal = true;
|
||||
}
|
||||
} else {
|
||||
log.vdebug("Shared folder name not found in database");
|
||||
// "what if" there is 'already' a local folder with this name
|
||||
// Check if in the database
|
||||
// If NOT in the database, but resides on disk, this could be a new local folder created after last sync but before this one
|
||||
// However we sync 'shared folders' before checking for local changes
|
||||
string localpath = expandTilde(cfg.getValueString("sync_dir")) ~ "/" ~ sharedFolderName;
|
||||
if (exists(localpath)) {
|
||||
// local path exists
|
||||
log.vdebug("Found shared folder name in local OneDrive sync_dir");
|
||||
itemLocalDirExists = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Shared Folder Evaluation Debugging
|
||||
log.vdebug("item in database: ", itemInDatabase);
|
||||
log.vdebug("path exists on disk: ", itemLocalDirExists);
|
||||
log.vdebug("database drive id matches defaultDriveId: ", itemPathIsLocal);
|
||||
log.vdebug("database data matches search data: ", ((databaseItem.driveId == searchResult["remoteItem"]["parentReference"]["driveId"].str) && (databaseItem.id == searchResult["remoteItem"]["id"].str)));
|
||||
|
||||
// Additional logging
|
||||
string sharedByName;
|
||||
string sharedByEmail;
|
||||
|
||||
// Extra details for verbose logging
|
||||
if ("sharedBy" in searchResult["remoteItem"]["shared"]) {
|
||||
if ("displayName" in searchResult["remoteItem"]["shared"]["sharedBy"]["user"]) {
|
||||
sharedByName = searchResult["remoteItem"]["shared"]["sharedBy"]["user"]["displayName"].str;
|
||||
}
|
||||
if ("email" in searchResult["remoteItem"]["shared"]["sharedBy"]["user"]) {
|
||||
sharedByEmail = searchResult["remoteItem"]["shared"]["sharedBy"]["user"]["email"].str;
|
||||
}
|
||||
}
|
||||
|
||||
if ( ((!itemInDatabase) || (!itemLocalDirExists)) || (((databaseItem.driveId == searchResult["remoteItem"]["parentReference"]["driveId"].str) && (databaseItem.id == searchResult["remoteItem"]["id"].str)) && (!itemPathIsLocal)) ) {
|
||||
// This shared folder does not exist in the database
|
||||
if (!cfg.getValueBool("monitor")) {
|
||||
log.log("Syncing this OneDrive Business Shared Folder: ", sharedFolderName);
|
||||
|
||||
// Shared Folder Evaluation Debugging
|
||||
log.vdebug("item in database: ", itemInDatabase);
|
||||
log.vdebug("path exists on disk: ", itemLocalDirExists);
|
||||
log.vdebug("database drive id matches defaultDriveId: ", itemPathIsLocal);
|
||||
log.vdebug("database data matches search data: ", ((databaseItem.driveId == searchResult["remoteItem"]["parentReference"]["driveId"].str) && (databaseItem.id == searchResult["remoteItem"]["id"].str)));
|
||||
|
||||
if ( ((!itemInDatabase) || (!itemLocalDirExists)) || (((databaseItem.driveId == searchResult["remoteItem"]["parentReference"]["driveId"].str) && (databaseItem.id == searchResult["remoteItem"]["id"].str)) && (!itemPathIsLocal)) ) {
|
||||
// This shared folder does not exist in the database
|
||||
if (!cfg.getValueBool("monitor")) {
|
||||
log.log("Syncing this OneDrive Business Shared Folder: ", sharedFolderName);
|
||||
} else {
|
||||
log.vlog("Syncing this OneDrive Business Shared Folder: ", sharedFolderName);
|
||||
}
|
||||
Item businessSharedFolder = makeItem(searchResult);
|
||||
|
||||
// Log who shared this to assist with sync data correlation
|
||||
if ((sharedByName != "") && (sharedByEmail != "")) {
|
||||
log.vlog("OneDrive Business Shared Folder - Shared By: ", sharedByName, " (", sharedByEmail, ")");
|
||||
} else {
|
||||
if (sharedByName != "") {
|
||||
log.vlog("OneDrive Business Shared Folder - Shared By: ", sharedByName);
|
||||
}
|
||||
}
|
||||
|
||||
// Do the actual sync
|
||||
applyDifferences(businessSharedFolder.remoteDriveId, businessSharedFolder.remoteId, performFullItemScan);
|
||||
// add this parent drive id to the array to search for, ready for next use
|
||||
string newDriveID = searchResult["remoteItem"]["parentReference"]["driveId"].str;
|
||||
// Keep the driveIDsArray with unique entries only
|
||||
if (!canFind(driveIDsArray, newDriveID)) {
|
||||
// Add this drive id to the array to search with
|
||||
driveIDsArray ~= newDriveID;
|
||||
}
|
||||
} else {
|
||||
log.vlog("Syncing this OneDrive Business Shared Folder: ", sharedFolderName);
|
||||
}
|
||||
Item businessSharedFolder = makeItem(searchResult);
|
||||
// Shared Folder Name Conflict ...
|
||||
log.log("WARNING: Skipping shared folder due to existing name conflict: ", sharedFolderName);
|
||||
log.log("WARNING: Skipping changes of Path ID: ", searchResult["remoteItem"]["id"].str);
|
||||
log.log("WARNING: To sync this shared folder, this shared folder needs to be renamed");
|
||||
|
||||
// Log who shared this to assist with conflict resolution
|
||||
if ((sharedByName != "") && (sharedByEmail != "")) {
|
||||
log.vlog("WARNING: Conflict Shared By: ", sharedByName, " (", sharedByEmail, ")");
|
||||
} else {
|
||||
if (sharedByName != "") {
|
||||
log.vlog("WARNING: Conflict Shared By: ", sharedByName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// not a folder, is this a file?
|
||||
if (isItemFile(searchResult)) {
|
||||
// shared item is a file
|
||||
string sharedFileName = searchResult["name"].str;
|
||||
// log that this is not supported
|
||||
log.vlog("WARNING: Not syncing this OneDrive Business Shared File: ", sharedFileName);
|
||||
|
||||
// Log who shared this to assist with sync data correlation
|
||||
if ((sharedByName != "") && (sharedByEmail != "")) {
|
||||
log.vlog("OneDrive Business Shared Folder - Shared By: ", sharedByName, " (", sharedByEmail, ")");
|
||||
log.vlog("OneDrive Business Shared File - Shared By: ", sharedByName, " (", sharedByEmail, ")");
|
||||
} else {
|
||||
if (sharedByName != "") {
|
||||
log.vlog("OneDrive Business Shared Folder - Shared By: ", sharedByName);
|
||||
log.vlog("OneDrive Business Shared File - Shared By: ", sharedByName);
|
||||
}
|
||||
}
|
||||
|
||||
// Do the actual sync
|
||||
applyDifferences(businessSharedFolder.remoteDriveId, businessSharedFolder.remoteId, performFullItemScan);
|
||||
// add this parent drive id to the array to search for, ready for next use
|
||||
string newDriveID = searchResult["remoteItem"]["parentReference"]["driveId"].str;
|
||||
// Keep the driveIDsArray with unique entries only
|
||||
if (!canFind(driveIDsArray, newDriveID)) {
|
||||
// Add this drive id to the array to search with
|
||||
driveIDsArray ~= newDriveID;
|
||||
}
|
||||
} else {
|
||||
// Shared Folder Name Conflict ...
|
||||
log.log("WARNING: Skipping shared folder due to existing name conflict: ", sharedFolderName);
|
||||
log.log("WARNING: Skipping changes of Path ID: ", searchResult["remoteItem"]["id"].str);
|
||||
log.log("WARNING: To sync this shared folder, this shared folder needs to be renamed");
|
||||
|
||||
// Log who shared this to assist with conflict resolution
|
||||
if ((sharedByName != "") && (sharedByEmail != "")) {
|
||||
log.vlog("WARNING: Conflict Shared By: ", sharedByName, " (", sharedByEmail, ")");
|
||||
} else {
|
||||
if (sharedByName != "") {
|
||||
log.vlog("WARNING: Conflict Shared By: ", sharedByName);
|
||||
}
|
||||
}
|
||||
// something else entirely
|
||||
log.log("WARNING: Not syncing this OneDrive Business Shared item: ", searchResult["name"].str);
|
||||
}
|
||||
}
|
||||
|
||||
// Was this shared folder on an external tenant?
|
||||
if (isExternalTenant) {
|
||||
// clear external tenant
|
||||
onedrive.clearExternalTenant();
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Log that an invalid JSON object was returned
|
||||
|
|
@ -2005,25 +2034,31 @@ final class SyncEngine
|
|||
// - full path + combination of any above two - /path/name*.txt
|
||||
// - full path to file - /path/to/file.txt
|
||||
|
||||
// need to compute the full path for this file
|
||||
path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name;
|
||||
|
||||
// The path that needs to be checked needs to include the '/'
|
||||
// This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched
|
||||
if (!startsWith(path, "/")){
|
||||
// Add '/' to the path
|
||||
path = '/' ~ path;
|
||||
// is the parent id in the database?
|
||||
if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){
|
||||
// need to compute the full path for this file
|
||||
path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name;
|
||||
|
||||
// The path that needs to be checked needs to include the '/'
|
||||
// This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched
|
||||
if (!startsWith(path, "/")){
|
||||
// Add '/' to the path
|
||||
path = '/' ~ path;
|
||||
}
|
||||
|
||||
log.vdebug("skip_file item to check: ", path);
|
||||
unwanted = selectiveSync.isFileNameExcluded(path);
|
||||
log.vdebug("Result: ", unwanted);
|
||||
if (unwanted) log.vlog("Skipping item - excluded by skip_file config: ", item.name);
|
||||
} else {
|
||||
// parent id is not in the database
|
||||
unwanted = true;
|
||||
log.vlog("Skipping file - parent path not present in local database");
|
||||
}
|
||||
|
||||
log.vdebug("skip_file item to check: ", path);
|
||||
unwanted = selectiveSync.isFileNameExcluded(path);
|
||||
log.vdebug("Result: ", unwanted);
|
||||
if (unwanted) log.vlog("Skipping item - excluded by skip_file config: ", item.name);
|
||||
}
|
||||
}
|
||||
|
||||
// check the item type
|
||||
|
||||
if (!unwanted) {
|
||||
if (isItemFile(driveItem)) {
|
||||
log.vdebug("The item we are syncing is a file");
|
||||
|
|
@ -2149,25 +2184,37 @@ final class SyncEngine
|
|||
if (cached && item.eTag != oldItem.eTag) {
|
||||
// Is the item in the local database
|
||||
if (itemdb.idInLocalDatabase(item.driveId, item.id)){
|
||||
log.vdebug("OneDrive item ID is present in local database");
|
||||
oldPath = itemdb.computePath(item.driveId, item.id);
|
||||
if (!isItemSynced(oldItem, oldPath)) {
|
||||
// Query DB for existing local item in specified path
|
||||
string itemSource = "database";
|
||||
if (!isItemSynced(oldItem, oldPath, itemSource)) {
|
||||
if (exists(oldPath)) {
|
||||
// Is the local file technically 'newer' based on UTC timestamp?
|
||||
SysTime localModifiedTime = timeLastModified(oldPath).toUTC();
|
||||
localModifiedTime.fracSecs = Duration.zero;
|
||||
item.mtime.fracSecs = Duration.zero;
|
||||
|
||||
if (localModifiedTime > item.mtime) {
|
||||
// local file is newer than item on OneDrive
|
||||
// debug the output of time comparison
|
||||
log.vdebug("localModifiedTime (local file): ", localModifiedTime);
|
||||
log.vdebug("item.mtime (OneDrive item): ", item.mtime);
|
||||
|
||||
// Compare file on disk modified time with modified time provided by OneDrive API
|
||||
if (localModifiedTime >= item.mtime) {
|
||||
// local file is newer or has the same time than the item on OneDrive
|
||||
log.vdebug("Skipping OneDrive change as this is determined to be unwanted due to local item modified time being newer or equal to item modified time from OneDrive");
|
||||
// no local rename
|
||||
// no download needed
|
||||
log.vlog("Local item modified time is newer based on UTC time conversion - keeping local item");
|
||||
log.vdebug("Skipping OneDrive change as this is determined to be unwanted due to local item modified time being newer than OneDrive item");
|
||||
if (localModifiedTime == item.mtime) {
|
||||
log.vlog("Local item modified time is equal to OneDrive item modified time based on UTC time conversion - keeping local item");
|
||||
} else {
|
||||
log.vlog("Local item modified time is newer than OneDrive item modified time based on UTC time conversion - keeping local item");
|
||||
}
|
||||
skippedItems ~= item.id;
|
||||
return;
|
||||
} else {
|
||||
// remote file is newer than local item
|
||||
log.vlog("Remote item modified time is newer based on UTC time conversion");
|
||||
log.vlog("Remote item modified time is newer based on UTC time conversion"); // correct message, remote item is newer
|
||||
auto ext = extension(oldPath);
|
||||
auto newPath = path.chomp(ext) ~ "-" ~ deviceName ~ ext;
|
||||
|
||||
|
|
@ -2225,6 +2272,14 @@ final class SyncEngine
|
|||
}
|
||||
// What was the item that was saved
|
||||
log.vdebug("item details: ", item);
|
||||
} else {
|
||||
// flag was tripped, which was it
|
||||
if (downloadFailed) {
|
||||
log.vdebug("Download or creation of local directory failed");
|
||||
}
|
||||
if (malwareDetected) {
|
||||
log.vdebug("OneDrive reported that file contained malware");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2233,7 +2288,9 @@ final class SyncEngine
|
|||
{
|
||||
if (exists(path)) {
|
||||
// path exists locally
|
||||
if (isItemSynced(item, path)) {
|
||||
// Query DB for new remote item in specified path
|
||||
string itemSource = "remote";
|
||||
if (isItemSynced(item, path, itemSource)) {
|
||||
// file details from OneDrive and local file details in database are in-sync
|
||||
log.vdebug("The item to sync is already present on the local file system and is in-sync with the local database");
|
||||
return;
|
||||
|
|
@ -2243,8 +2300,8 @@ final class SyncEngine
|
|||
SysTime localModifiedTime = timeLastModified(path).toUTC();
|
||||
SysTime itemModifiedTime = item.mtime;
|
||||
// HACK: reduce time resolution to seconds before comparing
|
||||
itemModifiedTime.fracSecs = Duration.zero;
|
||||
localModifiedTime.fracSecs = Duration.zero;
|
||||
itemModifiedTime.fracSecs = Duration.zero;
|
||||
|
||||
// is the local modified time greater than that from OneDrive?
|
||||
if (localModifiedTime > itemModifiedTime) {
|
||||
|
|
@ -2306,7 +2363,10 @@ final class SyncEngine
|
|||
}
|
||||
} else {
|
||||
// remote file is newer than local item
|
||||
log.vlog("Remote item modified time is newer based on UTC time conversion");
|
||||
log.vlog("Remote item modified time is newer based on UTC time conversion"); // correct message, remote item is newer
|
||||
log.vdebug("localModifiedTime (local file): ", localModifiedTime);
|
||||
log.vdebug("itemModifiedTime (OneDrive item): ", itemModifiedTime);
|
||||
|
||||
auto ext = extension(path);
|
||||
auto newPath = path.chomp(ext) ~ "-" ~ deviceName ~ ext;
|
||||
|
||||
|
|
@ -2405,7 +2465,23 @@ final class SyncEngine
|
|||
}
|
||||
|
||||
if (!dryRun) {
|
||||
mkdirRecurse(path);
|
||||
try {
|
||||
// Does the path exist locally?
|
||||
if (!exists(path)) {
|
||||
// Create the new directory
|
||||
log.vdebug("Requested path does not exist, creating directory structure: ", path);
|
||||
mkdirRecurse(path);
|
||||
// Configure the applicable permissions for the folder
|
||||
log.vdebug("Setting directory permissions for: ", path);
|
||||
path.setAttributes(cfg.returnRequiredDirectoryPermisions());
|
||||
}
|
||||
} catch (FileException e) {
|
||||
// display the error message
|
||||
displayFileSystemErrorMessage(e.msg);
|
||||
// flag that this failed
|
||||
downloadFailed = true;
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// we dont create the directory, but we need to track that we 'faked it'
|
||||
idsFaked ~= [item.driveId, item.id];
|
||||
|
|
@ -2431,7 +2507,9 @@ final class SyncEngine
|
|||
if (exists(newPath)) {
|
||||
Item localNewItem;
|
||||
if (itemdb.selectByPath(newPath, defaultDriveId, localNewItem)) {
|
||||
if (isItemSynced(localNewItem, newPath)) {
|
||||
// Query DB for new local item in specified path
|
||||
string itemSource = "database";
|
||||
if (isItemSynced(localNewItem, newPath, itemSource)) {
|
||||
log.vlog("Destination is in sync and will be overwritten");
|
||||
} else {
|
||||
// TODO: force remote sync by deleting local item
|
||||
|
|
@ -2457,6 +2535,7 @@ final class SyncEngine
|
|||
// handle changed time
|
||||
if (newItem.type == ItemType.file && oldItem.mtime != newItem.mtime) {
|
||||
try {
|
||||
log.vdebug("Calling setTimes() for this file: ", newPath);
|
||||
setTimes(newPath, newItem.mtime, newItem.mtime);
|
||||
} catch (FileException e) {
|
||||
// display the error message
|
||||
|
|
@ -2542,7 +2621,6 @@ final class SyncEngine
|
|||
log.vdebug("onedrive.downloadById(item.driveId, item.id, path, fileSize); generated a OneDriveException");
|
||||
// 408 = Request Time Out
|
||||
// 429 = Too Many Requests - need to delay
|
||||
|
||||
if (e.httpStatusCode == 408) {
|
||||
// 408 error handling - request time out
|
||||
// https://github.com/abraunegg/onedrive/issues/694
|
||||
|
|
@ -2614,6 +2692,12 @@ final class SyncEngine
|
|||
}
|
||||
}
|
||||
}
|
||||
} catch (FileException e) {
|
||||
// There was a file system error
|
||||
// display the error message
|
||||
displayFileSystemErrorMessage(e.msg);
|
||||
downloadFailed = true;
|
||||
return;
|
||||
} catch (std.exception.ErrnoException e) {
|
||||
// There was a file system error
|
||||
// display the error message
|
||||
|
|
@ -2632,6 +2716,7 @@ final class SyncEngine
|
|||
// downloaded matches either size or hash
|
||||
log.vdebug("Downloaded file matches reported size and or reported file hash");
|
||||
try {
|
||||
log.vdebug("Calling setTimes() for this file: ", path);
|
||||
setTimes(path, item.mtime, item.mtime);
|
||||
} catch (FileException e) {
|
||||
// display the error message
|
||||
|
|
@ -2667,7 +2752,7 @@ final class SyncEngine
|
|||
}
|
||||
|
||||
// returns true if the given item corresponds to the local one
|
||||
private bool isItemSynced(const ref Item item, const(string) path)
|
||||
private bool isItemSynced(const ref Item item, const(string) path, string itemSource)
|
||||
{
|
||||
if (!exists(path)) return false;
|
||||
final switch (item.type) {
|
||||
|
|
@ -2676,17 +2761,17 @@ final class SyncEngine
|
|||
SysTime localModifiedTime = timeLastModified(path).toUTC();
|
||||
SysTime itemModifiedTime = item.mtime;
|
||||
// HACK: reduce time resolution to seconds before comparing
|
||||
itemModifiedTime.fracSecs = Duration.zero;
|
||||
localModifiedTime.fracSecs = Duration.zero;
|
||||
itemModifiedTime.fracSecs = Duration.zero;
|
||||
if (localModifiedTime == itemModifiedTime) {
|
||||
return true;
|
||||
} else {
|
||||
log.vlog("The local item has a different modified time ", localModifiedTime, " remote is ", itemModifiedTime);
|
||||
log.vlog("The local item has a different modified time ", localModifiedTime, " when compared to ", itemSource, " modified time ", itemModifiedTime);
|
||||
}
|
||||
if (testFileHash(path, item)) {
|
||||
return true;
|
||||
} else {
|
||||
log.vlog("The local item has a different hash");
|
||||
log.vlog("The local item has a different hash when compared to ", itemSource, " item hash");
|
||||
}
|
||||
} else {
|
||||
log.vlog("The local item is a directory but should be a file");
|
||||
|
|
@ -2778,6 +2863,16 @@ final class SyncEngine
|
|||
logPath = path;
|
||||
}
|
||||
|
||||
// If we are using --upload-only & --sync-shared-folders there is a possability that a 'new' local folder might
|
||||
// be misinterpreted that it needs to be uploaded to the users default OneDrive DriveID rather than the requested / configured
|
||||
// Shared Business Folder. In --resync scenarios, the DB information that tells that this Business Shared Folder does not exist,
|
||||
// and in a --upload-only scenario will never exist, so the correct lookups are unable to be performed.
|
||||
if ((exists(cfg.businessSharedFolderFilePath)) && (syncBusinessFolders) && (cfg.getValueBool("upload_only"))){
|
||||
// business_shared_folders file exists, --sync-shared-folders is enabled, --upload-only is enabled
|
||||
log.vdebug("OneDrive Business --upload-only & --sync-shared-folders edge case triggered");
|
||||
handleUploadOnlyBusinessSharedFoldersEdgeCase();
|
||||
}
|
||||
|
||||
// Are we configured to use a National Cloud Deployment
|
||||
// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
|
||||
// Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query
|
||||
|
|
@ -2846,6 +2941,16 @@ final class SyncEngine
|
|||
logPath = path;
|
||||
}
|
||||
|
||||
// If we are using --upload-only & --sync-shared-folders there is a possability that a 'new' local folder might
|
||||
// be misinterpreted that it needs to be uploaded to the users default OneDrive DriveID rather than the requested / configured
|
||||
// Shared Business Folder. In --resync scenarios, the DB information that tells that this Business Shared Folder does not exist,
|
||||
// and in a --upload-only scenario will never exist, so the correct lookups are unable to be performed.
|
||||
if ((exists(cfg.businessSharedFolderFilePath)) && (syncBusinessFolders) && (cfg.getValueBool("upload_only"))){
|
||||
// business_shared_folders file exists, --sync-shared-folders is enabled, --upload-only is enabled
|
||||
log.vdebug("OneDrive Business --upload-only & --sync-shared-folders edge case triggered");
|
||||
handleUploadOnlyBusinessSharedFoldersEdgeCase();
|
||||
}
|
||||
|
||||
// Are we configured to use a National Cloud Deployment
|
||||
// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
|
||||
// Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query
|
||||
|
|
@ -2892,6 +2997,87 @@ final class SyncEngine
|
|||
}
|
||||
}
|
||||
|
||||
void handleUploadOnlyBusinessSharedFoldersEdgeCase() {
|
||||
// read in the business_shared_folders file contents
|
||||
string[] businessSharedFoldersList;
|
||||
// open file as read only
|
||||
auto file = File(cfg.businessSharedFolderFilePath, "r");
|
||||
auto range = file.byLine();
|
||||
foreach (line; range) {
|
||||
// Skip comments in file
|
||||
if (line.length == 0 || line[0] == ';' || line[0] == '#') continue;
|
||||
businessSharedFoldersList ~= buildNormalizedPath(line);
|
||||
}
|
||||
file.close();
|
||||
|
||||
// Query the GET /me/drive/sharedWithMe API
|
||||
JSONValue graphQuery = onedrive.getSharedWithMe();
|
||||
if (graphQuery.type() == JSONType.object) {
|
||||
if (count(graphQuery["value"].array) != 0) {
|
||||
// Shared items returned
|
||||
log.vdebug("onedrive.getSharedWithMe API Response: ", graphQuery);
|
||||
foreach (searchResult; graphQuery["value"].array) {
|
||||
// loop variables
|
||||
string sharedFolderName;
|
||||
string remoteParentDriveId;
|
||||
string remoteParentItemId;
|
||||
Item remoteItemRoot;
|
||||
Item remoteItem;
|
||||
|
||||
// is the shared item with us a 'folder' ?
|
||||
// we only handle folders, not files or other items
|
||||
if (isItemFolder(searchResult)) {
|
||||
// Debug response output
|
||||
log.vdebug("shared folder entry: ", searchResult);
|
||||
sharedFolderName = searchResult["name"].str;
|
||||
remoteParentDriveId = searchResult["remoteItem"]["parentReference"]["driveId"].str;
|
||||
remoteParentItemId = searchResult["remoteItem"]["parentReference"]["id"].str;
|
||||
|
||||
if (canFind(businessSharedFoldersList, sharedFolderName)) {
|
||||
// Shared Folder matches what is in the shared folder list
|
||||
log.vdebug("shared folder name matches business_shared_folders list item: ", sharedFolderName);
|
||||
// Actions:
|
||||
// 1. Add this remote item to the DB so that it can be queried
|
||||
// 2. Add remoteParentDriveId to driveIDsArray so we have a record of it
|
||||
|
||||
// Make JSON item DB compatible
|
||||
remoteItem = makeItem(searchResult);
|
||||
// Fix up entries, as we are manipulating the data
|
||||
remoteItem.driveId = remoteParentDriveId;
|
||||
remoteItem.eTag = "";
|
||||
remoteItem.cTag = "";
|
||||
remoteItem.parentId = defaultRootId;
|
||||
remoteItem.remoteDriveId = "";
|
||||
remoteItem.remoteId = "";
|
||||
|
||||
// Build the remote root DB item
|
||||
remoteItemRoot.driveId = remoteParentDriveId;
|
||||
remoteItemRoot.id = defaultRootId;
|
||||
remoteItemRoot.name = "root";
|
||||
remoteItemRoot.type = ItemType.dir;
|
||||
remoteItemRoot.mtime = remoteItem.mtime;
|
||||
remoteItemRoot.syncStatus = "Y";
|
||||
|
||||
// Add root remote item to the local database
|
||||
log.vdebug("Adding remote folder root to database: ", remoteItemRoot);
|
||||
itemdb.upsert(remoteItemRoot);
|
||||
|
||||
// Add shared folder item to the local database
|
||||
log.vdebug("Adding remote folder to database: ", remoteItem);
|
||||
itemdb.upsert(remoteItem);
|
||||
|
||||
// Keep the driveIDsArray with unique entries only
|
||||
if (!canFind(driveIDsArray, remoteParentDriveId)) {
|
||||
// Add this drive id to the array to search with
|
||||
driveIDsArray ~= remoteParentDriveId;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// scan the given directory for new items - for use with --monitor
|
||||
void scanForDifferencesFilesystemScan(const(string) path)
|
||||
{
|
||||
|
|
@ -2933,9 +3119,18 @@ final class SyncEngine
|
|||
// Is this item excluded by user configuration of skip_dir or skip_file?
|
||||
// Is this item a directory or 'remote' type? A 'remote' type is a folder DB tie so should be compared as directory for exclusion
|
||||
if ((item.type == ItemType.dir)||(item.type == ItemType.remote)) {
|
||||
// Do we need to check for .nosync? Only if --check-for-nosync was passed in
|
||||
if (cfg.getValueBool("check_nosync")) {
|
||||
if (exists(path ~ "/.nosync")) {
|
||||
log.vlog("Skipping item - .nosync found & --check-for-nosync enabled: ", path);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Is the path excluded?
|
||||
unwanted = selectiveSync.isDirNameExcluded(item.name);
|
||||
}
|
||||
|
||||
// Is this item a file?
|
||||
if (item.type == ItemType.file) {
|
||||
// Is the filename excluded?
|
||||
|
|
@ -3912,13 +4107,36 @@ final class SyncEngine
|
|||
} else {
|
||||
// parent is in database
|
||||
log.vlog("The parent for this path is in the local database - adding requested path (", path ,") to database");
|
||||
|
||||
// are we in a --dry-run scenario?
|
||||
if (!dryRun) {
|
||||
// get the live data
|
||||
auto res = onedrive.getPathDetails(path);
|
||||
JSONValue pathDetails;
|
||||
try {
|
||||
pathDetails = onedrive.getPathDetailsByDriveId(parent.driveId, path);
|
||||
} catch (OneDriveException e) {
|
||||
log.vdebug("pathDetails = onedrive.getPathDetailsByDriveId(parent.driveId, path) generated a OneDriveException");
|
||||
if (e.httpStatusCode == 404) {
|
||||
// The directory was not found
|
||||
log.error("ERROR: The requested single directory to sync was not found on OneDrive");
|
||||
return;
|
||||
}
|
||||
|
||||
if (e.httpStatusCode == 429) {
|
||||
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
|
||||
handleOneDriveThrottleRequest();
|
||||
// Retry original request by calling function again to avoid replicating any further error handling
|
||||
log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - calling onedrive.getPathDetailsByDriveId(parent.driveId, path);");
|
||||
pathDetails = onedrive.getPathDetailsByDriveId(parent.driveId, path);
|
||||
}
|
||||
|
||||
if (e.httpStatusCode >= 500) {
|
||||
// OneDrive returned a 'HTTP 5xx Server Side Error' - gracefully handling error - error message already logged
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Is the response a valid JSON object - validation checking done in saveItem
|
||||
saveItem(res);
|
||||
saveItem(pathDetails);
|
||||
} else {
|
||||
// need to fake this data
|
||||
auto fakeResponse = createFakeResponse(path);
|
||||
|
|
@ -4019,9 +4237,10 @@ final class SyncEngine
|
|||
// Does this 'file' already exist on OneDrive?
|
||||
try {
|
||||
// test if the local path exists on OneDrive
|
||||
fileDetailsFromOneDrive = onedrive.getPathDetails(path);
|
||||
fileDetailsFromOneDrive = onedrive.getPathDetailsByDriveId(parent.driveId, path);
|
||||
} catch (OneDriveException e) {
|
||||
log.vdebug("fileDetailsFromOneDrive = onedrive.getPathDetails(path); generated a OneDriveException");
|
||||
// A 404 is the expected response if the file was not present
|
||||
log.vdebug("fileDetailsFromOneDrive = onedrive.getPathDetailsByDriveId(parent.driveId, path); generated a OneDriveException");
|
||||
if (e.httpStatusCode == 401) {
|
||||
// OneDrive returned a 'HTTP/1.1 401 Unauthorized Error'
|
||||
log.vlog("Skipping item - OneDrive returned a 'HTTP 401 - Unauthorized' when attempting to query if file exists");
|
||||
|
|
@ -5507,8 +5726,31 @@ final class SyncEngine
|
|||
// 6. name
|
||||
// 7. parent reference
|
||||
|
||||
string fakeDriveId = defaultDriveId;
|
||||
string fakeRootId = defaultRootId;
|
||||
SysTime mtime = timeLastModified(path).toUTC();
|
||||
|
||||
// If the account type is Business, and if Shared Business Folders are being used
|
||||
// Need to update the 'fakeDriveId' & 'fakeRootId' with elements from the database
|
||||
// Otherwise some calls to validate objects fail as the actual driveId being used is invalid
|
||||
if (accountType == "business") {
|
||||
string parentPath = dirName(path);
|
||||
Item databaseItem;
|
||||
|
||||
if (parentPath != ".") {
|
||||
// Not a 'root' parent
|
||||
// For each driveid in the existing driveIDsArray
|
||||
foreach (searchDriveId; driveIDsArray) {
|
||||
log.vdebug("FakeResponse: searching database for: ", searchDriveId, " ", parentPath);
|
||||
if (itemdb.selectByPath(parentPath, searchDriveId, databaseItem)) {
|
||||
log.vdebug("FakeResponse: Found Database Item: ", databaseItem);
|
||||
fakeDriveId = databaseItem.driveId;
|
||||
fakeRootId = databaseItem.id;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// real id / eTag / cTag are different format for personal / business account
|
||||
auto sha1 = new SHA1Digest();
|
||||
ubyte[] hash1 = sha1.digest(path);
|
||||
|
|
@ -5527,9 +5769,9 @@ final class SyncEngine
|
|||
]),
|
||||
"name": JSONValue(baseName(path)),
|
||||
"parentReference": JSONValue([
|
||||
"driveId": JSONValue(defaultDriveId),
|
||||
"driveId": JSONValue(fakeDriveId),
|
||||
"driveType": JSONValue(accountType),
|
||||
"id": JSONValue(defaultRootId)
|
||||
"id": JSONValue(fakeRootId)
|
||||
]),
|
||||
"folder": JSONValue("")
|
||||
];
|
||||
|
|
@ -5548,9 +5790,9 @@ final class SyncEngine
|
|||
]),
|
||||
"name": JSONValue(baseName(path)),
|
||||
"parentReference": JSONValue([
|
||||
"driveId": JSONValue(defaultDriveId),
|
||||
"driveId": JSONValue(fakeDriveId),
|
||||
"driveType": JSONValue(accountType),
|
||||
"id": JSONValue(defaultRootId)
|
||||
"id": JSONValue(fakeRootId)
|
||||
]),
|
||||
"file": JSONValue([
|
||||
"hashes":JSONValue([
|
||||
|
|
@ -5920,44 +6162,49 @@ final class SyncEngine
|
|||
string sharedFolderName;
|
||||
string sharedByName;
|
||||
string sharedByEmail;
|
||||
|
||||
// Debug response output
|
||||
log.vdebug("shared folder entry: ", searchResult);
|
||||
sharedFolderName = searchResult["name"].str;
|
||||
|
||||
if ("sharedBy" in searchResult["remoteItem"]["shared"]) {
|
||||
// we have shared by details we can use
|
||||
if ("displayName" in searchResult["remoteItem"]["shared"]["sharedBy"]["user"]) {
|
||||
sharedByName = searchResult["remoteItem"]["shared"]["sharedBy"]["user"]["displayName"].str;
|
||||
// is the shared item with us a 'folder' ?
|
||||
// we only handle folders, not files or other items
|
||||
if (isItemFolder(searchResult)) {
|
||||
// Debug response output
|
||||
log.vdebug("shared folder entry: ", searchResult);
|
||||
sharedFolderName = searchResult["name"].str;
|
||||
|
||||
// configure who this was shared by
|
||||
if ("sharedBy" in searchResult["remoteItem"]["shared"]) {
|
||||
// we have shared by details we can use
|
||||
if ("displayName" in searchResult["remoteItem"]["shared"]["sharedBy"]["user"]) {
|
||||
sharedByName = searchResult["remoteItem"]["shared"]["sharedBy"]["user"]["displayName"].str;
|
||||
}
|
||||
if ("email" in searchResult["remoteItem"]["shared"]["sharedBy"]["user"]) {
|
||||
sharedByEmail = searchResult["remoteItem"]["shared"]["sharedBy"]["user"]["email"].str;
|
||||
}
|
||||
}
|
||||
if ("email" in searchResult["remoteItem"]["shared"]["sharedBy"]["user"]) {
|
||||
sharedByEmail = searchResult["remoteItem"]["shared"]["sharedBy"]["user"]["email"].str;
|
||||
// Output query result
|
||||
log.log("---------------------------------------");
|
||||
// Default output
|
||||
log.log("Shared Folder: ", sharedFolderName);
|
||||
if ((sharedByName != "") && (sharedByEmail != "")) {
|
||||
log.log("Shared By: ", sharedByName, " (", sharedByEmail, ")");
|
||||
} else {
|
||||
if (sharedByName != "") {
|
||||
log.log("Shared By: ", sharedByName);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Output query result
|
||||
log.log("---------------------------------------");
|
||||
// Default output
|
||||
log.log("Shared Folder: ", sharedFolderName);
|
||||
if ((sharedByName != "") && (sharedByEmail != "")) {
|
||||
log.log("Shared By: ", sharedByName, " (", sharedByEmail, ")");
|
||||
} else {
|
||||
if (sharedByName != "") {
|
||||
log.log("Shared By: ", sharedByName);
|
||||
// Tenant details
|
||||
if (searchResult["remoteItem"]["sharepointIds"]["tenantId"].str == myTenantID) {
|
||||
log.log("External Organisation: no");
|
||||
} else {
|
||||
log.log("External Organisation: yes");
|
||||
}
|
||||
|
||||
// Extra verbose output
|
||||
log.vlog("Item Id: ", searchResult["remoteItem"]["id"].str);
|
||||
log.vlog("Parent Drive Id: ", searchResult["remoteItem"]["parentReference"]["driveId"].str);
|
||||
if ("id" in searchResult["remoteItem"]["parentReference"]) {
|
||||
log.vlog("Parent Item Id: ", searchResult["remoteItem"]["parentReference"]["id"].str);
|
||||
}
|
||||
log.vlog("Tenant ID: ", searchResult["remoteItem"]["sharepointIds"]["tenantId"].str);
|
||||
}
|
||||
if (searchResult["remoteItem"]["sharepointIds"]["tenantId"].str == myTenantID) {
|
||||
log.log("External Organisation: no");
|
||||
} else {
|
||||
log.log("External Organisation: yes");
|
||||
}
|
||||
|
||||
// Extra verbose output
|
||||
log.vlog("Item Id: ", searchResult["remoteItem"]["id"].str);
|
||||
log.vlog("Parent Drive Id: ", searchResult["remoteItem"]["parentReference"]["driveId"].str);
|
||||
if ("id" in searchResult["remoteItem"]["parentReference"]) {
|
||||
log.vlog("Parent Item Id: ", searchResult["remoteItem"]["parentReference"]["id"].str);
|
||||
}
|
||||
log.vlog("Tenant ID: ", searchResult["remoteItem"]["sharepointIds"]["tenantId"].str);
|
||||
}
|
||||
}
|
||||
write("\n");
|
||||
|
|
|
|||
Loading…
Reference in a new issue