deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-04-25 11:00:57 +02:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into fix-issue-966

Browse source
This commit is contained in:
abraunegg 2022-06-30 07:17:58 +10:00
parent 47c3dcd849 ca984eba70
commit 0b147e6a12
7 changed files with 114 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
configure vendored
View file

@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for onedrive v2.4.19.
# Generated by GNU Autoconf 2.69 for onedrive v2.4.20-dev.
#
# Report bugs to <https://github.com/abraunegg/onedrive>.
#
@ -579,8 +579,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='onedrive'
PACKAGE_TARNAME='onedrive'
PACKAGE_VERSION='v2.4.19'
PACKAGE_STRING='onedrive v2.4.19'
PACKAGE_VERSION='v2.4.20-dev'
PACKAGE_STRING='onedrive v2.4.20-dev'
PACKAGE_BUGREPORT='https://github.com/abraunegg/onedrive'
PACKAGE_URL=''
@ -1219,7 +1219,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures onedrive v2.4.19 to adapt to many kinds of systems.
\`configure' configures onedrive v2.4.20-dev to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1280,7 +1280,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of onedrive v2.4.19:";;
short | recursive ) echo "Configuration of onedrive v2.4.20-dev:";;
esac
cat <<\_ACEOF
@ -1393,7 +1393,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
onedrive configure v2.4.19
onedrive configure v2.4.20-dev
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
@ -1410,7 +1410,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by onedrive $as_me v2.4.19, which was
It was created by onedrive $as_me v2.4.20-dev, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@ -3159,7 +3159,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by onedrive $as_me v2.4.19, which was
This file was extended by onedrive $as_me v2.4.20-dev, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -3212,7 +3212,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
onedrive config.status v2.4.19
onedrive config.status v2.4.20-dev
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"

2
configure.ac
View file

@ -9,7 +9,7 @@ dnl - commit the changed files (configure.ac, configure)
dnl - tag the release
AC_PREREQ([2.69])
AC_INIT([onedrive],[v2.4.19], [https://github.com/abraunegg/onedrive], [onedrive])
AC_INIT([onedrive],[v2.4.20-dev], [https://github.com/abraunegg/onedrive], [onedrive])
AC_CONFIG_SRCDIR([src/main.d])

8
docs/INSTALL.md
View file

@ -144,14 +144,14 @@ CentOS 6.x and RHEL 6.x reached End of Life status on November 30th 2020 and is
sudo yum groupinstall 'Development Tools'
sudo yum install libcurl-devel
sudo yum install sqlite-devel
curl -fsS https://dlang.org/install.sh | bash -s dmd
curl -fsS https://dlang.org/install.sh | bash -s dmd-2.099.0
```
For notifications the following is also necessary:
```text
sudo yum install libnotify-devel
```
### Dependencies: Fedora > Version 18 / CentOS 8.x / RHEL 8.x
### Dependencies: Fedora > Version 18 / CentOS 8.x / RHEL 8.x / RHEL 9.x
```text
sudo dnf groupinstall 'Development Tools'
sudo dnf install libcurl-devel
@ -296,7 +296,7 @@ sudo make install
## Upgrading the client
If you have installed the client from a distribution package, the client will be updated when the distribution package is updated by the package maintainer and will be updated to the new application version when you perform your package update.
If you have built the client from source, to upgrade your client, you must first uninstall your existing 'onedrive' binary (see above), then re-install the client by re-cloning, re-compiling and re-installing the client again to install the new version.
If you have built the client from source, to upgrade your client, you must first uninstall your existing 'onedrive' binary (see below), then re-install the client by re-cloning, re-compiling and re-installing the client again to install the new version.
To confirm you have the new version installed, use `onedrive --version` to determine the version that is now installed.
@ -314,7 +314,7 @@ If you are not upgrading your client, to remove your application state and confi
```
rm -rf ~/.config/onedrive
```
**Note:** If you are using the `--confdir option`, substitute `~/.config/onedrive` above for that directory.
**Note:** If you are using the `--confdir option`, substitute `~/.config/onedrive` for the correct directory storing your client configuration.
If you want to just delete the application key, but keep the items database:
```text

2
docs/known-issues.md
View file

@ -6,7 +6,7 @@ The below are known issues with this client:
**Description:**
When running the client in standalone mode (`--synchronize`) moving folders that are sucessfully synced around between subseqant standalone syncs causes a deletion & re-upload of data to occur.
When running the client in standalone mode (`--synchronize`) moving folders that are sucessfully synced around between subsequent standalone syncs causes a deletion & re-upload of data to occur.
**Explanation:**

7
src/config.d
View file

@ -147,7 +147,12 @@ final class Config
// display_sync_options = true | false
// - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging
boolValues["display_sync_options"] = false;
// force_children_scan = true | false
// - Force client to use /children rather than /delta to query changes on OneDrive
// - This option flags nationalCloudDeployment as true, forcing the client to act like it is using a National Cloud Deployment
boolValues["force_children_scan"] = false;
// EXPAND USERS HOME DIRECTORY
// Determine the users home directory.
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
// Check for HOME environment variable

9
src/main.d
View file

@ -1100,9 +1100,18 @@ int main(string[] args)
// value is configured, is it a valid value?
if ((cfg.getValueString("azure_ad_endpoint") == "USL4") || (cfg.getValueString("azure_ad_endpoint") == "USL5") || (cfg.getValueString("azure_ad_endpoint") == "DE") || (cfg.getValueString("azure_ad_endpoint") == "CN")) {
// valid entries to flag we are using a National Cloud Deployment
// National Cloud Deployments do not support /delta as a query
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
// Flag that we have a valid National Cloud Deployment that cannot use /delta queries
sync.setNationalCloudDeployment();
}
}
// Are we forcing to use /children scan instead of /delta to simulate National Cloud Deployment use of /children?
if (cfg.getValueBool("force_children_scan")) {
log.vdebug("Forcing client to use /children scan rather than /delta to simulate National Cloud Deployment use of /children");
sync.setNationalCloudDeployment();
}
// Do we need to validate the syncDir to check for the presence of a '.nosync' file
if (cfg.getValueBool("check_nomount")) {

132
src/sync.d
View file

@ -186,7 +186,7 @@ private Item makeItem(const ref JSONValue driveItem)
item.remoteId = driveItem["remoteItem"]["id"].str;
}
// National Cloud Deployments (US and DE) do not support /delta as a query
// National Cloud Deployments do not support /delta as a query
// Thus we need to track in the database that this item is in sync
// As we are making an item, set the syncStatus to Y
// ONLY when using a National Cloud Deployment, all the existing DB entries will get set to N
@ -1499,36 +1499,39 @@ final class SyncEngine
// To view changes correctly, we need to use the correct path id for the request
if (driveId == defaultDriveId) {
// The drive id matches our users default drive id
idToQuery = defaultRootId.dup;
log.vdebug("Configuring 'idToQuery' as defaultRootId duplicate");
idToQuery = defaultRootId.dup;
} else {
// The drive id does not match our users default drive id
// Potentially the 'path id' we are requesting the details of is a Shared Folder (remote item)
// Use the 'id' that was passed in (folderId)
idToQuery = id.dup;
log.vdebug("Configuring 'idToQuery' as 'id' duplicate");
idToQuery = id.dup;
}
// what path id are we going to query?
log.vdebug("Path object to query configured as 'idToQuery' = ", idToQuery);
long deltaChanges = 0;
// What query do we use?
// National Cloud Deployments (US and DE) do not support /delta as a query
// National Cloud Deployments do not support /delta as a query
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
// Are we running against a National Cloud Deployments that does not support /delta
if (nationalCloudDeployment) {
// Have to query /children rather than /delta
// National Cloud Deployment that does not support /delta query
// Have to query /children and build our own /delta response
nationalCloudChildrenScan = true;
log.vdebug("Using /children call to query drive for items to populate 'changes' and 'changesAvailable'");
// In a OneDrive Business Shared Folder scenario + nationalCloudDeployment, if ALL items are downgraded, then this leads to local file deletion
// Downgrade ONLY files associated with this driveId and idToQuery
log.vdebug("Downgrading all children for this driveId (" ~ driveId ~ ") and idToQuery (" ~ idToQuery ~ ") to an out-of-sync state");
// Before we get any data, flag any object in the database as out-of-sync for this driveID & ID
auto drivePathChildren = itemdb.selectChildren(driveId, idToQuery);
if (count(drivePathChildren) > 0) {
// Children to process and flag as out-of-sync
foreach (drivePathChild; drivePathChildren) {
// Flag any object in the database as out-of-sync for this driveID & ID
log.vdebug("Downgrading item as out-of-sync: ", drivePathChild.id);
itemdb.downgradeSyncStatusFlag(drivePathChild.driveId, drivePathChild.id);
}
}
@ -1804,6 +1807,47 @@ final class SyncEngine
}
}
// In some OneDrive Business scenarios, the shared folder /delta response lacks the 'root' drive details
// When this occurs, this creates the following error: A database statement execution error occurred: foreign key constraint failed
// Ensure we query independently the root details for this shared folder and ensure that it is added before we process the /delta response
// However, if we are using a National Cloud Deployment, these deployments do not support /delta, so we generate a /delta response via generateDeltaResponse()
// This specifically adds the root drive details to the self generated /delta response
if ((!nationalCloudDeployment) && (driveId!= defaultDriveId) && (syncBusinessFolders)) {
// fetch this driveId root details to ensure we add this to the database for this remote drive
JSONValue rootData;
try {
rootData = onedrive.getDriveIdRoot(driveId);
} catch (OneDriveException e) {
log.vdebug("rootData = onedrive.getDriveIdRoot(driveId) generated a OneDriveException");
// HTTP request returned status code 504 (Gateway Timeout) or 429 retry
if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) {
// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
if (e.httpStatusCode == 429) {
log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - retrying applicable request");
handleOneDriveThrottleRequest();
}
if (e.httpStatusCode == 504) {
log.vdebug("Retrying original request that generated the HTTP 504 (Gateway Timeout) - retrying applicable request");
Thread.sleep(dur!"seconds"(30));
}
// Retry original request by calling function again to avoid replicating any further error handling
rootData = onedrive.getDriveIdRoot(driveId);
} else {
// There was a HTTP 5xx Server Side Error
displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
// Must exit here
exit(-1);
}
}
// apply this root drive data
applyDifference(rootData, driveId, true);
}
// Process /delta response from OneDrive
// is changes a valid JSON response
if (changes.type() == JSONType.object) {
// Are there any changes to process?
@ -2496,9 +2540,13 @@ final class SyncEngine
SysTime remoteModifiedTime = item.mtime;
remoteModifiedTime.fracSecs = Duration.zero;
if (localModifiedTime != remoteModifiedTime) {
// Database update needed for this item because our record is out-of-date
log.vdebug("Updating local database with item details as timestamps of items are different");
// If the timestamp is different, or we are running on a National Cloud Deployment that does not support /delta queries - we have to update the DB with the details from OneDrive
// Unfortunatly because of the consequence of Nataional Cloud Deployments not supporting /delta queries, the application uses the local database to flag what is out-of-date / track changes
// This means that the constant disk writing to the database fix implemented with https://github.com/abraunegg/onedrive/pull/2004 cannot be utilised when using Nataional Cloud Deployments
// as all records are touched / updated when performing the OneDrive sync operations. The only way to change this, is for Microsoft to support /delta queries for Nataional Cloud Deployments
if ((localModifiedTime != remoteModifiedTime) || (nationalCloudDeployment)) {
// Database update needed for this item because our local record is out-of-date
log.vdebug("Updating local database with item details from OneDrive as local record needs to be updated");
itemdb.update(item);
}
} else {
@ -3188,27 +3236,9 @@ final class SyncEngine
}
// Are we configured to use a National Cloud Deployment
// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
// Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query
if ((nationalCloudDeployment) || (syncBusinessFolders)) {
if (nationalCloudDeployment) {
// Select items that have a out-of-sync flag set
foreach (driveId; driveIDsArray) {
// For each unique OneDrive driveID we know about
Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId);
foreach (item; outOfSyncItems) {
if (!dryRun) {
// clean up idsToDelete
idsToDelete.length = 0;
assumeSafeAppend(idsToDelete);
// flag to delete local file as it now is no longer in sync with OneDrive
log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive");
log.vdebug("item: ", item);
idsToDelete ~= [item.driveId, item.id];
// delete items in idsToDelete
if (idsToDelete.length > 0) deleteItems();
}
}
}
flagNationalCloudDeploymentOutOfSyncItems();
}
// scan for changes in the path provided
@ -3289,27 +3319,9 @@ final class SyncEngine
}
// Are we configured to use a National Cloud Deployment
// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
// Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query
if ((nationalCloudDeployment) || (syncBusinessFolders)) {
if (nationalCloudDeployment) {
// Select items that have a out-of-sync flag set
foreach (driveId; driveIDsArray) {
// For each unique OneDrive driveID we know about
Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId);
foreach (item; outOfSyncItems) {
if (!dryRun) {
// clean up idsToDelete
idsToDelete.length = 0;
assumeSafeAppend(idsToDelete);
// flag to delete local file as it now is no longer in sync with OneDrive
log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive");
log.vdebug("item: ", item);
idsToDelete ~= [item.driveId, item.id];
// delete items in idsToDelete
if (idsToDelete.length > 0) deleteItems();
}
}
}
flagNationalCloudDeploymentOutOfSyncItems();
}
// scan for changes in the path provided
@ -3350,6 +3362,30 @@ final class SyncEngine
}
}
void flagNationalCloudDeploymentOutOfSyncItems() {
// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
// Normally, this is done at the end of processing all /delta queries, however National Cloud Deployments do not support /delta as a query
// https://docs.microsoft.com/en-us/graph/deployments#supported-features
// Select items that have a out-of-sync flag set
foreach (driveId; driveIDsArray) {
// For each unique OneDrive driveID we know about
Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId);
foreach (item; outOfSyncItems) {
if (!dryRun) {
// clean up idsToDelete
idsToDelete.length = 0;
assumeSafeAppend(idsToDelete);
// flag to delete local file as it now is no longer in sync with OneDrive
log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive");
log.vdebug("item: ", item);
idsToDelete ~= [item.driveId, item.id];
// delete items in idsToDelete
if (idsToDelete.length > 0) deleteItems();
}
}
}
}
void handleUploadOnlyBusinessSharedFoldersEdgeCase() {
// read in the business_shared_folders file contents
string[] businessSharedFoldersList;