Merge branch 'master' into fix-issue-966

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for onedrive v2.4.19.
+# Generated by GNU Autoconf 2.69 for onedrive v2.4.20-dev.
 #
 # Report bugs to <https://github.com/abraunegg/onedrive>.
 #
@@ -579,8 +579,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='onedrive'
 PACKAGE_TARNAME='onedrive'
-PACKAGE_VERSION='v2.4.19'
+PACKAGE_VERSION='v2.4.20-dev'
-PACKAGE_STRING='onedrive v2.4.19'
+PACKAGE_STRING='onedrive v2.4.20-dev'
 PACKAGE_BUGREPORT='https://github.com/abraunegg/onedrive'
 PACKAGE_URL=''
 
@@ -1219,7 +1219,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures onedrive v2.4.19 to adapt to many kinds of systems.
+\`configure' configures onedrive v2.4.20-dev to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1280,7 +1280,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of onedrive v2.4.19:";;
+     short | recursive ) echo "Configuration of onedrive v2.4.20-dev:";;
    esac
   cat <<\_ACEOF
 
@@ -1393,7 +1393,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-onedrive configure v2.4.19
+onedrive configure v2.4.20-dev
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1410,7 +1410,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by onedrive $as_me v2.4.19, which was
+It was created by onedrive $as_me v2.4.20-dev, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -3159,7 +3159,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by onedrive $as_me v2.4.19, which was
+This file was extended by onedrive $as_me v2.4.20-dev, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -3212,7 +3212,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-onedrive config.status v2.4.19
+onedrive config.status v2.4.20-dev
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -9,7 +9,7 @@ dnl - commit the changed files (configure.ac, configure)
 dnl - tag the release
 
 AC_PREREQ([2.69])
-AC_INIT([onedrive],[v2.4.19], [https://github.com/abraunegg/onedrive], [onedrive])
+AC_INIT([onedrive],[v2.4.20-dev], [https://github.com/abraunegg/onedrive], [onedrive])
 AC_CONFIG_SRCDIR([src/main.d])
 
 

docs/INSTALL.md

@@ -144,14 +144,14 @@ CentOS 6.x and RHEL 6.x reached End of Life status on November 30th 2020 and is
 sudo yum groupinstall 'Development Tools'
 sudo yum install libcurl-devel
 sudo yum install sqlite-devel
-curl -fsS https://dlang.org/install.sh | bash -s dmd
+curl -fsS https://dlang.org/install.sh | bash -s dmd-2.099.0
 ```
 For notifications the following is also necessary:
 ```text
 sudo yum install libnotify-devel
 ```
 
-### Dependencies: Fedora > Version 18 / CentOS 8.x / RHEL 8.x
+### Dependencies: Fedora > Version 18 / CentOS 8.x / RHEL 8.x / RHEL 9.x
 ```text
 sudo dnf groupinstall 'Development Tools'
 sudo dnf install libcurl-devel
@@ -296,7 +296,7 @@ sudo make install
 ## Upgrading the client
 If you have installed the client from a distribution package, the client will be updated when the distribution package is updated by the package maintainer and will be updated to the new application version when you perform your package update.
 
-If you have built the client from source, to upgrade your client, you must first uninstall your existing 'onedrive' binary (see above), then re-install the client by re-cloning, re-compiling and re-installing the client again to install the new version.
+If you have built the client from source, to upgrade your client, you must first uninstall your existing 'onedrive' binary (see below), then re-install the client by re-cloning, re-compiling and re-installing the client again to install the new version.
 
 To confirm you have the new version installed, use `onedrive --version` to determine the version that is now installed.
 
@@ -314,7 +314,7 @@ If you are not upgrading your client, to remove your application state and confi
 ```
 rm -rf ~/.config/onedrive
 ```
-**Note:** If you are using the `--confdir option`, substitute `~/.config/onedrive` above for that directory.
+**Note:** If you are using the `--confdir option`, substitute `~/.config/onedrive` for the correct directory storing your client configuration.
 
 If you want to just delete the application key, but keep the items database:
 ```text

docs/known-issues.md

@@ -6,7 +6,7 @@ The below are known issues with this client:
 
 **Description:**
 
-When running the client in standalone mode (`--synchronize`) moving folders that are sucessfully synced around between subseqant standalone syncs causes a deletion & re-upload of data to occur.
+When running the client in standalone mode (`--synchronize`) moving folders that are sucessfully synced around between subsequent standalone syncs causes a deletion & re-upload of data to occur.
 
 **Explanation:**
 

src/config.d

@@ -147,7 +147,12 @@ final class Config
 		// display_sync_options = true | false
 		// - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging
 		boolValues["display_sync_options"] = false;
-
+		// force_children_scan = true | false
+		// - Force client to use /children rather than /delta to query changes on OneDrive
+		// - This option flags nationalCloudDeployment as true, forcing the client to act like it is using a National Cloud Deployment
+		boolValues["force_children_scan"] = false;
+		
+		// EXPAND USERS HOME DIRECTORY
 		// Determine the users home directory.
 		// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
 		// Check for HOME environment variable

src/main.d

@@ -1100,9 +1100,18 @@ int main(string[] args)
 		// value is configured, is it a valid value?
 		if ((cfg.getValueString("azure_ad_endpoint") == "USL4") || (cfg.getValueString("azure_ad_endpoint") == "USL5") || (cfg.getValueString("azure_ad_endpoint") == "DE") || (cfg.getValueString("azure_ad_endpoint") == "CN")) {
 			// valid entries to flag we are using a National Cloud Deployment
+			// National Cloud Deployments do not support /delta as a query
+			// https://docs.microsoft.com/en-us/graph/deployments#supported-features
+			// Flag that we have a valid National Cloud Deployment that cannot use /delta queries
 			sync.setNationalCloudDeployment();
 		}
 	}
+	
+	// Are we forcing to use /children scan instead of /delta to simulate National Cloud Deployment use of /children?
+	if (cfg.getValueBool("force_children_scan")) {
+		log.vdebug("Forcing client to use /children scan rather than /delta to simulate National Cloud Deployment use of /children");
+		sync.setNationalCloudDeployment();
+	}
 
 	// Do we need to validate the syncDir to check for the presence of a '.nosync' file
 	if (cfg.getValueBool("check_nomount")) {

src/sync.d

@@ -186,7 +186,7 @@ private Item makeItem(const ref JSONValue driveItem)
 		item.remoteId = driveItem["remoteItem"]["id"].str;
 	}
 	
-	// National Cloud Deployments (US and DE) do not support /delta as a query
+	// National Cloud Deployments do not support /delta as a query
 	// Thus we need to track in the database that this item is in sync
 	// As we are making an item, set the syncStatus to Y
 	// ONLY when using a National Cloud Deployment, all the existing DB entries will get set to N
@@ -1499,36 +1499,39 @@ final class SyncEngine
 			// To view changes correctly, we need to use the correct path id for the request
 			if (driveId == defaultDriveId) {
 				// The drive id matches our users default drive id
-				idToQuery = defaultRootId.dup;
 				log.vdebug("Configuring 'idToQuery' as defaultRootId duplicate");
+				idToQuery = defaultRootId.dup;
 			} else {
 				// The drive id does not match our users default drive id
 				// Potentially the 'path id' we are requesting the details of is a Shared Folder (remote item)
 				// Use the 'id' that was passed in (folderId)
-				idToQuery = id.dup;
 				log.vdebug("Configuring 'idToQuery' as 'id' duplicate");
+				idToQuery = id.dup;
 			}
 			// what path id are we going to query?
 			log.vdebug("Path object to query configured as 'idToQuery' = ", idToQuery);
 			long deltaChanges = 0;
 			
 			// What query do we use?
-			// National Cloud Deployments (US and DE) do not support /delta as a query
+			// National Cloud Deployments do not support /delta as a query
 			// https://docs.microsoft.com/en-us/graph/deployments#supported-features
 			// Are we running against a National Cloud Deployments that does not support /delta
 			if (nationalCloudDeployment) {
-				// Have to query /children rather than /delta
+				// National Cloud Deployment that does not support /delta query
+				// Have to query /children and build our own /delta response
 				nationalCloudChildrenScan = true;
 				log.vdebug("Using /children call to query drive for items to populate 'changes' and 'changesAvailable'");
 				// In a OneDrive Business Shared Folder scenario + nationalCloudDeployment, if ALL items are downgraded, then this leads to local file deletion
 				// Downgrade ONLY files associated with this driveId and idToQuery
 				log.vdebug("Downgrading all children for this driveId (" ~ driveId ~ ") and idToQuery (" ~ idToQuery ~ ") to an out-of-sync state");
+				
 				// Before we get any data, flag any object in the database as out-of-sync for this driveID & ID
 				auto drivePathChildren = itemdb.selectChildren(driveId, idToQuery);
 				if (count(drivePathChildren) > 0) {
 					// Children to process and flag as out-of-sync	
 					foreach (drivePathChild; drivePathChildren) {
 						// Flag any object in the database as out-of-sync for this driveID & ID
+						log.vdebug("Downgrading item as out-of-sync: ", drivePathChild.id);
 						itemdb.downgradeSyncStatusFlag(drivePathChild.driveId, drivePathChild.id);
 					}
 				}
@@ -1804,6 +1807,47 @@ final class SyncEngine
 				}
 			}
 			
+			// In some OneDrive Business scenarios, the shared folder /delta response lacks the 'root' drive details
+			// When this occurs, this creates the following error: A database statement execution error occurred: foreign key constraint failed
+			// Ensure we query independently the root details for this shared folder and ensure that it is added before we process the /delta response
+			
+			// However, if we are using a National Cloud Deployment, these deployments do not support /delta, so we generate a /delta response via generateDeltaResponse()
+			// This specifically adds the root drive details to the self generated /delta response
+			if ((!nationalCloudDeployment) && (driveId!= defaultDriveId) && (syncBusinessFolders)) {
+				// fetch this driveId root details to ensure we add this to the database for this remote drive
+				JSONValue rootData;
+				
+				try {
+					rootData = onedrive.getDriveIdRoot(driveId);
+				} catch (OneDriveException e) {
+					log.vdebug("rootData = onedrive.getDriveIdRoot(driveId) generated a OneDriveException");
+					// HTTP request returned status code 504 (Gateway Timeout) or 429 retry
+					if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) {
+						// HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed.
+						if (e.httpStatusCode == 429) {
+							log.vdebug("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - retrying applicable request");
+							handleOneDriveThrottleRequest();
+						}
+						if (e.httpStatusCode == 504) {
+							log.vdebug("Retrying original request that generated the HTTP 504 (Gateway Timeout) - retrying applicable request");
+							Thread.sleep(dur!"seconds"(30));
+						}
+						// Retry original request by calling function again to avoid replicating any further error handling
+						rootData = onedrive.getDriveIdRoot(driveId);
+						
+					} else {
+						// There was a HTTP 5xx Server Side Error
+						displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
+						// Must exit here
+						exit(-1);
+					}
+				}
+				
+				// apply this root drive data
+				applyDifference(rootData, driveId, true);
+			}
+			
+			// Process /delta response from OneDrive
 			// is changes a valid JSON response
 			if (changes.type() == JSONType.object) {
 				// Are there any changes to process?
@@ -2496,9 +2540,13 @@ final class SyncEngine
 				SysTime remoteModifiedTime = item.mtime;
 				remoteModifiedTime.fracSecs = Duration.zero;
 				
-				if (localModifiedTime != remoteModifiedTime) {
+				// If the timestamp is different, or we are running on a National Cloud Deployment that does not support /delta queries - we have to update the DB with the details from OneDrive
-					// Database update needed for this item because our record is out-of-date
+				// Unfortunatly because of the consequence of Nataional Cloud Deployments not supporting /delta queries, the application uses the local database to flag what is out-of-date / track changes
-					log.vdebug("Updating local database with item details as timestamps of items are different");
+				// This means that the constant disk writing to the database fix implemented with https://github.com/abraunegg/onedrive/pull/2004 cannot be utilised when using Nataional Cloud Deployments
+				// as all records are touched / updated when performing the OneDrive sync operations. The only way to change this, is for Microsoft to support /delta queries for Nataional Cloud Deployments
+				if ((localModifiedTime != remoteModifiedTime) || (nationalCloudDeployment)) {
+					// Database update needed for this item because our local record is out-of-date
+					log.vdebug("Updating local database with item details from OneDrive as local record needs to be updated");
 					itemdb.update(item);
 				}
 			} else {
@@ -3188,27 +3236,9 @@ final class SyncEngine
 		}
 		
 		// Are we configured to use a National Cloud Deployment
-		// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
+		if (nationalCloudDeployment) {
-		// Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query
-		if ((nationalCloudDeployment) || (syncBusinessFolders)) {
 			// Select items that have a out-of-sync flag set
-			foreach (driveId; driveIDsArray) {
+			flagNationalCloudDeploymentOutOfSyncItems();
-				// For each unique OneDrive driveID we know about
-				Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId);
-				foreach (item; outOfSyncItems) {
-					if (!dryRun) {
-						// clean up idsToDelete
-						idsToDelete.length = 0;
-						assumeSafeAppend(idsToDelete);
-						// flag to delete local file as it now is no longer in sync with OneDrive
-						log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive");
-						log.vdebug("item: ", item);
-						idsToDelete ~= [item.driveId, item.id];	
-						// delete items in idsToDelete
-						if (idsToDelete.length > 0) deleteItems();
-					}
-				}
-			}
 		}
 		
 		// scan for changes in the path provided
@@ -3289,27 +3319,9 @@ final class SyncEngine
 		}
 		
 		// Are we configured to use a National Cloud Deployment
-		// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
+		if (nationalCloudDeployment) {
-		// Normally, this is done at the end of processing all /delta queries, but National Cloud Deployments (US and DE) do not support /delta as a query
-		if ((nationalCloudDeployment) || (syncBusinessFolders)) {
 			// Select items that have a out-of-sync flag set
-			foreach (driveId; driveIDsArray) {
+			flagNationalCloudDeploymentOutOfSyncItems();
-				// For each unique OneDrive driveID we know about
-				Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId);
-				foreach (item; outOfSyncItems) {
-					if (!dryRun) {
-						// clean up idsToDelete
-						idsToDelete.length = 0;
-						assumeSafeAppend(idsToDelete);
-						// flag to delete local file as it now is no longer in sync with OneDrive
-						log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive");
-						log.vdebug("item: ", item);
-						idsToDelete ~= [item.driveId, item.id];	
-						// delete items in idsToDelete
-						if (idsToDelete.length > 0) deleteItems();
-					}
-				}
-			}
 		}
 		
 		// scan for changes in the path provided
@@ -3350,6 +3362,30 @@ final class SyncEngine
 		}
 	}
 	
+	void flagNationalCloudDeploymentOutOfSyncItems() {
+		// Any entry in the DB than is flagged as out-of-sync needs to be cleaned up locally first before we scan the entire DB
+		// Normally, this is done at the end of processing all /delta queries, however National Cloud Deployments do not support /delta as a query
+		// https://docs.microsoft.com/en-us/graph/deployments#supported-features
+		// Select items that have a out-of-sync flag set
+		foreach (driveId; driveIDsArray) {
+			// For each unique OneDrive driveID we know about
+			Item[] outOfSyncItems = itemdb.selectOutOfSyncItems(driveId);
+			foreach (item; outOfSyncItems) {
+				if (!dryRun) {
+					// clean up idsToDelete
+					idsToDelete.length = 0;
+					assumeSafeAppend(idsToDelete);
+					// flag to delete local file as it now is no longer in sync with OneDrive
+					log.vdebug("Flagging to delete local item as it now is no longer in sync with OneDrive");
+					log.vdebug("item: ", item);
+					idsToDelete ~= [item.driveId, item.id];	
+					// delete items in idsToDelete
+					if (idsToDelete.length > 0) deleteItems();
+				}
+			}
+		}
+	}
+	
 	void handleUploadOnlyBusinessSharedFoldersEdgeCase() {
 		// read in the business_shared_folders file contents
 		string[] businessSharedFoldersList;