Implement FR #2544: Allow synchronizing smallest files first (#3114)

* Implement 'transfer_order' configuration option to allow the user to determine what order files are transferred in

docs/application-config-options.md

@@ -57,6 +57,7 @@ Before reading this document, please ensure you are running application version
   - [sync_file_permissions](#sync_file_permissions)
   - [sync_root_files](#sync_root_files)
   - [threads](#threads)
+  - [transfer_order](#transfer_order)
   - [upload_only](#upload_only)
   - [user_agent](#user_agent)
   - [webhook_enabled](#webhook_enabled)
@@ -887,6 +888,32 @@ _**Config Example:**_ `threads = "16"`
 > [!WARNING]
 > Increasing the threads beyond the default will lead to increased system utilisation and local TCP port use, which may lead to unpredictable behaviour and/or may lead application stability issues.
 
+### transfer_order
+_**Description:**_ This configuration option controls the transfer order of files between your local system and Microsoft OneDrive.
+
+_**Value Type:**_ String
+
+_**Default Value:**_ `default`
+
+_**Config Example:**_
+#### Transfer by size, smallest first
+```
+transfer_order = "size_asc"
+```
+
+#### Transfer by size, largest first
+```
+transfer_order = "size_dsc"
+```
+#### Transfer by file name sorted A to Z
+```
+transfer_order = "name_asc"
+```
+#### Transfer by file name sorted Z to A
+```
+transfer_order = "name_dsc"
+```
+
 ### upload_only
 _**Description:**_ This setting forces the client to only upload data to Microsoft OneDrive and replicate the locate state online. By default, this will also remove content online, that has been removed locally.
 

src/config.d

@@ -231,9 +231,17 @@ class ApplicationConfig {
 		//     AD Endpoint:    https://login.chinacloudapi.cn
 		//     Graph Endpoint: 	https://microsoftgraph.chinacloudapi.cn
 		stringValues["azure_ad_endpoint"] = "";
-		
 		// Support single-tenant applications that are not able to use the "common" multiplexer
 		stringValues["azure_tenant_id"] = "";
+		
+		// Support synchronising files based on user desire
+		// - default = whatever order these came in as, processed essentially FIFO
+		// - size_asc = file size ascending
+		// - size_dsc = file size descending
+		// - name_asc = file name ascending
+		// - name_dsc = file name descending
+		stringValues["transfer_order"] = "default";
+				
 		// - Store how many times was --verbose added
 		longValues["verbose"] = verbosityCount; 
 		// - The amount of time (seconds) between monitor sync loops
@@ -836,6 +844,23 @@ class ApplicationConfig {
 						default:
 							addLogEntry("Unknown Azure AD Endpoint - using Global Azure AD Endpoints");
 					}
+				} else if (key == "transfer_order") {
+					switch (value) {
+						case "size_asc":
+							addLogEntry("Files will be transferred sorted by ascending size (smallest first)");
+							break;
+						case "size_dsc":
+							addLogEntry("Files will be transferred sorted by descending size (largest first)");
+							break;
+						case "name_asc":
+							addLogEntry("Files will be transferred sorted by ascending name (A -> Z)");
+							break;
+						case "name_dsc":
+							addLogEntry("Files will be transferred sorted by descending name (Z -> A)");
+							break;
+						default:
+							addLogEntry("Files will be transferred in original order that they were received (FIFO)");
+					}
 				} else if (key == "application_id") {
 					string tempApplicationId = strip(value);
 					if (tempApplicationId.empty) {
@@ -1411,6 +1436,7 @@ class ApplicationConfig {
 		addLogEntry("Config option 'resync'                       = " ~ to!string(getValueBool("resync")));
 		addLogEntry("Config option 'resync_auth'                  = " ~ to!string(getValueBool("resync_auth")));
 		addLogEntry("Config option 'cleanup_local_files'          = " ~ to!string(getValueBool("cleanup_local_files")));
+		addLogEntry("Config option 'transfer_order'               = " ~ getValueString("transfer_order"));
 
 		// data integrity
 		addLogEntry("Config option 'classify_as_big_delete'       = " ~ to!string(getValueLong("classify_as_big_delete")));

src/sync.d

@@ -27,6 +27,8 @@ import std.uri;
 import std.utf;
 import std.math;
 
+import std.typecons;
+
 // What other modules that we have created do we need to import?
 import config;
 import log;
@@ -3186,6 +3188,28 @@ class SyncEngine {
 		long batchCount = (fileJSONItemsToDownload.length + batchSize - 1) / batchSize;
 		long batchesProcessed = 0;
 		
+		// Transfer order
+		string transferOrder = appConfig.getValueString("transfer_order");
+		
+		// Has the user configured to specify the transfer order of files?
+		if (transferOrder != "default") {
+			// If we have more than 1 item to download, sort the items
+			if (count(fileJSONItemsToDownload) > 1) {
+			
+				// Perform sorting based on transferOrder
+				if (transferOrder == "size_asc") {
+					fileJSONItemsToDownload.sort!((a, b) => a["size"].integer < b["size"].integer); // sort the array by ascending size
+				} else if (transferOrder == "size_dsc") {
+					fileJSONItemsToDownload.sort!((a, b) => a["size"].integer > b["size"].integer); // sort the array by descending size
+				} else if (transferOrder == "name_asc") {
+					fileJSONItemsToDownload.sort!((a, b) => a["name"].str < b["name"].str); // sort the array by ascending name
+				} else if (transferOrder == "name_dsc") {
+					fileJSONItemsToDownload.sort!((a, b) => a["name"].str > b["name"].str); // sort the array by descending name
+				}
+			}
+		}
+		
+		// Process fileJSONItemsToDownload
 		foreach (chunk; fileJSONItemsToDownload.chunks(batchSize)) {
 			// send an array containing 'appConfig.getValueLong("threads")' JSON items to download
 			downloadOneDriveItemsInParallel(chunk);
@@ -7404,7 +7428,37 @@ class SyncEngine {
 		long batchCount = (newLocalFilesToUploadToOneDrive.length + batchSize - 1) / batchSize;
 		long batchesProcessed = 0;
 		
+		// Transfer order
+		string transferOrder = appConfig.getValueString("transfer_order");
+		
+		// Has the user configured to specify the transfer order of files?
+		if (transferOrder != "default") {
+			// If we have more than 1 item to upload, sort the items
+			if (count(newLocalFilesToUploadToOneDrive) > 1) {
+				// Create an array of tuples (file path, file size)
+				auto fileInfo = newLocalFilesToUploadToOneDrive
+					.map!(file => tuple(file, getSize(file))) // Get file size for each file that needs to be uploaded
+					.array;
+
+				// Perform sorting based on transferOrder
+				if (transferOrder == "size_asc") {
+					fileInfo.sort!((a, b) => a[1] < b[1]); // sort the array by ascending size
+				} else if (transferOrder == "size_dsc") {
+					fileInfo.sort!((a, b) => a[1] > b[1]); // sort the array by descending size
+				} else if (transferOrder == "name_asc") {
+					fileInfo.sort!((a, b) => a[0] < b[0]); // sort the array by ascending name
+				} else if (transferOrder == "name_dsc") {
+					fileInfo.sort!((a, b) => a[0] > b[0]); // sort the array by descending name
+				}
+				
+				// Extract sorted file paths
+				newLocalFilesToUploadToOneDrive = fileInfo.map!(t => t[0]).array;
+			}
+		}
+		
+		// Process newLocalFilesToUploadToOneDrive
 		foreach (chunk; newLocalFilesToUploadToOneDrive.chunks(batchSize)) {
+			// send an array containing 'appConfig.getValueLong("threads")' local files to upload
 			uploadNewLocalFileItemsInParallel(chunk);
 		}