Merge 23c0326822 into 0f012b9f82

docs/application-config-options.md

@@ -206,7 +206,7 @@ _**Description:**_ This setting controls the timeout duration, in seconds, for w
 
 _**Value Type:**_ Integer
 
-_**Default Value:**_ 240
+_**Default Value:**_ 60
 
 _**Config Example:**_ `data_timeout = "300"`
 

docs/usage.md

@@ -420,6 +420,18 @@ Documents/latest_report.docx
 #
 # Include all Work/Project directories or files, inside 'Work' folder(s), anywhere in OneDrive
 Work/Project*
+# Include the 'Blog' directory, but exclude 'Parent' and any other children of the parent
+# .
+# ├── Parent
+# │   ├── Blog
+# │   │   ├── random_files
+# │   │   │   ├── CZ9aZRM7U1j7pM21fH0MfP2gywlX7bqW
+# │   │   │   └── k4GptfTBE2z2meRFqjf54tnvSXcXe30Y
+# │   │   └── random_images
+# │   │       ├── cAuQMfX7qsMIOmzyQYdELikZwsXeCYsL
+# │   │       └── GqjZuo7UBB0qjYM2WUcZXOvToAhCQ29M
+# │   └── other_stuffs
+/Parent/Blog/*
 #
 # Include the 'Blog' directory, but exclude 'Parent' and any other children of the parent
 # .

src/clientSideFiltering.d

@@ -30,6 +30,13 @@ class ClientSideFiltering {
 		this.appConfig = appConfig;
 	}
 	
+	~this() {
+		object.destroy(appConfig);
+		object.destroy(paths);
+		object.destroy(fileMask);
+		object.destroy(directoryMask);
+	}
+	
 	// Initialise the required items
 	bool initialise() {
 		// Log what is being done

src/config.d

@@ -61,17 +61,17 @@ class ApplicationConfig {
 	// HTTP Struct items, used for configuring HTTP()
 	// Curl Timeout Handling
 	// libcurl dns_cache_timeout timeout
-	immutable int defaultDnsTimeout = 60;
+	immutable int defaultDnsTimeout = 60; // in seconds
 	// Connect timeout for HTTP|HTTPS connections
 	// Controls CURLOPT_CONNECTTIMEOUT
-	immutable int defaultConnectTimeout = 10;
-	// Default data timeout for HTTP
+	immutable int defaultConnectTimeout = 10; // in seconds
+	// Default data timeout for HTTP operations
 	// curl.d has a default of: _defaultDataTimeout = dur!"minutes"(2);
-	immutable int defaultDataTimeout = 240;
+	immutable int defaultDataTimeout = 60; // in seconds
 	// Maximum time any operation is allowed to take
 	// This includes dns resolution, connecting, data transfer, etc.
 	// Controls CURLOPT_TIMEOUT
-	immutable int defaultOperationTimeout = 3600;
+	immutable int defaultOperationTimeout = 3600; // in seconds
 	// Specify what IP protocol version should be used when communicating with OneDrive
 	immutable int defaultIpProtocol = 0; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
 	// Specify how many redirects should be allowed
@@ -682,11 +682,21 @@ class ApplicationConfig {
 		}
 		
 		auto file = File(filename, "r");
-		scope(exit) file.close();
-		scope(failure) file.close();
-
+		string lineBuffer;
+		scope(exit) {
+			file.close();
+			object.destroy(file);
+			object.destroy(lineBuffer);
+		}
+		
+		scope(failure) {
+			file.close();
+			object.destroy(file);
+			object.destroy(lineBuffer);
+		}
+		
 		foreach (line; file.byLine()) {
-			string lineBuffer = stripLeft(line).to!string;
+			lineBuffer = stripLeft(line).to!string;
 			if (lineBuffer.empty || lineBuffer[0] == ';' || lineBuffer[0] == '#') continue;
 			auto c = lineBuffer.matchFirst(configRegex);
 			if (c.empty) {

src/curlEngine.d

@@ -20,16 +20,22 @@ class CurlResponse {
 	const(char)[][const(char)[]] requestHeaders;
 	const(char)[] postBody;
 
+	bool hasResponse;
 	string[string] responseHeaders;
 	HTTP.StatusLine statusLine;
 	char[] content;
 
+	this() {
+		reset();
+	}
+
 	void reset() {
 		method = HTTP.Method.undefined;
 		url = null;
 		requestHeaders = null;
 		postBody = null;
 		
+		hasResponse = false;
 		responseHeaders = null;
 		object.destroy(statusLine);
 		content = null;
@@ -56,6 +62,7 @@ class CurlResponse {
 	};
 
 	void update(HTTP *http) {
+		hasResponse = true;
 		this.responseHeaders = http.responseHeaders();
 		this.statusLine = http.statusLine;
 	}
@@ -65,14 +72,14 @@ class CurlResponse {
 	}
 
 	// Return the current value of retryAfterValue
-	ulong getRetryAfterValue() {
-		ulong delayBeforeRetry;
+	int getRetryAfterValue() {
+		int delayBeforeRetry;
 		// is retry-after in the response headers
 		if ("retry-after" in responseHeaders) {
 			// Set the retry-after value
 			addLogEntry("curlEngine.http.perform() => Received a 'Retry-After' Header Response with the following value: " ~ to!string(responseHeaders["retry-after"]), ["debug"]);
 			addLogEntry("curlEngine.http.perform() => Setting retryAfterValue to: " ~ responseHeaders["retry-after"], ["debug"]);
-			delayBeforeRetry = to!ulong(responseHeaders["retry-after"]);
+			delayBeforeRetry = to!int(responseHeaders["retry-after"]);
 		} else {
 			// Use a 120 second delay as a default given header value was zero
 			// This value is based on log files and data when determining correct process for 429 response handling
@@ -81,24 +88,33 @@ class CurlResponse {
 			addLogEntry("HTTP Response Header retry-after value was 0 - Using a preconfigured default of: " ~ to!string(delayBeforeRetry), ["debug"]);
 		}
 		
-		return delayBeforeRetry; // default to 60 seconds
+		return delayBeforeRetry;
 	}
 
-	const string parseHeaders(const(string[string]) headers) {
+	const string parseHeaders(const(immutable(char)[][immutable(char)[]]) headers) {
 		string responseHeadersStr = "";
-		foreach (const(char)[] header; headers.byKey()) {
-			responseHeadersStr ~= "> " ~ header ~ ": " ~ headers[header] ~ "\n";
+		// Ensure headers is not null and iterate over keys safely.
+		if (headers !is null) {
+			foreach (const(char)[] header; headers.byKey()) {
+				// Check if the key actually exists before accessing it to avoid RangeError.
+				if (auto val = header in headers) { // 'in' checks for the key and returns a pointer to the value if found.
+					responseHeadersStr ~= "> " ~ header ~ ": " ~ *val ~ "\n"; // Dereference pointer to get the value.
+				}
+			}
 		}
 		return responseHeadersStr;
 	}
 
-
 	const string parseHeaders(const(const(char)[][const(char)[]]) headers) {
 		string responseHeadersStr = "";
 		foreach (string header; headers.byKey()) {
-			if (header == "Authorization")
+			if (header == "Authorization") {
 				continue;
-			responseHeadersStr ~= "< " ~ header ~ ": " ~ headers[header] ~ "\n";
+			}
+			// Use the 'in' operator to safely check if the key exists in the associative array.
+			if (auto val = header in headers) {
+				responseHeadersStr ~= "< " ~ header ~ ": " ~ *val ~ "\n";
+			}
 		}
 		return responseHeadersStr;
 	}
@@ -136,8 +152,10 @@ class CurlResponse {
 	override string toString() const {
 		string str = "Curl debugging: \n";
 		str ~= dumpDebug();
-		str ~= "Curl response: \n";
-		str ~= dumpResponse();
+		if (hasResponse) {
+			str ~= "Curl response: \n";
+			str ~= dumpResponse();
+		}
 		return str;
 	}
 
@@ -158,52 +176,62 @@ class CurlResponse {
 
 class CurlEngine {
 
-	__gshared CurlEngine[] curlEnginePool;
+	__gshared static CurlEngine[] curlEnginePool;  // __gshared is used for thread-shared static variables
+    
+    HTTP http;
+    bool keepAlive;
+    ulong dnsTimeout;
+    CurlResponse response;
+    File uploadFile;
 
-	static CurlEngine get() {
-		synchronized(CurlEngine.classinfo) {
-			if (curlEnginePool.empty) {
-				return new CurlEngine;
-			} else {
-				CurlEngine curlEngine = curlEnginePool[$-1];
-				curlEnginePool.popBack();
-				return curlEngine;
-			}
-		}
-	}
+    static CurlEngine getCurlInstance() {
+        synchronized (CurlEngine.classinfo) {
+            if (curlEnginePool.empty) {
+                return new CurlEngine;  // Constructs a new CurlEngine with a fresh HTTP instance
+            } else {
+                CurlEngine curlEngine = curlEnginePool[$ - 1];
+                curlEnginePool = curlEnginePool[0 .. $ - 1];  
+                return curlEngine;
+            }
+        }
+    }
 
-	static releaseAll() {
-		synchronized(CurlEngine.classinfo) {
-			foreach(curlEngine; curlEnginePool) {
-				curlEngine.shutdown();
+    static void releaseAll() {
+		synchronized (CurlEngine.classinfo) {
+            foreach (CurlEngine curlEngine; curlEnginePool) {
+				curlEngine.cleanUp(); // Cleanup instance by resetting values
+                curlEngine.shutdown();  // Assume proper cleanup of any resources used by HTTP
 				object.destroy(curlEngine);
-			}
-			curlEnginePool = null;
-		}
-	}
+            }
+            curlEnginePool.length = 0;
+        }
+		
+		// Cleanup curlEnginePool
+		object.destroy(curlEnginePool);
+    }
 
-	void release() {
-		cleanUp();
-		synchronized(CurlEngine.classinfo) {
-			curlEnginePool ~= this;
-		}
-	}
+    this() {
+        http = HTTP();   // Directly initializes HTTP using its default constructor
+        response = null; // Initialize as null
+    }
 
-	HTTP http;
-	bool keepAlive;
-	ulong dnsTimeout;
-	CurlResponse response;
+    ~this() {
+        // The destructor should only clean up resources owned directly by this instance
+        // Avoid modifying or destroying shared/static resources here
+		if (uploadFile.isOpen())
+            uploadFile.close();
 
-	this() {	
-		http = HTTP();
-		response = new CurlResponse();
-	}
-
-	~this() {
-		object.destroy(http);
-		object.destroy(response);
-	}
+		// Cleanup curlEnginePool
+		object.destroy(curlEnginePool);
+    }
 	
+	void release() {
+        cleanUp();
+        synchronized (CurlEngine.classinfo) {
+            curlEnginePool ~= this;
+        }
+    }
+
 	void initialise(ulong dnsTimeout, ulong connectTimeout, ulong dataTimeout, ulong operationTimeout, int maxRedirects, bool httpsDebug, string userAgent, bool httpProtocol, ulong userRateLimit, ulong protocolVersion, bool keepAlive=true) {
 		//   Setting this to false ensures that when we close the curl instance, any open sockets are closed - which we need to do when running 
 		//   multiple threads and API instances at the same time otherwise we run out of local files | sockets pretty quickly
@@ -287,12 +315,24 @@ class CurlEngine {
 		}
 	}
 
+	void setResponseHolder(CurlResponse response) {
+		if (response is null) {
+			// Create a response instance if it doesn't already exist
+			if (this.response is null)
+				this.response = new CurlResponse();
+		} else {
+			this.response = response;
+		}
+	}
+
 	void addRequestHeader(const(char)[] name, const(char)[] value) {
+		setResponseHolder(null);
 		http.addRequestHeader(name, value);
 		response.addRequestHeader(name, value);
 	}
 
 	void connect(HTTP.Method method, const(char)[] url) {
+		setResponseHolder(null);
 		if (!keepAlive)
 			addRequestHeader("Connection", "close");
 		http.method = method;
@@ -301,6 +341,7 @@ class CurlEngine {
 	}
 
 	void setContent(const(char)[] contentType, const(char)[] sendData) {
+		setResponseHolder(null);
 		addRequestHeader("Content-Type", contentType);
 		if (sendData) {
 			http.contentLength = sendData.length;
@@ -316,9 +357,25 @@ class CurlEngine {
 		}
 	}
 
-	void setFile(File* file, ulong offsetSize) {
+	void setFile(string filepath, string contentRange, ulong offset, ulong offsetSize) {
+		setResponseHolder(null);
+		// open file as read-only in binary mode
+		uploadFile = File(filepath, "rb");
+
+		if (contentRange.empty) {
+			offsetSize = uploadFile.size();
+		} else {
+			addRequestHeader("Content-Range", contentRange);
+			uploadFile.seek(offset);
+		}
+
+		// Setup progress bar to display
+		http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) {
+			return 0;
+		};
+		
 		addRequestHeader("Content-Type", "application/octet-stream");
-		http.onSend = data => file.rawRead(data).length;
+		http.onSend = data => uploadFile.rawRead(data).length;
 		http.contentLength = offsetSize;
 	}
 
@@ -326,6 +383,7 @@ class CurlEngine {
 		scope(exit) {
 			cleanUp();
 		}
+		setResponseHolder(null);
 		http.onReceive = (ubyte[] data) {
 			response.content ~= data;
 			// HTTP Server Response Code Debugging if --https-debug is being used
@@ -334,14 +392,11 @@ class CurlEngine {
 		};
 		http.perform();
 		response.update(&http);
-		return response.dup;
+		return response;
 	}
 
 	CurlResponse download(string originalFilename, string downloadFilename) {
-		// Threshold for displaying download bar
-		long thresholdFileSize = 4 * 2^^20; // 4 MiB
-		
-		CurlResponse response = new CurlResponse();
+		setResponseHolder(null);
 		// open downloadFilename as write in binary mode
 		auto file = File(downloadFilename, "wb");
 
@@ -379,7 +434,13 @@ class CurlEngine {
 			return 0;
 		};
 		http.contentLength = 0;
-		response.reset();
+		response = null;
+
+		// close file if open
+		if (uploadFile.isOpen()){
+			// close open file
+			uploadFile.close();
+		}
 	}
 
 	void shutdown() {

src/log.d

@@ -100,7 +100,8 @@ class LogBuffer {
             // Use dnotify's functionality for GUI notifications, if GUI notifications is enabled
 			version(Notifications) {
 				try {
-					auto n = new Notification("Log Notification", message, "IGNORED");
+					auto n = new Notification("OneDrive Client for Linux", message, "IGNORED");
+					//n.timeout = 5;
 					n.show();
 				} catch (NotificationError e) {
 					sendGUINotification = false;

src/main.d

@@ -1123,12 +1123,6 @@ void performStandardExitProcess(string scopeCaller = null) {
 		object.destroy(oneDriveWebhook);
 	}
 	
-	// Shutdown the sync engine
-	if (syncEngineInstance !is null) {
-		addLogEntry("Shutdown Sync Engine instance", ["debug"]);
-		object.destroy(syncEngineInstance);
-	}
-	
 	// Shutdown the client side filtering objects
 	if (selectiveSync !is null) {
 		addLogEntry("Shutdown Client Side Filtering instance", ["debug"]);
@@ -1151,6 +1145,12 @@ void performStandardExitProcess(string scopeCaller = null) {
 		object.destroy(filesystemMonitor);
 	}
 	
+	// Shutdown the sync engine
+	if (syncEngineInstance !is null) {
+		addLogEntry("Shutdown Sync Engine instance", ["debug"]);
+		object.destroy(syncEngineInstance);
+	}
+	
 	// Shutdown the database
 	if (itemDB !is null) {
 		addLogEntry("Shutdown Database instance", ["debug"]);
@@ -1406,23 +1406,14 @@ auto assumeNoGC(T) (T t) if (isFunctionPointer!T || isDelegate!T) {
 	return cast(SetFunctionAttributes!(T, functionLinkage!T, attrs)) t;
 }
 
-// Catch CTRL-C
+// Catch CTRL-C if user pressed this 
 extern(C) nothrow @nogc @system void exitHandler(int value) {
 	try {
 		assumeNoGC ( () {
-			addLogEntry("Got termination signal, performing clean up");
-			// Wait for all parallel jobs that depend on the database to complete
-			addLogEntry("Waiting for any existing upload|download process to complete");
-			taskPool.finish(true);
-			// Was itemDb initialised?
-			if (itemDB.isDatabaseInitialised()) {
-				// Make sure the .wal file is incorporated into the main db before we exit
-				addLogEntry("Shutting down DB connection and merging temporary data");
-				itemDB.performVacuum();
-				object.destroy(itemDB);
-			}
-			performStandardExitProcess();
+			// Force kill any running threads as ^C was used
+			taskPool.finish(false);
 		})();
 	} catch(Exception e) {}
-	exit(0);
+	// Exit with the exitHandler value
+	exit(value);
 }

src/util.d

@@ -201,42 +201,54 @@ Regex!char wild2regex(const(char)[] pattern) {
     return regex(str, "i");
 }
 
-// Test Internet access to Microsoft OneDrive
+// Test Internet access to Microsoft OneDrive using a simple HTTP HEAD request
 bool testInternetReachability(ApplicationConfig appConfig) {
-    CurlEngine curlEngine;
-    bool result = false;
+    auto http = HTTP();
+    http.url = "https://login.microsoftonline.com";
+    
+    // Configure timeouts based on application configuration
+    http.dnsTimeout = dur!"seconds"(appConfig.getValueLong("dns_timeout"));
+    http.connectTimeout = dur!"seconds"(appConfig.getValueLong("connect_timeout"));
+    http.dataTimeout = dur!"seconds"(appConfig.getValueLong("data_timeout"));
+    http.operationTimeout = dur!"seconds"(appConfig.getValueLong("operation_timeout"));
+
+    // Set IP protocol version
+    http.handle.set(CurlOption.ipresolve, appConfig.getValueLong("ip_protocol_version"));
+
+    // Set HTTP method to HEAD for minimal data transfer
+    http.method = HTTP.Method.head;
+
+    // Execute the request and handle exceptions
     try {
-		// Use preconfigured object with all the correct http values assigned
-		curlEngine = CurlEngine.get();
-		curlEngine.initialise(appConfig.getValueLong("dns_timeout"), appConfig.getValueLong("connect_timeout"), appConfig.getValueLong("data_timeout"), appConfig.getValueLong("operation_timeout"), appConfig.defaultMaxRedirects, appConfig.getValueBool("debug_https"), appConfig.getValueString("user_agent"), appConfig.getValueBool("force_http_11"), appConfig.getValueLong("rate_limit"), appConfig.getValueLong("ip_protocol_version"));
+        addLogEntry("Attempting to contact Microsoft OneDrive Login Service");
+        http.perform();
 
-		// Configure the remaining items required
-		// URL to use
-		// HTTP connection test method
+        // Check response for HTTP status code
+        if (http.statusLine.code >= 200 && http.statusLine.code < 400) {
+            addLogEntry("Successfully reached Microsoft OneDrive Login Service");
+        } else {
+            addLogEntry("Failed to reach Microsoft OneDrive Login Service. HTTP status code: " ~ to!string(http.statusLine.code));
+            throw new Exception("HTTP Request Failed with Status Code: " ~ to!string(http.statusLine.code));
+        }
 
-		curlEngine.connect(HTTP.Method.head, "https://login.microsoftonline.com");
-		addLogEntry("Attempting to contact Microsoft OneDrive Login Service", ["debug"]);
-		curlEngine.http.perform();
-		addLogEntry("Shutting down HTTP engine as successfully reached OneDrive Login Service", ["debug"]);
-		result = true;
+        http.shutdown();
+        return true;
     } catch (SocketException e) {
-        addLogEntry("HTTP Socket Issue", ["debug"]);
-        addLogEntry("Cannot connect to Microsoft OneDrive Login Service - Socket Issue");
+        addLogEntry("Cannot connect to Microsoft OneDrive Service - Socket Issue: " ~ e.msg);
         displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
+        http.shutdown();
+        return false;
     } catch (CurlException e) {
-        addLogEntry("No Network Connection", ["debug"]);
-        addLogEntry("Cannot connect to Microsoft OneDrive Login Service - Network Connection Issue");
+        addLogEntry("Cannot connect to Microsoft OneDrive Service - Network Connection Issue: " ~ e.msg);
         displayOneDriveErrorMessage(e.msg, getFunctionName!({}));
-    } 
-
-	// Shutdown engine
-	curlEngine.http.shutdown();
-	curlEngine.releaseAll();
-	object.destroy(curlEngine);
-	curlEngine = null;
-
-	// Return test result
-    return result;
+        http.shutdown();
+        return false;
+    } catch (Exception e) {
+        addLogEntry("Unexpected error occurred: " ~ e.toString());
+        displayOneDriveErrorMessage(e.toString(), getFunctionName!({}));
+        http.shutdown();
+        return false;
+    }
 }
 
 // Retry Internet access test to Microsoft OneDrive
@@ -489,7 +501,6 @@ bool isValidUTF16(string path) {
     return true;
 }
 
-
 // Does the path contain any HTML URL encoded items (e.g., '%20' for space)
 bool containsURLEncodedItems(string path) {
     // Check for null or empty string
@@ -636,7 +647,7 @@ void displayFileSystemErrorMessage(string message, string callingFunction) {
     addLogEntry("  Error Message:    " ~ errorMessage);
     
     // Log the calling function
-    addLogEntry("  Calling Function: " ~ callingFunction, ["verbose"]);
+    addLogEntry("  Calling Function: " ~ callingFunction);
 
     try {
         // Safely check for disk space
@@ -659,6 +670,15 @@ void displayPosixErrorMessage(string message) {
 	addLogEntry("  Error Message:    " ~ message);
 }
 
+// Display the Error Message
+void displayGeneralErrorMessage(Exception e, string callingFunction=__FUNCTION__, int lineno=__LINE__) {
+	addLogEntry(); // used rather than writeln
+	addLogEntry("ERROR: Encounter " ~ e.classinfo.name ~ ":");
+	addLogEntry("  Error Message:    " ~ e.msg);
+	addLogEntry("  Calling Function:    " ~ callingFunction);
+	addLogEntry("  Line number:    " ~ to!string(lineno));
+}
+
 // Get the function name that is being called to assist with identifying where an error is being generated
 string getFunctionName(alias func)() {
     return __traits(identifier, __traits(parent, func)) ~ "()\n";