Implement FR #2719: Add information about file creator/last editor as extended file attributes (#3113)

* Implement feature request to add information about file creator/last editor as extended file attributes
* On a --resync , if the data exists on disk, and 'write_xattr_data' has been enabled, write the xattr data out

.github/actions/spelling/allow.txt

@@ -110,17 +110,17 @@ estr
 eventfd
 evt
 fasynchronous
+fbc
 fcf
 fcgid
 fcgienv
-FFFD
 FCGX
-fbc
 fcontext
 fedoraproject
 fefefe
 fexceptions
 ffat
+FFFD
 fhandler
 flto
 fstack
@@ -131,6 +131,7 @@ gdc
 gdk
 gerror
 getenforce
+getxattr
 gfortran
 GFree
 Gibi
@@ -192,9 +193,9 @@ libgdk
 libgio
 libglib
 libgobject
+libinotify
 liblphobos
 libm
-libinotify
 libnotify
 libsqlite
 Lighttpd
@@ -337,6 +338,7 @@ semanage
 sendfd
 setsebool
 settime
+setxattr
 sev
 sfn
 sharedstatedir

Makefile.in

@@ -86,7 +86,8 @@ SOURCES = \
 	src/sqlite.d \
 	src/clientSideFiltering.d \
 	src/monitor.d \
-	src/arsd/cgi.d
+	src/arsd/cgi.d \
+	src/xattr.d
 
 ifeq ($(NOTIFICATIONS),yes)
 SOURCES += src/notifications/notify.d src/notifications/dnotify.d

docs/application-config-options.md

@@ -67,6 +67,7 @@ Before reading this document, please ensure you are running application version
   - [webhook_listening_port](#webhook_listening_port)
   - [webhook_public_url](#webhook_public_url)
   - [webhook_renewal_interval](#webhook_renewal_interval)
+  - [write_xattr_data](#write_xattr_data)
 - [Command Line Interface (CLI) Only Options](#command-line-interface-cli-only-options)
   - [CLI Option: --auth-files](#cli-option---auth-files)
   - [CLI Option: --auth-response](#cli-option---auth-response)
@@ -1055,6 +1056,24 @@ _**Default Value:**_ 60
 
 _**Config Example:**_ `webhook_retry_interval = "120"`
 
+### write_xattr_data
+_**Description:**_ This setting enables writing xattr values detailing the 'createdBy' and 'lastModifiedBy' information provided by the OneDrive API
+
+_**Value Type:**_ Boolean
+
+_**Default Value:**_ False
+
+_**Config Example:**_ `write_xattr_data = "false"` or `write_xattr_data = "true"`
+
+_**CLI Option Use:**_ *None - this is a config file option only*
+
+_**xattr Data Example:**_
+```
+user.onedrive.createdBy="Account Display Name"
+user.onedrive.lastModifiedBy="Account Display Name"
+```
+
+
 ## Command Line Interface (CLI) Only Options
 
 ### CLI Option: --auth-files

src/config.d

@@ -364,6 +364,11 @@ class ApplicationConfig {
 		// - Enable the calculation of transfer metrics (duration,speed) for the transfer of a file
 		boolValues["display_transfer_metrics"] = false;
 		
+		// Enable writing extended attributes about a file to xattr values
+		// - file creator
+		// - file last modifier
+		boolValues["write_xattr_data"] = false;
+
 		// Diable setting the permissions for directories and files, using the inherited permissions
 		boolValues["disable_permission_set"] = false;
 		
@@ -1453,6 +1458,7 @@ class ApplicationConfig {
 		addLogEntry("Config option 'sync_file_permissions'        = " ~ to!string(getValueLong("sync_file_permissions")));
 		addLogEntry("Config option 'space_reservation'            = " ~ to!string(getValueLong("space_reservation")));
 		addLogEntry("Config option 'permanent_delete'             = " ~ to!string(getValueBool("permanent_delete")));
+		addLogEntry("Config option 'write_xattr_data'             = " ~ to!string(getValueBool("write_xattr_data")));
 		
 		// curl operations
 		addLogEntry("Config option 'application_id'               = " ~ getValueString("application_id"));

src/onedrive.d

@@ -527,7 +527,7 @@ class OneDriveApi {
 			url = itemByPathUrl ~ encodeComponent(path) ~ ":/";
 		}
 		// Add select clause
-		url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
+		url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,createdBy,lastModifiedBy";
 		return get(url);
 	}
 	
@@ -536,7 +536,7 @@ class OneDriveApi {
 	JSONValue getPathDetailsById(string driveId, string id) {
 		string url;
 		url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
-		url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,webUrl,lastModifiedBy,lastModifiedDateTime";
+		url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,createdBy,lastModifiedBy,webUrl,lastModifiedDateTime";
 		return get(url);
 	}
 	
@@ -561,7 +561,7 @@ class OneDriveApi {
 		// https://learn.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online
 		// Required format: /drives/{drive-id}/root:/{item-path}:
 		url = driveByIdUrl ~ driveId ~ "/root:/" ~ encodeComponent(path) ~ ":";
-		url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
+		url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,createdBy,lastModifiedBy";
 		return get(url);
 	}
 	
@@ -582,7 +582,7 @@ class OneDriveApi {
 		if (deltaLink.empty) {
 			url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/delta";
 			// Reduce what we ask for in the response - which reduces the data transferred back to us, and reduces what is held in memory during initial JSON processing
-			url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
+			url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,createdBy,lastModifiedBy";
 		} else {
 			url = deltaLink;
 		}
@@ -601,7 +601,7 @@ class OneDriveApi {
 		// configure URL to query
 		if (nextLink.empty) {
 			url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/children";
-			url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
+			url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,createdBy,lastModifiedBy";
 		} else {
 			url = nextLink;
 		}

src/sync.d

@@ -36,6 +36,7 @@ import util;
 import onedrive;
 import itemdb;
 import clientSideFiltering;
+import xattr;
 
 class JsonResponseException: Exception {
 	@safe pure this(string inputMessage) {
@@ -2636,6 +2637,11 @@ class SyncEngine {
 					createRequiredSharedFolderDatabaseRecords(onedriveJSONItem);
 				}
 				
+				// Did the user configure to save xattr data about this file?
+				if (appConfig.getValueBool("write_xattr_data")) {
+					writeXattrData(newItemPath, onedriveJSONItem);
+				}
+				
 				// Display function processing time if configured to do so
 				if (appConfig.getValueBool("display_processing_time") && debugLogging) {
 					// Combine module name & running Function
@@ -2735,6 +2741,11 @@ class SyncEngine {
 						// Add item to database
 						itemDB.upsert(newDatabaseItem);
 						
+						// Did the user configure to save xattr data about this file?
+						if (appConfig.getValueBool("write_xattr_data")) {
+							writeXattrData(newItemPath, onedriveJSONItem);
+						}
+						
 						// Display function processing time if configured to do so
 						if (appConfig.getValueBool("display_processing_time") && debugLogging) {
 							// Combine module name & running Function
@@ -3641,6 +3652,11 @@ class SyncEngine {
 					// Remove 'newItemPath' from 'fileDownloadFailures' as this is no longer a failed download
 					fileDownloadFailures = fileDownloadFailures.filter!(item => item != newItemPath).array;
 				}
+				
+				// Did the user configure to save xattr data about this file?
+				if (appConfig.getValueBool("write_xattr_data")) {
+					writeXattrData(newItemPath, onedriveJSONItem);
+				}
 			} else {
 				// Output download failed
 				addLogEntry("Downloading file: " ~ newItemPath ~ " ... failed!", ["info", "notify"]);
@@ -3658,6 +3674,53 @@ class SyncEngine {
 		}
 	}
 	
+	// Write xattr data if configured to do so
+	void writeXattrData(string filePath, JSONValue onedriveJSONItem) {
+		// Function Start Time
+		SysTime functionStartTime;
+		string logKey;
+		string thisFunctionName = format("%s.%s", strip(__MODULE__) , strip(getFunctionName!({})));
+		// Only set this if we are generating performance processing times
+		if (appConfig.getValueBool("display_processing_time") && debugLogging) {
+			functionStartTime = Clock.currTime();
+			logKey = generateAlphanumericString();
+			displayFunctionProcessingStart(thisFunctionName, logKey);
+		}
+		
+		// This function will write the following xattr attributes based on the JSON data received from Microsoft onedrive
+		// - createdBy using the 'displayName' value
+		// - lastModifiedBy using the 'displayName' value
+		
+		string createdBy;
+		string lastModifiedBy;
+		
+		// Configure 'createdBy' from the JSON data
+		if (hasCreatedByUserDisplayName(onedriveJSONItem)) {
+			createdBy = onedriveJSONItem["createdBy"]["user"]["displayName"].str;
+		} else {
+			// required data not in JSON data
+			createdBy = "Unknown";
+		}
+		
+		// Configure 'lastModifiedBy' from the JSON data
+		if (hasLastModifiedByUserDisplayName(onedriveJSONItem)) {
+			lastModifiedBy = onedriveJSONItem["lastModifiedBy"]["user"]["displayName"].str;
+		} else {
+			// required data not in JSON data
+			lastModifiedBy = "Unknown";
+		}
+		
+		// Set the xattr values
+		setXAttr(filePath, "user.onedrive.createdBy", createdBy);
+		setXAttr(filePath, "user.onedrive.lastModifiedBy", lastModifiedBy);
+		
+		// Display function processing time if configured to do so
+		if (appConfig.getValueBool("display_processing_time") && debugLogging) {
+			// Combine module name & running Function
+			displayFunctionProcessingTime(thisFunctionName, functionStartTime, Clock.currTime(), logKey);
+		}	
+	}
+	
 	// Test if the given item is in-sync. Returns true if the given item corresponds to the local one
 	bool isItemSynced(Item item, string path, string itemSource) {
 		// Function Start Time

src/util.d

@@ -1168,6 +1168,46 @@ bool hasName(const ref JSONValue item) {
 	return ("name" in item) != null;
 }
 
+bool hasCreatedBy(const ref JSONValue item) {
+	return ("createdBy" in item) != null;
+}
+
+bool hasCreatedByUser(const ref JSONValue item) {
+	return ("user" in item["createdBy"]) != null;
+}
+
+bool hasCreatedByUserDisplayName(const ref JSONValue item) {
+	if (hasCreatedBy(item)) {
+		if (hasCreatedByUser(item)) {
+			return ("displayName" in item["createdBy"]["user"]) != null;
+		} else {
+			return false;
+		}
+	} else {
+		return false;
+	}
+}
+
+bool hasLastModifiedBy(const ref JSONValue item) {
+	return ("lastModifiedBy" in item) != null;
+}
+
+bool hasLastModifiedByUser(const ref JSONValue item) {
+	return ("user" in item["lastModifiedBy"]) != null;
+}
+
+bool hasLastModifiedByUserDisplayName(const ref JSONValue item) {
+	if (hasLastModifiedBy(item)) {
+		if (hasLastModifiedByUser(item)) {
+			return ("displayName" in item["lastModifiedBy"]["user"]) != null;
+		} else {
+			return false;
+		}
+	} else {
+		return false;
+	}
+}
+
 // Convert bytes to GB
 string byteToGibiByte(ulong bytes) {
     if (bytes == 0) {

src/xattr.d

@@ -0,0 +1,51 @@
+module xattr;
+
+import core.sys.posix.sys.types;
+import core.stdc.errno;
+import core.stdc.stdlib;
+import core.stdc.string;
+import core.stdc.stdio;
+import std.string;
+import std.conv;
+
+extern (C) {
+    int setxattr(const(char)* path, const(char)* name, const(void)* value, size_t size, int flags);
+    ssize_t getxattr(const(char)* path, const(char)* name, void* value, size_t size);
+}
+
+class XAttrException : Exception {
+    this(string message) {
+        super(message);
+    }
+}
+
+// Sets an extended attribute for a given file.
+// Throws `XAttrException` on failure.
+void setXAttr(string filePath, string attrName, string attrValue) {
+    int result = setxattr(filePath.toStringz(), attrName.toStringz(), cast(const(void)*)attrValue.ptr, attrValue.length, 0);
+    if (result != 0) {
+        throw new XAttrException("Failed to set xattr '" ~ attrName ~ "' on '" ~ filePath ~ "': " ~ to!string(strerror(errno)));
+    }
+}
+
+// Retrieves an extended attribute value from a file.
+// Returns the attribute value as a string.
+// Throws `XAttrException` if the attribute cannot be read.
+string getXAttr(string filePath, string attrName) {
+    // First, determine the required buffer size
+    ssize_t size = getxattr(filePath.toStringz(), attrName.toStringz(), null, 0);
+    if (size == -1) {
+        throw new XAttrException("Failed to determine xattr size for '" ~ attrName ~ "' on '" ~ filePath ~ "': " ~ to!string(strerror(errno)));
+    }
+
+    // Allocate buffer
+    char[] buffer = new char[size];
+
+    // Read the attribute value
+    ssize_t result = getxattr(filePath.toStringz(), attrName.toStringz(), cast(void*)buffer.ptr, buffer.length);
+    if (result == -1) {
+        throw new XAttrException("Failed to read xattr '" ~ attrName ~ "' from '" ~ filePath ~ "': " ~ to!string(strerror(errno)));
+    }
+
+    return buffer[0 .. result].idup;
+}