Resolve 'Key not found: lastModifiedDateTime'

* Fix 'Key not found: lastModifiedDateTime'
* Handling of Tombstoned items in the database
* Add for debugging purposes the rootID, driveID and Account Type
* Code cleanup - remove old commented out debugging items that are not
needed

src/sync.d

@@ -44,7 +44,9 @@ private Item makeItem(const ref JSONValue driveItem)
 		name: "name" in driveItem ? driveItem["name"].str : null, // name may be missing for deleted files in OneDrive Biz
 		eTag: "eTag" in driveItem ? driveItem["eTag"].str : null, // eTag is not returned for the root in OneDrive Biz
 		cTag: "cTag" in driveItem ? driveItem["cTag"].str : null, // cTag is missing in old files (and all folders in OneDrive Biz)
-		mtime: "fileSystemInfo" in driveItem ? SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str) : SysTime(0),
+		// OneDrive API Change: https://github.com/OneDrive/onedrive-api-docs/issues/834
+		// Fixes issue 'Key not found: lastModifiedDateTime' (#334, #337)
+		mtime: ("fileSystemInfo" in driveItem && "lastModifiedDateTime" in driveItem["fileSystemInfo"])? SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str) : SysTime(0),
 	};
 
 	if (isItemFile(driveItem)) {
@@ -117,6 +119,10 @@ final class SyncEngine
 	private string[2][] idsToDelete;
 	// default drive id
 	private string defaultDriveId;
+	// type of OneDrive account 
+	private string accountType;
+	// default root id
+	private string defaultRootId;
 
 	this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync)
 	{
@@ -130,6 +136,17 @@ final class SyncEngine
 
 	void init()
 	{
+		// Set accountType, defaultDriveId & defaultRootId once and reuse where possible
+		auto oneDriveDetails = onedrive.getDefaultDrive();
+		accountType = oneDriveDetails["driveType"].str;
+		defaultDriveId = oneDriveDetails["id"].str;
+		defaultRootId = onedrive.getDefaultRoot["id"].str;
+		
+		// display accountType, defaultDriveId & defaultRootId
+		log.vlog("Account Type: ", accountType);
+		log.vlog("Default Drive ID: ", defaultDriveId);
+		log.vlog("Default Root ID: ", defaultRootId);
+		
 		// check if there is an interrupted upload session
 		if (session.restore()) {
 			log.log("Continuing the upload session ...");
@@ -142,8 +159,9 @@ final class SyncEngine
 	void applyDifferences()
 	{
 		// Set defaults for the root folder
-		string driveId = defaultDriveId = onedrive.getDefaultDrive()["id"].str;
+		string driveId = defaultDriveId;
-		string rootId = onedrive.getDefaultRoot["id"].str;
+		string rootId = defaultRootId;
+		
 		applyDifferences(driveId, rootId);
 
 		// check all remote folders
@@ -156,8 +174,9 @@ final class SyncEngine
 	void applyDifferencesSingleDirectory(string path)
 	{
 		// test if the path we are going to sync from actually exists on OneDrive
+		JSONValue onedrivePathDetails;
 		try {
-			onedrive.getPathDetails(path);
+			onedrivePathDetails = onedrive.getPathDetails(path);
 		} catch (OneDriveException e) {
 			if (e.httpStatusCode == 404) {
 				// The directory was not found 
@@ -167,13 +186,11 @@ final class SyncEngine
 		} 
 		// OK - it should exist, get the driveId and rootId for this folder
 		log.vlog("Checking for differences from OneDrive ...");
-		JSONValue onedrivePathDetails = onedrive.getPathDetails(path); // Returns a JSON String for the OneDrive Path
 		
 		// If the OneDrive Root is not in the local database, creating a remote folder will fail
 		checkDatabaseForOneDriveRoot();
 		
-		// Configure the defaults
+		// Configure driveID and folderId
-		defaultDriveId = onedrive.getDefaultDrive()["id"].str;
 		string driveId = onedrivePathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1
 		string folderId = onedrivePathDetails["id"].str; // Should give something like 12345ABCDE1234A1!101
 		
@@ -188,8 +205,6 @@ final class SyncEngine
 		JSONValue rootPathDetails = onedrive.getDefaultRoot(); // Returns a JSON Value
 		Item rootPathItem = makeItem(rootPathDetails);
 		
-		// configure driveId and rootId for the OneDrive Root
-		
 		// Set defaults for the root folder
 		string driveId = rootPathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1
 		string rootId = rootPathDetails["id"].str; // Should give something like 12345ABCDE1234A1!101
@@ -219,10 +234,6 @@ final class SyncEngine
 	// delete a directory on OneDrive without syncing
 	auto deleteDirectoryNoSync(string path)
 	{
-		// Set defaults for the root folder
-		defaultDriveId = onedrive.getDefaultDrive()["id"].str;
-		string rootId = onedrive.getDefaultRoot["id"].str;
-		
 		// Attempt to delete the requested path within OneDrive without performing a sync
 		log.vlog("Attempting to delete the requested path within OneDrive");
 		
@@ -276,9 +287,6 @@ final class SyncEngine
 		string deltaLink = itemdb.getDeltaLink(driveId, id);
 		log.vlog("Applying changes of Path ID: " ~ id);
 		
-		// Get the OneDrive Root ID
-		string oneDriveRootId = onedrive.getDefaultRoot["id"].str;
-		
 		for (;;) {
 			try {
 				changes = onedrive.viewChangesById(driveId, id, deltaLink);
@@ -291,13 +299,8 @@ final class SyncEngine
 					throw e;
 				}
 			}
-			foreach (item; changes["value"].array) {			
+			foreach (item; changes["value"].array) {
-				// Test is this is the OneDrive Root - not say a single folder root sync
+				bool isRoot = (id == defaultRootId); // fix for https://github.com/skilion/onedrive/issues/269
-				bool isRoot = false;
-				if ((id == oneDriveRootId) && (item["name"].str == "root")) { // fix for https://github.com/skilion/onedrive/issues/269
-					// This IS the OneDrive Root
-					isRoot = true;
-				}
 				// Apply the change
 				applyDifference(item, driveId, isRoot);
 			}
@@ -319,25 +322,29 @@ final class SyncEngine
 	// process the change of a single DriveItem
 	private void applyDifference(JSONValue driveItem, string driveId, bool isRoot)
 	{
+		bool unwanted = false;
 		Item item = makeItem(driveItem);
-		//log.vlog("Processing item to apply differences");
+		
-
+		//if (isItemRoot(driveItem) || !item.parentId || isRoot) {
-		if (isItemRoot(driveItem) || !item.parentId || isRoot) {
+		// Post 'Key not found: lastModifiedDateTime' makeItem change, if the DB has items & OneDrive returns a tombstone, !item.parentID will always return true - thus we get in a loop
+		// Remove !item.parentId and || qualifier - these two are better at flagging is this the OneDrive root
+		if (isItemRoot(driveItem) && isRoot) { 
 			log.vlog("Adding OneDrive Root to the local database");
 			item.parentId = null; // ensures that it has no parent
 			item.driveId = driveId; // HACK: makeItem() cannot set the driveId propery of the root
-			
-			// What parent.driveId and parent.id are we using?
-			//log.vlog("Parent Drive ID: ", item.driveId);
-			//log.vlog("Parent ID:       ", item.parentId);
 			itemdb.upsert(item);
 			return;
 		}
-
+		
-		bool unwanted;
 		unwanted |= skippedItems.find(item.parentId).length != 0;
 		unwanted |= selectiveSync.isNameExcluded(item.name);
 
+		// what if this item's state is deleted - ie - OneDrive now returns a tombstoned item with no lastModifiedDateTime entry
+		if (isItemDeleted(driveItem)) {
+			log.vlog("This remote item is in a deleted state");
+			unwanted = true;
+		}
+		
 		// check the item type
 		if (!unwanted) {
 			if (isItemFile(driveItem)) {
@@ -353,7 +360,7 @@ final class SyncEngine
 			}
 		}
 
-		// check for selective sync
+		// check for selective path sync
 		string path;
 		if (!unwanted) {
 			path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name;
@@ -363,7 +370,7 @@ final class SyncEngine
 
 		// skip unwanted items early
 		if (unwanted) {
-			log.vlog("Filtered out");
+			log.vlog("Skipping item: ", item.id);
 			skippedItems ~= item.id;
 			return;
 		}
@@ -567,12 +574,6 @@ final class SyncEngine
 		// Make sure the OneDrive Root is in the database
 		checkDatabaseForOneDriveRoot();
 	
-		// make sure defaultDriveId is set
-		if (defaultDriveId == ""){
-			// defaultDriveId is not set ... odd ..
-			defaultDriveId = onedrive.getDefaultDrive()["id"].str;
-		}
-	
 		// scan for changes
 		log.vlog("Uploading differences of ", path);
 		Item item;
@@ -714,11 +715,6 @@ final class SyncEngine
 		if(encodeComponent(path).length < 430){
 			// path is less than 430 characters
 
-			if (defaultDriveId == ""){
-				// defaultDriveId is not set ... odd ..
-				defaultDriveId = onedrive.getDefaultDrive()["id"].str;
-			}
-			
 			// skip unexisting symbolic links
 			if (isSymlink(path) && !exists(readLink(path))) {
 				log.vlog("Skipping item - symbolic link: ", path);
@@ -780,39 +776,26 @@ final class SyncEngine
 			// If this is null or empty - we cant query the database properly
 			if ((parent.driveId == "") && (parent.id == "")){
 				// These are both empty .. not good
-				//log.vlog("WHOOPS: Well this is odd - parent.driveId & parent.id are empty - we have to query OneDrive for some values for the parent");
-				
 				// What path to use?
 				string parentPath = dirName(path);		// will be either . or something else
-				//log.vlog("WHOOPS FIX: Query OneDrive path details for parent: ", parentPath);
-				
 				if (parentPath == "."){
 					// We cant create this directory, as this would essentially equal the users OneDrive root:/
 					checkDatabaseForOneDriveRoot();
 				}
 				
+				JSONValue onedrivePathDetails;
 				try {
-					onedrive.getPathDetails(parentPath);
+					onedrivePathDetails = onedrive.getPathDetails(parentPath);
 				} catch (OneDriveException e) {
 					if (e.httpStatusCode == 404) {
 						// Parent does not exist ... need to create parent
 						uploadCreateDir(parentPath);
 					}
 				}
-				
+							
-				// Get the Parent Path Details
-				JSONValue onedrivePathDetails = onedrive.getPathDetails(parentPath); // Returns a JSON String for the OneDrive Path
-				
-				// JSON Response
-				//log.vlog("WHOOPS JSON Response: ", onedrivePathDetails);
-				
 				// configure the data
 				parent.driveId = onedrivePathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1
 				parent.id = onedrivePathDetails["id"].str; // This item's ID. Should give something like 12345ABCDE1234A1!101
-				
-				// What parent.driveId and parent.id did we find?
-				//log.vlog("Using Parent DriveID: ", parent.driveId);
-				//log.vlog("Using Parent ID:      ", parent.id);
 			}
 		
 			// test if the path we are going to create already exists on OneDrive
@@ -860,11 +843,6 @@ final class SyncEngine
 	{
 		Item parent;
 		
-		if (defaultDriveId == ""){
-			// defaultDriveId is not set ... odd ..
-			defaultDriveId = onedrive.getDefaultDrive()["id"].str;
-		}
-		
 		// Check the database for the parent
 		enforce(itemdb.selectByPath(dirName(path), defaultDriveId, parent), "The parent item is not in the local database");
 		
@@ -954,20 +932,15 @@ final class SyncEngine
 		if ((item.driveId == "") && (item.id == "") && (item.eTag == "")){
 			// These are empty ... we cannot delete if this is empty ....
 			JSONValue onedrivePathDetails = onedrive.getPathDetails(path); // Returns a JSON String for the OneDrive Path
-			//log.vlog("WHOOPS JSON Response: ", onedrivePathDetails);
 			item.driveId = onedrivePathDetails["parentReference"]["driveId"].str; // Should give something like 12345abcde1234a1
 			item.id = onedrivePathDetails["id"].str; // This item's ID. Should give something like 12345ABCDE1234A1!101
 			item.eTag = onedrivePathDetails["eTag"].str; // Should be something like aNjM2NjJFRUVGQjY2NjJFMSE5MzUuMA
-		
-			//log.vlog("item.driveId = ", item.driveId);
-			//log.vlog("item.id = ", item.id);
-			//log.vlog("item.eTag = ", item.eTag);
 		}
 			
 		try {
 			onedrive.deleteById(item.driveId, item.id, item.eTag);
 		} catch (OneDriveException e) {
-			if (e.httpStatusCode == 404) log.log(e.msg);
+			if (e.httpStatusCode == 404) log.log("OneDrive reported: The resource could not be found.");
 			else throw e;
 		}
 		itemdb.deleteById(item.driveId, item.id);