Resolve: Key not found: fileSystemInfo when syncing shared folders

* Resolve #11 where shared folders were unable to be sync'd due to fileSystemInfo data being within the remoteItem object
* Initial work on resolving #2, but fix not validated or complete
This commit is contained in:
abraunegg 2018-06-09 21:43:39 +10:00
parent 2576b69a88
commit e7267e597b
4 changed files with 69 additions and 37 deletions

View file

@ -141,7 +141,7 @@ final class ItemDatabase
return false; return false;
} }
// returns if an item id is in the database // returns true if an item id is in the database
bool idInLocalDatabase(const(string) driveId, const(string)id) bool idInLocalDatabase(const(string) driveId, const(string)id)
{ {
selectItemByIdStmt.bind(1, driveId); selectItemByIdStmt.bind(1, driveId);

View file

@ -11,7 +11,7 @@ private immutable {
string redirectUrl = "https://login.microsoftonline.com/common/oauth2/nativeclient"; string redirectUrl = "https://login.microsoftonline.com/common/oauth2/nativeclient";
string tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token"; string tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
string driveUrl = "https://graph.microsoft.com/v1.0/me/drive"; string driveUrl = "https://graph.microsoft.com/v1.0/me/drive";
string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/"; string itemByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/"; string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
} }
@ -54,6 +54,7 @@ final class OneDriveApi
this.cfg = cfg; this.cfg = cfg;
http = HTTP(); http = HTTP();
http.dnsTimeout = (dur!"seconds"(5)); http.dnsTimeout = (dur!"seconds"(5));
//http.setUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko";
if (debugHttp) { if (debugHttp) {
http.verbose = true; http.verbose = true;
.debugResponse = true; .debugResponse = true;
@ -190,12 +191,11 @@ final class OneDriveApi
// Return the details of the specified id // Return the details of the specified id
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getPathDetailsById(const(char)[] id) JSONValue getPathDetailsById(string driveId, string id)
{ {
checkAccessTokenExpired(); checkAccessTokenExpired();
const(char)[] url; const(char)[] url;
// string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/"; url = itemByIdUrl ~ driveId ~ "/items/" ~ id;
url = itemByIdUrl ~ id;
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference"; url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
return get(url); return get(url);
} }
@ -212,12 +212,14 @@ final class OneDriveApi
} }
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
JSONValue createUploadSession(const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null) JSONValue createUploadSession(const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null, JSONValue item = null)
{ {
checkAccessTokenExpired(); checkAccessTokenExpired();
const(char)[] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/createUploadSession"; const(char)[] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/createUploadSession";
if (eTag) http.addRequestHeader("If-Match", eTag); if (eTag) http.addRequestHeader("If-Match", eTag);
http.addRequestHeader("Content-Type", "application/json");
return post(url, null); return post(url, null);
//return post(url, item.toString());
} }
// https://dev.onedrive.com/items/upload_large_files.htm // https://dev.onedrive.com/items/upload_large_files.htm

View file

@ -58,8 +58,13 @@ private Item makeItem(const ref JSONValue driveItem)
item.mtime = SysTime(0); item.mtime = SysTime(0);
} else { } else {
// Item is not in a deleted state // Item is not in a deleted state
if (isItemRemote(driveItem)) {
// Resolve https://github.com/abraunegg/onedrive/issues/11
item.mtime = SysTime.fromISOExtString(driveItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str);
} else {
item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str); item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str);
} }
}
if (isItemFile(driveItem)) { if (isItemFile(driveItem)) {
item.type = ItemType.file; item.type = ItemType.file;
@ -262,7 +267,7 @@ final class SyncEngine
} catch (OneDriveException e) { } catch (OneDriveException e) {
if (e.httpStatusCode == 404) { if (e.httpStatusCode == 404) {
// The directory was not found on OneDrive - no need to delete it // The directory was not found on OneDrive - no need to delete it
log.vlog("The requested directory to create was not found on OneDrive - skipping removing the remote directory as it doesnt exist"); log.vlog("The requested directory to create was not found on OneDrive - skipping removing the remote directory as it doesn't exist");
return; return;
} }
} }
@ -275,7 +280,7 @@ final class SyncEngine
} else { } else {
// the folder was in the local database // the folder was in the local database
// Handle the deletion and saving any update to the local database // Handle the deletion and saving any update to the local database
log.vlog("The requested directory to delete was found in the local database. Processing the delection normally"); log.vlog("The requested directory to delete was found in the local database. Processing the deletion normally");
deleteByPath(path); deleteByPath(path);
} }
} }
@ -300,26 +305,39 @@ final class SyncEngine
// download the new changes of a specific item // download the new changes of a specific item
// id is the root of the drive or a shared folder // id is the root of the drive or a shared folder
private void applyDifferences(string driveId, const(char)[] id) private void applyDifferences(string driveId, string id)
{ {
log.vlog("Applying changes of Path ID: " ~ id); log.vlog("Applying changes of Path ID: " ~ id);
JSONValue changes; JSONValue changes;
string syncFolderName;
string deltaLink = itemdb.getDeltaLink(driveId, id); string deltaLink = itemdb.getDeltaLink(driveId, id);
JSONValue idDetails = onedrive.getPathDetailsById(id);
// Set the name of this folder // Query the name of this folder id
string syncFolderName;
JSONValue idDetails = parseJSON("{}");
try {
idDetails = onedrive.getPathDetailsById(driveId, id);
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
// id was not found - possibly a remote (shared) folder
log.vlog("No details returned for given Path ID");
return;
}
}
if (("id" in idDetails) != null) {
// valid response from onedrive.getPathDetailsById(id) a JSON item object present
if ((idDetails["id"].str == id) && (isItemFolder(idDetails))){ if ((idDetails["id"].str == id) && (isItemFolder(idDetails))){
syncFolderName = idDetails["name"].str; syncFolderName = idDetails["name"].str;
} }
}
for (;;) { for (;;) {
try { try {
// Due to differences in OneDrive API's between personal and business we need to get changes only from defaultRootId // Due to differences in OneDrive API's between personal and business we need to get changes only from defaultRootId
// If we used the 'id' passed in & when using --single-directory with a business account we get: // If we used the 'id' passed in & when using --single-directory with a business account we get:
// 'HTTP request returned status code 501 (Not Implemented): view.delta can only be called on the root.' // 'HTTP request returned status code 501 (Not Implemented): view.delta can only be called on the root.'
// To view changes correctly, we need to use 'defaultRootId' // To view changes correctly, we need to use 'defaultRootId' or the 'id' if this is different from 'defaultRootId'
changes = onedrive.viewChangesById(driveId, defaultRootId, deltaLink); changes = onedrive.viewChangesById(driveId, id, deltaLink);
} catch (OneDriveException e) { } catch (OneDriveException e) {
if (e.httpStatusCode == 410) { if (e.httpStatusCode == 410) {
@ -341,7 +359,7 @@ final class SyncEngine
// HTTP request returned status code 504 (Gateway Timeout) // HTTP request returned status code 504 (Gateway Timeout)
// Retry // Retry
//log.vlog("OneDrive returned a 'HTTP 504 - Gateway Timeout' - gracefully handling error"); //log.vlog("OneDrive returned a 'HTTP 504 - Gateway Timeout' - gracefully handling error");
changes = onedrive.viewChangesById(driveId, defaultRootId, deltaLink); changes = onedrive.viewChangesById(driveId, id, deltaLink);
} }
else throw e; else throw e;
@ -354,10 +372,12 @@ final class SyncEngine
// Thus we cannot name check for 'root' below on deleted items // Thus we cannot name check for 'root' below on deleted items
if(!isItemDeleted(item)){ if(!isItemDeleted(item)){
// This is not a deleted item // This is not a deleted item
// Test is this is the OneDrive Root? // Test is this is the OneDrive Users Root?
// Test is this a Shared Folder - which should be classified as a 'root'
// Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls // Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls
if ((id == defaultRootId) && (item["name"].str == "root")) { // fix for https://github.com/skilion/onedrive/issues/269 string sharedDriveRootPath = "/drives/" ~ item["parentReference"]["driveId"].str ~ "/root:";
// This IS the OneDrive Root if ( ((id == defaultRootId) && (item["name"].str == "root")) || (item["parentReference"]["path"].str == sharedDriveRootPath) ) {
// This change has to be processed as a 'root' item
isRoot = true; isRoot = true;
} }
} }
@ -369,7 +389,7 @@ final class SyncEngine
} else { } else {
// What is this item's path? // What is this item's path?
thisItemPath = item["parentReference"]["path"].str; thisItemPath = item["parentReference"]["path"].str;
// Check this item's path to see if this is a change on the path we want // Check this item's id to see if this is a change we want to process
if ( (item["id"].str == id) || (item["parentReference"]["id"].str == id) || (canFind(thisItemPath, syncFolderName)) ){ if ( (item["id"].str == id) || (item["parentReference"]["id"].str == id) || (canFind(thisItemPath, syncFolderName)) ){
// This is a change we want to apply // This is a change we want to apply
applyDifference(item, driveId, isRoot); applyDifference(item, driveId, isRoot);
@ -380,10 +400,11 @@ final class SyncEngine
// This is a corner edge case - https://github.com/skilion/onedrive/issues/341 // This is a corner edge case - https://github.com/skilion/onedrive/issues/341
JSONValue oneDriveMovedNotDeleted; JSONValue oneDriveMovedNotDeleted;
try { try {
oneDriveMovedNotDeleted = onedrive.getPathDetailsById(item["id"].str); oneDriveMovedNotDeleted = onedrive.getPathDetailsById(driveId, item["id"].str);
} catch (OneDriveException e) { } catch (OneDriveException e) {
if (e.httpStatusCode == 404) { if (e.httpStatusCode == 404) {
// No .. that ID is GONE // No .. that ID is GONE
log.vlog("Remote change discarded - item cannot be found");
return; return;
} }
} }
@ -442,7 +463,7 @@ final class SyncEngine
} else if (isItemFolder(driveItem)) { } else if (isItemFolder(driveItem)) {
//log.vlog("The item we are syncing is a folder"); //log.vlog("The item we are syncing is a folder");
} else if (isItemRemote(driveItem)) { } else if (isItemRemote(driveItem)) {
//log.vlog("The item we are syncing is a remote item"); //log.vlog("The item we are syncing is a shared item");
assert(isItemFolder(driveItem["remoteItem"]), "The remote item is not a folder"); assert(isItemFolder(driveItem["remoteItem"]), "The remote item is not a folder");
} else { } else {
log.vlog("This item type (", item.name, ") is not supported"); log.vlog("This item type (", item.name, ") is not supported");
@ -459,6 +480,7 @@ final class SyncEngine
path = buildNormalizedPath(path); path = buildNormalizedPath(path);
unwanted = selectiveSync.isPathExcluded(path); unwanted = selectiveSync.isPathExcluded(path);
} else { } else {
// is this a remote item?
unwanted = true; unwanted = true;
} }
} }
@ -476,7 +498,7 @@ final class SyncEngine
// check if the item is going to be deleted // check if the item is going to be deleted
if (isItemDeleted(driveItem)) { if (isItemDeleted(driveItem)) {
log.vlog("This item is marked for deletion:", item.name); //log.vlog("This item is marked for deletion:", item.name);
if (cached) { if (cached) {
// flag to delete // flag to delete
idsToDelete ~= [item.driveId, item.id]; idsToDelete ~= [item.driveId, item.id];
@ -514,13 +536,6 @@ final class SyncEngine
} else { } else {
itemdb.insert(item); itemdb.insert(item);
} }
// sync remote folder
// https://github.com/OneDrive/onedrive-api-docs/issues/764
/*if (isItemRemote(driveItem)) {
log.log("Syncing remote folder: ", path);
applyDifferences(item.remoteDriveId, item.remoteId);
}*/
} }
// download an item that was not synced before // download an item that was not synced before

View file

@ -23,7 +23,22 @@ struct UploadSession
JSONValue upload(string localPath, const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null) JSONValue upload(string localPath, const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null)
{ {
session = onedrive.createUploadSession(parentDriveId, parentId, filename, eTag); // Fix https://github.com/abraunegg/onedrive/issues/2
// More Details https://github.com/OneDrive/onedrive-api-docs/issues/778
SysTime localFileLastModifiedTime = timeLastModified(localPath).toUTC();
localFileLastModifiedTime.fracSecs = Duration.zero;
JSONValue fileSystemInfo = [
"item": JSONValue([
"@name.conflictBehavior": JSONValue("replace"),
"fileSystemInfo": JSONValue([
"lastModifiedDateTime": localFileLastModifiedTime.toISOExtString()
])
])
];
session = onedrive.createUploadSession(parentDriveId, parentId, filename, eTag, fileSystemInfo);
session["localPath"] = localPath; session["localPath"] = localPath;
save(); save();
return upload(); return upload();