deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-10 09:46:40 +02:00
Code Issues Projects Releases Wiki Activity

Resolve: Key not found: fileSystemInfo when syncing shared folders

Browse source 
* Resolve #11 where shared folders were unable to be sync'd due to fileSystemInfo data being within the remoteItem object
* Initial work on resolving #2, but fix not validated or complete
This commit is contained in:
abraunegg 2018-06-09 21:43:39 +10:00
parent 2576b69a88
commit e7267e597b
4 changed files with 69 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/itemdb.d
View file

@ -141,7 +141,7 @@ final class ItemDatabase
return false;
}
// returns if an item id is in the database
// returns true if an item id is in the database
bool idInLocalDatabase(const(string) driveId, const(string)id)
{
selectItemByIdStmt.bind(1, driveId);

12
src/onedrive.d
View file

@ -11,7 +11,7 @@ private immutable {
string redirectUrl = "https://login.microsoftonline.com/common/oauth2/nativeclient";
string tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
string driveUrl = "https://graph.microsoft.com/v1.0/me/drive";
string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/";
string itemByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
}
@ -54,6 +54,7 @@ final class OneDriveApi
this.cfg = cfg;
http = HTTP();
http.dnsTimeout = (dur!"seconds"(5));
//http.setUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko";
if (debugHttp) {
http.verbose = true;
.debugResponse = true;
@ -190,12 +191,11 @@ final class OneDriveApi
// Return the details of the specified id
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
JSONValue getPathDetailsById(const(char)[] id)
JSONValue getPathDetailsById(string driveId, string id)
{
checkAccessTokenExpired();
const(char)[] url;
// string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/";
url = itemByIdUrl ~ id;
url = itemByIdUrl ~ driveId ~ "/items/" ~ id;
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
return get(url);
}
@ -212,12 +212,14 @@ final class OneDriveApi
}
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
JSONValue createUploadSession(const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null)
JSONValue createUploadSession(const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null, JSONValue item = null)
{
checkAccessTokenExpired();
const(char)[] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/createUploadSession";
if (eTag) http.addRequestHeader("If-Match", eTag);
http.addRequestHeader("Content-Type", "application/json");
return post(url, null);
//return post(url, item.toString());
}
// https://dev.onedrive.com/items/upload_large_files.htm

75
src/sync.d
View file

@ -58,7 +58,12 @@ private Item makeItem(const ref JSONValue driveItem)
item.mtime = SysTime(0);
} else {
// Item is not in a deleted state
item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str);
if (isItemRemote(driveItem)) {
// Resolve https://github.com/abraunegg/onedrive/issues/11
item.mtime = SysTime.fromISOExtString(driveItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str);
} else {
item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str);
}
}
if (isItemFile(driveItem)) {
@ -262,7 +267,7 @@ final class SyncEngine
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
// The directory was not found on OneDrive - no need to delete it
log.vlog("The requested directory to create was not found on OneDrive - skipping removing the remote directory as it doesnt exist");
log.vlog("The requested directory to create was not found on OneDrive - skipping removing the remote directory as it doesn't exist");
return;
}
}
@ -275,7 +280,7 @@ final class SyncEngine
} else {
// the folder was in the local database
// Handle the deletion and saving any update to the local database
log.vlog("The requested directory to delete was found in the local database. Processing the delection normally");
log.vlog("The requested directory to delete was found in the local database. Processing the deletion normally");
deleteByPath(path);
}
}
@ -300,26 +305,39 @@ final class SyncEngine
// download the new changes of a specific item
// id is the root of the drive or a shared folder
private void applyDifferences(string driveId, const(char)[] id)
private void applyDifferences(string driveId, string id)
{
log.vlog("Applying changes of Path ID: " ~ id);
JSONValue changes;
string syncFolderName;
string deltaLink = itemdb.getDeltaLink(driveId, id);
JSONValue idDetails = onedrive.getPathDetailsById(id);
// Set the name of this folder
if ((idDetails["id"].str == id) && (isItemFolder(idDetails))){
syncFolderName = idDetails["name"].str;
// Query the name of this folder id
string syncFolderName;
JSONValue idDetails = parseJSON("{}");
try {
idDetails = onedrive.getPathDetailsById(driveId, id);
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
// id was not found - possibly a remote (shared) folder
log.vlog("No details returned for given Path ID");
return;
}
}
if (("id" in idDetails) != null) {
// valid response from onedrive.getPathDetailsById(id) a JSON item object present
if ((idDetails["id"].str == id) && (isItemFolder(idDetails))){
syncFolderName = idDetails["name"].str;
}
}
for (;;) {
try {
// Due to differences in OneDrive API's between personal and business we need to get changes only from defaultRootId
// If we used the 'id' passed in & when using --single-directory with a business account we get:
// 'HTTP request returned status code 501 (Not Implemented): view.delta can only be called on the root.'
// To view changes correctly, we need to use 'defaultRootId'
changes = onedrive.viewChangesById(driveId, defaultRootId, deltaLink);
// To view changes correctly, we need to use 'defaultRootId' or the 'id' if this is different from 'defaultRootId'
changes = onedrive.viewChangesById(driveId, id, deltaLink);
} catch (OneDriveException e) {
if (e.httpStatusCode == 410) {
@ -341,7 +359,7 @@ final class SyncEngine
// HTTP request returned status code 504 (Gateway Timeout)
// Retry
//log.vlog("OneDrive returned a 'HTTP 504 - Gateway Timeout' - gracefully handling error");
changes = onedrive.viewChangesById(driveId, defaultRootId, deltaLink);
changes = onedrive.viewChangesById(driveId, id, deltaLink);
}
else throw e;
@ -354,10 +372,12 @@ final class SyncEngine
// Thus we cannot name check for 'root' below on deleted items
if(!isItemDeleted(item)){
// This is not a deleted item
// Test is this is the OneDrive Root?
// Test is this is the OneDrive Users Root?
// Test is this a Shared Folder - which should be classified as a 'root'
// Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls
if ((id == defaultRootId) && (item["name"].str == "root")) { // fix for https://github.com/skilion/onedrive/issues/269
// This IS the OneDrive Root
string sharedDriveRootPath = "/drives/" ~ item["parentReference"]["driveId"].str ~ "/root:";
if ( ((id == defaultRootId) && (item["name"].str == "root")) || (item["parentReference"]["path"].str == sharedDriveRootPath) ) {
// This change has to be processed as a 'root' item
isRoot = true;
}
}
@ -369,7 +389,7 @@ final class SyncEngine
} else {
// What is this item's path?
thisItemPath = item["parentReference"]["path"].str;
// Check this item's path to see if this is a change on the path we want
// Check this item's id to see if this is a change we want to process
if ( (item["id"].str == id) || (item["parentReference"]["id"].str == id) || (canFind(thisItemPath, syncFolderName)) ){
// This is a change we want to apply
applyDifference(item, driveId, isRoot);
@ -380,10 +400,11 @@ final class SyncEngine
// This is a corner edge case - https://github.com/skilion/onedrive/issues/341
JSONValue oneDriveMovedNotDeleted;
try {
oneDriveMovedNotDeleted = onedrive.getPathDetailsById(item["id"].str);
oneDriveMovedNotDeleted = onedrive.getPathDetailsById(driveId, item["id"].str);
} catch (OneDriveException e) {
if (e.httpStatusCode == 404) {
// No .. that ID is GONE
log.vlog("Remote change discarded - item cannot be found");
return;
}
}
@ -430,11 +451,11 @@ final class SyncEngine
itemdb.upsert(item);
return;
}
bool unwanted;
unwanted |= skippedItems.find(item.parentId).length != 0;
unwanted |= selectiveSync.isNameExcluded(item.name);
// check the item type
if (!unwanted) {
if (isItemFile(driveItem)) {
@ -442,7 +463,7 @@ final class SyncEngine
} else if (isItemFolder(driveItem)) {
//log.vlog("The item we are syncing is a folder");
} else if (isItemRemote(driveItem)) {
//log.vlog("The item we are syncing is a remote item");
//log.vlog("The item we are syncing is a shared item");
assert(isItemFolder(driveItem["remoteItem"]), "The remote item is not a folder");
} else {
log.vlog("This item type (", item.name, ") is not supported");
@ -454,11 +475,12 @@ final class SyncEngine
string path;
if (!unwanted) {
// Is the item in the local database
if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){
if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){
path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name;
path = buildNormalizedPath(path);
unwanted = selectiveSync.isPathExcluded(path);
} else {
// is this a remote item?
unwanted = true;
}
}
@ -476,7 +498,7 @@ final class SyncEngine
// check if the item is going to be deleted
if (isItemDeleted(driveItem)) {
log.vlog("This item is marked for deletion:", item.name);
//log.vlog("This item is marked for deletion:", item.name);
if (cached) {
// flag to delete
idsToDelete ~= [item.driveId, item.id];
@ -514,13 +536,6 @@ final class SyncEngine
} else {
itemdb.insert(item);
}
// sync remote folder
// https://github.com/OneDrive/onedrive-api-docs/issues/764
/*if (isItemRemote(driveItem)) {
log.log("Syncing remote folder: ", path);
applyDifferences(item.remoteDriveId, item.remoteId);
}*/
}
// download an item that was not synced before

17
src/upload.d
View file

@ -23,7 +23,22 @@ struct UploadSession
JSONValue upload(string localPath, const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null)
{
session = onedrive.createUploadSession(parentDriveId, parentId, filename, eTag);
// Fix https://github.com/abraunegg/onedrive/issues/2
// More Details https://github.com/OneDrive/onedrive-api-docs/issues/778
SysTime localFileLastModifiedTime = timeLastModified(localPath).toUTC();
localFileLastModifiedTime.fracSecs = Duration.zero;
JSONValue fileSystemInfo = [
"item": JSONValue([
"@name.conflictBehavior": JSONValue("replace"),
"fileSystemInfo": JSONValue([
"lastModifiedDateTime": localFileLastModifiedTime.toISOExtString()
])
])
];
session = onedrive.createUploadSession(parentDriveId, parentId, filename, eTag, fileSystemInfo);
session["localPath"] = localPath;
save();
return upload();