mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-13 19:32:13 +02:00
Resolve: Key not found: fileSystemInfo when syncing shared folders
* Resolve #11 where shared folders were unable to be sync'd due to fileSystemInfo data being within the remoteItem object * Initial work on resolving #2, but fix not validated or complete
This commit is contained in:
parent
2576b69a88
commit
e7267e597b
|
@ -141,7 +141,7 @@ final class ItemDatabase
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// returns if an item id is in the database
|
// returns true if an item id is in the database
|
||||||
bool idInLocalDatabase(const(string) driveId, const(string)id)
|
bool idInLocalDatabase(const(string) driveId, const(string)id)
|
||||||
{
|
{
|
||||||
selectItemByIdStmt.bind(1, driveId);
|
selectItemByIdStmt.bind(1, driveId);
|
||||||
|
|
|
@ -11,7 +11,7 @@ private immutable {
|
||||||
string redirectUrl = "https://login.microsoftonline.com/common/oauth2/nativeclient";
|
string redirectUrl = "https://login.microsoftonline.com/common/oauth2/nativeclient";
|
||||||
string tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
|
string tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
|
||||||
string driveUrl = "https://graph.microsoft.com/v1.0/me/drive";
|
string driveUrl = "https://graph.microsoft.com/v1.0/me/drive";
|
||||||
string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/";
|
string itemByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
|
||||||
string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
|
string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
|
||||||
string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
|
string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
|
||||||
}
|
}
|
||||||
|
@ -54,6 +54,7 @@ final class OneDriveApi
|
||||||
this.cfg = cfg;
|
this.cfg = cfg;
|
||||||
http = HTTP();
|
http = HTTP();
|
||||||
http.dnsTimeout = (dur!"seconds"(5));
|
http.dnsTimeout = (dur!"seconds"(5));
|
||||||
|
//http.setUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko";
|
||||||
if (debugHttp) {
|
if (debugHttp) {
|
||||||
http.verbose = true;
|
http.verbose = true;
|
||||||
.debugResponse = true;
|
.debugResponse = true;
|
||||||
|
@ -190,12 +191,11 @@ final class OneDriveApi
|
||||||
|
|
||||||
// Return the details of the specified id
|
// Return the details of the specified id
|
||||||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
|
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get
|
||||||
JSONValue getPathDetailsById(const(char)[] id)
|
JSONValue getPathDetailsById(string driveId, string id)
|
||||||
{
|
{
|
||||||
checkAccessTokenExpired();
|
checkAccessTokenExpired();
|
||||||
const(char)[] url;
|
const(char)[] url;
|
||||||
// string itemByIdUrl = "https://graph.microsoft.com/v1.0/me/drive/items/";
|
url = itemByIdUrl ~ driveId ~ "/items/" ~ id;
|
||||||
url = itemByIdUrl ~ id;
|
|
||||||
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
|
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference";
|
||||||
return get(url);
|
return get(url);
|
||||||
}
|
}
|
||||||
|
@ -212,12 +212,14 @@ final class OneDriveApi
|
||||||
}
|
}
|
||||||
|
|
||||||
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
|
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
|
||||||
JSONValue createUploadSession(const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null)
|
JSONValue createUploadSession(const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null, JSONValue item = null)
|
||||||
{
|
{
|
||||||
checkAccessTokenExpired();
|
checkAccessTokenExpired();
|
||||||
const(char)[] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/createUploadSession";
|
const(char)[] url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/createUploadSession";
|
||||||
if (eTag) http.addRequestHeader("If-Match", eTag);
|
if (eTag) http.addRequestHeader("If-Match", eTag);
|
||||||
|
http.addRequestHeader("Content-Type", "application/json");
|
||||||
return post(url, null);
|
return post(url, null);
|
||||||
|
//return post(url, item.toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
// https://dev.onedrive.com/items/upload_large_files.htm
|
// https://dev.onedrive.com/items/upload_large_files.htm
|
||||||
|
|
61
src/sync.d
61
src/sync.d
|
@ -58,8 +58,13 @@ private Item makeItem(const ref JSONValue driveItem)
|
||||||
item.mtime = SysTime(0);
|
item.mtime = SysTime(0);
|
||||||
} else {
|
} else {
|
||||||
// Item is not in a deleted state
|
// Item is not in a deleted state
|
||||||
|
if (isItemRemote(driveItem)) {
|
||||||
|
// Resolve https://github.com/abraunegg/onedrive/issues/11
|
||||||
|
item.mtime = SysTime.fromISOExtString(driveItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str);
|
||||||
|
} else {
|
||||||
item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str);
|
item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (isItemFile(driveItem)) {
|
if (isItemFile(driveItem)) {
|
||||||
item.type = ItemType.file;
|
item.type = ItemType.file;
|
||||||
|
@ -262,7 +267,7 @@ final class SyncEngine
|
||||||
} catch (OneDriveException e) {
|
} catch (OneDriveException e) {
|
||||||
if (e.httpStatusCode == 404) {
|
if (e.httpStatusCode == 404) {
|
||||||
// The directory was not found on OneDrive - no need to delete it
|
// The directory was not found on OneDrive - no need to delete it
|
||||||
log.vlog("The requested directory to create was not found on OneDrive - skipping removing the remote directory as it doesnt exist");
|
log.vlog("The requested directory to create was not found on OneDrive - skipping removing the remote directory as it doesn't exist");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -275,7 +280,7 @@ final class SyncEngine
|
||||||
} else {
|
} else {
|
||||||
// the folder was in the local database
|
// the folder was in the local database
|
||||||
// Handle the deletion and saving any update to the local database
|
// Handle the deletion and saving any update to the local database
|
||||||
log.vlog("The requested directory to delete was found in the local database. Processing the delection normally");
|
log.vlog("The requested directory to delete was found in the local database. Processing the deletion normally");
|
||||||
deleteByPath(path);
|
deleteByPath(path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -300,26 +305,39 @@ final class SyncEngine
|
||||||
|
|
||||||
// download the new changes of a specific item
|
// download the new changes of a specific item
|
||||||
// id is the root of the drive or a shared folder
|
// id is the root of the drive or a shared folder
|
||||||
private void applyDifferences(string driveId, const(char)[] id)
|
private void applyDifferences(string driveId, string id)
|
||||||
{
|
{
|
||||||
log.vlog("Applying changes of Path ID: " ~ id);
|
log.vlog("Applying changes of Path ID: " ~ id);
|
||||||
JSONValue changes;
|
JSONValue changes;
|
||||||
string syncFolderName;
|
|
||||||
string deltaLink = itemdb.getDeltaLink(driveId, id);
|
string deltaLink = itemdb.getDeltaLink(driveId, id);
|
||||||
JSONValue idDetails = onedrive.getPathDetailsById(id);
|
|
||||||
|
|
||||||
// Set the name of this folder
|
// Query the name of this folder id
|
||||||
|
string syncFolderName;
|
||||||
|
JSONValue idDetails = parseJSON("{}");
|
||||||
|
try {
|
||||||
|
idDetails = onedrive.getPathDetailsById(driveId, id);
|
||||||
|
} catch (OneDriveException e) {
|
||||||
|
if (e.httpStatusCode == 404) {
|
||||||
|
// id was not found - possibly a remote (shared) folder
|
||||||
|
log.vlog("No details returned for given Path ID");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (("id" in idDetails) != null) {
|
||||||
|
// valid response from onedrive.getPathDetailsById(id) a JSON item object present
|
||||||
if ((idDetails["id"].str == id) && (isItemFolder(idDetails))){
|
if ((idDetails["id"].str == id) && (isItemFolder(idDetails))){
|
||||||
syncFolderName = idDetails["name"].str;
|
syncFolderName = idDetails["name"].str;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for (;;) {
|
for (;;) {
|
||||||
try {
|
try {
|
||||||
// Due to differences in OneDrive API's between personal and business we need to get changes only from defaultRootId
|
// Due to differences in OneDrive API's between personal and business we need to get changes only from defaultRootId
|
||||||
// If we used the 'id' passed in & when using --single-directory with a business account we get:
|
// If we used the 'id' passed in & when using --single-directory with a business account we get:
|
||||||
// 'HTTP request returned status code 501 (Not Implemented): view.delta can only be called on the root.'
|
// 'HTTP request returned status code 501 (Not Implemented): view.delta can only be called on the root.'
|
||||||
// To view changes correctly, we need to use 'defaultRootId'
|
// To view changes correctly, we need to use 'defaultRootId' or the 'id' if this is different from 'defaultRootId'
|
||||||
changes = onedrive.viewChangesById(driveId, defaultRootId, deltaLink);
|
changes = onedrive.viewChangesById(driveId, id, deltaLink);
|
||||||
|
|
||||||
} catch (OneDriveException e) {
|
} catch (OneDriveException e) {
|
||||||
if (e.httpStatusCode == 410) {
|
if (e.httpStatusCode == 410) {
|
||||||
|
@ -341,7 +359,7 @@ final class SyncEngine
|
||||||
// HTTP request returned status code 504 (Gateway Timeout)
|
// HTTP request returned status code 504 (Gateway Timeout)
|
||||||
// Retry
|
// Retry
|
||||||
//log.vlog("OneDrive returned a 'HTTP 504 - Gateway Timeout' - gracefully handling error");
|
//log.vlog("OneDrive returned a 'HTTP 504 - Gateway Timeout' - gracefully handling error");
|
||||||
changes = onedrive.viewChangesById(driveId, defaultRootId, deltaLink);
|
changes = onedrive.viewChangesById(driveId, id, deltaLink);
|
||||||
}
|
}
|
||||||
|
|
||||||
else throw e;
|
else throw e;
|
||||||
|
@ -354,10 +372,12 @@ final class SyncEngine
|
||||||
// Thus we cannot name check for 'root' below on deleted items
|
// Thus we cannot name check for 'root' below on deleted items
|
||||||
if(!isItemDeleted(item)){
|
if(!isItemDeleted(item)){
|
||||||
// This is not a deleted item
|
// This is not a deleted item
|
||||||
// Test is this is the OneDrive Root?
|
// Test is this is the OneDrive Users Root?
|
||||||
|
// Test is this a Shared Folder - which should be classified as a 'root'
|
||||||
// Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls
|
// Use the global's as initialised via init() rather than performing unnecessary additional HTTPS calls
|
||||||
if ((id == defaultRootId) && (item["name"].str == "root")) { // fix for https://github.com/skilion/onedrive/issues/269
|
string sharedDriveRootPath = "/drives/" ~ item["parentReference"]["driveId"].str ~ "/root:";
|
||||||
// This IS the OneDrive Root
|
if ( ((id == defaultRootId) && (item["name"].str == "root")) || (item["parentReference"]["path"].str == sharedDriveRootPath) ) {
|
||||||
|
// This change has to be processed as a 'root' item
|
||||||
isRoot = true;
|
isRoot = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -369,7 +389,7 @@ final class SyncEngine
|
||||||
} else {
|
} else {
|
||||||
// What is this item's path?
|
// What is this item's path?
|
||||||
thisItemPath = item["parentReference"]["path"].str;
|
thisItemPath = item["parentReference"]["path"].str;
|
||||||
// Check this item's path to see if this is a change on the path we want
|
// Check this item's id to see if this is a change we want to process
|
||||||
if ( (item["id"].str == id) || (item["parentReference"]["id"].str == id) || (canFind(thisItemPath, syncFolderName)) ){
|
if ( (item["id"].str == id) || (item["parentReference"]["id"].str == id) || (canFind(thisItemPath, syncFolderName)) ){
|
||||||
// This is a change we want to apply
|
// This is a change we want to apply
|
||||||
applyDifference(item, driveId, isRoot);
|
applyDifference(item, driveId, isRoot);
|
||||||
|
@ -380,10 +400,11 @@ final class SyncEngine
|
||||||
// This is a corner edge case - https://github.com/skilion/onedrive/issues/341
|
// This is a corner edge case - https://github.com/skilion/onedrive/issues/341
|
||||||
JSONValue oneDriveMovedNotDeleted;
|
JSONValue oneDriveMovedNotDeleted;
|
||||||
try {
|
try {
|
||||||
oneDriveMovedNotDeleted = onedrive.getPathDetailsById(item["id"].str);
|
oneDriveMovedNotDeleted = onedrive.getPathDetailsById(driveId, item["id"].str);
|
||||||
} catch (OneDriveException e) {
|
} catch (OneDriveException e) {
|
||||||
if (e.httpStatusCode == 404) {
|
if (e.httpStatusCode == 404) {
|
||||||
// No .. that ID is GONE
|
// No .. that ID is GONE
|
||||||
|
log.vlog("Remote change discarded - item cannot be found");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -442,7 +463,7 @@ final class SyncEngine
|
||||||
} else if (isItemFolder(driveItem)) {
|
} else if (isItemFolder(driveItem)) {
|
||||||
//log.vlog("The item we are syncing is a folder");
|
//log.vlog("The item we are syncing is a folder");
|
||||||
} else if (isItemRemote(driveItem)) {
|
} else if (isItemRemote(driveItem)) {
|
||||||
//log.vlog("The item we are syncing is a remote item");
|
//log.vlog("The item we are syncing is a shared item");
|
||||||
assert(isItemFolder(driveItem["remoteItem"]), "The remote item is not a folder");
|
assert(isItemFolder(driveItem["remoteItem"]), "The remote item is not a folder");
|
||||||
} else {
|
} else {
|
||||||
log.vlog("This item type (", item.name, ") is not supported");
|
log.vlog("This item type (", item.name, ") is not supported");
|
||||||
|
@ -459,6 +480,7 @@ final class SyncEngine
|
||||||
path = buildNormalizedPath(path);
|
path = buildNormalizedPath(path);
|
||||||
unwanted = selectiveSync.isPathExcluded(path);
|
unwanted = selectiveSync.isPathExcluded(path);
|
||||||
} else {
|
} else {
|
||||||
|
// is this a remote item?
|
||||||
unwanted = true;
|
unwanted = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -476,7 +498,7 @@ final class SyncEngine
|
||||||
|
|
||||||
// check if the item is going to be deleted
|
// check if the item is going to be deleted
|
||||||
if (isItemDeleted(driveItem)) {
|
if (isItemDeleted(driveItem)) {
|
||||||
log.vlog("This item is marked for deletion:", item.name);
|
//log.vlog("This item is marked for deletion:", item.name);
|
||||||
if (cached) {
|
if (cached) {
|
||||||
// flag to delete
|
// flag to delete
|
||||||
idsToDelete ~= [item.driveId, item.id];
|
idsToDelete ~= [item.driveId, item.id];
|
||||||
|
@ -514,13 +536,6 @@ final class SyncEngine
|
||||||
} else {
|
} else {
|
||||||
itemdb.insert(item);
|
itemdb.insert(item);
|
||||||
}
|
}
|
||||||
|
|
||||||
// sync remote folder
|
|
||||||
// https://github.com/OneDrive/onedrive-api-docs/issues/764
|
|
||||||
/*if (isItemRemote(driveItem)) {
|
|
||||||
log.log("Syncing remote folder: ", path);
|
|
||||||
applyDifferences(item.remoteDriveId, item.remoteId);
|
|
||||||
}*/
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// download an item that was not synced before
|
// download an item that was not synced before
|
||||||
|
|
17
src/upload.d
17
src/upload.d
|
@ -23,7 +23,22 @@ struct UploadSession
|
||||||
|
|
||||||
JSONValue upload(string localPath, const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null)
|
JSONValue upload(string localPath, const(char)[] parentDriveId, const(char)[] parentId, const(char)[] filename, const(char)[] eTag = null)
|
||||||
{
|
{
|
||||||
session = onedrive.createUploadSession(parentDriveId, parentId, filename, eTag);
|
// Fix https://github.com/abraunegg/onedrive/issues/2
|
||||||
|
// More Details https://github.com/OneDrive/onedrive-api-docs/issues/778
|
||||||
|
|
||||||
|
SysTime localFileLastModifiedTime = timeLastModified(localPath).toUTC();
|
||||||
|
localFileLastModifiedTime.fracSecs = Duration.zero;
|
||||||
|
|
||||||
|
JSONValue fileSystemInfo = [
|
||||||
|
"item": JSONValue([
|
||||||
|
"@name.conflictBehavior": JSONValue("replace"),
|
||||||
|
"fileSystemInfo": JSONValue([
|
||||||
|
"lastModifiedDateTime": localFileLastModifiedTime.toISOExtString()
|
||||||
|
])
|
||||||
|
])
|
||||||
|
];
|
||||||
|
|
||||||
|
session = onedrive.createUploadSession(parentDriveId, parentId, filename, eTag, fileSystemInfo);
|
||||||
session["localPath"] = localPath;
|
session["localPath"] = localPath;
|
||||||
save();
|
save();
|
||||||
return upload();
|
return upload();
|
||||||
|
|
Loading…
Reference in a new issue