mirror of
https://github.com/abraunegg/onedrive
synced 2024-06-15 20:25:18 +02:00
* Resolve 'The parent item is not in the local database' where data cannot be uploaded / saved to the local database because there are folders in the same path with the same name due to case sensitivity issues
This commit is contained in:
parent
b9faa5cd1f
commit
b2c9e041be
304
src/sync.d
304
src/sync.d
|
@ -216,7 +216,7 @@ final class SyncEngine
|
||||||
} catch (OneDriveException e) {
|
} catch (OneDriveException e) {
|
||||||
if (e.httpStatusCode == 404) {
|
if (e.httpStatusCode == 404) {
|
||||||
// The directory was not found
|
// The directory was not found
|
||||||
log.vlog("ERROR: The requested single directory to sync was not found on OneDrive");
|
log.error("ERROR: The requested single directory to sync was not found on OneDrive");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1087,9 +1087,10 @@ final class SyncEngine
|
||||||
parent.id = onedrivePathDetails["id"].str; // This item's ID. Should give something like 12345ABCDE1234A1!101
|
parent.id = onedrivePathDetails["id"].str; // This item's ID. Should give something like 12345ABCDE1234A1!101
|
||||||
}
|
}
|
||||||
|
|
||||||
|
JSONValue response;
|
||||||
// test if the path we are going to create already exists on OneDrive
|
// test if the path we are going to create already exists on OneDrive
|
||||||
try {
|
try {
|
||||||
onedrive.getPathDetails(path);
|
response = onedrive.getPathDetails(path);
|
||||||
} catch (OneDriveException e) {
|
} catch (OneDriveException e) {
|
||||||
if (e.httpStatusCode == 404) {
|
if (e.httpStatusCode == 404) {
|
||||||
// The directory was not found
|
// The directory was not found
|
||||||
|
@ -1103,7 +1104,6 @@ final class SyncEngine
|
||||||
];
|
];
|
||||||
|
|
||||||
// Submit the creation request
|
// Submit the creation request
|
||||||
JSONValue response;
|
|
||||||
// Fix for https://github.com/skilion/onedrive/issues/356
|
// Fix for https://github.com/skilion/onedrive/issues/356
|
||||||
try {
|
try {
|
||||||
response = onedrive.createById(parent.driveId, parent.id, driveItem);
|
response = onedrive.createById(parent.driveId, parent.id, driveItem);
|
||||||
|
@ -1121,19 +1121,33 @@ final class SyncEngine
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
log.vlog("The requested directory to create was found on OneDrive - skipping creating the directory: ", path );
|
|
||||||
|
|
||||||
// Check that this path is in the database
|
// https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file
|
||||||
if (!itemdb.selectById(parent.driveId, parent.id, parent)){
|
// Do not assume case sensitivity. For example, consider the names OSCAR, Oscar, and oscar to be the same,
|
||||||
// parent for 'path' is NOT in the database
|
// even though some file systems (such as a POSIX-compliant file system) may consider them as different.
|
||||||
log.vlog("The parent for this path is not in the local database - need to add parent to local database");
|
// Note that NTFS supports POSIX semantics for case sensitivity but this is not the default behavior.
|
||||||
string parentPath = dirName(path);
|
|
||||||
uploadCreateDir(parentPath);
|
if (response["name"].str == baseName(path)){
|
||||||
|
// OneDrive 'name' matches local path name
|
||||||
|
log.vlog("The requested directory to create was found on OneDrive - skipping creating the directory: ", path );
|
||||||
|
// Check that this path is in the database
|
||||||
|
if (!itemdb.selectById(parent.driveId, parent.id, parent)){
|
||||||
|
// parent for 'path' is NOT in the database
|
||||||
|
log.vlog("The parent for this path is not in the local database - need to add parent to local database");
|
||||||
|
string parentPath = dirName(path);
|
||||||
|
uploadCreateDir(parentPath);
|
||||||
|
} else {
|
||||||
|
// parent is in database
|
||||||
|
log.vlog("The parent for this path is in the local database - adding requested path (", path ,") to database");
|
||||||
|
auto res = onedrive.getPathDetails(path);
|
||||||
|
saveItem(res);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
// parent is in database
|
// They are the "same" name wise but different in case sensitivity
|
||||||
log.vlog("The parent for this path is in the local database - adding requested path (", path ,") to database");
|
log.error("ERROR: A local directory has the same name as another local directory.");
|
||||||
auto res = onedrive.getPathDetails(path);
|
log.error("ERROR: To resolve, rename this local directory: ", absolutePath(path));
|
||||||
saveItem(res);
|
log.log("Skipping: ", absolutePath(path));
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1143,162 +1157,166 @@ final class SyncEngine
|
||||||
Item parent;
|
Item parent;
|
||||||
|
|
||||||
// Check the database for the parent
|
// Check the database for the parent
|
||||||
enforce(itemdb.selectByPath(dirName(path), defaultDriveId, parent), "The parent item is not in the local database");
|
//enforce(itemdb.selectByPath(dirName(path), defaultDriveId, parent), "The parent item is not in the local database");
|
||||||
|
if (itemdb.selectByPath(dirName(path), defaultDriveId, parent)) {
|
||||||
|
// Maximum file size upload
|
||||||
|
// https://support.microsoft.com/en-au/help/3125202/restrictions-and-limitations-when-you-sync-files-and-folders
|
||||||
|
// 1. OneDrive Business say's 15GB
|
||||||
|
// 2. Another article updated April 2018 says 20GB:
|
||||||
|
// https://answers.microsoft.com/en-us/onedrive/forum/odoptions-oddesktop-sdwin10/personal-onedrive-file-upload-size-max/a3621fc9-b766-4a99-99f8-bcc01ccb025f
|
||||||
|
|
||||||
// Maximum file size upload
|
// Use smaller size for now
|
||||||
// https://support.microsoft.com/en-au/help/3125202/restrictions-and-limitations-when-you-sync-files-and-folders
|
auto maxUploadFileSize = 16106127360; // 15GB
|
||||||
// 1. OneDrive Business say's 15GB
|
//auto maxUploadFileSize = 21474836480; // 20GB
|
||||||
// 2. Another article updated April 2018 says 20GB:
|
auto thisFileSize = getSize(path);
|
||||||
// https://answers.microsoft.com/en-us/onedrive/forum/odoptions-oddesktop-sdwin10/personal-onedrive-file-upload-size-max/a3621fc9-b766-4a99-99f8-bcc01ccb025f
|
|
||||||
|
|
||||||
// Use smaller size for now
|
// Can we read the file - as a permissions issue or file corruption will cause a failure
|
||||||
auto maxUploadFileSize = 16106127360; // 15GB
|
// https://github.com/abraunegg/onedrive/issues/113
|
||||||
//auto maxUploadFileSize = 21474836480; // 20GB
|
if (readLocalFile(path)){
|
||||||
auto thisFileSize = getSize(path);
|
// able to read the file
|
||||||
|
if (thisFileSize <= maxUploadFileSize){
|
||||||
|
// Resolves: https://github.com/skilion/onedrive/issues/121, https://github.com/skilion/onedrive/issues/294, https://github.com/skilion/onedrive/issues/329
|
||||||
|
|
||||||
// Can we read the file - as a permissions issue or file corruption will cause a failure
|
// To avoid a 409 Conflict error - does the file actually exist on OneDrive already?
|
||||||
// https://github.com/abraunegg/onedrive/issues/113
|
JSONValue fileDetailsFromOneDrive;
|
||||||
if (readLocalFile(path)){
|
|
||||||
// able to read the file
|
|
||||||
if (thisFileSize <= maxUploadFileSize){
|
|
||||||
// Resolves: https://github.com/skilion/onedrive/issues/121, https://github.com/skilion/onedrive/issues/294, https://github.com/skilion/onedrive/issues/329
|
|
||||||
|
|
||||||
// To avoid a 409 Conflict error - does the file actually exist on OneDrive already?
|
// Does this 'file' already exist on OneDrive?
|
||||||
JSONValue fileDetailsFromOneDrive;
|
try {
|
||||||
|
// test if the local path exists on OneDrive
|
||||||
|
fileDetailsFromOneDrive = onedrive.getPathDetails(path);
|
||||||
|
} catch (OneDriveException e) {
|
||||||
|
if (e.httpStatusCode == 404) {
|
||||||
|
// The file was not found on OneDrive, need to upload it
|
||||||
|
write("Uploading file ", path, " ...");
|
||||||
|
JSONValue response;
|
||||||
|
|
||||||
// Does this 'file' already exist on OneDrive?
|
// Resolve https://github.com/abraunegg/onedrive/issues/37
|
||||||
try {
|
if (thisFileSize == 0){
|
||||||
// test if the local path exists on OneDrive
|
// We can only upload zero size files via simpleFileUpload regardless of account type
|
||||||
fileDetailsFromOneDrive = onedrive.getPathDetails(path);
|
// https://github.com/OneDrive/onedrive-api-docs/issues/53
|
||||||
} catch (OneDriveException e) {
|
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
||||||
if (e.httpStatusCode == 404) {
|
writeln(" done.");
|
||||||
// The file was not found on OneDrive, need to upload it
|
} else {
|
||||||
write("Uploading file ", path, " ...");
|
// File is not a zero byte file
|
||||||
JSONValue response;
|
// Are we using OneDrive Personal or OneDrive Business?
|
||||||
|
// To solve 'Multiple versions of file shown on website after single upload' (https://github.com/abraunegg/onedrive/issues/2)
|
||||||
// Resolve https://github.com/abraunegg/onedrive/issues/37
|
// check what 'account type' this is as this issue only affects OneDrive Business so we need some extra logic here
|
||||||
if (thisFileSize == 0){
|
if (accountType == "personal"){
|
||||||
// We can only upload zero size files via simpleFileUpload regardless of account type
|
// Original file upload logic
|
||||||
// https://github.com/OneDrive/onedrive-api-docs/issues/53
|
if (getSize(path) <= thresholdFileSize) {
|
||||||
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
try {
|
||||||
writeln(" done.");
|
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
||||||
} else {
|
} catch (OneDriveException e) {
|
||||||
// File is not a zero byte file
|
if (e.httpStatusCode == 504) {
|
||||||
// Are we using OneDrive Personal or OneDrive Business?
|
// HTTP request returned status code 504 (Gateway Timeout)
|
||||||
// To solve 'Multiple versions of file shown on website after single upload' (https://github.com/abraunegg/onedrive/issues/2)
|
// Try upload as a session
|
||||||
// check what 'account type' this is as this issue only affects OneDrive Business so we need some extra logic here
|
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||||
if (accountType == "personal"){
|
}
|
||||||
// Original file upload logic
|
else throw e;
|
||||||
if (getSize(path) <= thresholdFileSize) {
|
|
||||||
try {
|
|
||||||
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
|
||||||
} catch (OneDriveException e) {
|
|
||||||
if (e.httpStatusCode == 504) {
|
|
||||||
// HTTP request returned status code 504 (Gateway Timeout)
|
|
||||||
// Try upload as a session
|
|
||||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
|
||||||
}
|
}
|
||||||
else throw e;
|
writeln(" done.");
|
||||||
}
|
} else {
|
||||||
|
writeln("");
|
||||||
|
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||||
writeln(" done.");
|
writeln(" done.");
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
|
// OneDrive Business Account - always use a session to upload
|
||||||
writeln("");
|
writeln("");
|
||||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||||
writeln(" done.");
|
writeln(" done.");
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log action to log file
|
||||||
|
log.fileOnly("Uploading file ", path, " ... done.");
|
||||||
|
|
||||||
|
// The file was uploaded
|
||||||
|
ulong uploadFileSize = response["size"].integer;
|
||||||
|
|
||||||
|
// In some cases the file that was uploaded was not complete, but 'completed' without errors on OneDrive
|
||||||
|
// This has been seen with PNG / JPG files mainly, which then contributes to generating a 412 error when we attempt to update the metadata
|
||||||
|
// Validate here that the file uploaded, at least in size, matches in the response to what the size is on disk
|
||||||
|
if (thisFileSize != uploadFileSize){
|
||||||
|
// OK .. the uploaded file does not match
|
||||||
|
log.log("Uploaded file size does not match local file - upload failure - retrying");
|
||||||
|
// Delete uploaded bad file
|
||||||
|
onedrive.deleteById(response["parentReference"]["driveId"].str, response["id"].str, response["eTag"].str);
|
||||||
|
// Re-upload
|
||||||
|
uploadNewFile(path);
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
if ((accountType == "personal") || (thisFileSize == 0)){
|
||||||
|
// Update the item's metadata on OneDrive
|
||||||
|
string id = response["id"].str;
|
||||||
|
string cTag = response["cTag"].str;
|
||||||
|
SysTime mtime = timeLastModified(path).toUTC();
|
||||||
|
// use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded
|
||||||
|
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
// OneDrive Business Account - always use a session to upload
|
||||||
|
// The session includes a Request Body element containing lastModifiedDateTime
|
||||||
|
// which negates the need for a modify event against OneDrive
|
||||||
|
saveItem(response);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.vlog("Requested file to upload exists on OneDrive - local database is out of sync for this file: ", path);
|
||||||
|
|
||||||
|
// Is the local file newer than the uploaded file?
|
||||||
|
SysTime localFileModifiedTime = timeLastModified(path).toUTC();
|
||||||
|
SysTime remoteFileModifiedTime = SysTime.fromISOExtString(fileDetailsFromOneDrive["fileSystemInfo"]["lastModifiedDateTime"].str);
|
||||||
|
localFileModifiedTime.fracSecs = Duration.zero;
|
||||||
|
|
||||||
|
if (localFileModifiedTime > remoteFileModifiedTime){
|
||||||
|
// local file is newer
|
||||||
|
log.vlog("Requested file to upload is newer than existing file on OneDrive");
|
||||||
|
write("Uploading file ", path, " ...");
|
||||||
|
JSONValue response;
|
||||||
|
|
||||||
|
if (accountType == "personal"){
|
||||||
|
// OneDrive Personal account upload handling
|
||||||
|
if (getSize(path) <= thresholdFileSize) {
|
||||||
|
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
||||||
|
writeln(" done.");
|
||||||
} else {
|
} else {
|
||||||
// OneDrive Business Account - always use a session to upload
|
|
||||||
writeln("");
|
writeln("");
|
||||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||||
writeln(" done.");
|
writeln(" done.");
|
||||||
}
|
}
|
||||||
|
string id = response["id"].str;
|
||||||
|
string cTag = response["cTag"].str;
|
||||||
|
SysTime mtime = timeLastModified(path).toUTC();
|
||||||
|
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
|
||||||
|
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
||||||
|
} else {
|
||||||
|
// OneDrive Business account upload handling
|
||||||
|
writeln("");
|
||||||
|
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||||
|
writeln(" done.");
|
||||||
|
saveItem(response);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Log action to log file
|
// Log action to log file
|
||||||
log.fileOnly("Uploading file ", path, " ... done.");
|
log.fileOnly("Uploading file ", path, " ... done.");
|
||||||
|
|
||||||
// The file was uploaded
|
|
||||||
ulong uploadFileSize = response["size"].integer;
|
|
||||||
|
|
||||||
// In some cases the file that was uploaded was not complete, but 'completed' without errors on OneDrive
|
|
||||||
// This has been seen with PNG / JPG files mainly, which then contributes to generating a 412 error when we attempt to update the metadata
|
|
||||||
// Validate here that the file uploaded, at least in size, matches in the response to what the size is on disk
|
|
||||||
if (thisFileSize != uploadFileSize){
|
|
||||||
// OK .. the uploaded file does not match
|
|
||||||
log.log("Uploaded file size does not match local file - upload failure - retrying");
|
|
||||||
// Delete uploaded bad file
|
|
||||||
onedrive.deleteById(response["parentReference"]["driveId"].str, response["id"].str, response["eTag"].str);
|
|
||||||
// Re-upload
|
|
||||||
uploadNewFile(path);
|
|
||||||
return;
|
|
||||||
} else {
|
|
||||||
if ((accountType == "personal") || (thisFileSize == 0)){
|
|
||||||
// Update the item's metadata on OneDrive
|
|
||||||
string id = response["id"].str;
|
|
||||||
string cTag = response["cTag"].str;
|
|
||||||
SysTime mtime = timeLastModified(path).toUTC();
|
|
||||||
// use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded
|
|
||||||
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
|
||||||
return;
|
|
||||||
} else {
|
|
||||||
// OneDrive Business Account - always use a session to upload
|
|
||||||
// The session includes a Request Body element containing lastModifiedDateTime
|
|
||||||
// which negates the need for a modify event against OneDrive
|
|
||||||
saveItem(response);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.vlog("Requested file to upload exists on OneDrive - local database is out of sync for this file: ", path);
|
|
||||||
|
|
||||||
// Is the local file newer than the uploaded file?
|
|
||||||
SysTime localFileModifiedTime = timeLastModified(path).toUTC();
|
|
||||||
SysTime remoteFileModifiedTime = SysTime.fromISOExtString(fileDetailsFromOneDrive["fileSystemInfo"]["lastModifiedDateTime"].str);
|
|
||||||
localFileModifiedTime.fracSecs = Duration.zero;
|
|
||||||
|
|
||||||
if (localFileModifiedTime > remoteFileModifiedTime){
|
|
||||||
// local file is newer
|
|
||||||
log.vlog("Requested file to upload is newer than existing file on OneDrive");
|
|
||||||
write("Uploading file ", path, " ...");
|
|
||||||
JSONValue response;
|
|
||||||
|
|
||||||
if (accountType == "personal"){
|
|
||||||
// OneDrive Personal account upload handling
|
|
||||||
if (getSize(path) <= thresholdFileSize) {
|
|
||||||
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
|
||||||
writeln(" done.");
|
|
||||||
} else {
|
|
||||||
writeln("");
|
|
||||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
|
||||||
writeln(" done.");
|
|
||||||
}
|
|
||||||
string id = response["id"].str;
|
|
||||||
string cTag = response["cTag"].str;
|
|
||||||
SysTime mtime = timeLastModified(path).toUTC();
|
|
||||||
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
|
|
||||||
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
|
||||||
} else {
|
} else {
|
||||||
// OneDrive Business account upload handling
|
// Save the details of the file that we got from OneDrive
|
||||||
writeln("");
|
log.vlog("Updating the local database with details for this file: ", path);
|
||||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
saveItem(fileDetailsFromOneDrive);
|
||||||
writeln(" done.");
|
|
||||||
saveItem(response);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Log action to log file
|
|
||||||
log.fileOnly("Uploading file ", path, " ... done.");
|
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
// Save the details of the file that we got from OneDrive
|
// Skip file - too large
|
||||||
log.vlog("Updating the local database with details for this file: ", path);
|
log.log("Skipping uploading this new file as it exceeds the maximum size allowed by OneDrive: ", path);
|
||||||
saveItem(fileDetailsFromOneDrive);
|
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
// Skip file - too large
|
|
||||||
log.log("Skipping uploading this new file as it exceeds the maximum size allowed by OneDrive: ", path);
|
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
log.log("Skipping uploading this new file as parent path is not in the database: ", path);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue