mirror of
https://github.com/abraunegg/onedrive
synced 2024-05-02 14:12:52 +02:00
Add file access check for file upload (#116)
* Add a check to validate that we can read the file by reading the first 10MB when uploading a new file or resuming an upload
This commit is contained in:
parent
6a5ab5607a
commit
e0b1b595e4
237
src/sync.d
237
src/sync.d
|
@ -1124,144 +1124,149 @@ final class SyncEngine
|
|||
//auto maxUploadFileSize = 21474836480; // 20GB
|
||||
auto thisFileSize = getSize(path);
|
||||
|
||||
if (thisFileSize <= maxUploadFileSize){
|
||||
// Resolves: https://github.com/skilion/onedrive/issues/121, https://github.com/skilion/onedrive/issues/294, https://github.com/skilion/onedrive/issues/329
|
||||
|
||||
// To avoid a 409 Conflict error - does the file actually exist on OneDrive already?
|
||||
JSONValue fileDetailsFromOneDrive;
|
||||
// Can we read the file - as a permissions issue or file corruption will cause a failure
|
||||
// https://github.com/abraunegg/onedrive/issues/113
|
||||
if (readLocalFile(path)){
|
||||
// able to read the file
|
||||
if (thisFileSize <= maxUploadFileSize){
|
||||
// Resolves: https://github.com/skilion/onedrive/issues/121, https://github.com/skilion/onedrive/issues/294, https://github.com/skilion/onedrive/issues/329
|
||||
|
||||
// Does this 'file' already exist on OneDrive?
|
||||
try {
|
||||
// test if the local path exists on OneDrive
|
||||
fileDetailsFromOneDrive = onedrive.getPathDetails(path);
|
||||
} catch (OneDriveException e) {
|
||||
if (e.httpStatusCode == 404) {
|
||||
// The file was not found on OneDrive, need to upload it
|
||||
write("Uploading file ", path, " ...");
|
||||
JSONValue response;
|
||||
|
||||
// Resolve https://github.com/abraunegg/onedrive/issues/37
|
||||
if (thisFileSize == 0){
|
||||
// We can only upload zero size files via simpleFileUpload regardless of account type
|
||||
// https://github.com/OneDrive/onedrive-api-docs/issues/53
|
||||
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
||||
writeln(" done.");
|
||||
} else {
|
||||
// File is not a zero byte file
|
||||
// Are we using OneDrive Personal or OneDrive Business?
|
||||
// To solve 'Multiple versions of file shown on website after single upload' (https://github.com/abraunegg/onedrive/issues/2)
|
||||
// check what 'account type' this is as this issue only affects OneDrive Business so we need some extra logic here
|
||||
if (accountType == "personal"){
|
||||
// Original file upload logic
|
||||
if (getSize(path) <= thresholdFileSize) {
|
||||
try {
|
||||
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
||||
} catch (OneDriveException e) {
|
||||
if (e.httpStatusCode == 504) {
|
||||
// HTTP request returned status code 504 (Gateway Timeout)
|
||||
// Try upload as a session
|
||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||
// To avoid a 409 Conflict error - does the file actually exist on OneDrive already?
|
||||
JSONValue fileDetailsFromOneDrive;
|
||||
|
||||
// Does this 'file' already exist on OneDrive?
|
||||
try {
|
||||
// test if the local path exists on OneDrive
|
||||
fileDetailsFromOneDrive = onedrive.getPathDetails(path);
|
||||
} catch (OneDriveException e) {
|
||||
if (e.httpStatusCode == 404) {
|
||||
// The file was not found on OneDrive, need to upload it
|
||||
write("Uploading file ", path, " ...");
|
||||
JSONValue response;
|
||||
|
||||
// Resolve https://github.com/abraunegg/onedrive/issues/37
|
||||
if (thisFileSize == 0){
|
||||
// We can only upload zero size files via simpleFileUpload regardless of account type
|
||||
// https://github.com/OneDrive/onedrive-api-docs/issues/53
|
||||
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
||||
writeln(" done.");
|
||||
} else {
|
||||
// File is not a zero byte file
|
||||
// Are we using OneDrive Personal or OneDrive Business?
|
||||
// To solve 'Multiple versions of file shown on website after single upload' (https://github.com/abraunegg/onedrive/issues/2)
|
||||
// check what 'account type' this is as this issue only affects OneDrive Business so we need some extra logic here
|
||||
if (accountType == "personal"){
|
||||
// Original file upload logic
|
||||
if (getSize(path) <= thresholdFileSize) {
|
||||
try {
|
||||
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
||||
} catch (OneDriveException e) {
|
||||
if (e.httpStatusCode == 504) {
|
||||
// HTTP request returned status code 504 (Gateway Timeout)
|
||||
// Try upload as a session
|
||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||
}
|
||||
else throw e;
|
||||
}
|
||||
else throw e;
|
||||
}
|
||||
writeln(" done.");
|
||||
} else {
|
||||
writeln("");
|
||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||
writeln(" done.");
|
||||
}
|
||||
} else {
|
||||
// OneDrive Business Account - always use a session to upload
|
||||
writeln("");
|
||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||
writeln(" done.");
|
||||
}
|
||||
}
|
||||
|
||||
// Log action to log file
|
||||
log.fileOnly("Uploading file ", path, " ... done.");
|
||||
|
||||
// The file was uploaded
|
||||
ulong uploadFileSize = response["size"].integer;
|
||||
|
||||
// In some cases the file that was uploaded was not complete, but 'completed' without errors on OneDrive
|
||||
// This has been seen with PNG / JPG files mainly, which then contributes to generating a 412 error when we attempt to update the metadata
|
||||
// Validate here that the file uploaded, at least in size, matches in the response to what the size is on disk
|
||||
if (thisFileSize != uploadFileSize){
|
||||
// OK .. the uploaded file does not match
|
||||
log.log("Uploaded file size does not match local file - upload failure - retrying");
|
||||
// Delete uploaded bad file
|
||||
onedrive.deleteById(response["parentReference"]["driveId"].str, response["id"].str, response["eTag"].str);
|
||||
// Re-upload
|
||||
uploadNewFile(path);
|
||||
return;
|
||||
} else {
|
||||
if ((accountType == "personal") || (thisFileSize == 0)){
|
||||
// Update the item's metadata on OneDrive
|
||||
string id = response["id"].str;
|
||||
string cTag = response["cTag"].str;
|
||||
SysTime mtime = timeLastModified(path).toUTC();
|
||||
// use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded
|
||||
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
||||
return;
|
||||
} else {
|
||||
// OneDrive Business Account - always use a session to upload
|
||||
// The session includes a Request Body element containing lastModifiedDateTime
|
||||
// which negates the need for a modify event against OneDrive
|
||||
saveItem(response);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.vlog("Requested file to upload exists on OneDrive - local database is out of sync for this file: ", path);
|
||||
|
||||
// Is the local file newer than the uploaded file?
|
||||
SysTime localFileModifiedTime = timeLastModified(path).toUTC();
|
||||
SysTime remoteFileModifiedTime = SysTime.fromISOExtString(fileDetailsFromOneDrive["fileSystemInfo"]["lastModifiedDateTime"].str);
|
||||
localFileModifiedTime.fracSecs = Duration.zero;
|
||||
|
||||
if (localFileModifiedTime > remoteFileModifiedTime){
|
||||
// local file is newer
|
||||
log.vlog("Requested file to upload is newer than existing file on OneDrive");
|
||||
write("Uploading file ", path, " ...");
|
||||
JSONValue response;
|
||||
|
||||
if (accountType == "personal"){
|
||||
// OneDrive Personal account upload handling
|
||||
if (getSize(path) <= thresholdFileSize) {
|
||||
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
||||
writeln(" done.");
|
||||
} else {
|
||||
// OneDrive Business Account - always use a session to upload
|
||||
writeln("");
|
||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||
writeln(" done.");
|
||||
}
|
||||
string id = response["id"].str;
|
||||
string cTag = response["cTag"].str;
|
||||
SysTime mtime = timeLastModified(path).toUTC();
|
||||
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
|
||||
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
||||
} else {
|
||||
// OneDrive Business account upload handling
|
||||
writeln("");
|
||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||
writeln(" done.");
|
||||
saveItem(response);
|
||||
}
|
||||
|
||||
// Log action to log file
|
||||
log.fileOnly("Uploading file ", path, " ... done.");
|
||||
|
||||
// The file was uploaded
|
||||
ulong uploadFileSize = response["size"].integer;
|
||||
|
||||
// In some cases the file that was uploaded was not complete, but 'completed' without errors on OneDrive
|
||||
// This has been seen with PNG / JPG files mainly, which then contributes to generating a 412 error when we attempt to update the metadata
|
||||
// Validate here that the file uploaded, at least in size, matches in the response to what the size is on disk
|
||||
if (thisFileSize != uploadFileSize){
|
||||
// OK .. the uploaded file does not match
|
||||
log.log("Uploaded file size does not match local file - upload failure - retrying");
|
||||
// Delete uploaded bad file
|
||||
onedrive.deleteById(response["parentReference"]["driveId"].str, response["id"].str, response["eTag"].str);
|
||||
// Re-upload
|
||||
uploadNewFile(path);
|
||||
return;
|
||||
} else {
|
||||
if ((accountType == "personal") || (thisFileSize == 0)){
|
||||
// Update the item's metadata on OneDrive
|
||||
string id = response["id"].str;
|
||||
string cTag = response["cTag"].str;
|
||||
SysTime mtime = timeLastModified(path).toUTC();
|
||||
// use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded
|
||||
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
||||
return;
|
||||
} else {
|
||||
// OneDrive Business Account - always use a session to upload
|
||||
// The session includes a Request Body element containing lastModifiedDateTime
|
||||
// which negates the need for a modify event against OneDrive
|
||||
saveItem(response);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.vlog("Requested file to upload exists on OneDrive - local database is out of sync for this file: ", path);
|
||||
|
||||
// Is the local file newer than the uploaded file?
|
||||
SysTime localFileModifiedTime = timeLastModified(path).toUTC();
|
||||
SysTime remoteFileModifiedTime = SysTime.fromISOExtString(fileDetailsFromOneDrive["fileSystemInfo"]["lastModifiedDateTime"].str);
|
||||
localFileModifiedTime.fracSecs = Duration.zero;
|
||||
|
||||
if (localFileModifiedTime > remoteFileModifiedTime){
|
||||
// local file is newer
|
||||
log.vlog("Requested file to upload is newer than existing file on OneDrive");
|
||||
write("Uploading file ", path, " ...");
|
||||
JSONValue response;
|
||||
|
||||
if (accountType == "personal"){
|
||||
// OneDrive Personal account upload handling
|
||||
if (getSize(path) <= thresholdFileSize) {
|
||||
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
|
||||
writeln(" done.");
|
||||
} else {
|
||||
writeln("");
|
||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||
writeln(" done.");
|
||||
}
|
||||
string id = response["id"].str;
|
||||
string cTag = response["cTag"].str;
|
||||
SysTime mtime = timeLastModified(path).toUTC();
|
||||
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
|
||||
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
|
||||
} else {
|
||||
// OneDrive Business account upload handling
|
||||
writeln("");
|
||||
response = session.upload(path, parent.driveId, parent.id, baseName(path));
|
||||
writeln(" done.");
|
||||
saveItem(response);
|
||||
// Save the details of the file that we got from OneDrive
|
||||
log.vlog("Updating the local database with details for this file: ", path);
|
||||
saveItem(fileDetailsFromOneDrive);
|
||||
}
|
||||
|
||||
// Log action to log file
|
||||
log.fileOnly("Uploading file ", path, " ... done.");
|
||||
|
||||
} else {
|
||||
// Save the details of the file that we got from OneDrive
|
||||
log.vlog("Updating the local database with details for this file: ", path);
|
||||
saveItem(fileDetailsFromOneDrive);
|
||||
// Skip file - too large
|
||||
log.log("Skipping uploading this new file as it exceeds the maximum size allowed by OneDrive: ", path);
|
||||
}
|
||||
} else {
|
||||
// Skip file - too large
|
||||
log.log("Skipping uploading this new file as it exceeds the maximum size allowed by OneDrive: ", path);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
44
src/upload.d
44
src/upload.d
|
@ -1,7 +1,6 @@
|
|||
import std.algorithm, std.conv, std.datetime, std.file, std.json;
|
||||
import std.stdio, core.thread;
|
||||
import progress;
|
||||
import onedrive;
|
||||
import progress, onedrive, util;
|
||||
static import log;
|
||||
|
||||
private long fragmentSize = 10 * 2^^20; // 10 MiB
|
||||
|
@ -63,25 +62,34 @@ struct UploadSession
|
|||
log.vlog("The file does not exist anymore");
|
||||
return false;
|
||||
}
|
||||
// request the session status
|
||||
JSONValue response;
|
||||
try {
|
||||
response = onedrive.requestUploadStatus(session["uploadUrl"].str);
|
||||
} catch (OneDriveException e) {
|
||||
if (e.httpStatusCode == 400) {
|
||||
log.vlog("Upload session not found");
|
||||
return false;
|
||||
} else {
|
||||
throw e;
|
||||
// Can we read the file - as a permissions issue or file corruption will cause a failure on resume
|
||||
// https://github.com/abraunegg/onedrive/issues/113
|
||||
if (readLocalFile(session["localPath"].str)){
|
||||
// able to read the file
|
||||
// request the session status
|
||||
JSONValue response;
|
||||
try {
|
||||
response = onedrive.requestUploadStatus(session["uploadUrl"].str);
|
||||
} catch (OneDriveException e) {
|
||||
if (e.httpStatusCode == 400) {
|
||||
log.vlog("Upload session not found");
|
||||
return false;
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
session["expirationDateTime"] = response["expirationDateTime"];
|
||||
session["nextExpectedRanges"] = response["nextExpectedRanges"];
|
||||
if (session["nextExpectedRanges"].array.length == 0) {
|
||||
log.vlog("The upload session is completed");
|
||||
session["expirationDateTime"] = response["expirationDateTime"];
|
||||
session["nextExpectedRanges"] = response["nextExpectedRanges"];
|
||||
if (session["nextExpectedRanges"].array.length == 0) {
|
||||
log.vlog("The upload session is completed");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
// unable to read the local file
|
||||
remove(sessionFilePath);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
17
src/util.d
17
src/util.d
|
@ -12,6 +12,7 @@ import std.string;
|
|||
import std.algorithm;
|
||||
import std.uri;
|
||||
import qxor;
|
||||
static import log;
|
||||
|
||||
private string deviceName;
|
||||
|
||||
|
@ -129,6 +130,22 @@ bool testNetwork()
|
|||
}
|
||||
}
|
||||
|
||||
// Can we read the file - as a permissions issue or file corruption will cause a failure
|
||||
// https://github.com/abraunegg/onedrive/issues/113
|
||||
// returns true if file can be accessed
|
||||
bool readLocalFile(string path)
|
||||
{
|
||||
try {
|
||||
// attempt to read the first 10MB of the file
|
||||
read(path,10000000);
|
||||
} catch (std.file.FileException e) {
|
||||
// unable to read the new local file
|
||||
log.log("Skipping uploading this file as it cannot be read (file permissions or file corruption): ", path);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// calls globMatch for each string in pattern separated by '|'
|
||||
bool multiGlobMatch(const(char)[] path, const(char)[] pattern)
|
||||
{
|
||||
|
|
Loading…
Reference in a new issue