deblan-mirror/abraunegg-onedrive
deblan-mirror/abraunegg-onedrive
1
0
Fork 0
mirror of https://github.com/abraunegg/onedrive synced 2024-05-18 13:46:41 +02:00
Code Issues Projects Releases Wiki Activity

Add file access check for file upload (#116)

Browse source 
* Add a check to validate that we can read the file by reading the first 10MB when uploading a new file or resuming an upload
This commit is contained in:
abraunegg 2018-08-14 18:30:13 +10:00 committed by GitHub
parent 6a5ab5607a
commit e0b1b595e4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 164 additions and 134 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

237
src/sync.d
View file

@ -1124,144 +1124,149 @@ final class SyncEngine
//auto maxUploadFileSize = 21474836480; // 20GB //auto maxUploadFileSize = 21474836480; // 20GB
auto thisFileSize = getSize(path); auto thisFileSize = getSize(path);
if (thisFileSize <= maxUploadFileSize){ // Can we read the file - as a permissions issue or file corruption will cause a failure
// Resolves: https://github.com/skilion/onedrive/issues/121, https://github.com/skilion/onedrive/issues/294, https://github.com/skilion/onedrive/issues/329 // https://github.com/abraunegg/onedrive/issues/113
if (readLocalFile(path)){
// To avoid a 409 Conflict error - does the file actually exist on OneDrive already? // able to read the file
JSONValue fileDetailsFromOneDrive; if (thisFileSize <= maxUploadFileSize){
// Resolves: https://github.com/skilion/onedrive/issues/121, https://github.com/skilion/onedrive/issues/294, https://github.com/skilion/onedrive/issues/329
// Does this 'file' already exist on OneDrive? // To avoid a 409 Conflict error - does the file actually exist on OneDrive already?
try { JSONValue fileDetailsFromOneDrive;
// test if the local path exists on OneDrive
fileDetailsFromOneDrive = onedrive.getPathDetails(path); // Does this 'file' already exist on OneDrive?
} catch (OneDriveException e) { try {
if (e.httpStatusCode == 404) { // test if the local path exists on OneDrive
// The file was not found on OneDrive, need to upload it fileDetailsFromOneDrive = onedrive.getPathDetails(path);
write("Uploading file ", path, " ..."); } catch (OneDriveException e) {
JSONValue response; if (e.httpStatusCode == 404) {
// The file was not found on OneDrive, need to upload it
// Resolve https://github.com/abraunegg/onedrive/issues/37 write("Uploading file ", path, " ...");
if (thisFileSize == 0){ JSONValue response;
// We can only upload zero size files via simpleFileUpload regardless of account type
// https://github.com/OneDrive/onedrive-api-docs/issues/53 // Resolve https://github.com/abraunegg/onedrive/issues/37
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path)); if (thisFileSize == 0){
writeln(" done."); // We can only upload zero size files via simpleFileUpload regardless of account type
} else { // https://github.com/OneDrive/onedrive-api-docs/issues/53
// File is not a zero byte file response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
// Are we using OneDrive Personal or OneDrive Business? writeln(" done.");
// To solve 'Multiple versions of file shown on website after single upload' (https://github.com/abraunegg/onedrive/issues/2) } else {
// check what 'account type' this is as this issue only affects OneDrive Business so we need some extra logic here // File is not a zero byte file
if (accountType == "personal"){ // Are we using OneDrive Personal or OneDrive Business?
// Original file upload logic // To solve 'Multiple versions of file shown on website after single upload' (https://github.com/abraunegg/onedrive/issues/2)
if (getSize(path) <= thresholdFileSize) { // check what 'account type' this is as this issue only affects OneDrive Business so we need some extra logic here
try { if (accountType == "personal"){
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path)); // Original file upload logic
} catch (OneDriveException e) { if (getSize(path) <= thresholdFileSize) {
if (e.httpStatusCode == 504) { try {
// HTTP request returned status code 504 (Gateway Timeout) response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
// Try upload as a session } catch (OneDriveException e) {
response = session.upload(path, parent.driveId, parent.id, baseName(path)); if (e.httpStatusCode == 504) {
// HTTP request returned status code 504 (Gateway Timeout)
// Try upload as a session
response = session.upload(path, parent.driveId, parent.id, baseName(path));
}
else throw e;
} }
else throw e; writeln(" done.");
} } else {
writeln("");
response = session.upload(path, parent.driveId, parent.id, baseName(path));
writeln(" done."); writeln(" done.");
}
} else { } else {
// OneDrive Business Account - always use a session to upload
writeln(""); writeln("");
response = session.upload(path, parent.driveId, parent.id, baseName(path)); response = session.upload(path, parent.driveId, parent.id, baseName(path));
writeln(" done."); writeln(" done.");
} }
}
// Log action to log file
log.fileOnly("Uploading file ", path, " ... done.");
// The file was uploaded
ulong uploadFileSize = response["size"].integer;
// In some cases the file that was uploaded was not complete, but 'completed' without errors on OneDrive
// This has been seen with PNG / JPG files mainly, which then contributes to generating a 412 error when we attempt to update the metadata
// Validate here that the file uploaded, at least in size, matches in the response to what the size is on disk
if (thisFileSize != uploadFileSize){
// OK .. the uploaded file does not match
log.log("Uploaded file size does not match local file - upload failure - retrying");
// Delete uploaded bad file
onedrive.deleteById(response["parentReference"]["driveId"].str, response["id"].str, response["eTag"].str);
// Re-upload
uploadNewFile(path);
return;
} else {
if ((accountType == "personal") || (thisFileSize == 0)){
// Update the item's metadata on OneDrive
string id = response["id"].str;
string cTag = response["cTag"].str;
SysTime mtime = timeLastModified(path).toUTC();
// use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
return;
} else {
// OneDrive Business Account - always use a session to upload
// The session includes a Request Body element containing lastModifiedDateTime
// which negates the need for a modify event against OneDrive
saveItem(response);
return;
}
}
}
}
log.vlog("Requested file to upload exists on OneDrive - local database is out of sync for this file: ", path);
// Is the local file newer than the uploaded file?
SysTime localFileModifiedTime = timeLastModified(path).toUTC();
SysTime remoteFileModifiedTime = SysTime.fromISOExtString(fileDetailsFromOneDrive["fileSystemInfo"]["lastModifiedDateTime"].str);
localFileModifiedTime.fracSecs = Duration.zero;
if (localFileModifiedTime > remoteFileModifiedTime){
// local file is newer
log.vlog("Requested file to upload is newer than existing file on OneDrive");
write("Uploading file ", path, " ...");
JSONValue response;
if (accountType == "personal"){
// OneDrive Personal account upload handling
if (getSize(path) <= thresholdFileSize) {
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
writeln(" done.");
} else { } else {
// OneDrive Business Account - always use a session to upload
writeln(""); writeln("");
response = session.upload(path, parent.driveId, parent.id, baseName(path)); response = session.upload(path, parent.driveId, parent.id, baseName(path));
writeln(" done."); writeln(" done.");
} }
string id = response["id"].str;
string cTag = response["cTag"].str;
SysTime mtime = timeLastModified(path).toUTC();
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
} else {
// OneDrive Business account upload handling
writeln("");
response = session.upload(path, parent.driveId, parent.id, baseName(path));
writeln(" done.");
saveItem(response);
} }
// Log action to log file // Log action to log file
log.fileOnly("Uploading file ", path, " ... done."); log.fileOnly("Uploading file ", path, " ... done.");
// The file was uploaded
ulong uploadFileSize = response["size"].integer;
// In some cases the file that was uploaded was not complete, but 'completed' without errors on OneDrive
// This has been seen with PNG / JPG files mainly, which then contributes to generating a 412 error when we attempt to update the metadata
// Validate here that the file uploaded, at least in size, matches in the response to what the size is on disk
if (thisFileSize != uploadFileSize){
// OK .. the uploaded file does not match
log.log("Uploaded file size does not match local file - upload failure - retrying");
// Delete uploaded bad file
onedrive.deleteById(response["parentReference"]["driveId"].str, response["id"].str, response["eTag"].str);
// Re-upload
uploadNewFile(path);
return;
} else {
if ((accountType == "personal") || (thisFileSize == 0)){
// Update the item's metadata on OneDrive
string id = response["id"].str;
string cTag = response["cTag"].str;
SysTime mtime = timeLastModified(path).toUTC();
// use the cTag instead of the eTag because OneDrive may update the metadata of files AFTER they have been uploaded
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
return;
} else {
// OneDrive Business Account - always use a session to upload
// The session includes a Request Body element containing lastModifiedDateTime
// which negates the need for a modify event against OneDrive
saveItem(response);
return;
}
}
}
}
log.vlog("Requested file to upload exists on OneDrive - local database is out of sync for this file: ", path);
// Is the local file newer than the uploaded file?
SysTime localFileModifiedTime = timeLastModified(path).toUTC();
SysTime remoteFileModifiedTime = SysTime.fromISOExtString(fileDetailsFromOneDrive["fileSystemInfo"]["lastModifiedDateTime"].str);
localFileModifiedTime.fracSecs = Duration.zero;
if (localFileModifiedTime > remoteFileModifiedTime){
// local file is newer
log.vlog("Requested file to upload is newer than existing file on OneDrive");
write("Uploading file ", path, " ...");
JSONValue response;
if (accountType == "personal"){
// OneDrive Personal account upload handling
if (getSize(path) <= thresholdFileSize) {
response = onedrive.simpleUpload(path, parent.driveId, parent.id, baseName(path));
writeln(" done.");
} else {
writeln("");
response = session.upload(path, parent.driveId, parent.id, baseName(path));
writeln(" done.");
}
string id = response["id"].str;
string cTag = response["cTag"].str;
SysTime mtime = timeLastModified(path).toUTC();
// use the cTag instead of the eTag because Onedrive may update the metadata of files AFTER they have been uploaded
uploadLastModifiedTime(parent.driveId, id, cTag, mtime);
} else { } else {
// OneDrive Business account upload handling // Save the details of the file that we got from OneDrive
writeln(""); log.vlog("Updating the local database with details for this file: ", path);
response = session.upload(path, parent.driveId, parent.id, baseName(path)); saveItem(fileDetailsFromOneDrive);
writeln(" done.");
saveItem(response);
} }
// Log action to log file
log.fileOnly("Uploading file ", path, " ... done.");
} else { } else {
// Save the details of the file that we got from OneDrive // Skip file - too large
log.vlog("Updating the local database with details for this file: ", path); log.log("Skipping uploading this new file as it exceeds the maximum size allowed by OneDrive: ", path);
saveItem(fileDetailsFromOneDrive);
} }
} else {
// Skip file - too large
log.log("Skipping uploading this new file as it exceeds the maximum size allowed by OneDrive: ", path);
} }
} }

44
src/upload.d
View file

@ -1,7 +1,6 @@
import std.algorithm, std.conv, std.datetime, std.file, std.json; import std.algorithm, std.conv, std.datetime, std.file, std.json;
import std.stdio, core.thread; import std.stdio, core.thread;
import progress; import progress, onedrive, util;
import onedrive;
static import log; static import log;
private long fragmentSize = 10 * 2^^20; // 10 MiB private long fragmentSize = 10 * 2^^20; // 10 MiB
@ -63,25 +62,34 @@ struct UploadSession
log.vlog("The file does not exist anymore"); log.vlog("The file does not exist anymore");
return false; return false;
} }
// request the session status // Can we read the file - as a permissions issue or file corruption will cause a failure on resume
JSONValue response; // https://github.com/abraunegg/onedrive/issues/113
try { if (readLocalFile(session["localPath"].str)){
response = onedrive.requestUploadStatus(session["uploadUrl"].str); // able to read the file
} catch (OneDriveException e) { // request the session status
if (e.httpStatusCode == 400) { JSONValue response;
log.vlog("Upload session not found"); try {
return false; response = onedrive.requestUploadStatus(session["uploadUrl"].str);
} else { } catch (OneDriveException e) {
throw e; if (e.httpStatusCode == 400) {
log.vlog("Upload session not found");
return false;
} else {
throw e;
}
} }
} session["expirationDateTime"] = response["expirationDateTime"];
session["expirationDateTime"] = response["expirationDateTime"]; session["nextExpectedRanges"] = response["nextExpectedRanges"];
session["nextExpectedRanges"] = response["nextExpectedRanges"]; if (session["nextExpectedRanges"].array.length == 0) {
if (session["nextExpectedRanges"].array.length == 0) { log.vlog("The upload session is completed");
log.vlog("The upload session is completed"); return false;
}
return true;
} else {
// unable to read the local file
remove(sessionFilePath);
return false; return false;
} }
return true;
} }
return false; return false;
} }

17
src/util.d
View file

@ -12,6 +12,7 @@ import std.string;
import std.algorithm; import std.algorithm;
import std.uri; import std.uri;
import qxor; import qxor;
static import log;
private string deviceName; private string deviceName;
@ -129,6 +130,22 @@ bool testNetwork()
} }
} }
// Can we read the file - as a permissions issue or file corruption will cause a failure
// https://github.com/abraunegg/onedrive/issues/113
// returns true if file can be accessed
bool readLocalFile(string path)
{
try {
// attempt to read the first 10MB of the file
read(path,10000000);
} catch (std.file.FileException e) {
// unable to read the new local file
log.log("Skipping uploading this file as it cannot be read (file permissions or file corruption): ", path);
return false;
}
return true;
}
// calls globMatch for each string in pattern separated by '|' // calls globMatch for each string in pattern separated by '|'
bool multiGlobMatch(const(char)[] path, const(char)[] pattern) bool multiGlobMatch(const(char)[] path, const(char)[] pattern)
{ {