deblan-mirror/abraunegg-onedrive
deblan-mirror
/
abraunegg-onedrive
mirror of https://github.com/abraunegg/onedrive
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add --disable-download-validation (#1686) 

* Add --disable-download-validation to allow downloading files from OneDrive where SharePoint and the OneDrive API mis-represents the values in the API as compared to the actual HTTP server response sent to the client. In the API JSON response we get a "size" value of X, but the HTTP Server Content Length reports a size of Y. When this occurs, the download will report as failed. This was seen as part understanding the cause of https://github.com/abraunegg/onedrive/discussions/1667
This commit is contained in:
abraunegg 2021-11-23 07:34:16 +11:00 committed by GitHub
parent 5d7e4532ef
commit 2901534171
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 79 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/config.d
View File

@ -59,6 +59,7 @@ final class Config
boolValues["check_nosync"] = false;
boolValues["download_only"] = false;
boolValues["disable_notifications"] = false;
boolValues["disable_download_validation"] = false;
boolValues["disable_upload_validation"] = false;
boolValues["enable_logging"] = false;
boolValues["force_http_2"] = false;
@ -118,6 +119,9 @@ final class Config
longValues["sync_file_permissions"] = defaultFilePermissionMode;
// Configure download / upload rate limits
longValues["rate_limit"] = 0;
// maximum time an operation is allowed to take
// This includes dns resolution, connecting, data transfer, etc.
longValues["operation_timeout"] = 3600;
// Webhook options
boolValues["webhook_enabled"] = false;
@ -140,10 +144,6 @@ final class Config
// - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging
boolValues["display_sync_options"] = false;
// maximum time an operation is allowed to take
// This includes dns resolution, connecting, data transfer, etc.
longValues["operation_timeout"] = 3600;
// Determine the users home directory.
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
// Check for HOME environment variable
@ -340,6 +340,9 @@ final class Config
"disable-notifications",
"Do not use desktop notifications in monitor mode.",
&boolValues["disable_notifications"],
"disable-download-validation",
"Disable download validation when downloading from OneDrive",
&boolValues["disable_download_validation"],
"disable-upload-validation",
"Disable upload validation when uploading to OneDrive",
&boolValues["disable_upload_validation"],

3
src/main.d
View File

@ -870,6 +870,9 @@ int main(string[] args)
// Do we configure to disable the upload validation routine
if (cfg.getValueBool("disable_upload_validation")) sync.setDisableUploadValidation();
// Do we configure to disable the download validation routine
if (cfg.getValueBool("disable_download_validation")) sync.setDisableDownloadValidation();
// Has the user enabled to bypass data preservation of renaming local files when there is a conflict?
if (cfg.getValueBool("bypass_data_preservation")) {

96
src/sync.d
View File

@ -27,6 +27,9 @@ private bool uploadOnly = false;
// Do we configure to disable the upload validation routine
private bool disableUploadValidation = false;
// Do we configure to disable the download validation routine
private bool disableDownloadValidation = false;
private bool isItemFolder(const ref JSONValue item)
{
return ("folder" in item) != null;
@ -451,6 +454,7 @@ final class SyncEngine
// If account type is documentLibrary - then most likely this is a SharePoint repository
// and files 'may' be modified after upload. See: https://github.com/abraunegg/onedrive/issues/205
if(accountType == "documentLibrary") {
// set this flag for SharePoint regardless of --disable-upload-validation being used
setDisableUploadValidation();
}
@ -556,6 +560,18 @@ final class SyncEngine
log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments");
}
// Configure disableDownloadValidation if function is called
// By default, disableDownloadValidation = false;
// Meaning we will always validate our downloads
// However, when downloading files from SharePoint, the OneDrive API will not advise the correct file size
// which means that the application thinks the file download has failed as the size is different / hash is different
// See: https://github.com/abraunegg/onedrive/discussions/1667
void setDisableDownloadValidation()
{
disableDownloadValidation = true;
log.vdebug("Flagging to disable download validation checks due to user request");
}
// Issue #658 Handling
// If an existing folder is moved into a sync_list valid path (where it previously was out of scope due to sync_list),
// then set this flag to true, so that on the second 'true-up' sync, we force a rescan of the OneDrive path to capture any 'files'
@ -2891,36 +2907,62 @@ final class SyncEngine
}
// file has to have downloaded in order to set the times / data for the file
if (exists(path)) {
// A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on
// does the file hash OneDrive reports match what we have locally?
string quickXorHash = computeQuickXorHash(path);
string sha1Hash = computeSha1Hash(path);
// When downloading some files from SharePoint, the OneDrive API reports one file size, but the SharePoint HTTP Server sends a totally different byte count
// for the same file
// we have implemented --disable-download-validation to disable these checks
if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash) || (OneDriveFileHash == sha1Hash)) {
// downloaded matches either size or hash
log.vdebug("Downloaded file matches reported size and or reported file hash");
try {
log.vdebug("Calling setTimes() for this file: ", path);
setTimes(path, item.mtime, item.mtime);
} catch (FileException e) {
// display the error message
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
if (!disableDownloadValidation) {
// A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on
// does the file hash OneDrive reports match what we have locally?
string quickXorHash = computeQuickXorHash(path);
string sha1Hash = computeSha1Hash(path);
if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash) || (OneDriveFileHash == sha1Hash)) {
// downloaded matches either size or hash
log.vdebug("Downloaded file matches reported size and or reported file hash");
try {
log.vdebug("Calling setTimes() for this file: ", path);
setTimes(path, item.mtime, item.mtime);
} catch (FileException e) {
// display the error message
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
}
} else {
// size error?
if (getSize(path) != fileSize) {
// downloaded file size does not match
log.vdebug("File size on disk: ", getSize(path));
log.vdebug("OneDrive API reported size: ", fileSize);
log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why.");
}
// hash error?
if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) {
// downloaded file hash does not match
log.vdebug("Actual file hash: ", OneDriveFileHash);
log.vdebug("OneDrive API reported hash: ", quickXorHash);
log.error("ERROR: File download hash mis-match. Increase logging verbosity to determine why.");
}
// add some workaround messaging
if (accountType == "documentLibrary"){
// It has been seen where SharePoint / OneDrive API reports one size via the JSON
// but the content length and file size written to disk is totally different - example:
// From JSON: "size": 17133
// From HTTPS Server: < Content-Length: 19340
// with no logical reason for the difference, except for a 302 redirect before file download
log.error("INFO: It is most likely that a SharePoint OneDrive API issue is the root cause. Add --disable-download-validation to work around this issue but downloaded data integrity cannot be guaranteed.");
} else {
// other account types
log.error("INFO: Potentially add --disable-download-validation to work around this issue but downloaded data integrity cannot be guaranteed.");
}
// we do not want this local file to remain on the local file system
safeRemove(path);
downloadFailed = true;
return;
}
} else {
// size error?
if (getSize(path) != fileSize) {
// downloaded file size does not match
log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why.");
}
// hash error?
if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) {
// downloaded file hash does not match
log.error("ERROR: File download hash mis-match. Increase logging verbosity to determine why.");
}
// we do not want this local file to remain on the local file system
safeRemove(path);
downloadFailed = true;
return;
// download checks have been disabled
log.vdebug("Downloaded file validation disabled due to --disable-download-validation ");
}
} else {
log.error("ERROR: File failed to download. Increase logging verbosity to determine why.");