mirror of
https://github.com/abraunegg/onedrive
synced 2024-05-23 16:12:12 +02:00
Add --disable-download-validation (#1686)
* Add --disable-download-validation to allow downloading files from OneDrive where SharePoint and the OneDrive API mis-represents the values in the API as compared to the actual HTTP server response sent to the client. In the API JSON response we get a "size" value of X, but the HTTP Server Content Length reports a size of Y. When this occurs, the download will report as failed. This was seen as part understanding the cause of https://github.com/abraunegg/onedrive/discussions/1667
This commit is contained in:
parent
5d7e4532ef
commit
2901534171
11
src/config.d
11
src/config.d
|
@ -59,6 +59,7 @@ final class Config
|
|||
boolValues["check_nosync"] = false;
|
||||
boolValues["download_only"] = false;
|
||||
boolValues["disable_notifications"] = false;
|
||||
boolValues["disable_download_validation"] = false;
|
||||
boolValues["disable_upload_validation"] = false;
|
||||
boolValues["enable_logging"] = false;
|
||||
boolValues["force_http_2"] = false;
|
||||
|
@ -118,6 +119,9 @@ final class Config
|
|||
longValues["sync_file_permissions"] = defaultFilePermissionMode;
|
||||
// Configure download / upload rate limits
|
||||
longValues["rate_limit"] = 0;
|
||||
// maximum time an operation is allowed to take
|
||||
// This includes dns resolution, connecting, data transfer, etc.
|
||||
longValues["operation_timeout"] = 3600;
|
||||
|
||||
// Webhook options
|
||||
boolValues["webhook_enabled"] = false;
|
||||
|
@ -140,10 +144,6 @@ final class Config
|
|||
// - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging
|
||||
boolValues["display_sync_options"] = false;
|
||||
|
||||
// maximum time an operation is allowed to take
|
||||
// This includes dns resolution, connecting, data transfer, etc.
|
||||
longValues["operation_timeout"] = 3600;
|
||||
|
||||
// Determine the users home directory.
|
||||
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
|
||||
// Check for HOME environment variable
|
||||
|
@ -340,6 +340,9 @@ final class Config
|
|||
"disable-notifications",
|
||||
"Do not use desktop notifications in monitor mode.",
|
||||
&boolValues["disable_notifications"],
|
||||
"disable-download-validation",
|
||||
"Disable download validation when downloading from OneDrive",
|
||||
&boolValues["disable_download_validation"],
|
||||
"disable-upload-validation",
|
||||
"Disable upload validation when uploading to OneDrive",
|
||||
&boolValues["disable_upload_validation"],
|
||||
|
|
|
@ -871,6 +871,9 @@ int main(string[] args)
|
|||
// Do we configure to disable the upload validation routine
|
||||
if (cfg.getValueBool("disable_upload_validation")) sync.setDisableUploadValidation();
|
||||
|
||||
// Do we configure to disable the download validation routine
|
||||
if (cfg.getValueBool("disable_download_validation")) sync.setDisableDownloadValidation();
|
||||
|
||||
// Has the user enabled to bypass data preservation of renaming local files when there is a conflict?
|
||||
if (cfg.getValueBool("bypass_data_preservation")) {
|
||||
log.log("WARNING: Application has been configured to bypass local data preservation in the event of file conflict.");
|
||||
|
|
42
src/sync.d
42
src/sync.d
|
@ -27,6 +27,9 @@ private bool uploadOnly = false;
|
|||
// Do we configure to disable the upload validation routine
|
||||
private bool disableUploadValidation = false;
|
||||
|
||||
// Do we configure to disable the download validation routine
|
||||
private bool disableDownloadValidation = false;
|
||||
|
||||
private bool isItemFolder(const ref JSONValue item)
|
||||
{
|
||||
return ("folder" in item) != null;
|
||||
|
@ -451,6 +454,7 @@ final class SyncEngine
|
|||
// If account type is documentLibrary - then most likely this is a SharePoint repository
|
||||
// and files 'may' be modified after upload. See: https://github.com/abraunegg/onedrive/issues/205
|
||||
if(accountType == "documentLibrary") {
|
||||
// set this flag for SharePoint regardless of --disable-upload-validation being used
|
||||
setDisableUploadValidation();
|
||||
}
|
||||
|
||||
|
@ -556,6 +560,18 @@ final class SyncEngine
|
|||
log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments");
|
||||
}
|
||||
|
||||
// Configure disableDownloadValidation if function is called
|
||||
// By default, disableDownloadValidation = false;
|
||||
// Meaning we will always validate our downloads
|
||||
// However, when downloading files from SharePoint, the OneDrive API will not advise the correct file size
|
||||
// which means that the application thinks the file download has failed as the size is different / hash is different
|
||||
// See: https://github.com/abraunegg/onedrive/discussions/1667
|
||||
void setDisableDownloadValidation()
|
||||
{
|
||||
disableDownloadValidation = true;
|
||||
log.vdebug("Flagging to disable download validation checks due to user request");
|
||||
}
|
||||
|
||||
// Issue #658 Handling
|
||||
// If an existing folder is moved into a sync_list valid path (where it previously was out of scope due to sync_list),
|
||||
// then set this flag to true, so that on the second 'true-up' sync, we force a rescan of the OneDrive path to capture any 'files'
|
||||
|
@ -2891,6 +2907,11 @@ final class SyncEngine
|
|||
}
|
||||
// file has to have downloaded in order to set the times / data for the file
|
||||
if (exists(path)) {
|
||||
// When downloading some files from SharePoint, the OneDrive API reports one file size, but the SharePoint HTTP Server sends a totally different byte count
|
||||
// for the same file
|
||||
// we have implemented --disable-download-validation to disable these checks
|
||||
|
||||
if (!disableDownloadValidation) {
|
||||
// A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on
|
||||
// does the file hash OneDrive reports match what we have locally?
|
||||
string quickXorHash = computeQuickXorHash(path);
|
||||
|
@ -2910,18 +2931,39 @@ final class SyncEngine
|
|||
// size error?
|
||||
if (getSize(path) != fileSize) {
|
||||
// downloaded file size does not match
|
||||
log.vdebug("File size on disk: ", getSize(path));
|
||||
log.vdebug("OneDrive API reported size: ", fileSize);
|
||||
log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why.");
|
||||
}
|
||||
// hash error?
|
||||
if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) {
|
||||
// downloaded file hash does not match
|
||||
log.vdebug("Actual file hash: ", OneDriveFileHash);
|
||||
log.vdebug("OneDrive API reported hash: ", quickXorHash);
|
||||
log.error("ERROR: File download hash mis-match. Increase logging verbosity to determine why.");
|
||||
}
|
||||
// add some workaround messaging
|
||||
if (accountType == "documentLibrary"){
|
||||
// It has been seen where SharePoint / OneDrive API reports one size via the JSON
|
||||
// but the content length and file size written to disk is totally different - example:
|
||||
// From JSON: "size": 17133
|
||||
// From HTTPS Server: < Content-Length: 19340
|
||||
// with no logical reason for the difference, except for a 302 redirect before file download
|
||||
log.error("INFO: It is most likely that a SharePoint OneDrive API issue is the root cause. Add --disable-download-validation to work around this issue but downloaded data integrity cannot be guaranteed.");
|
||||
} else {
|
||||
// other account types
|
||||
log.error("INFO: Potentially add --disable-download-validation to work around this issue but downloaded data integrity cannot be guaranteed.");
|
||||
}
|
||||
|
||||
// we do not want this local file to remain on the local file system
|
||||
safeRemove(path);
|
||||
downloadFailed = true;
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// download checks have been disabled
|
||||
log.vdebug("Downloaded file validation disabled due to --disable-download-validation ");
|
||||
}
|
||||
} else {
|
||||
log.error("ERROR: File failed to download. Increase logging verbosity to determine why.");
|
||||
downloadFailed = true;
|
||||
|
|
Loading…
Reference in a new issue