mirror of
https://github.com/abraunegg/onedrive
synced 2024-05-23 16:12:12 +02:00
Add --disable-download-validation (#1686)
* Add --disable-download-validation to allow downloading files from OneDrive where SharePoint and the OneDrive API mis-represents the values in the API as compared to the actual HTTP server response sent to the client. In the API JSON response we get a "size" value of X, but the HTTP Server Content Length reports a size of Y. When this occurs, the download will report as failed. This was seen as part understanding the cause of https://github.com/abraunegg/onedrive/discussions/1667
This commit is contained in:
parent
5d7e4532ef
commit
2901534171
11
src/config.d
11
src/config.d
|
@ -59,6 +59,7 @@ final class Config
|
||||||
boolValues["check_nosync"] = false;
|
boolValues["check_nosync"] = false;
|
||||||
boolValues["download_only"] = false;
|
boolValues["download_only"] = false;
|
||||||
boolValues["disable_notifications"] = false;
|
boolValues["disable_notifications"] = false;
|
||||||
|
boolValues["disable_download_validation"] = false;
|
||||||
boolValues["disable_upload_validation"] = false;
|
boolValues["disable_upload_validation"] = false;
|
||||||
boolValues["enable_logging"] = false;
|
boolValues["enable_logging"] = false;
|
||||||
boolValues["force_http_2"] = false;
|
boolValues["force_http_2"] = false;
|
||||||
|
@ -118,6 +119,9 @@ final class Config
|
||||||
longValues["sync_file_permissions"] = defaultFilePermissionMode;
|
longValues["sync_file_permissions"] = defaultFilePermissionMode;
|
||||||
// Configure download / upload rate limits
|
// Configure download / upload rate limits
|
||||||
longValues["rate_limit"] = 0;
|
longValues["rate_limit"] = 0;
|
||||||
|
// maximum time an operation is allowed to take
|
||||||
|
// This includes dns resolution, connecting, data transfer, etc.
|
||||||
|
longValues["operation_timeout"] = 3600;
|
||||||
|
|
||||||
// Webhook options
|
// Webhook options
|
||||||
boolValues["webhook_enabled"] = false;
|
boolValues["webhook_enabled"] = false;
|
||||||
|
@ -140,10 +144,6 @@ final class Config
|
||||||
// - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging
|
// - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging
|
||||||
boolValues["display_sync_options"] = false;
|
boolValues["display_sync_options"] = false;
|
||||||
|
|
||||||
// maximum time an operation is allowed to take
|
|
||||||
// This includes dns resolution, connecting, data transfer, etc.
|
|
||||||
longValues["operation_timeout"] = 3600;
|
|
||||||
|
|
||||||
// Determine the users home directory.
|
// Determine the users home directory.
|
||||||
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
|
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
|
||||||
// Check for HOME environment variable
|
// Check for HOME environment variable
|
||||||
|
@ -340,6 +340,9 @@ final class Config
|
||||||
"disable-notifications",
|
"disable-notifications",
|
||||||
"Do not use desktop notifications in monitor mode.",
|
"Do not use desktop notifications in monitor mode.",
|
||||||
&boolValues["disable_notifications"],
|
&boolValues["disable_notifications"],
|
||||||
|
"disable-download-validation",
|
||||||
|
"Disable download validation when downloading from OneDrive",
|
||||||
|
&boolValues["disable_download_validation"],
|
||||||
"disable-upload-validation",
|
"disable-upload-validation",
|
||||||
"Disable upload validation when uploading to OneDrive",
|
"Disable upload validation when uploading to OneDrive",
|
||||||
&boolValues["disable_upload_validation"],
|
&boolValues["disable_upload_validation"],
|
||||||
|
|
|
@ -870,6 +870,9 @@ int main(string[] args)
|
||||||
|
|
||||||
// Do we configure to disable the upload validation routine
|
// Do we configure to disable the upload validation routine
|
||||||
if (cfg.getValueBool("disable_upload_validation")) sync.setDisableUploadValidation();
|
if (cfg.getValueBool("disable_upload_validation")) sync.setDisableUploadValidation();
|
||||||
|
|
||||||
|
// Do we configure to disable the download validation routine
|
||||||
|
if (cfg.getValueBool("disable_download_validation")) sync.setDisableDownloadValidation();
|
||||||
|
|
||||||
// Has the user enabled to bypass data preservation of renaming local files when there is a conflict?
|
// Has the user enabled to bypass data preservation of renaming local files when there is a conflict?
|
||||||
if (cfg.getValueBool("bypass_data_preservation")) {
|
if (cfg.getValueBool("bypass_data_preservation")) {
|
||||||
|
|
96
src/sync.d
96
src/sync.d
|
@ -27,6 +27,9 @@ private bool uploadOnly = false;
|
||||||
// Do we configure to disable the upload validation routine
|
// Do we configure to disable the upload validation routine
|
||||||
private bool disableUploadValidation = false;
|
private bool disableUploadValidation = false;
|
||||||
|
|
||||||
|
// Do we configure to disable the download validation routine
|
||||||
|
private bool disableDownloadValidation = false;
|
||||||
|
|
||||||
private bool isItemFolder(const ref JSONValue item)
|
private bool isItemFolder(const ref JSONValue item)
|
||||||
{
|
{
|
||||||
return ("folder" in item) != null;
|
return ("folder" in item) != null;
|
||||||
|
@ -451,6 +454,7 @@ final class SyncEngine
|
||||||
// If account type is documentLibrary - then most likely this is a SharePoint repository
|
// If account type is documentLibrary - then most likely this is a SharePoint repository
|
||||||
// and files 'may' be modified after upload. See: https://github.com/abraunegg/onedrive/issues/205
|
// and files 'may' be modified after upload. See: https://github.com/abraunegg/onedrive/issues/205
|
||||||
if(accountType == "documentLibrary") {
|
if(accountType == "documentLibrary") {
|
||||||
|
// set this flag for SharePoint regardless of --disable-upload-validation being used
|
||||||
setDisableUploadValidation();
|
setDisableUploadValidation();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -556,6 +560,18 @@ final class SyncEngine
|
||||||
log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments");
|
log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Configure disableDownloadValidation if function is called
|
||||||
|
// By default, disableDownloadValidation = false;
|
||||||
|
// Meaning we will always validate our downloads
|
||||||
|
// However, when downloading files from SharePoint, the OneDrive API will not advise the correct file size
|
||||||
|
// which means that the application thinks the file download has failed as the size is different / hash is different
|
||||||
|
// See: https://github.com/abraunegg/onedrive/discussions/1667
|
||||||
|
void setDisableDownloadValidation()
|
||||||
|
{
|
||||||
|
disableDownloadValidation = true;
|
||||||
|
log.vdebug("Flagging to disable download validation checks due to user request");
|
||||||
|
}
|
||||||
|
|
||||||
// Issue #658 Handling
|
// Issue #658 Handling
|
||||||
// If an existing folder is moved into a sync_list valid path (where it previously was out of scope due to sync_list),
|
// If an existing folder is moved into a sync_list valid path (where it previously was out of scope due to sync_list),
|
||||||
// then set this flag to true, so that on the second 'true-up' sync, we force a rescan of the OneDrive path to capture any 'files'
|
// then set this flag to true, so that on the second 'true-up' sync, we force a rescan of the OneDrive path to capture any 'files'
|
||||||
|
@ -2891,36 +2907,62 @@ final class SyncEngine
|
||||||
}
|
}
|
||||||
// file has to have downloaded in order to set the times / data for the file
|
// file has to have downloaded in order to set the times / data for the file
|
||||||
if (exists(path)) {
|
if (exists(path)) {
|
||||||
// A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on
|
// When downloading some files from SharePoint, the OneDrive API reports one file size, but the SharePoint HTTP Server sends a totally different byte count
|
||||||
// does the file hash OneDrive reports match what we have locally?
|
// for the same file
|
||||||
string quickXorHash = computeQuickXorHash(path);
|
// we have implemented --disable-download-validation to disable these checks
|
||||||
string sha1Hash = computeSha1Hash(path);
|
|
||||||
|
|
||||||
if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash) || (OneDriveFileHash == sha1Hash)) {
|
if (!disableDownloadValidation) {
|
||||||
// downloaded matches either size or hash
|
// A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on
|
||||||
log.vdebug("Downloaded file matches reported size and or reported file hash");
|
// does the file hash OneDrive reports match what we have locally?
|
||||||
try {
|
string quickXorHash = computeQuickXorHash(path);
|
||||||
log.vdebug("Calling setTimes() for this file: ", path);
|
string sha1Hash = computeSha1Hash(path);
|
||||||
setTimes(path, item.mtime, item.mtime);
|
|
||||||
} catch (FileException e) {
|
if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash) || (OneDriveFileHash == sha1Hash)) {
|
||||||
// display the error message
|
// downloaded matches either size or hash
|
||||||
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
log.vdebug("Downloaded file matches reported size and or reported file hash");
|
||||||
|
try {
|
||||||
|
log.vdebug("Calling setTimes() for this file: ", path);
|
||||||
|
setTimes(path, item.mtime, item.mtime);
|
||||||
|
} catch (FileException e) {
|
||||||
|
// display the error message
|
||||||
|
displayFileSystemErrorMessage(e.msg, getFunctionName!({}));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// size error?
|
||||||
|
if (getSize(path) != fileSize) {
|
||||||
|
// downloaded file size does not match
|
||||||
|
log.vdebug("File size on disk: ", getSize(path));
|
||||||
|
log.vdebug("OneDrive API reported size: ", fileSize);
|
||||||
|
log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why.");
|
||||||
|
}
|
||||||
|
// hash error?
|
||||||
|
if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) {
|
||||||
|
// downloaded file hash does not match
|
||||||
|
log.vdebug("Actual file hash: ", OneDriveFileHash);
|
||||||
|
log.vdebug("OneDrive API reported hash: ", quickXorHash);
|
||||||
|
log.error("ERROR: File download hash mis-match. Increase logging verbosity to determine why.");
|
||||||
|
}
|
||||||
|
// add some workaround messaging
|
||||||
|
if (accountType == "documentLibrary"){
|
||||||
|
// It has been seen where SharePoint / OneDrive API reports one size via the JSON
|
||||||
|
// but the content length and file size written to disk is totally different - example:
|
||||||
|
// From JSON: "size": 17133
|
||||||
|
// From HTTPS Server: < Content-Length: 19340
|
||||||
|
// with no logical reason for the difference, except for a 302 redirect before file download
|
||||||
|
log.error("INFO: It is most likely that a SharePoint OneDrive API issue is the root cause. Add --disable-download-validation to work around this issue but downloaded data integrity cannot be guaranteed.");
|
||||||
|
} else {
|
||||||
|
// other account types
|
||||||
|
log.error("INFO: Potentially add --disable-download-validation to work around this issue but downloaded data integrity cannot be guaranteed.");
|
||||||
|
}
|
||||||
|
|
||||||
|
// we do not want this local file to remain on the local file system
|
||||||
|
safeRemove(path);
|
||||||
|
downloadFailed = true;
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// size error?
|
// download checks have been disabled
|
||||||
if (getSize(path) != fileSize) {
|
log.vdebug("Downloaded file validation disabled due to --disable-download-validation ");
|
||||||
// downloaded file size does not match
|
|
||||||
log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why.");
|
|
||||||
}
|
|
||||||
// hash error?
|
|
||||||
if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) {
|
|
||||||
// downloaded file hash does not match
|
|
||||||
log.error("ERROR: File download hash mis-match. Increase logging verbosity to determine why.");
|
|
||||||
}
|
|
||||||
// we do not want this local file to remain on the local file system
|
|
||||||
safeRemove(path);
|
|
||||||
downloadFailed = true;
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.error("ERROR: File failed to download. Increase logging verbosity to determine why.");
|
log.error("ERROR: File failed to download. Increase logging verbosity to determine why.");
|
||||||
|
|
Loading…
Reference in a new issue