diff --git a/src/config.d b/src/config.d index 2972aa19..40d51527 100644 --- a/src/config.d +++ b/src/config.d @@ -59,6 +59,7 @@ final class Config boolValues["check_nosync"] = false; boolValues["download_only"] = false; boolValues["disable_notifications"] = false; + boolValues["disable_download_validation"] = false; boolValues["disable_upload_validation"] = false; boolValues["enable_logging"] = false; boolValues["force_http_2"] = false; @@ -118,6 +119,9 @@ final class Config longValues["sync_file_permissions"] = defaultFilePermissionMode; // Configure download / upload rate limits longValues["rate_limit"] = 0; + // maximum time an operation is allowed to take + // This includes dns resolution, connecting, data transfer, etc. + longValues["operation_timeout"] = 3600; // Webhook options boolValues["webhook_enabled"] = false; @@ -140,10 +144,6 @@ final class Config // - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging boolValues["display_sync_options"] = false; - // maximum time an operation is allowed to take - // This includes dns resolution, connecting, data transfer, etc. - longValues["operation_timeout"] = 3600; - // Determine the users home directory. // Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts // Check for HOME environment variable @@ -340,6 +340,9 @@ final class Config "disable-notifications", "Do not use desktop notifications in monitor mode.", &boolValues["disable_notifications"], + "disable-download-validation", + "Disable download validation when downloading from OneDrive", + &boolValues["disable_download_validation"], "disable-upload-validation", "Disable upload validation when uploading to OneDrive", &boolValues["disable_upload_validation"], diff --git a/src/main.d b/src/main.d index bd0740b9..827b7697 100644 --- a/src/main.d +++ b/src/main.d @@ -870,6 +870,9 @@ int main(string[] args) // Do we configure to disable the upload validation routine if (cfg.getValueBool("disable_upload_validation")) sync.setDisableUploadValidation(); + + // Do we configure to disable the download validation routine + if (cfg.getValueBool("disable_download_validation")) sync.setDisableDownloadValidation(); // Has the user enabled to bypass data preservation of renaming local files when there is a conflict? if (cfg.getValueBool("bypass_data_preservation")) { diff --git a/src/sync.d b/src/sync.d index 95e293a9..54c9df25 100644 --- a/src/sync.d +++ b/src/sync.d @@ -27,6 +27,9 @@ private bool uploadOnly = false; // Do we configure to disable the upload validation routine private bool disableUploadValidation = false; +// Do we configure to disable the download validation routine +private bool disableDownloadValidation = false; + private bool isItemFolder(const ref JSONValue item) { return ("folder" in item) != null; @@ -451,6 +454,7 @@ final class SyncEngine // If account type is documentLibrary - then most likely this is a SharePoint repository // and files 'may' be modified after upload. See: https://github.com/abraunegg/onedrive/issues/205 if(accountType == "documentLibrary") { + // set this flag for SharePoint regardless of --disable-upload-validation being used setDisableUploadValidation(); } @@ -556,6 +560,18 @@ final class SyncEngine log.vdebug("documentLibrary account type - flagging to disable upload validation checks due to Microsoft SharePoint file modification enrichments"); } + // Configure disableDownloadValidation if function is called + // By default, disableDownloadValidation = false; + // Meaning we will always validate our downloads + // However, when downloading files from SharePoint, the OneDrive API will not advise the correct file size + // which means that the application thinks the file download has failed as the size is different / hash is different + // See: https://github.com/abraunegg/onedrive/discussions/1667 + void setDisableDownloadValidation() + { + disableDownloadValidation = true; + log.vdebug("Flagging to disable download validation checks due to user request"); + } + // Issue #658 Handling // If an existing folder is moved into a sync_list valid path (where it previously was out of scope due to sync_list), // then set this flag to true, so that on the second 'true-up' sync, we force a rescan of the OneDrive path to capture any 'files' @@ -2891,36 +2907,62 @@ final class SyncEngine } // file has to have downloaded in order to set the times / data for the file if (exists(path)) { - // A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on - // does the file hash OneDrive reports match what we have locally? - string quickXorHash = computeQuickXorHash(path); - string sha1Hash = computeSha1Hash(path); + // When downloading some files from SharePoint, the OneDrive API reports one file size, but the SharePoint HTTP Server sends a totally different byte count + // for the same file + // we have implemented --disable-download-validation to disable these checks - if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash) || (OneDriveFileHash == sha1Hash)) { - // downloaded matches either size or hash - log.vdebug("Downloaded file matches reported size and or reported file hash"); - try { - log.vdebug("Calling setTimes() for this file: ", path); - setTimes(path, item.mtime, item.mtime); - } catch (FileException e) { - // display the error message - displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + if (!disableDownloadValidation) { + // A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on + // does the file hash OneDrive reports match what we have locally? + string quickXorHash = computeQuickXorHash(path); + string sha1Hash = computeSha1Hash(path); + + if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash) || (OneDriveFileHash == sha1Hash)) { + // downloaded matches either size or hash + log.vdebug("Downloaded file matches reported size and or reported file hash"); + try { + log.vdebug("Calling setTimes() for this file: ", path); + setTimes(path, item.mtime, item.mtime); + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + } + } else { + // size error? + if (getSize(path) != fileSize) { + // downloaded file size does not match + log.vdebug("File size on disk: ", getSize(path)); + log.vdebug("OneDrive API reported size: ", fileSize); + log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why."); + } + // hash error? + if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) { + // downloaded file hash does not match + log.vdebug("Actual file hash: ", OneDriveFileHash); + log.vdebug("OneDrive API reported hash: ", quickXorHash); + log.error("ERROR: File download hash mis-match. Increase logging verbosity to determine why."); + } + // add some workaround messaging + if (accountType == "documentLibrary"){ + // It has been seen where SharePoint / OneDrive API reports one size via the JSON + // but the content length and file size written to disk is totally different - example: + // From JSON: "size": 17133 + // From HTTPS Server: < Content-Length: 19340 + // with no logical reason for the difference, except for a 302 redirect before file download + log.error("INFO: It is most likely that a SharePoint OneDrive API issue is the root cause. Add --disable-download-validation to work around this issue but downloaded data integrity cannot be guaranteed."); + } else { + // other account types + log.error("INFO: Potentially add --disable-download-validation to work around this issue but downloaded data integrity cannot be guaranteed."); + } + + // we do not want this local file to remain on the local file system + safeRemove(path); + downloadFailed = true; + return; } } else { - // size error? - if (getSize(path) != fileSize) { - // downloaded file size does not match - log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why."); - } - // hash error? - if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) { - // downloaded file hash does not match - log.error("ERROR: File download hash mis-match. Increase logging verbosity to determine why."); - } - // we do not want this local file to remain on the local file system - safeRemove(path); - downloadFailed = true; - return; + // download checks have been disabled + log.vdebug("Downloaded file validation disabled due to --disable-download-validation "); } } else { log.error("ERROR: File failed to download. Increase logging verbosity to determine why.");