From 3a6ac4e5ecffdd6a133b8546b00d6c844098b2fc Mon Sep 17 00:00:00 2001 From: Nachtalb Date: Thu, 1 Apr 2021 16:46:45 +0200 Subject: [PATCH 1/5] Support animated webp images We need to remove the metadata without breaking the animation. For that we use sharp which incooperates libvips (binaries for most common distros included). This also decreases client side upload complexity as we remove the metadata on the serverside. Sharp: https://sharp.pixelplumbing.com/ libvips: https://libvips.github.io/libvips/ --- client/js/upload.js | 41 +--------- package.json | 1 + src/plugins/uploader.js | 30 +++++++- webpack.config.js | 1 + yarn.lock | 166 +++++++++++++++++++++++++++++++++++++--- 5 files changed, 186 insertions(+), 53 deletions(-) diff --git a/client/js/upload.js b/client/js/upload.js index 3da21a44..a651eeff 100644 --- a/client/js/upload.js +++ b/client/js/upload.js @@ -137,46 +137,7 @@ class Uploader { // This issue only happens if The Lounge is proxied through other software // as it may buffer the upload before the upload request will be processed by The Lounge. this.tokenKeepAlive = setInterval(() => socket.emit("upload:ping", token), 40 * 1000); - - if ( - store.state.settings.uploadCanvas && - file.type.startsWith("image/") && - !file.type.includes("svg") && - file.type !== "image/gif" - ) { - this.renderImage(file, (newFile) => this.performUpload(token, newFile)); - } else { - this.performUpload(token, file); - } - } - - renderImage(file, callback) { - const fileReader = new FileReader(); - - fileReader.onabort = () => callback(file); - fileReader.onerror = () => fileReader.abort(); - - fileReader.onload = () => { - const img = new Image(); - - img.onerror = () => callback(file); - - img.onload = () => { - const canvas = document.createElement("canvas"); - canvas.width = img.width; - canvas.height = img.height; - const ctx = canvas.getContext("2d"); - ctx.drawImage(img, 0, 0); - - canvas.toBlob((blob) => { - callback(new File([blob], file.name)); - }, file.type); - }; - - img.src = fileReader.result; - }; - - fileReader.readAsDataURL(file); + this.performUpload(token, file); } performUpload(token, file) { diff --git a/package.json b/package.json index c3d297b0..06ebe332 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "read": "1.0.7", "read-chunk": "3.2.0", "semver": "7.3.4", + "sharp": "0.28.0", "socket.io": "3.1.2", "tlds": "1.216.0", "ua-parser-js": "0.7.23", diff --git a/src/plugins/uploader.js b/src/plugins/uploader.js index 3b5dd9ca..cd616380 100644 --- a/src/plugins/uploader.js +++ b/src/plugins/uploader.js @@ -11,6 +11,7 @@ const crypto = require("crypto"); const isUtf8 = require("is-utf8"); const log = require("../log"); const contentDisposition = require("content-disposition"); +const sharp = require("sharp"); // Map of allowed mime types to their respecive default filenames // that will be rendered in browser without forcing them to be downloaded @@ -217,7 +218,7 @@ class Uploader { streamWriter = fs.createWriteStream(destPath); streamWriter.on("error", abortWithError); - busboyInstance.on("file", (fieldname, fileStream, filename) => { + busboyInstance.on("file", (fieldname, fileStream, filename, encoding, contentType) => { uploadUrl = `${randomName}/${encodeURIComponent(filename)}`; if (Helper.config.fileUpload.baseUrl) { @@ -226,18 +227,39 @@ class Uploader { uploadUrl = `uploads/${uploadUrl}`; } + // Sharps prebuilt libvips does not include gif support, but that is not a problem, + // as GIFs don't support EXIF metadata or anything alike + const isImage = contentType.startsWith("image/") && !contentType.endsWith("gif"); + // if the busboy data stream errors out or goes over the file size limit // abort the processing with an error fileStream.on("error", abortWithError); fileStream.on("limit", () => { - fileStream.unpipe(streamWriter); + if (!isImage) { + fileStream.unpipe(streamWriter); + } + fileStream.on("readable", fileStream.read.bind(fileStream)); abortWithError(Error("File size limit reached")); }); - // Attempt to write the stream to file - fileStream.pipe(streamWriter); + if (isImage) { + const chunks = []; + fileStream + .on("data", (chunk) => { + chunks.push(chunk); + }) + .on("end", () => { + const buffer = Buffer.concat(chunks); + sharp(buffer, {animated: true, pages: -1, sequentialRead: true}) + .toFile(destPath) // Removes metadata by default https://sharp.pixelplumbing.com/api-output#tofile + .catch(abortWithError); + }); + } else { + // Attempt to write the stream to file + fileStream.pipe(streamWriter); + } }); busboyInstance.on("finish", () => { diff --git a/webpack.config.js b/webpack.config.js index 4475e52d..e9ed55e1 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -85,6 +85,7 @@ const config = { }, externals: { json3: "JSON", // socket.io uses json3.js, but we do not target any browsers that need it + sharp: "commonjs sharp", }, plugins: [ new VueLoaderPlugin(), diff --git a/yarn.lock b/yarn.lock index 53a32848..bd51c311 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1894,6 +1894,11 @@ base64-arraybuffer@0.1.4: resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-0.1.4.tgz#9818c79e059b1355f97e0428a017c838e90ba812" integrity sha1-mBjHngWbE1X5fgQooBfIOOkLqBI= +base64-js@^1.3.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + base64id@2.0.0, base64id@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/base64id/-/base64id-2.0.0.tgz#2770ac6bc47d312af97a8bf9a634342e0cd25cb6" @@ -1921,6 +1926,15 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== +bl@^4.0.3: + version "4.1.0" + resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" + integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w== + dependencies: + buffer "^5.5.0" + inherits "^2.0.4" + readable-stream "^3.4.0" + block-stream@*: version "0.0.9" resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a" @@ -2000,6 +2014,14 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== +buffer@^5.5.0: + version "5.7.1" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" + integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.1.13" + busboy@0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/busboy/-/busboy-0.3.1.tgz#170899274c5bf38aae27d5c62b71268cd585fd1b" @@ -2340,7 +2362,7 @@ color-string@^1.5.4: color-name "^1.0.0" simple-swizzle "^0.2.2" -color@^3.0.0: +color@^3.0.0, color@^3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/color/-/color-3.1.3.tgz#ca67fb4e7b97d611dcde39eceed422067d91596e" integrity sha512-xgXAcTHa2HeFCGLE9Xs/R82hujGtu9Jd9x4NW3T34+OMs7VoPsjwzRczKHvTAHeJwWFwX5j15+MgAppE8ztObQ== @@ -2804,6 +2826,13 @@ decompress-response@^3.3.0: dependencies: mimic-response "^1.0.0" +decompress-response@^4.2.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-4.2.1.tgz#414023cc7a302da25ce2ec82d0d5238ccafd8986" + integrity sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw== + dependencies: + mimic-response "^2.0.0" + decompress-response@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc" @@ -2872,7 +2901,7 @@ destroy@~1.0.4: resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA= -detect-libc@^1.0.2: +detect-libc@^1.0.2, detect-libc@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" integrity sha1-+hN8S9aY7fVc1c0CrFWfkaTEups= @@ -3038,7 +3067,7 @@ encodeurl@~1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= -end-of-stream@^1.1.0: +end-of-stream@^1.1.0, end-of-stream@^1.4.1: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== @@ -3391,6 +3420,11 @@ execall@^2.0.0: dependencies: clone-regexp "^2.1.0" +expand-template@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" + integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== + express@4.17.1: version "4.17.1" resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134" @@ -3650,6 +3684,11 @@ fromentries@^1.2.0: resolved "https://registry.yarnpkg.com/fromentries/-/fromentries-1.3.2.tgz#e4bca6808816bf8f93b52750f1127f5a6fd86e3a" integrity sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg== +fs-constants@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" + integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== + fs-minipass@^1.2.5: version "1.2.7" resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-1.2.7.tgz#ccff8570841e7fe4265693da88936c55aed7f7c7" @@ -3776,6 +3815,11 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" +github-from-package@0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" + integrity sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4= + glob-parent@^5.0.0, glob-parent@^5.1.0, glob-parent@^5.1.1, glob-parent@~5.1.0: version "5.1.1" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.1.tgz#b6c1ef417c4e5663ea498f1c45afac6916bbc229" @@ -4156,7 +4200,7 @@ icss-utils@^5.0.0, icss-utils@^5.1.0: resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae" integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA== -ieee754@^1.2.1: +ieee754@^1.1.13, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== @@ -4230,7 +4274,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.0, inherits@~2.0.3: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -5141,6 +5185,11 @@ mimic-response@^1.0.0, mimic-response@^1.0.1: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ== +mimic-response@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-2.1.0.tgz#d13763d35f613d09ec37ebb30bac0469c0ee8f43" + integrity sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA== + mimic-response@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" @@ -5181,7 +5230,7 @@ minimist-options@4.1.0: is-plain-obj "^1.1.0" kind-of "^6.0.3" -minimist@^1.2.0, minimist@^1.2.5: +minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5: version "1.2.5" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== @@ -5201,6 +5250,11 @@ minizlib@^1.2.1: dependencies: minipass "^2.9.0" +mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: + version "0.5.3" + resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" + integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== + "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1: version "0.5.5" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" @@ -5300,6 +5354,11 @@ nanoid@^3.1.20: resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.20.tgz#badc263c6b1dcf14b71efaa85f6ab4c1d6cfc788" integrity sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw== +napi-build-utils@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806" + integrity sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg== + natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" @@ -5340,7 +5399,14 @@ nise@^4.0.4: just-extend "^4.0.2" path-to-regexp "^1.7.0" -node-addon-api@^3.0.0: +node-abi@^2.7.0: + version "2.21.0" + resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.21.0.tgz#c2dc9ebad6f4f53d6ea9b531e7b8faad81041d48" + integrity sha512-smhrivuPqEM3H5LmnY3KU6HfYv0u4QklgAxfFyRNujKUzbUcYZ+Jc2EhukB9SRcD2VpqhxM7n/MIcp1Ua1/JMg== + dependencies: + semver "^5.4.1" + +node-addon-api@^3.0.0, node-addon-api@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.1.0.tgz#98b21931557466c6729e51cb77cd39c965f42239" integrity sha512-flmrDNB06LIl5lywUz7YlNGZH/5p0M7W28k8hzd9Lshtdh1wshD2Y+U4h9LD6KObOy1f+fEVdgprPrEymjM5uw== @@ -5396,6 +5462,11 @@ node-releases@^1.1.70: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.70.tgz#66e0ed0273aa65666d7fe78febe7634875426a08" integrity sha512-Slf2s69+2/uAD79pVVQo8uSiC34+g8GWY8UH2Qtqv34ZfhYrxpYpfzs9Js9d6O0mbDmALuxaTlplnBTnSELcrw== +noop-logger@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/noop-logger/-/noop-logger-0.1.1.tgz#94a2b1633c4f1317553007d8966fd0e841b6a4c2" + integrity sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI= + "nopt@2 || 3": version "3.0.6" resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9" @@ -5511,7 +5582,7 @@ npm-run-path@^4.0.0, npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" -"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.2: +"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.1, npmlog@^4.0.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg== @@ -6632,6 +6703,27 @@ postcss@^8.2.6: nanoid "^3.1.20" source-map "^0.6.1" +prebuild-install@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-6.0.1.tgz#5902172f7a40eb67305b96c2a695db32636ee26d" + integrity sha512-7GOJrLuow8yeiyv75rmvZyeMGzl8mdEX5gY69d6a6bHWmiPevwqFw+tQavhK0EYMaSg3/KD24cWqeQv1EWsqDQ== + dependencies: + detect-libc "^1.0.3" + expand-template "^2.0.3" + github-from-package "0.0.0" + minimist "^1.2.3" + mkdirp-classic "^0.5.3" + napi-build-utils "^1.0.1" + node-abi "^2.7.0" + noop-logger "^0.1.1" + npmlog "^4.0.1" + pump "^3.0.0" + rc "^1.2.7" + simple-get "^3.0.3" + tar-fs "^2.0.0" + tunnel-agent "^0.6.0" + which-pm-runs "^1.0.0" + precond@0.2: version "0.2.3" resolved "https://registry.yarnpkg.com/precond/-/precond-0.2.3.tgz#aa9591bcaa24923f1e0f4849d240f47efc1075ac" @@ -6868,7 +6960,7 @@ readable-stream@^2.0.6: string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@^3.1.1, readable-stream@^3.6.0: +readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== @@ -7211,6 +7303,13 @@ semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@^7.3.5: + version "7.3.5" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" + integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ== + dependencies: + lru-cache "^6.0.0" + semver@~5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" @@ -7276,6 +7375,20 @@ shallow-clone@^3.0.0: dependencies: kind-of "^6.0.2" +sharp@0.28.0: + version "0.28.0" + resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.28.0.tgz#93297cec530b3709e11677cf41565d9a654075a0" + integrity sha512-kGTaWLNMCkLYxkH2Pv7s+5LQBnWQ4mRKXs1XD19AWOxShWvU8b78qaWqTR/4ryNcPORO+qBoBnFF/Lzda5HgkQ== + dependencies: + color "^3.1.3" + detect-libc "^1.0.3" + node-addon-api "^3.1.0" + prebuild-install "^6.0.1" + semver "^7.3.5" + simple-get "^3.1.0" + tar-fs "^2.1.1" + tunnel-agent "^0.6.0" + shebang-command@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" @@ -7315,6 +7428,20 @@ signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3: resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA== +simple-concat@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.1.tgz#f46976082ba35c2263f1c8ab5edfe26c41c9552f" + integrity sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q== + +simple-get@^3.0.3, simple-get@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-3.1.0.tgz#b45be062435e50d159540b576202ceec40b9c6b3" + integrity sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA== + dependencies: + decompress-response "^4.2.0" + once "^1.3.1" + simple-concat "^1.0.0" + simple-swizzle@^0.2.2: version "0.2.2" resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz#a4da6b635ffcccca33f70d17cb92592de95e557a" @@ -7825,6 +7952,27 @@ tapable@^2.1.1, tapable@^2.2.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.0.tgz#5c373d281d9c672848213d0e037d1c4165ab426b" integrity sha512-FBk4IesMV1rBxX2tfiK8RAmogtWn53puLOQlvO8XuwlgxcYbP4mVPS9Ph4aeamSyyVjOl24aYWAuc8U5kCVwMw== +tar-fs@^2.0.0, tar-fs@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" + integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== + dependencies: + chownr "^1.1.1" + mkdirp-classic "^0.5.2" + pump "^3.0.0" + tar-stream "^2.1.4" + +tar-stream@^2.1.4: + version "2.2.0" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" + integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== + dependencies: + bl "^4.0.3" + end-of-stream "^1.4.1" + fs-constants "^1.0.0" + inherits "^2.0.3" + readable-stream "^3.1.1" + tar@^2.0.0: version "2.2.2" resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.2.tgz#0ca8848562c7299b8b446ff6a4d60cdbb23edc40" From 846da41b01e5532c19bdda29c9896012f6b77fdd Mon Sep 17 00:00:00 2001 From: Nachtalb Date: Thu, 1 Apr 2021 18:23:54 +0200 Subject: [PATCH 2/5] Rename uploadCanvas to a more appropriate removeImageMetadata Including backwards compatibility --- client/components/Windows/Settings.vue | 13 ++++++------- client/js/settings.js | 2 +- client/js/store-settings.js | 9 +++++++++ src/client.js | 5 +++++ 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/client/components/Windows/Settings.vue b/client/components/Windows/Settings.vue index 81bec2dd..9f2cfdbd 100644 --- a/client/components/Windows/Settings.vue +++ b/client/components/Windows/Settings.vue @@ -242,15 +242,14 @@
diff --git a/src/plugins/uploader.js b/src/plugins/uploader.js index cd616380..a878ebdd 100644 --- a/src/plugins/uploader.js +++ b/src/plugins/uploader.js @@ -251,8 +251,12 @@ class Uploader { chunks.push(chunk); }) .on("end", () => { - const buffer = Buffer.concat(chunks); - sharp(buffer, {animated: true, pages: -1, sequentialRead: true}) + sharp(Buffer.concat(chunks), { + animated: true, + pages: -1, + sequentialRead: true, + }) + .rotate() // auto-orient based on the EXIF Orientation tag .toFile(destPath) // Removes metadata by default https://sharp.pixelplumbing.com/api-output#tofile .catch(abortWithError); }); From 89390b3fc5270b3db178fd7daf3e4fb245895dba Mon Sep 17 00:00:00 2001 From: Nachtalb Date: Tue, 13 Apr 2021 20:39:45 +0200 Subject: [PATCH 4/5] Ensure proper error handling when processing of file fails --- src/plugins/uploader.js | 66 ++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/src/plugins/uploader.js b/src/plugins/uploader.js index a878ebdd..61555619 100644 --- a/src/plugins/uploader.js +++ b/src/plugins/uploader.js @@ -148,6 +148,19 @@ class Uploader { } }; + const successfullCompletion = () => { + doneCallback(); + + if (!uploadUrl) { + return res.status(400).json({error: "Missing file"}); + } + + // upload was done, send the generated file url to the client + res.status(200).json({ + url: uploadUrl, + }); + }; + const abortWithError = (err) => { doneCallback(); @@ -214,10 +227,6 @@ class Uploader { return abortWithError(err); } - // Open a file stream for writing - streamWriter = fs.createWriteStream(destPath); - streamWriter.on("error", abortWithError); - busboyInstance.on("file", (fieldname, fileStream, filename, encoding, contentType) => { uploadUrl = `${randomName}/${encodeURIComponent(filename)}`; @@ -245,40 +254,35 @@ class Uploader { }); if (isImage) { - const chunks = []; - fileStream - .on("data", (chunk) => { - chunks.push(chunk); - }) - .on("end", () => { - sharp(Buffer.concat(chunks), { - animated: true, - pages: -1, - sequentialRead: true, - }) - .rotate() // auto-orient based on the EXIF Orientation tag - .toFile(destPath) // Removes metadata by default https://sharp.pixelplumbing.com/api-output#tofile - .catch(abortWithError); + let sharpInstance = sharp({ + animated: true, + pages: -1, + sequentialRead: true, + }); + + sharpInstance + .rotate() // auto-orient based on the EXIF Orientation tag + .toFile(destPath, (err) => { + // Removes metadata by default https://sharp.pixelplumbing.com/api-output#tofile if no `withMetadata` is present + if (err) { + abortWithError(err); + } else { + successfullCompletion(); + } }); + + fileStream.pipe(sharpInstance); } else { + // Open a file stream for writing + streamWriter = fs.createWriteStream(destPath); + streamWriter.on("error", abortWithError); + streamWriter.on("finish", successfullCompletion); + // Attempt to write the stream to file fileStream.pipe(streamWriter); } }); - busboyInstance.on("finish", () => { - doneCallback(); - - if (!uploadUrl) { - return res.status(400).json({error: "Missing file"}); - } - - // upload was done, send the generated file url to the client - res.status(200).json({ - url: uploadUrl, - }); - }); - // pipe request body to busboy for processing return req.pipe(busboyInstance); } From e010fe47cc2dfeb30408801f0bacf72515d88b9f Mon Sep 17 00:00:00 2001 From: Nachtalb Date: Tue, 13 Apr 2021 20:45:16 +0200 Subject: [PATCH 5/5] Respect metadata removal switch --- client/js/upload.js | 1 + src/plugins/uploader.js | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/client/js/upload.js b/client/js/upload.js index a651eeff..faec57e6 100644 --- a/client/js/upload.js +++ b/client/js/upload.js @@ -180,6 +180,7 @@ class Uploader { }; const formData = new FormData(); + formData.append("removeMetadata", store.state.settings.removeImageMetadata); formData.append("file", file); this.xhr.open("POST", `uploads/new/${token}`); this.xhr.send(formData); diff --git a/src/plugins/uploader.js b/src/plugins/uploader.js index 61555619..249aa579 100644 --- a/src/plugins/uploader.js +++ b/src/plugins/uploader.js @@ -130,6 +130,7 @@ class Uploader { let destDir; let destPath; let streamWriter; + let removeMetadata; const doneCallback = () => { // detach the stream and drain any remaining data @@ -207,6 +208,11 @@ class Uploader { busboyInstance.on("partsLimit", () => abortWithError(Error("Parts limit reached"))); busboyInstance.on("filesLimit", () => abortWithError(Error("Files limit reached"))); busboyInstance.on("fieldsLimit", () => abortWithError(Error("Fields limit reached"))); + busboyInstance.on("field", (fieldname, val) => { + if (fieldname === "removeMetadata") { + removeMetadata = val === "true"; + } + }); // generate a random output filename for the file // we use do/while loop to prevent the rare case of generating a file name @@ -260,6 +266,10 @@ class Uploader { sequentialRead: true, }); + if (!removeMetadata) { + sharpInstance = sharpInstance.withMetadata(); + } + sharpInstance .rotate() // auto-orient based on the EXIF Orientation tag .toFile(destPath, (err) => {