A platform for community discussion. Free, open, simple.
DEV: Make WASM the only client-side JPEG/PNG decode path (#40955)
Rafael dos Santos Silva22902d023302
diff --git a/config/locales/server.en.yml b/config/locales/server.en.ymlindex 7b3e843be69..281440c155e 100644--- a/config/locales/server.en.yml+++ b/config/locales/server.en.yml@@ -2301,7 +2301,6 @@ en:composer_media_optimization_image_resize_width_target: "Images with widths larger than {{setting:composer_media_optimization_image_resize_dimensions_threshold}} will be resized to this width. Must be >= than {{setting:composer_media_optimization_image_resize_dimensions_threshold}}."composer_media_optimization_image_encode_quality: "JPG encode quality used in the re-encode process."composer_media_optimization_image_convert_enabled: "Enables client-side conversion of JXL, HEIC, and animated GIF images before upload. JXL and HEIC are converted to JPEG, animated GIFs larger than the threshold are converted to animated WEBP."- composer_media_optimization_image_wasm_decode_enabled: "Decode JPEG and PNG images in WASM inside the Web Worker instead of using browser canvas APIs. Avoids iOS canvas pixel limits and createImageBitmap flakiness, at the cost of a larger worker bundle."composer_media_optimization_gif_conversion_threshold: "Minimum GIF file size in bytes to trigger client-side conversion to animated WEBP."video_conversion_enabled: "Enable video conversion for uploaded video files. This allows videos to be converted to web-compatible formats."diff --git a/config/site_settings.yml b/config/site_settings.ymlindex 459f34735de..5b0645ec7cd 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -2639,10 +2639,6 @@ files:learn_more_url: "https://meta.discourse.org/t/-/402705"allow_enabled_for:- everyone- composer_media_optimization_image_wasm_decode_enabled:- default: false- hidden: true- client: truecomposer_media_optimization_gif_conversion_threshold:default: 512000hidden: truediff --git a/frontend/discourse/app/instance-initializers/register-media-optimization-upload-processor.js b/frontend/discourse/app/instance-initializers/register-media-optimization-upload-processor.jsindex 5523cb4cf39..5f84cb15a87 100644--- a/frontend/discourse/app/instance-initializers/register-media-optimization-upload-processor.js+++ b/frontend/discourse/app/instance-initializers/register-media-optimization-upload-processor.js@@ -15,14 +15,6 @@ export default {return;}- // Restrict feature to browsers that support OffscreenCanvas- if (typeof OffscreenCanvas === "undefined") {- return;- }- if (!("createImageBitmap" in self)) {- return;- }-addComposerUploadPreProcessor(UppyMediaOptimization,({ isMobileDevice }) => {diff --git a/frontend/discourse/app/lib/media-optimization-utils.js b/frontend/discourse/app/lib/media-optimization-utils.jsdeleted file mode 100644index e3f426422ff..00000000000--- a/frontend/discourse/app/lib/media-optimization-utils.js+++ /dev/null@@ -1,103 +0,0 @@-async function fileToDrawable(file, isIOS) {- if (!isIOS) {- return await createImageBitmap(file);- } else {- // iOS has performance issues with createImageBitmap on large images- // this workaround partially borrowed from https://github.com/Donaldcwl/browser-image-compression/blob/master/lib/utils.js- const dataUrl = URL.createObjectURL(file);- return await loadImage(dataUrl);- }-}--function drawableToImageData(drawable, isIOS) {- const width = drawable.width,- height = drawable.height,- sx = 0,- sy = 0,- sw = width,- sh = height;-- let canvas = new OffscreenCanvas(width, height);-- // Check if the canvas is too large- // iOS _still_ enforces a max pixel count of 16,777,216 per canvas- const maxLimit = 4096;- const maximumPixelCount = maxLimit * maxLimit;-- if (isIOS && width * height > maximumPixelCount) {- const ratio = Math.min(maxLimit / width, maxLimit / height);-- canvas.width = Math.floor(width * ratio);- canvas.height = Math.floor(height * ratio);- }-- // Draw image onto canvas- const ctx = canvas.getContext("2d");- if (!ctx) {- throw "Could not create canvas context";- }-- ctx.drawImage(drawable, sx, sy, sw, sh, 0, 0, canvas.width, canvas.height);- const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);-- // iOS strikes again, need to clear canvas to free up memory- if (isIOS) {- canvas.width = 1;- canvas.height = 1;- ctx && ctx.clearRect(0, 0, 1, 1);- }-- return imageData;-}--function isTransparent(type, imageData) {- if (!/(\.|\/)(png|webp)$/i.test(type)) {- return false;- }-- for (let i = 0; i < imageData.data.length; i += 4) {- if (imageData.data[i + 3] < 255) {- return true;- }- }-- return false;-}--function jpegDecodeFailure(type, imageData) {- if (!/(\.|\/)jpe?g$/i.test(type)) {- return false;- }-- return imageData.data[3] === 0;-}--export async function fileToImageData(file, isIOS) {- const drawable = await fileToDrawable(file, isIOS);- const imageData = drawableToImageData(drawable, isIOS);-- if (isTransparent(file.type, imageData)) {- throw "Image has transparent pixels, won't convert to JPEG!";- }-- if (jpegDecodeFailure(file.type, imageData)) {- throw "JPEG image has transparent pixel, decode failed!";- }-- return imageData;-}--function loadImage(src) {- return new Promise((resolve, reject) => {- const img = new Image();- img.onload = () => {- URL.revokeObjectURL(src);- resolve(img);- };- img.onerror = (e) => {- URL.revokeObjectURL(src);- reject(e);- };- img.src = src;- });-}diff --git a/frontend/discourse/app/services/media-optimization-worker.js b/frontend/discourse/app/services/media-optimization-worker.jsindex 6fb64f73ef7..52abce4a82b 100644--- a/frontend/discourse/app/services/media-optimization-worker.js+++ b/frontend/discourse/app/services/media-optimization-worker.js@@ -2,7 +2,10 @@ import Service, { service } from "@ember/service";import { Promise } from "rsvp";import workerUrl from "virtual:dynamic-chunk-url:discourse/workers/media-optimization/entrypoint";import { disableImplicitInjections } from "discourse/lib/implicit-injections";-import { fileToImageData } from "discourse/lib/media-optimization-utils";+import {+ authorizedExtensions,+ authorizesAllExtensions,+} from "discourse/lib/uploads";const CONVERT_FORMAT_REGEX = /(\.|\/)(jxl|hei[cf])$/i;const ANIMATED_GIF_REGEX = /(\.|\/)(gif)$/i;@@ -13,8 +16,9 @@ const OPTIMIZABLE_REGEX = /(\.|\/)(jpe?g|png)$/i;** 1. optimizeImage called* 2. worker started if one isn't already running, message handlers registered- * 3. "compress"/"convert" message posted to the worker- * 4. worker posts back the optimized file (or an error), resolving the upload+ * 3. "convert"/"convertAnimated" message posted to the worker+ * 4. worker posts back the optimized file (or a skip notice/error), resolving+ * the upload** The worker is a module worker bundled with its codecs, so it is ready as soon* as it is created (posted messages queue until its module evaluates). A worker@@ -24,8 +28,8 @@ const OPTIMIZABLE_REGEX = /(\.|\/)(jpe?g|png)$/i;@disableImplicitInjectionsexport default class MediaOptimizationWorkerService extends Service {@service appEvents;+ @service currentUser;@service siteSettings;- @service capabilities;worker = null;currentComposerUploadData = null;@@ -45,9 +49,6 @@ export default class MediaOptimizationWorkerService extends Service {const isConvertFormat = CONVERT_FORMAT_REGEX.test(typeOrName);const isAnimatedGif = ANIMATED_GIF_REGEX.test(typeOrName);const isOptimizable = OPTIMIZABLE_REGEX.test(typeOrName);- const wasmDecodeOptimizable =- isOptimizable &&- this.siteSettings.composer_media_optimization_image_wasm_decode_enabled;if (!isConvertFormat && !isAnimatedGif && !isOptimizable) {return Promise.resolve();@@ -117,15 +118,15 @@ export default class MediaOptimizationWorkerService extends Service {debug_mode: this.siteSettings.composer_media_optimization_debug_mode,};- if (isConvertFormat || wasmDecodeOptimizable) {- let arrayBuffer;- try {- arrayBuffer = await file.data.arrayBuffer();- } catch (error) {- this.logIfDebug(error);- return resolve();- }+ let arrayBuffer;+ try {+ arrayBuffer = await file.data.arrayBuffer();+ } catch (error) {+ this.logIfDebug(error);+ return resolve();+ }+ if (isConvertFormat || isOptimizable) {this.worker.postMessage({type: "convert",@@ -138,15 +139,7 @@ export default class MediaOptimizationWorkerService extends Service {},[arrayBuffer]);- } else if (isAnimatedGif) {- let arrayBuffer;- try {- arrayBuffer = await file.data.arrayBuffer();- } catch (error) {- this.logIfDebug(error);- return resolve();- }-+ } else {this.worker.postMessage({type: "convertAnimated",@@ -158,28 +151,6 @@ export default class MediaOptimizationWorkerService extends Service {},[arrayBuffer]);- } else {- let imageData;- try {- imageData = await fileToImageData(file.data, this.capabilities.isIOS);- } catch (error) {- this.logIfDebug(error);- return resolve();- }-- this.worker.postMessage(- {- type: "compress",- fileId: file.id,- file: imageData.data.buffer,- fileName: file.name,- width: imageData.width,- height: imageData.height,- originalFileSize: file.size,- settings,- },- [imageData.data.buffer]- );}this.workerPendingCount++;@@ -240,8 +211,6 @@ export default class MediaOptimizationWorkerService extends Service {switch (outputType) {case "image/jpeg":return baseName + ".jpg";- case "image/png":- return baseName + ".png";case "image/webp":return baseName + ".webp";default:@@ -249,27 +218,45 @@ export default class MediaOptimizationWorkerService extends Service {}}+ _extensionAuthorized(fileName) {+ const extension = fileName.split(".").pop().toLowerCase();+ const staff = this.currentUser?.staff;+ return (+ authorizesAllExtensions(staff, this.siteSettings) ||+ authorizedExtensions(staff, this.siteSettings).includes(extension)+ );+ }+registerMessageHandler() {this.worker.onmessage = (workerMessage) => {switch (workerMessage.data.type) {case "file":- const outputType = workerMessage.data.outputType || "image/jpeg";+ const outputType = workerMessage.data.outputType;const outputFileName = this._renameForOutputType(workerMessage.data.fileName,outputType);- const optimizedFile = new File(- [workerMessage.data.file],- outputFileName,- {- type: outputType,- }- );- this.logIfDebug(- `Finished optimization of ${optimizedFile.name}, new size is ${optimizedFile.size} bytes`- );- this.promiseResolvers[workerMessage.data.fileId](optimizedFile);+ if (this._extensionAuthorized(outputFileName)) {+ const optimizedFile = new File(+ [workerMessage.data.file],+ outputFileName,+ {+ type: outputType,+ }+ );+ this.logIfDebug(+ `Finished optimization of ${optimizedFile.name}, new size is ${optimizedFile.size} bytes`+ );++ this.promiseResolvers[workerMessage.data.fileId](optimizedFile);+ } else {+ this.logIfDebug(+ `Optimizing ${workerMessage.data.fileName} would produce ${outputFileName}, which is not an authorized extension here; keeping the original`+ );++ this.promiseResolvers[workerMessage.data.fileId]();+ }this.workerDoneCount++;this.workerPendingCount--;@@ -293,7 +280,7 @@ export default class MediaOptimizationWorkerService extends Service {break;case "skipped":this.logIfDebug(- `Conversion skipped for ${workerMessage.data.fileName} (output not smaller than original)`+ `Conversion skipped for ${workerMessage.data.fileName}, keeping the original`);this.promiseResolvers[workerMessage.data.fileId]();diff --git a/frontend/discourse/app/workers/media-optimization/codecs.js b/frontend/discourse/app/workers/media-optimization/codecs.jsindex 2370997984f..a2c9f06921e 100644--- a/frontend/discourse/app/workers/media-optimization/codecs.js+++ b/frontend/discourse/app/workers/media-optimization/codecs.js@@ -73,7 +73,35 @@ function hasTransparency(imageData) {return false;}-async function maybeResize(imageData, width, height, settings) {+function isAnimatedPng(fileBuffer) {+ // APNGs declare an acTL chunk before the first IDAT; our PNG decoder only+ // returns the first frame, so these must be detected up front and skipped.+ const view = new DataView(fileBuffer);+ let offset = 8; // PNG signature++ while (offset + 8 <= view.byteLength) {+ const length = view.getUint32(offset);+ const type = String.fromCharCode(+ view.getUint8(offset + 4),+ view.getUint8(offset + 5),+ view.getUint8(offset + 6),+ view.getUint8(offset + 7)+ );++ if (type === "acTL") {+ return true;+ }+ if (type === "IDAT" || type === "IEND") {+ return false;+ }++ offset += 12 + length; // length + type + data + crc+ }++ return false;+}++async function maybeResize(imageData, width, height, settings, transparent) {if (width > settings.resize_threshold) {try {const targetDimensions = resizeWithAspect(@@ -88,7 +116,9 @@ async function maybeResize(imageData, width, height, settings) {premultiply: settings.resize_pre_multiply,linearRGB: settings.resize_linear_rgb,});- if (resizeResult.data[3] !== 255) {+ // An opaque source must stay opaque: a transparent top-left pixel after+ // resizing indicates corruption. Transparent sources can't use this check.+ if (!transparent && resizeResult.data[3] !== 255) {throw "Image corrupted during resize. Falling back to the original for encode";}logIfDebug(@@ -106,58 +136,6 @@ async function maybeResize(imageData, width, height, settings) {}}-export async function optimize(- imageData,- fileName,- width,- height,- originalFileSize,- settings-) {- const mozJpegOptions = buildMozJpegOptions(settings);-- const initialSize = imageData.byteLength;- logIfDebug(- `Received imageData ${initialSize} bytes (raw uncompressed pixels) from original file size of ${originalFileSize} bytes (compressed) for ${fileName}`- );-- const wrappedImageData = new ImageData(- new Uint8ClampedArray(imageData),- width,- height- );- const resized = await maybeResize(wrappedImageData, width, height, settings);- const finalWidth = resized.width;- const finalHeight = resized.height;-- const result = await encodeJpeg(- resized instanceof ImageData- ? resized- : new ImageData(- new Uint8ClampedArray(resized.data),- finalWidth,- finalHeight- ),- mozJpegOptions- );-- const finalSize = result.byteLength;- logIfDebug(- `Post-reencode size for ${fileName} is ${finalSize} bytes (compressed JPEG), original was ${originalFileSize} bytes`- );- const compressionFromOriginal = (originalFileSize / finalSize).toFixed(1);- const compressionFromRaw = (initialSize / finalSize).toFixed(1);- logIfDebug(- `Compressed ${compressionFromOriginal}x vs original file, ${compressionFromRaw}x vs raw pixels for ${fileName}`- );-- if (finalSize < 20000) {- throw "Final size suspiciously small, discarding optimizations";- }-- return result;-}-export async function convert(fileBuffer,fileName,@@ -175,6 +153,12 @@ export async function convert(} else if (/jpe?g$/i.test(fileType)) {imageData = await decodeJpeg(fileBuffer, { preserveOrientation: true });} else if (/png$/i.test(fileType)) {+ if (isAnimatedPng(fileBuffer)) {+ logIfDebug(+ `${fileName} is an animated PNG, skipping conversion to keep the animation`+ );+ return null;+ }imageData = await decodePng(fileBuffer);} else {throw `Unsupported file type for conversion: ${fileType}`;@@ -190,7 +174,8 @@ export async function convert(imageData,imageData.width,imageData.height,- settings+ settings,+ transparent);if (transparent) {@@ -204,6 +189,18 @@ export async function convert(logIfDebug(`Converted ${fileName} from ${originalFileSize} bytes to ${finalSize} bytes WEBP`);++ if (finalSize < 20000) {+ throw "Final size suspiciously small, discarding conversion";+ }++ if (finalSize >= originalFileSize) {+ logIfDebug(+ `WEBP (${finalSize} bytes) is not smaller than the original (${originalFileSize} bytes), skipping conversion`+ );+ return null;+ }+return { data: result, outputType: "image/webp" };}diff --git a/frontend/discourse/app/workers/media-optimization/entrypoint.js b/frontend/discourse/app/workers/media-optimization/entrypoint.jsindex f63a766c126..20f9b75bfe0 100644--- a/frontend/discourse/app/workers/media-optimization/entrypoint.js+++ b/frontend/discourse/app/workers/media-optimization/entrypoint.js@@ -1,42 +1,10 @@// Built by rolldown as a standalone chunk and started as a module worker via a// blob bootstrap (see the media-optimization-worker service), so it inherits the// host document CSP. The codecs are bundled straight into this chunk.-import { convert, convertAnimated, optimize } from "./codecs.js";+import { convert, convertAnimated } from "./codecs.js";self.onmessage = async function (e) {switch (e.data.type) {- case "compress":- try {- globalThis.debugMode = e.data.settings.debug_mode;-- let optimized = await optimize(- e.data.file,- e.data.fileName,- e.data.width,- e.data.height,- e.data.originalFileSize,- e.data.settings- );- postMessage(- {- type: "file",- file: optimized,- fileName: e.data.fileName,- fileId: e.data.fileId,- },- [optimized]- );- } catch (error) {- // eslint-disable-next-line no-console- console.error(error);- postMessage({- type: "error",- file: e.data.file,- fileName: e.data.fileName,- fileId: e.data.fileId,- });- }- break;case "convert":try {globalThis.debugMode = e.data.settings.debug_mode;@@ -48,16 +16,24 @@ self.onmessage = async function (e) {e.data.originalFileSize,e.data.settings);- postMessage(- {- type: "file",- file: converted.data,+ if (converted) {+ postMessage(+ {+ type: "file",+ file: converted.data,+ fileName: e.data.fileName,+ fileId: e.data.fileId,+ outputType: converted.outputType,+ },+ [converted.data]+ );+ } else {+ postMessage({+ type: "skipped",fileName: e.data.fileName,fileId: e.data.fileId,- outputType: converted.outputType,- },- [converted.data]- );+ });+ }} catch (error) {// eslint-disable-next-line no-consoleconsole.error(error);diff --git a/spec/system/composer_image_optimization_spec.rb b/spec/system/composer_image_optimization_spec.rbindex 66b82d0d9cd..a94f8385953 100644--- a/spec/system/composer_image_optimization_spec.rb+++ b/spec/system/composer_image_optimization_spec.rb@@ -79,8 +79,9 @@ describe "Composer image optimization for uploads using media-optimization-workefile = file_from_fixtures("large_and_unoptimized.png", "images")converted_file_path = file.path.gsub("unoptimized", "notransparent")- # We do a conversion here to simulate a PNG with no transparency,- # the fixture has transparency, and PNGs with transparency are not optimized.+ # We do a conversion here to simulate a PNG with no transparency, the+ # fixture has transparency and transparent PNGs are converted to WEBP+ # rather than JPEG (see the transparent PNG test below).Discourse::Utils.execute_command(*["convert", file.path, "-background", "white", "-alpha", "remove", converted_file_path],timeout: 3,@@ -95,12 +96,31 @@ describe "Composer image optimization for uploads using media-optimization-workeUpload.find_by(original_filename: File.basename(converted_file_path).gsub("png", "jpg"))# Original large_and_unoptimized PNG with no transparency is 312_889 bytes- expect(upload.filesize).to eq(285_809)+ expect(upload.filesize).to eq(271_246)ensureFileUtils.rm(converted_file_path)end- it "does not optimize a png with transparent pixels in the composer" do+ it "converts a png with transparent pixels to webp in the composer" do+ visit "/new-topic"+ expect(composer).to be_opened++ file = file_from_fixtures("large_and_unoptimized.png", "images")+ attach_file("file-uploader", [file.path], make_visible: true)++ expect(composer).to have_no_in_progress_uploads+ expect(composer.preview).to have_css(".image-wrapper", count: 1)++ upload = Upload.find_by(original_filename: File.basename(file.path).gsub("png", "webp"))++ # Original large_and_unoptimized.png is 421_730 bytes and 2032px wide;+ # transparent PNGs are resized and re-encoded as WEBP instead of JPEG.+ expect(upload.filesize).to eq(126_626)+ end++ it "keeps the original transparent png when webp is not an authorized extension" do+ SiteSetting.authorized_extensions = "jpg|jpeg|png|gif"+visit "/new-topic"expect(composer).to be_opened@@ -112,8 +132,39 @@ describe "Composer image optimization for uploads using media-optimization-workeupload = Upload.find_by(original_filename: File.basename(file.path))- # Original large_and_unoptimized.png is 421_730 bytes+ # Original large_and_unoptimized.png is 421_730 bytes, uploaded untouched+ # because converting it to WEBP would produce an unauthorized extension.expect(upload.filesize).to eq(421_730)end++ it "keeps an animated png untouched in the composer" do+ visit "/new-topic"+ expect(composer).to be_opened++ file = file_from_fixtures("large_and_unoptimized.png", "images")+ apng_path = file.path.gsub("unoptimized", "animated")++ # Emulate an APNG by inserting an acTL chunk right after IHDR; the+ # optimizer's decoder would only keep the first frame, so it must skip+ # animated PNGs entirely.+ png = File.binread(file.path)+ actl_data = [2, 0].pack("N2")+ actl_chunk =+ [actl_data.bytesize].pack("N") + "acTL" + actl_data ++ [Zlib.crc32("acTL" + actl_data)].pack("N")+ ihdr_end = 8 + 4 + 4 + 13 + 4 # signature + IHDR length/type/data/crc+ File.binwrite(apng_path, png[0...ihdr_end] + actl_chunk + png[ihdr_end..])++ attach_file("file-uploader", [apng_path], make_visible: true)++ expect(composer).to have_no_in_progress_uploads+ expect(composer.preview).to have_css(".image-wrapper", count: 1)++ upload = Upload.find_by(original_filename: File.basename(apng_path))++ expect(upload.filesize).to eq(File.size(apng_path))+ ensure+ FileUtils.rm(apng_path)+ endendend