Skip to content

A platform for community discussion. Free, open, simple.

DEV: Make WASM the only client-side JPEG/PNG decode path (#40955)

Rafael dos Santos Silva22902d023302
diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index 7b3e843be69..281440c155e 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -2301,7 +2301,6 @@ en:
composer_media_optimization_image_resize_width_target: "Images with widths larger than {{setting:composer_media_optimization_image_resize_dimensions_threshold}} will be resized to this width. Must be >= than {{setting:composer_media_optimization_image_resize_dimensions_threshold}}."
composer_media_optimization_image_encode_quality: "JPG encode quality used in the re-encode process."
composer_media_optimization_image_convert_enabled: "Enables client-side conversion of JXL, HEIC, and animated GIF images before upload. JXL and HEIC are converted to JPEG, animated GIFs larger than the threshold are converted to animated WEBP."
- composer_media_optimization_image_wasm_decode_enabled: "Decode JPEG and PNG images in WASM inside the Web Worker instead of using browser canvas APIs. Avoids iOS canvas pixel limits and createImageBitmap flakiness, at the cost of a larger worker bundle."
composer_media_optimization_gif_conversion_threshold: "Minimum GIF file size in bytes to trigger client-side conversion to animated WEBP."
video_conversion_enabled: "Enable video conversion for uploaded video files. This allows videos to be converted to web-compatible formats."
diff --git a/config/site_settings.yml b/config/site_settings.yml
index 459f34735de..5b0645ec7cd 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -2639,10 +2639,6 @@ files:
learn_more_url: "https://meta.discourse.org/t/-/402705"
allow_enabled_for:
- everyone
- composer_media_optimization_image_wasm_decode_enabled:
- default: false
- hidden: true
- client: true
composer_media_optimization_gif_conversion_threshold:
default: 512000
hidden: true
diff --git a/frontend/discourse/app/instance-initializers/register-media-optimization-upload-processor.js b/frontend/discourse/app/instance-initializers/register-media-optimization-upload-processor.js
index 5523cb4cf39..5f84cb15a87 100644
--- a/frontend/discourse/app/instance-initializers/register-media-optimization-upload-processor.js
+++ b/frontend/discourse/app/instance-initializers/register-media-optimization-upload-processor.js
@@ -15,14 +15,6 @@ export default {
return;
}
- // Restrict feature to browsers that support OffscreenCanvas
- if (typeof OffscreenCanvas === "undefined") {
- return;
- }
- if (!("createImageBitmap" in self)) {
- return;
- }
-
addComposerUploadPreProcessor(
UppyMediaOptimization,
({ isMobileDevice }) => {
diff --git a/frontend/discourse/app/lib/media-optimization-utils.js b/frontend/discourse/app/lib/media-optimization-utils.js
deleted file mode 100644
index e3f426422ff..00000000000
--- a/frontend/discourse/app/lib/media-optimization-utils.js
+++ /dev/null
@@ -1,103 +0,0 @@
-async function fileToDrawable(file, isIOS) {
- if (!isIOS) {
- return await createImageBitmap(file);
- } else {
- // iOS has performance issues with createImageBitmap on large images
- // this workaround partially borrowed from https://github.com/Donaldcwl/browser-image-compression/blob/master/lib/utils.js
- const dataUrl = URL.createObjectURL(file);
- return await loadImage(dataUrl);
- }
-}
-
-function drawableToImageData(drawable, isIOS) {
- const width = drawable.width,
- height = drawable.height,
- sx = 0,
- sy = 0,
- sw = width,
- sh = height;
-
- let canvas = new OffscreenCanvas(width, height);
-
- // Check if the canvas is too large
- // iOS _still_ enforces a max pixel count of 16,777,216 per canvas
- const maxLimit = 4096;
- const maximumPixelCount = maxLimit * maxLimit;
-
- if (isIOS && width * height > maximumPixelCount) {
- const ratio = Math.min(maxLimit / width, maxLimit / height);
-
- canvas.width = Math.floor(width * ratio);
- canvas.height = Math.floor(height * ratio);
- }
-
- // Draw image onto canvas
- const ctx = canvas.getContext("2d");
- if (!ctx) {
- throw "Could not create canvas context";
- }
-
- ctx.drawImage(drawable, sx, sy, sw, sh, 0, 0, canvas.width, canvas.height);
- const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
-
- // iOS strikes again, need to clear canvas to free up memory
- if (isIOS) {
- canvas.width = 1;
- canvas.height = 1;
- ctx && ctx.clearRect(0, 0, 1, 1);
- }
-
- return imageData;
-}
-
-function isTransparent(type, imageData) {
- if (!/(\.|\/)(png|webp)$/i.test(type)) {
- return false;
- }
-
- for (let i = 0; i < imageData.data.length; i += 4) {
- if (imageData.data[i + 3] < 255) {
- return true;
- }
- }
-
- return false;
-}
-
-function jpegDecodeFailure(type, imageData) {
- if (!/(\.|\/)jpe?g$/i.test(type)) {
- return false;
- }
-
- return imageData.data[3] === 0;
-}
-
-export async function fileToImageData(file, isIOS) {
- const drawable = await fileToDrawable(file, isIOS);
- const imageData = drawableToImageData(drawable, isIOS);
-
- if (isTransparent(file.type, imageData)) {
- throw "Image has transparent pixels, won't convert to JPEG!";
- }
-
- if (jpegDecodeFailure(file.type, imageData)) {
- throw "JPEG image has transparent pixel, decode failed!";
- }
-
- return imageData;
-}
-
-function loadImage(src) {
- return new Promise((resolve, reject) => {
- const img = new Image();
- img.onload = () => {
- URL.revokeObjectURL(src);
- resolve(img);
- };
- img.onerror = (e) => {
- URL.revokeObjectURL(src);
- reject(e);
- };
- img.src = src;
- });
-}
diff --git a/frontend/discourse/app/services/media-optimization-worker.js b/frontend/discourse/app/services/media-optimization-worker.js
index 6fb64f73ef7..52abce4a82b 100644
--- a/frontend/discourse/app/services/media-optimization-worker.js
+++ b/frontend/discourse/app/services/media-optimization-worker.js
@@ -2,7 +2,10 @@ import Service, { service } from "@ember/service";
import { Promise } from "rsvp";
import workerUrl from "virtual:dynamic-chunk-url:discourse/workers/media-optimization/entrypoint";
import { disableImplicitInjections } from "discourse/lib/implicit-injections";
-import { fileToImageData } from "discourse/lib/media-optimization-utils";
+import {
+ authorizedExtensions,
+ authorizesAllExtensions,
+} from "discourse/lib/uploads";
const CONVERT_FORMAT_REGEX = /(\.|\/)(jxl|hei[cf])$/i;
const ANIMATED_GIF_REGEX = /(\.|\/)(gif)$/i;
@@ -13,8 +16,9 @@ const OPTIMIZABLE_REGEX = /(\.|\/)(jpe?g|png)$/i;
*
* 1. optimizeImage called
* 2. worker started if one isn't already running, message handlers registered
- * 3. "compress"/"convert" message posted to the worker
- * 4. worker posts back the optimized file (or an error), resolving the upload
+ * 3. "convert"/"convertAnimated" message posted to the worker
+ * 4. worker posts back the optimized file (or a skip notice/error), resolving
+ * the upload
*
* The worker is a module worker bundled with its codecs, so it is ready as soon
* as it is created (posted messages queue until its module evaluates). A worker
@@ -24,8 +28,8 @@ const OPTIMIZABLE_REGEX = /(\.|\/)(jpe?g|png)$/i;
@disableImplicitInjections
export default class MediaOptimizationWorkerService extends Service {
@service appEvents;
+ @service currentUser;
@service siteSettings;
- @service capabilities;
worker = null;
currentComposerUploadData = null;
@@ -45,9 +49,6 @@ export default class MediaOptimizationWorkerService extends Service {
const isConvertFormat = CONVERT_FORMAT_REGEX.test(typeOrName);
const isAnimatedGif = ANIMATED_GIF_REGEX.test(typeOrName);
const isOptimizable = OPTIMIZABLE_REGEX.test(typeOrName);
- const wasmDecodeOptimizable =
- isOptimizable &&
- this.siteSettings.composer_media_optimization_image_wasm_decode_enabled;
if (!isConvertFormat && !isAnimatedGif && !isOptimizable) {
return Promise.resolve();
@@ -117,15 +118,15 @@ export default class MediaOptimizationWorkerService extends Service {
debug_mode: this.siteSettings.composer_media_optimization_debug_mode,
};
- if (isConvertFormat || wasmDecodeOptimizable) {
- let arrayBuffer;
- try {
- arrayBuffer = await file.data.arrayBuffer();
- } catch (error) {
- this.logIfDebug(error);
- return resolve();
- }
+ let arrayBuffer;
+ try {
+ arrayBuffer = await file.data.arrayBuffer();
+ } catch (error) {
+ this.logIfDebug(error);
+ return resolve();
+ }
+ if (isConvertFormat || isOptimizable) {
this.worker.postMessage(
{
type: "convert",
@@ -138,15 +139,7 @@ export default class MediaOptimizationWorkerService extends Service {
},
[arrayBuffer]
);
- } else if (isAnimatedGif) {
- let arrayBuffer;
- try {
- arrayBuffer = await file.data.arrayBuffer();
- } catch (error) {
- this.logIfDebug(error);
- return resolve();
- }
-
+ } else {
this.worker.postMessage(
{
type: "convertAnimated",
@@ -158,28 +151,6 @@ export default class MediaOptimizationWorkerService extends Service {
},
[arrayBuffer]
);
- } else {
- let imageData;
- try {
- imageData = await fileToImageData(file.data, this.capabilities.isIOS);
- } catch (error) {
- this.logIfDebug(error);
- return resolve();
- }
-
- this.worker.postMessage(
- {
- type: "compress",
- fileId: file.id,
- file: imageData.data.buffer,
- fileName: file.name,
- width: imageData.width,
- height: imageData.height,
- originalFileSize: file.size,
- settings,
- },
- [imageData.data.buffer]
- );
}
this.workerPendingCount++;
@@ -240,8 +211,6 @@ export default class MediaOptimizationWorkerService extends Service {
switch (outputType) {
case "image/jpeg":
return baseName + ".jpg";
- case "image/png":
- return baseName + ".png";
case "image/webp":
return baseName + ".webp";
default:
@@ -249,27 +218,45 @@ export default class MediaOptimizationWorkerService extends Service {
}
}
+ _extensionAuthorized(fileName) {
+ const extension = fileName.split(".").pop().toLowerCase();
+ const staff = this.currentUser?.staff;
+ return (
+ authorizesAllExtensions(staff, this.siteSettings) ||
+ authorizedExtensions(staff, this.siteSettings).includes(extension)
+ );
+ }
+
registerMessageHandler() {
this.worker.onmessage = (workerMessage) => {
switch (workerMessage.data.type) {
case "file":
- const outputType = workerMessage.data.outputType || "image/jpeg";
+ const outputType = workerMessage.data.outputType;
const outputFileName = this._renameForOutputType(
workerMessage.data.fileName,
outputType
);
- const optimizedFile = new File(
- [workerMessage.data.file],
- outputFileName,
- {
- type: outputType,
- }
- );
- this.logIfDebug(
- `Finished optimization of ${optimizedFile.name}, new size is ${optimizedFile.size} bytes`
- );
- this.promiseResolvers[workerMessage.data.fileId](optimizedFile);
+ if (this._extensionAuthorized(outputFileName)) {
+ const optimizedFile = new File(
+ [workerMessage.data.file],
+ outputFileName,
+ {
+ type: outputType,
+ }
+ );
+ this.logIfDebug(
+ `Finished optimization of ${optimizedFile.name}, new size is ${optimizedFile.size} bytes`
+ );
+
+ this.promiseResolvers[workerMessage.data.fileId](optimizedFile);
+ } else {
+ this.logIfDebug(
+ `Optimizing ${workerMessage.data.fileName} would produce ${outputFileName}, which is not an authorized extension here; keeping the original`
+ );
+
+ this.promiseResolvers[workerMessage.data.fileId]();
+ }
this.workerDoneCount++;
this.workerPendingCount--;
@@ -293,7 +280,7 @@ export default class MediaOptimizationWorkerService extends Service {
break;
case "skipped":
this.logIfDebug(
- `Conversion skipped for ${workerMessage.data.fileName} (output not smaller than original)`
+ `Conversion skipped for ${workerMessage.data.fileName}, keeping the original`
);
this.promiseResolvers[workerMessage.data.fileId]();
diff --git a/frontend/discourse/app/workers/media-optimization/codecs.js b/frontend/discourse/app/workers/media-optimization/codecs.js
index 2370997984f..a2c9f06921e 100644
--- a/frontend/discourse/app/workers/media-optimization/codecs.js
+++ b/frontend/discourse/app/workers/media-optimization/codecs.js
@@ -73,7 +73,35 @@ function hasTransparency(imageData) {
return false;
}
-async function maybeResize(imageData, width, height, settings) {
+function isAnimatedPng(fileBuffer) {
+ // APNGs declare an acTL chunk before the first IDAT; our PNG decoder only
+ // returns the first frame, so these must be detected up front and skipped.
+ const view = new DataView(fileBuffer);
+ let offset = 8; // PNG signature
+
+ while (offset + 8 <= view.byteLength) {
+ const length = view.getUint32(offset);
+ const type = String.fromCharCode(
+ view.getUint8(offset + 4),
+ view.getUint8(offset + 5),
+ view.getUint8(offset + 6),
+ view.getUint8(offset + 7)
+ );
+
+ if (type === "acTL") {
+ return true;
+ }
+ if (type === "IDAT" || type === "IEND") {
+ return false;
+ }
+
+ offset += 12 + length; // length + type + data + crc
+ }
+
+ return false;
+}
+
+async function maybeResize(imageData, width, height, settings, transparent) {
if (width > settings.resize_threshold) {
try {
const targetDimensions = resizeWithAspect(
@@ -88,7 +116,9 @@ async function maybeResize(imageData, width, height, settings) {
premultiply: settings.resize_pre_multiply,
linearRGB: settings.resize_linear_rgb,
});
- if (resizeResult.data[3] !== 255) {
+ // An opaque source must stay opaque: a transparent top-left pixel after
+ // resizing indicates corruption. Transparent sources can't use this check.
+ if (!transparent && resizeResult.data[3] !== 255) {
throw "Image corrupted during resize. Falling back to the original for encode";
}
logIfDebug(
@@ -106,58 +136,6 @@ async function maybeResize(imageData, width, height, settings) {
}
}
-export async function optimize(
- imageData,
- fileName,
- width,
- height,
- originalFileSize,
- settings
-) {
- const mozJpegOptions = buildMozJpegOptions(settings);
-
- const initialSize = imageData.byteLength;
- logIfDebug(
- `Received imageData ${initialSize} bytes (raw uncompressed pixels) from original file size of ${originalFileSize} bytes (compressed) for ${fileName}`
- );
-
- const wrappedImageData = new ImageData(
- new Uint8ClampedArray(imageData),
- width,
- height
- );
- const resized = await maybeResize(wrappedImageData, width, height, settings);
- const finalWidth = resized.width;
- const finalHeight = resized.height;
-
- const result = await encodeJpeg(
- resized instanceof ImageData
- ? resized
- : new ImageData(
- new Uint8ClampedArray(resized.data),
- finalWidth,
- finalHeight
- ),
- mozJpegOptions
- );
-
- const finalSize = result.byteLength;
- logIfDebug(
- `Post-reencode size for ${fileName} is ${finalSize} bytes (compressed JPEG), original was ${originalFileSize} bytes`
- );
- const compressionFromOriginal = (originalFileSize / finalSize).toFixed(1);
- const compressionFromRaw = (initialSize / finalSize).toFixed(1);
- logIfDebug(
- `Compressed ${compressionFromOriginal}x vs original file, ${compressionFromRaw}x vs raw pixels for ${fileName}`
- );
-
- if (finalSize < 20000) {
- throw "Final size suspiciously small, discarding optimizations";
- }
-
- return result;
-}
-
export async function convert(
fileBuffer,
fileName,
@@ -175,6 +153,12 @@ export async function convert(
} else if (/jpe?g$/i.test(fileType)) {
imageData = await decodeJpeg(fileBuffer, { preserveOrientation: true });
} else if (/png$/i.test(fileType)) {
+ if (isAnimatedPng(fileBuffer)) {
+ logIfDebug(
+ `${fileName} is an animated PNG, skipping conversion to keep the animation`
+ );
+ return null;
+ }
imageData = await decodePng(fileBuffer);
} else {
throw `Unsupported file type for conversion: ${fileType}`;
@@ -190,7 +174,8 @@ export async function convert(
imageData,
imageData.width,
imageData.height,
- settings
+ settings,
+ transparent
);
if (transparent) {
@@ -204,6 +189,18 @@ export async function convert(
logIfDebug(
`Converted ${fileName} from ${originalFileSize} bytes to ${finalSize} bytes WEBP`
);
+
+ if (finalSize < 20000) {
+ throw "Final size suspiciously small, discarding conversion";
+ }
+
+ if (finalSize >= originalFileSize) {
+ logIfDebug(
+ `WEBP (${finalSize} bytes) is not smaller than the original (${originalFileSize} bytes), skipping conversion`
+ );
+ return null;
+ }
+
return { data: result, outputType: "image/webp" };
}
diff --git a/frontend/discourse/app/workers/media-optimization/entrypoint.js b/frontend/discourse/app/workers/media-optimization/entrypoint.js
index f63a766c126..20f9b75bfe0 100644
--- a/frontend/discourse/app/workers/media-optimization/entrypoint.js
+++ b/frontend/discourse/app/workers/media-optimization/entrypoint.js
@@ -1,42 +1,10 @@
// Built by rolldown as a standalone chunk and started as a module worker via a
// blob bootstrap (see the media-optimization-worker service), so it inherits the
// host document CSP. The codecs are bundled straight into this chunk.
-import { convert, convertAnimated, optimize } from "./codecs.js";
+import { convert, convertAnimated } from "./codecs.js";
self.onmessage = async function (e) {
switch (e.data.type) {
- case "compress":
- try {
- globalThis.debugMode = e.data.settings.debug_mode;
-
- let optimized = await optimize(
- e.data.file,
- e.data.fileName,
- e.data.width,
- e.data.height,
- e.data.originalFileSize,
- e.data.settings
- );
- postMessage(
- {
- type: "file",
- file: optimized,
- fileName: e.data.fileName,
- fileId: e.data.fileId,
- },
- [optimized]
- );
- } catch (error) {
- // eslint-disable-next-line no-console
- console.error(error);
- postMessage({
- type: "error",
- file: e.data.file,
- fileName: e.data.fileName,
- fileId: e.data.fileId,
- });
- }
- break;
case "convert":
try {
globalThis.debugMode = e.data.settings.debug_mode;
@@ -48,16 +16,24 @@ self.onmessage = async function (e) {
e.data.originalFileSize,
e.data.settings
);
- postMessage(
- {
- type: "file",
- file: converted.data,
+ if (converted) {
+ postMessage(
+ {
+ type: "file",
+ file: converted.data,
+ fileName: e.data.fileName,
+ fileId: e.data.fileId,
+ outputType: converted.outputType,
+ },
+ [converted.data]
+ );
+ } else {
+ postMessage({
+ type: "skipped",
fileName: e.data.fileName,
fileId: e.data.fileId,
- outputType: converted.outputType,
- },
- [converted.data]
- );
+ });
+ }
} catch (error) {
// eslint-disable-next-line no-console
console.error(error);
diff --git a/spec/system/composer_image_optimization_spec.rb b/spec/system/composer_image_optimization_spec.rb
index 66b82d0d9cd..a94f8385953 100644
--- a/spec/system/composer_image_optimization_spec.rb
+++ b/spec/system/composer_image_optimization_spec.rb
@@ -79,8 +79,9 @@ describe "Composer image optimization for uploads using media-optimization-worke
file = file_from_fixtures("large_and_unoptimized.png", "images")
converted_file_path = file.path.gsub("unoptimized", "notransparent")
- # We do a conversion here to simulate a PNG with no transparency,
- # the fixture has transparency, and PNGs with transparency are not optimized.
+ # We do a conversion here to simulate a PNG with no transparency, the
+ # fixture has transparency and transparent PNGs are converted to WEBP
+ # rather than JPEG (see the transparent PNG test below).
Discourse::Utils.execute_command(
*["convert", file.path, "-background", "white", "-alpha", "remove", converted_file_path],
timeout: 3,
@@ -95,12 +96,31 @@ describe "Composer image optimization for uploads using media-optimization-worke
Upload.find_by(original_filename: File.basename(converted_file_path).gsub("png", "jpg"))
# Original large_and_unoptimized PNG with no transparency is 312_889 bytes
- expect(upload.filesize).to eq(285_809)
+ expect(upload.filesize).to eq(271_246)
ensure
FileUtils.rm(converted_file_path)
end
- it "does not optimize a png with transparent pixels in the composer" do
+ it "converts a png with transparent pixels to webp in the composer" do
+ visit "/new-topic"
+ expect(composer).to be_opened
+
+ file = file_from_fixtures("large_and_unoptimized.png", "images")
+ attach_file("file-uploader", [file.path], make_visible: true)
+
+ expect(composer).to have_no_in_progress_uploads
+ expect(composer.preview).to have_css(".image-wrapper", count: 1)
+
+ upload = Upload.find_by(original_filename: File.basename(file.path).gsub("png", "webp"))
+
+ # Original large_and_unoptimized.png is 421_730 bytes and 2032px wide;
+ # transparent PNGs are resized and re-encoded as WEBP instead of JPEG.
+ expect(upload.filesize).to eq(126_626)
+ end
+
+ it "keeps the original transparent png when webp is not an authorized extension" do
+ SiteSetting.authorized_extensions = "jpg|jpeg|png|gif"
+
visit "/new-topic"
expect(composer).to be_opened
@@ -112,8 +132,39 @@ describe "Composer image optimization for uploads using media-optimization-worke
upload = Upload.find_by(original_filename: File.basename(file.path))
- # Original large_and_unoptimized.png is 421_730 bytes
+ # Original large_and_unoptimized.png is 421_730 bytes, uploaded untouched
+ # because converting it to WEBP would produce an unauthorized extension.
expect(upload.filesize).to eq(421_730)
end
+
+ it "keeps an animated png untouched in the composer" do
+ visit "/new-topic"
+ expect(composer).to be_opened
+
+ file = file_from_fixtures("large_and_unoptimized.png", "images")
+ apng_path = file.path.gsub("unoptimized", "animated")
+
+ # Emulate an APNG by inserting an acTL chunk right after IHDR; the
+ # optimizer's decoder would only keep the first frame, so it must skip
+ # animated PNGs entirely.
+ png = File.binread(file.path)
+ actl_data = [2, 0].pack("N2")
+ actl_chunk =
+ [actl_data.bytesize].pack("N") + "acTL" + actl_data +
+ [Zlib.crc32("acTL" + actl_data)].pack("N")
+ ihdr_end = 8 + 4 + 4 + 13 + 4 # signature + IHDR length/type/data/crc
+ File.binwrite(apng_path, png[0...ihdr_end] + actl_chunk + png[ihdr_end..])
+
+ attach_file("file-uploader", [apng_path], make_visible: true)
+
+ expect(composer).to have_no_in_progress_uploads
+ expect(composer.preview).to have_css(".image-wrapper", count: 1)
+
+ upload = Upload.find_by(original_filename: File.basename(apng_path))
+
+ expect(upload.filesize).to eq(File.size(apng_path))
+ ensure
+ FileUtils.rm(apng_path)
+ end
end
end