A platform for community discussion. Free, open, simple.
DEV: Refactor worker and WASM loading for media-optimization-worker (#40793)
David Taylora32f09021f57
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rbindex 96f6cdf764e..0b0955642db 100644--- a/app/helpers/application_helper.rb+++ b/app/helpers/application_helper.rb@@ -984,11 +984,6 @@ module ApplicationHelperdisable_custom_css: loading_admin?,highlight_js_path: HighlightJs.path,svg_sprite_path: SvgSprite.path(theme_id),- media_optimization_bundle:- script_asset_path(- EmberCli.script_chunks["media-optimization-bundle"]&.first ||- "/media-optimization-bundle.js",- ),enable_js_error_reporting: GlobalSetting.enable_js_error_reporting,color_scheme_is_dark: dark_color_scheme?,user_color_scheme_id: user_scheme_id || -1,diff --git a/frontend/discourse/app/initializers/discourse-bootstrap.js b/frontend/discourse/app/initializers/discourse-bootstrap.jsindex 4046dec3ec5..cc82eef4063 100644--- a/frontend/discourse/app/initializers/discourse-bootstrap.js+++ b/frontend/discourse/app/initializers/discourse-bootstrap.js@@ -80,7 +80,6 @@ export default {session.highlightJsPath = setupData.highlightJsPath;session.svgSpritePath = setupData.svgSpritePath;- session.mediaOptimizationBundle = setupData.mediaOptimizationBundle;session.userColorSchemeId = parseInt(setupData.userColorSchemeId, 10);session.userDarkSchemeId = parseInt(setupData.userDarkSchemeId, 10);diff --git a/frontend/discourse/app/services/media-optimization-worker.js b/frontend/discourse/app/services/media-optimization-worker.jsindex c273e6565b6..8fe42fe7e41 100644--- a/frontend/discourse/app/services/media-optimization-worker.js+++ b/frontend/discourse/app/services/media-optimization-worker.js@@ -1,40 +1,33 @@import Service, { service } from "@ember/service";import { Promise } from "rsvp";-import { getAbsoluteURL } from "discourse/lib/get-url";+import workerUrl from "virtual:dynamic-chunk-url:discourse/workers/media-optimization/entrypoint";import { disableImplicitInjections } from "discourse/lib/implicit-injections";import { fileToImageData } from "discourse/lib/media-optimization-utils";-const WORKER_INSTALL_TIMEOUT_MS = 3000;-const CONVERT_FORMAT_REGEX = /(\.|\/)(jxl|hei[cf])$/i;const ANIMATED_GIF_REGEX = /(\.|\/)(gif)$/i;const OPTIMIZABLE_REGEX = /(\.|\/)(jpe?g|png)$/i;/**- * This worker follows a particular promise/callback flow to ensure- * that the media-optimization-worker is installed and has its libraries- * loaded before optimizations can happen. The flow:+ * Optimizes composer image uploads in a worker. The flow:** 1. optimizeImage called- * 2. worker initialized and started- * 3. message handlers for worker registered- * 4. "install" message posted to worker- * 5. "installed" message received from worker- * 6. optimizeImage continues, posting "compress" message to worker+ * 2. worker started if one isn't already running, message handlers registered+ * 3. "compress"/"convert" message posted to the worker+ * 4. worker posts back the optimized file (or an error), resolving the upload*- * When the worker is being installed, all other calls to optimizeImage- * will wait for the "installed" message to be handled before continuing- * with any image optimization work.+ * The worker is a module worker bundled with its codecs, so it is ready as soon+ * as it is created (posted messages queue until its module evaluates). A worker+ * that fails to boot surfaces via onerror, and in-flight uploads continue+ * unoptimized.*/@disableImplicitInjectionsexport default class MediaOptimizationWorkerService extends Service {@service appEvents;@service siteSettings;@service capabilities;- @service session;worker = null;- workerUrl = getAbsoluteURL("/javascripts/media-optimization-worker.js");currentComposerUploadData = null;promiseResolvers = null;workerDoneCount = 0;@@ -93,7 +86,10 @@ export default class MediaOptimizationWorkerService extends Service {}}- await this.ensureAvailableWorker();+ this.ensureAvailableWorker();+ if (!this.worker) {+ return Promise.resolve();+ }// eslint-disable-next-line no-async-promise-executorreturn new Promise(async (resolve) => {@@ -120,7 +116,6 @@ export default class MediaOptimizationWorkerService extends Service {? this.siteSettings.composer_media_optimization_image_encode_quality: this.siteSettings.image_quality,debug_mode: this.siteSettings.composer_media_optimization_debug_mode,- mediaOptimizationBundle: this.session.mediaOptimizationBundle,};if (isConvertFormat || wasmDecodeOptimizable) {@@ -192,65 +187,48 @@ export default class MediaOptimizationWorkerService extends Service {});}- async ensureAvailableWorker() {- if (this.worker && this.workerInstalled) {- return Promise.resolve();- }- if (this.installPromise) {- return this.installPromise;- }- return this.install();- }-- async install() {- this.installPromise = new Promise((resolve, reject) => {- this.afterInstalled = resolve;- this.failedInstall = reject;- this.logIfDebug("Installing worker");+ ensureAvailableWorker() {+ if (!this.worker) {this.startWorker();- this.registerMessageHandler();- this.worker.postMessage({- type: "install",- settings: {- mediaOptimizationBundle: this.session.mediaOptimizationBundle,- },- });-- // new Worker(workerUrl) can hang indefinitely in some cases, so we wait- // and see if the worker manages to install in a reasonable time. If not,- // we reject the install promise and clean up.- setTimeout(() => {- if (!this.workerInstalled) {- this.failInstall("Worker install timed out!");- }- }, WORKER_INSTALL_TIMEOUT_MS);-- this.appEvents.on("composer:closed", this, "stopWorker");- });-- return this.installPromise;+ }}startWorker() {this.logIfDebug("Starting media-optimization-worker");+ // Module workers queue posted messages until their module has evaluated, so+ // the worker is usable immediately; a failure to boot surfaces via onerror.try {- // TODO come up with a workaround for FF that lacks type: module support- this.worker = new Worker(this.workerUrl);+ const blobUrl = URL.createObjectURL(+ new Blob([`import ${JSON.stringify(workerUrl)};`], {+ type: "text/javascript",+ })+ );+ try {+ this.worker = new Worker(blobUrl, { type: "module" });+ } finally {+ URL.revokeObjectURL(blobUrl);+ }} catch (err) {- this.failInstall("Error starting worker:", err);+ this.logIfDebug("Error starting media-optimization-worker", err);+ return;}+ this.registerMessageHandler();+ this.worker.onerror = (err) => this.handleWorkerError(err);+ this.appEvents.on("composer:closed", this, "stopWorker");}- failInstall(message, err = null) {- this.logIfDebug(message, err);- this.failedInstall(err);- this.cleanupInstallPromises();+ handleWorkerError(err) {+ this.logIfDebug("media-optimization-worker error", err);+ this.stopWorker();+ // Resolve any in-flight optimizations so their uploads continue unoptimized.+ Object.values(this.promiseResolvers ?? {}).forEach((resolve) =>+ resolve?.()+ );}stopWorker() {if (this.worker) {this.logIfDebug("Stopping media-optimization-worker...");- this.workerInstalled = false;this.worker.terminate();this.worker = null;this.workerDoneCount = 0;@@ -323,30 +301,12 @@ export default class MediaOptimizationWorkerService extends Service {this.workerDoneCount++;this.workerPendingCount--;break;- case "installed":- this.logIfDebug("Worker installed");- this.workerInstalled = true;- this.afterInstalled();- this.cleanupInstallPromises();- break;- case "installFailed":- this.failInstall(- "Worker failed to install.",- workerMessage.data.errorMessage- );- break;default:this.logIfDebug(`Sorry, we are out of ${workerMessage}`);}};}- cleanupInstallPromises() {- this.afterInstalled = null;- this.failedInstall = null;- this.installPromise = null;- }-logIfDebug(...messages) {if (this.siteSettings.composer_media_optimization_debug_mode) {// eslint-disable-next-line no-consolediff --git a/frontend/discourse/media-optimization-bundle.js b/frontend/discourse/app/workers/media-optimization/codecs.jssimilarity index 98%rename from frontend/discourse/media-optimization-bundle.jsrename to frontend/discourse/app/workers/media-optimization/codecs.jsindex be0273bad8c..2370997984f 100644--- a/frontend/discourse/media-optimization-bundle.js+++ b/frontend/discourse/app/workers/media-optimization/codecs.js@@ -106,7 +106,7 @@ async function maybeResize(imageData, width, height, settings) {}}-globalThis.optimize = async function (+export async function optimize(imageData,fileName,width,@@ -156,9 +156,9 @@ globalThis.optimize = async function (}return result;-};+}-globalThis.convert = async function (+export async function convert(fileBuffer,fileName,fileType,@@ -219,9 +219,9 @@ globalThis.convert = async function (}return { data: result, outputType: "image/jpeg" };-};+}-globalThis.convertAnimated = async function (+export async function convertAnimated(fileBuffer,fileName,originalFileSize,@@ -251,4 +251,4 @@ globalThis.convertAnimated = async function (}return { data: result, outputType: "image/webp" };-};+}diff --git a/public/javascripts/media-optimization-worker.js b/frontend/discourse/app/workers/media-optimization/entrypoint.jssimilarity index 76%rename from public/javascripts/media-optimization-worker.jsrename to frontend/discourse/app/workers/media-optimization/entrypoint.jsindex 987e96c63ae..f63a766c126 100644--- a/public/javascripts/media-optimization-worker.js+++ b/frontend/discourse/app/workers/media-optimization/entrypoint.js@@ -1,12 +1,15 @@-// The media optimization bundle uses webpack's import-scripts chunk loader,-// so worker-only dynamic chunks can be loaded without a DOM shim.-onmessage = async function (e) {+// Built by rolldown as a standalone chunk and started as a module worker via a+// blob bootstrap (see the media-optimization-worker service), so it inherits the+// host document CSP. The codecs are bundled straight into this chunk.+import { convert, convertAnimated, optimize } from "./codecs.js";++self.onmessage = async function (e) {switch (e.data.type) {case "compress":try {globalThis.debugMode = e.data.settings.debug_mode;- let optimized = await globalThis.optimize(+ let optimized = await optimize(e.data.file,e.data.fileName,e.data.width,@@ -24,6 +27,7 @@ onmessage = async function (e) {[optimized]);} catch (error) {+ // eslint-disable-next-line no-consoleconsole.error(error);postMessage({type: "error",@@ -37,7 +41,7 @@ onmessage = async function (e) {try {globalThis.debugMode = e.data.settings.debug_mode;- let converted = await globalThis.convert(+ let converted = await convert(e.data.file,e.data.fileName,e.data.fileType,@@ -55,6 +59,7 @@ onmessage = async function (e) {[converted.data]);} catch (error) {+ // eslint-disable-next-line no-consoleconsole.error(error);postMessage({type: "error",@@ -68,7 +73,7 @@ onmessage = async function (e) {try {globalThis.debugMode = e.data.settings.debug_mode;- let animatedResult = await globalThis.convertAnimated(+ let animatedResult = await convertAnimated(e.data.file,e.data.fileName,e.data.originalFileSize,@@ -93,6 +98,7 @@ onmessage = async function (e) {});}} catch (error) {+ // eslint-disable-next-line no-consoleconsole.error(error);postMessage({type: "error",@@ -102,20 +108,8 @@ onmessage = async function (e) {});}break;- case "install":- try {- await loadLibs(e.data.settings);- postMessage({ type: "installed" });- } catch (error) {- console.error(error);- postMessage({ type: "installFailed", errorMessage: error.message });- }- break;default:- logIfDebug(`Sorry, we are out of ${e}.`);+ // eslint-disable-next-line no-console+ console.error(`Unexpected message type: ${e.data.type}`);}};--async function loadLibs(settings) {- return import(settings.mediaOptimizationBundle);-}diff --git a/frontend/discourse/lib/dynamic-chunk-url-plugin.mjs b/frontend/discourse/lib/dynamic-chunk-url-plugin.mjsnew file mode 100644index 00000000000..a4cf63aabd6--- /dev/null+++ b/frontend/discourse/lib/dynamic-chunk-url-plugin.mjs@@ -0,0 +1,49 @@+import { relative } from "path";++const PREFIX = "virtual:dynamic-chunk-url:";+const RESOLVED_PREFIX = "\0" + PREFIX;++/*+ * Importing `virtual:dynamic-chunk-url:<specifier>` emits <specifier> as its own+ * bundled, hashed chunk and resolves to that chunk's final URL (a string). The+ * specifier is resolved like a normal import (relative to the importer).+ *+ * For example, a worker can be started from a blob bootstrap that imports the real+ * chunk by absolute URL, so the worker inherits the host document CSP.+ */+export default function dynamicChunkUrlPlugin() {+ return {+ name: "dynamic-chunk-url",+ resolveId: {+ filter: { id: /^virtual:dynamic-chunk-url:/ },+ async handler(source, importer) {+ const target = source.slice(PREFIX.length);+ const resolved = await this.resolve(target, importer, {+ skipSelf: true,+ });+ if (!resolved) {+ this.error(`dynamic-chunk-url: could not resolve "${target}"`);+ }+ return { id: RESOLVED_PREFIX + resolved.id };+ },+ },+ load: {+ filter: { id: /^\0virtual:dynamic-chunk-url:/ },+ handler(id) {+ const target = id.slice(RESOLVED_PREFIX.length);+ const refId = this.emitFile({+ type: "chunk",+ id: target,+ name: relative(process.cwd(), target)+ .replace(/\.[^.]+$/, "")+ .replace(/[/\\]/g, "-"),+ preserveSignature: "strict",+ });+ return {+ code: `export default import.meta.ROLLUP_FILE_URL_${refId};`,+ moduleType: "js",+ };+ },+ },+ };+}diff --git a/frontend/discourse/rolldown.config.mjs b/frontend/discourse/rolldown.config.mjsindex f5fddd83074..6811b526eed 100644--- a/frontend/discourse/rolldown.config.mjs+++ b/frontend/discourse/rolldown.config.mjs@@ -1,6 +1,7 @@import * as fs from "fs";import { basename, relative } from "path";import { viteAliasPlugin, viteImportGlobPlugin } from "rolldown/experimental";+import dynamicChunkUrlPlugin from "./lib/dynamic-chunk-url-plugin.mjs";import writeResolverConfig from "./lib/embroider-vite-resolver-options.mjs";import maybeBabel from "./lib/maybe-babel.mjs";import optimizedEmber from "./lib/optimized-ember.mjs";@@ -8,7 +9,7 @@ import wrapTestModulesPlugin from "./lib/wrap-test-modules-plugin.mjs";writeResolverConfig({- staticAppPaths: ["static", "admin"],+ staticAppPaths: ["static", "admin", "workers"],splitAtRoutes: [{ type: "string", value: "wizard" }],},{@@ -85,7 +86,6 @@ export function buildConfig({ devMode } = {}) {input: {discourse: "discourse.js",vendor: "vendor.js",- "media-optimization-bundle": "media-optimization-bundle.js",...(!isProduction || process.env.FORCE_BUILD_TESTS? {"test-entrypoint": "tests/test-entrypoint.js",@@ -112,6 +112,7 @@ export function buildConfig({ devMode } = {}) {preserveEntrySignatures: "strict",plugins: [viteAliasPlugin({ entries: aliases }),+ dynamicChunkUrlPlugin(),optimizedEmber(),viteImportGlobPlugin(),maybeBabel({diff --git a/lib/content_security_policy/default.rb b/lib/content_security_policy/default.rbindex 9666d794717..47eb39118c0 100644--- a/lib/content_security_policy/default.rb+++ b/lib/content_security_policy/default.rb@@ -12,7 +12,7 @@ class ContentSecurityPolicydirectives[:base_uri] = [:self]directives[:object_src] = [:none]directives[:script_src] = script_src- directives[:worker_src] = []+ directives[:worker_src] = [:self, "blob:"]directives[:frame_ancestors] = frame_ancestors if restrict_embed?directives[:manifest_src] = ["'self'"]end@@ -21,7 +21,7 @@ class ContentSecurityPolicyprivatedef script_src- ["'strict-dynamic'"]+ %w['strict-dynamic' 'wasm-unsafe-eval']enddef frame_ancestorsdiff --git a/spec/integration/content_security_policy_spec.rb b/spec/integration/content_security_policy_spec.rbindex 8788a750220..5a38cabcc68 100644--- a/spec/integration/content_security_policy_spec.rb+++ b/spec/integration/content_security_policy_spec.rb@@ -13,7 +13,7 @@ RSpec.describe "content security policy integration" doexpect(response.headers["Content-Security-Policy"]).to be_presentexpect(response.headers["Content-Security-Policy"]).to match(- /script-src 'nonce-[^']+' 'strict-dynamic';/,+ /script-src 'nonce-[^']+' 'strict-dynamic' 'wasm-unsafe-eval'; worker-src 'self' blob:;/,)endenddiff --git a/spec/lib/content_security_policy_spec.rb b/spec/lib/content_security_policy_spec.rbindex c6377256935..45cae13536c 100644--- a/spec/lib/content_security_policy_spec.rb+++ b/spec/lib/content_security_policy_spec.rb@@ -34,9 +34,14 @@ RSpec.describe ContentSecurityPolicy doexpect(script_srcs).to include("'strict-dynamic'")end- it "does not set worker-src" do+ it "allows wasm compilation" do+ script_srcs = parse(policy)["script-src"]+ expect(script_srcs).to include("'wasm-unsafe-eval'")+ end++ it "sets worker-src to self and blob:" doworker_src = parse(policy)["worker-src"]- expect(worker_src).to eq(nil)+ expect(worker_src).to contain_exactly("'self'", "blob:")endend