Skip to content

A platform for community discussion. Free, open, simple.

DEV: Refactor worker and WASM loading for media-optimization-worker (#40793)

David Taylora32f09021f57
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index 96f6cdf764e..0b0955642db 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -984,11 +984,6 @@ module ApplicationHelper
disable_custom_css: loading_admin?,
highlight_js_path: HighlightJs.path,
svg_sprite_path: SvgSprite.path(theme_id),
- media_optimization_bundle:
- script_asset_path(
- EmberCli.script_chunks["media-optimization-bundle"]&.first ||
- "/media-optimization-bundle.js",
- ),
enable_js_error_reporting: GlobalSetting.enable_js_error_reporting,
color_scheme_is_dark: dark_color_scheme?,
user_color_scheme_id: user_scheme_id || -1,
diff --git a/frontend/discourse/app/initializers/discourse-bootstrap.js b/frontend/discourse/app/initializers/discourse-bootstrap.js
index 4046dec3ec5..cc82eef4063 100644
--- a/frontend/discourse/app/initializers/discourse-bootstrap.js
+++ b/frontend/discourse/app/initializers/discourse-bootstrap.js
@@ -80,7 +80,6 @@ export default {
session.highlightJsPath = setupData.highlightJsPath;
session.svgSpritePath = setupData.svgSpritePath;
- session.mediaOptimizationBundle = setupData.mediaOptimizationBundle;
session.userColorSchemeId = parseInt(setupData.userColorSchemeId, 10);
session.userDarkSchemeId = parseInt(setupData.userDarkSchemeId, 10);
diff --git a/frontend/discourse/app/services/media-optimization-worker.js b/frontend/discourse/app/services/media-optimization-worker.js
index c273e6565b6..8fe42fe7e41 100644
--- a/frontend/discourse/app/services/media-optimization-worker.js
+++ b/frontend/discourse/app/services/media-optimization-worker.js
@@ -1,40 +1,33 @@
import Service, { service } from "@ember/service";
import { Promise } from "rsvp";
-import { getAbsoluteURL } from "discourse/lib/get-url";
+import workerUrl from "virtual:dynamic-chunk-url:discourse/workers/media-optimization/entrypoint";
import { disableImplicitInjections } from "discourse/lib/implicit-injections";
import { fileToImageData } from "discourse/lib/media-optimization-utils";
-const WORKER_INSTALL_TIMEOUT_MS = 3000;
-
const CONVERT_FORMAT_REGEX = /(\.|\/)(jxl|hei[cf])$/i;
const ANIMATED_GIF_REGEX = /(\.|\/)(gif)$/i;
const OPTIMIZABLE_REGEX = /(\.|\/)(jpe?g|png)$/i;
/**
- * This worker follows a particular promise/callback flow to ensure
- * that the media-optimization-worker is installed and has its libraries
- * loaded before optimizations can happen. The flow:
+ * Optimizes composer image uploads in a worker. The flow:
*
* 1. optimizeImage called
- * 2. worker initialized and started
- * 3. message handlers for worker registered
- * 4. "install" message posted to worker
- * 5. "installed" message received from worker
- * 6. optimizeImage continues, posting "compress" message to worker
+ * 2. worker started if one isn't already running, message handlers registered
+ * 3. "compress"/"convert" message posted to the worker
+ * 4. worker posts back the optimized file (or an error), resolving the upload
*
- * When the worker is being installed, all other calls to optimizeImage
- * will wait for the "installed" message to be handled before continuing
- * with any image optimization work.
+ * The worker is a module worker bundled with its codecs, so it is ready as soon
+ * as it is created (posted messages queue until its module evaluates). A worker
+ * that fails to boot surfaces via onerror, and in-flight uploads continue
+ * unoptimized.
*/
@disableImplicitInjections
export default class MediaOptimizationWorkerService extends Service {
@service appEvents;
@service siteSettings;
@service capabilities;
- @service session;
worker = null;
- workerUrl = getAbsoluteURL("/javascripts/media-optimization-worker.js");
currentComposerUploadData = null;
promiseResolvers = null;
workerDoneCount = 0;
@@ -93,7 +86,10 @@ export default class MediaOptimizationWorkerService extends Service {
}
}
- await this.ensureAvailableWorker();
+ this.ensureAvailableWorker();
+ if (!this.worker) {
+ return Promise.resolve();
+ }
// eslint-disable-next-line no-async-promise-executor
return new Promise(async (resolve) => {
@@ -120,7 +116,6 @@ export default class MediaOptimizationWorkerService extends Service {
? this.siteSettings.composer_media_optimization_image_encode_quality
: this.siteSettings.image_quality,
debug_mode: this.siteSettings.composer_media_optimization_debug_mode,
- mediaOptimizationBundle: this.session.mediaOptimizationBundle,
};
if (isConvertFormat || wasmDecodeOptimizable) {
@@ -192,65 +187,48 @@ export default class MediaOptimizationWorkerService extends Service {
});
}
- async ensureAvailableWorker() {
- if (this.worker && this.workerInstalled) {
- return Promise.resolve();
- }
- if (this.installPromise) {
- return this.installPromise;
- }
- return this.install();
- }
-
- async install() {
- this.installPromise = new Promise((resolve, reject) => {
- this.afterInstalled = resolve;
- this.failedInstall = reject;
- this.logIfDebug("Installing worker");
+ ensureAvailableWorker() {
+ if (!this.worker) {
this.startWorker();
- this.registerMessageHandler();
- this.worker.postMessage({
- type: "install",
- settings: {
- mediaOptimizationBundle: this.session.mediaOptimizationBundle,
- },
- });
-
- // new Worker(workerUrl) can hang indefinitely in some cases, so we wait
- // and see if the worker manages to install in a reasonable time. If not,
- // we reject the install promise and clean up.
- setTimeout(() => {
- if (!this.workerInstalled) {
- this.failInstall("Worker install timed out!");
- }
- }, WORKER_INSTALL_TIMEOUT_MS);
-
- this.appEvents.on("composer:closed", this, "stopWorker");
- });
-
- return this.installPromise;
+ }
}
startWorker() {
this.logIfDebug("Starting media-optimization-worker");
+ // Module workers queue posted messages until their module has evaluated, so
+ // the worker is usable immediately; a failure to boot surfaces via onerror.
try {
- // TODO come up with a workaround for FF that lacks type: module support
- this.worker = new Worker(this.workerUrl);
+ const blobUrl = URL.createObjectURL(
+ new Blob([`import ${JSON.stringify(workerUrl)};`], {
+ type: "text/javascript",
+ })
+ );
+ try {
+ this.worker = new Worker(blobUrl, { type: "module" });
+ } finally {
+ URL.revokeObjectURL(blobUrl);
+ }
} catch (err) {
- this.failInstall("Error starting worker:", err);
+ this.logIfDebug("Error starting media-optimization-worker", err);
+ return;
}
+ this.registerMessageHandler();
+ this.worker.onerror = (err) => this.handleWorkerError(err);
+ this.appEvents.on("composer:closed", this, "stopWorker");
}
- failInstall(message, err = null) {
- this.logIfDebug(message, err);
- this.failedInstall(err);
- this.cleanupInstallPromises();
+ handleWorkerError(err) {
+ this.logIfDebug("media-optimization-worker error", err);
+ this.stopWorker();
+ // Resolve any in-flight optimizations so their uploads continue unoptimized.
+ Object.values(this.promiseResolvers ?? {}).forEach((resolve) =>
+ resolve?.()
+ );
}
stopWorker() {
if (this.worker) {
this.logIfDebug("Stopping media-optimization-worker...");
- this.workerInstalled = false;
this.worker.terminate();
this.worker = null;
this.workerDoneCount = 0;
@@ -323,30 +301,12 @@ export default class MediaOptimizationWorkerService extends Service {
this.workerDoneCount++;
this.workerPendingCount--;
break;
- case "installed":
- this.logIfDebug("Worker installed");
- this.workerInstalled = true;
- this.afterInstalled();
- this.cleanupInstallPromises();
- break;
- case "installFailed":
- this.failInstall(
- "Worker failed to install.",
- workerMessage.data.errorMessage
- );
- break;
default:
this.logIfDebug(`Sorry, we are out of ${workerMessage}`);
}
};
}
- cleanupInstallPromises() {
- this.afterInstalled = null;
- this.failedInstall = null;
- this.installPromise = null;
- }
-
logIfDebug(...messages) {
if (this.siteSettings.composer_media_optimization_debug_mode) {
// eslint-disable-next-line no-console
diff --git a/frontend/discourse/media-optimization-bundle.js b/frontend/discourse/app/workers/media-optimization/codecs.js
similarity index 98%
rename from frontend/discourse/media-optimization-bundle.js
rename to frontend/discourse/app/workers/media-optimization/codecs.js
index be0273bad8c..2370997984f 100644
--- a/frontend/discourse/media-optimization-bundle.js
+++ b/frontend/discourse/app/workers/media-optimization/codecs.js
@@ -106,7 +106,7 @@ async function maybeResize(imageData, width, height, settings) {
}
}
-globalThis.optimize = async function (
+export async function optimize(
imageData,
fileName,
width,
@@ -156,9 +156,9 @@ globalThis.optimize = async function (
}
return result;
-};
+}
-globalThis.convert = async function (
+export async function convert(
fileBuffer,
fileName,
fileType,
@@ -219,9 +219,9 @@ globalThis.convert = async function (
}
return { data: result, outputType: "image/jpeg" };
-};
+}
-globalThis.convertAnimated = async function (
+export async function convertAnimated(
fileBuffer,
fileName,
originalFileSize,
@@ -251,4 +251,4 @@ globalThis.convertAnimated = async function (
}
return { data: result, outputType: "image/webp" };
-};
+}
diff --git a/public/javascripts/media-optimization-worker.js b/frontend/discourse/app/workers/media-optimization/entrypoint.js
similarity index 76%
rename from public/javascripts/media-optimization-worker.js
rename to frontend/discourse/app/workers/media-optimization/entrypoint.js
index 987e96c63ae..f63a766c126 100644
--- a/public/javascripts/media-optimization-worker.js
+++ b/frontend/discourse/app/workers/media-optimization/entrypoint.js
@@ -1,12 +1,15 @@
-// The media optimization bundle uses webpack's import-scripts chunk loader,
-// so worker-only dynamic chunks can be loaded without a DOM shim.
-onmessage = async function (e) {
+// Built by rolldown as a standalone chunk and started as a module worker via a
+// blob bootstrap (see the media-optimization-worker service), so it inherits the
+// host document CSP. The codecs are bundled straight into this chunk.
+import { convert, convertAnimated, optimize } from "./codecs.js";
+
+self.onmessage = async function (e) {
switch (e.data.type) {
case "compress":
try {
globalThis.debugMode = e.data.settings.debug_mode;
- let optimized = await globalThis.optimize(
+ let optimized = await optimize(
e.data.file,
e.data.fileName,
e.data.width,
@@ -24,6 +27,7 @@ onmessage = async function (e) {
[optimized]
);
} catch (error) {
+ // eslint-disable-next-line no-console
console.error(error);
postMessage({
type: "error",
@@ -37,7 +41,7 @@ onmessage = async function (e) {
try {
globalThis.debugMode = e.data.settings.debug_mode;
- let converted = await globalThis.convert(
+ let converted = await convert(
e.data.file,
e.data.fileName,
e.data.fileType,
@@ -55,6 +59,7 @@ onmessage = async function (e) {
[converted.data]
);
} catch (error) {
+ // eslint-disable-next-line no-console
console.error(error);
postMessage({
type: "error",
@@ -68,7 +73,7 @@ onmessage = async function (e) {
try {
globalThis.debugMode = e.data.settings.debug_mode;
- let animatedResult = await globalThis.convertAnimated(
+ let animatedResult = await convertAnimated(
e.data.file,
e.data.fileName,
e.data.originalFileSize,
@@ -93,6 +98,7 @@ onmessage = async function (e) {
});
}
} catch (error) {
+ // eslint-disable-next-line no-console
console.error(error);
postMessage({
type: "error",
@@ -102,20 +108,8 @@ onmessage = async function (e) {
});
}
break;
- case "install":
- try {
- await loadLibs(e.data.settings);
- postMessage({ type: "installed" });
- } catch (error) {
- console.error(error);
- postMessage({ type: "installFailed", errorMessage: error.message });
- }
- break;
default:
- logIfDebug(`Sorry, we are out of ${e}.`);
+ // eslint-disable-next-line no-console
+ console.error(`Unexpected message type: ${e.data.type}`);
}
};
-
-async function loadLibs(settings) {
- return import(settings.mediaOptimizationBundle);
-}
diff --git a/frontend/discourse/lib/dynamic-chunk-url-plugin.mjs b/frontend/discourse/lib/dynamic-chunk-url-plugin.mjs
new file mode 100644
index 00000000000..a4cf63aabd6
--- /dev/null
+++ b/frontend/discourse/lib/dynamic-chunk-url-plugin.mjs
@@ -0,0 +1,49 @@
+import { relative } from "path";
+
+const PREFIX = "virtual:dynamic-chunk-url:";
+const RESOLVED_PREFIX = "\0" + PREFIX;
+
+/*
+ * Importing `virtual:dynamic-chunk-url:<specifier>` emits <specifier> as its own
+ * bundled, hashed chunk and resolves to that chunk's final URL (a string). The
+ * specifier is resolved like a normal import (relative to the importer).
+ *
+ * For example, a worker can be started from a blob bootstrap that imports the real
+ * chunk by absolute URL, so the worker inherits the host document CSP.
+ */
+export default function dynamicChunkUrlPlugin() {
+ return {
+ name: "dynamic-chunk-url",
+ resolveId: {
+ filter: { id: /^virtual:dynamic-chunk-url:/ },
+ async handler(source, importer) {
+ const target = source.slice(PREFIX.length);
+ const resolved = await this.resolve(target, importer, {
+ skipSelf: true,
+ });
+ if (!resolved) {
+ this.error(`dynamic-chunk-url: could not resolve "${target}"`);
+ }
+ return { id: RESOLVED_PREFIX + resolved.id };
+ },
+ },
+ load: {
+ filter: { id: /^\0virtual:dynamic-chunk-url:/ },
+ handler(id) {
+ const target = id.slice(RESOLVED_PREFIX.length);
+ const refId = this.emitFile({
+ type: "chunk",
+ id: target,
+ name: relative(process.cwd(), target)
+ .replace(/\.[^.]+$/, "")
+ .replace(/[/\\]/g, "-"),
+ preserveSignature: "strict",
+ });
+ return {
+ code: `export default import.meta.ROLLUP_FILE_URL_${refId};`,
+ moduleType: "js",
+ };
+ },
+ },
+ };
+}
diff --git a/frontend/discourse/rolldown.config.mjs b/frontend/discourse/rolldown.config.mjs
index f5fddd83074..6811b526eed 100644
--- a/frontend/discourse/rolldown.config.mjs
+++ b/frontend/discourse/rolldown.config.mjs
@@ -1,6 +1,7 @@
import * as fs from "fs";
import { basename, relative } from "path";
import { viteAliasPlugin, viteImportGlobPlugin } from "rolldown/experimental";
+import dynamicChunkUrlPlugin from "./lib/dynamic-chunk-url-plugin.mjs";
import writeResolverConfig from "./lib/embroider-vite-resolver-options.mjs";
import maybeBabel from "./lib/maybe-babel.mjs";
import optimizedEmber from "./lib/optimized-ember.mjs";
@@ -8,7 +9,7 @@ import wrapTestModulesPlugin from "./lib/wrap-test-modules-plugin.mjs";
writeResolverConfig(
{
- staticAppPaths: ["static", "admin"],
+ staticAppPaths: ["static", "admin", "workers"],
splitAtRoutes: [{ type: "string", value: "wizard" }],
},
{
@@ -85,7 +86,6 @@ export function buildConfig({ devMode } = {}) {
input: {
discourse: "discourse.js",
vendor: "vendor.js",
- "media-optimization-bundle": "media-optimization-bundle.js",
...(!isProduction || process.env.FORCE_BUILD_TESTS
? {
"test-entrypoint": "tests/test-entrypoint.js",
@@ -112,6 +112,7 @@ export function buildConfig({ devMode } = {}) {
preserveEntrySignatures: "strict",
plugins: [
viteAliasPlugin({ entries: aliases }),
+ dynamicChunkUrlPlugin(),
optimizedEmber(),
viteImportGlobPlugin(),
maybeBabel({
diff --git a/lib/content_security_policy/default.rb b/lib/content_security_policy/default.rb
index 9666d794717..47eb39118c0 100644
--- a/lib/content_security_policy/default.rb
+++ b/lib/content_security_policy/default.rb
@@ -12,7 +12,7 @@ class ContentSecurityPolicy
directives[:base_uri] = [:self]
directives[:object_src] = [:none]
directives[:script_src] = script_src
- directives[:worker_src] = []
+ directives[:worker_src] = [:self, "blob:"]
directives[:frame_ancestors] = frame_ancestors if restrict_embed?
directives[:manifest_src] = ["'self'"]
end
@@ -21,7 +21,7 @@ class ContentSecurityPolicy
private
def script_src
- ["'strict-dynamic'"]
+ %w['strict-dynamic' 'wasm-unsafe-eval']
end
def frame_ancestors
diff --git a/spec/integration/content_security_policy_spec.rb b/spec/integration/content_security_policy_spec.rb
index 8788a750220..5a38cabcc68 100644
--- a/spec/integration/content_security_policy_spec.rb
+++ b/spec/integration/content_security_policy_spec.rb
@@ -13,7 +13,7 @@ RSpec.describe "content security policy integration" do
expect(response.headers["Content-Security-Policy"]).to be_present
expect(response.headers["Content-Security-Policy"]).to match(
- /script-src 'nonce-[^']+' 'strict-dynamic';/,
+ /script-src 'nonce-[^']+' 'strict-dynamic' 'wasm-unsafe-eval'; worker-src 'self' blob:;/,
)
end
end
diff --git a/spec/lib/content_security_policy_spec.rb b/spec/lib/content_security_policy_spec.rb
index c6377256935..45cae13536c 100644
--- a/spec/lib/content_security_policy_spec.rb
+++ b/spec/lib/content_security_policy_spec.rb
@@ -34,9 +34,14 @@ RSpec.describe ContentSecurityPolicy do
expect(script_srcs).to include("'strict-dynamic'")
end
- it "does not set worker-src" do
+ it "allows wasm compilation" do
+ script_srcs = parse(policy)["script-src"]
+ expect(script_srcs).to include("'wasm-unsafe-eval'")
+ end
+
+ it "sets worker-src to self and blob:" do
worker_src = parse(policy)["worker-src"]
- expect(worker_src).to eq(nil)
+ expect(worker_src).to contain_exactly("'self'", "blob:")
end
end