Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
feat(engine): render WebGPU compositions on SwiftShader behind an opt-in (#5133)
Miguel Ángel4cf5cf9b53d1
diff --git a/packages/engine/src/config.test.ts b/packages/engine/src/config.test.tsindex 4098b4cc87..9f8c9b95c8 100644--- a/packages/engine/src/config.test.ts+++ b/packages/engine/src/config.test.ts@@ -85,6 +85,12 @@ describe("resolveConfig", () => {expect(config.enableBrowserPool).toBe(true);});+ it("keeps software WebGPU off unless PRODUCER_ALLOW_SOFTWARE_WEBGPU opts in", () => {+ expect(resolveConfig().allowSoftwareWebGpu).toBe(false);+ setEnv("PRODUCER_ALLOW_SOFTWARE_WEBGPU", "true");+ expect(resolveConfig().allowSoftwareWebGpu).toBe(true);+ });+it("lets env vars opt out of default streaming encode", () => {setEnv("PRODUCER_ENABLE_STREAMING_ENCODE", "false");diff --git a/packages/engine/src/config.ts b/packages/engine/src/config.tsindex fd6c4af07f..46e2abe257 100644--- a/packages/engine/src/config.ts+++ b/packages/engine/src/config.ts@@ -51,6 +51,11 @@ export interface EngineConfig {* Cost: one extra Chrome launch (~1-2 s) per process; result cached.*/browserGpuMode: "software" | "hardware" | "auto";+ /**+ * Render a `data-requires-webgpu` composition on SwiftShader when no GPU is in use, instead of refusing it.+ * Off by default: software WebGPU is slow and skips the compositor workaround for HF#3049.+ */+ allowSoftwareWebGpu: boolean;enableBrowserPool: boolean;browserTimeout: number;protocolTimeout: number;@@ -287,6 +292,7 @@ export const DEFAULT_CONFIG: EngineConfig = {disableGpu: false,browserGpuMode: "software",+ allowSoftwareWebGpu: false,enableBrowserPool: true,browserTimeout: 120_000,protocolTimeout: 300_000,@@ -340,6 +346,7 @@ const OPTIONAL_ENGINE_CONFIG_FIELDS = [const BOOLEAN_ENGINE_CONFIG_FIELDS = ["disableGpu",+ "allowSoftwareWebGpu","enableBrowserPool","forceScreenshot","staticFrameDedup",@@ -840,6 +847,10 @@ export function resolveConfig(overrides?: Partial<EngineConfig>): EngineConfig {chromePath: env("PRODUCER_HEADLESS_SHELL_PATH"),disableGpu: envBool("PRODUCER_DISABLE_GPU", DEFAULT_CONFIG.disableGpu),browserGpuMode: envBrowserGpuMode(),+ allowSoftwareWebGpu: envBool(+ "PRODUCER_ALLOW_SOFTWARE_WEBGPU",+ DEFAULT_CONFIG.allowSoftwareWebGpu,+ ),enableBrowserPool: envBool("PRODUCER_ENABLE_BROWSER_POOL", DEFAULT_CONFIG.enableBrowserPool),browserTimeout: envNum("PRODUCER_PUPPETEER_LAUNCH_TIMEOUT_MS", DEFAULT_CONFIG.browserTimeout),protocolTimeout: envNum(diff --git a/packages/engine/src/services/browserManager.test.ts b/packages/engine/src/services/browserManager.test.tsindex c1fd8bc204..08bb47d726 100644--- a/packages/engine/src/services/browserManager.test.ts+++ b/packages/engine/src/services/browserManager.test.ts@@ -20,6 +20,7 @@ import {buildChromeArgs,compositionRequiresWebGpu,assertWebGpuAdapterAvailable,+ usesSoftwareWebGpu,WebGpuUnavailableError,drainBrowserPool,forceReleaseBrowser,@@ -208,6 +209,37 @@ describe("buildChromeArgs browser GPU mode", () => {it("is byte-identical to the requiresWebGpu-absent case for a non-declaring composition", () => {expect(buildChromeArgs({ ...base, requiresWebGpu: false })).toEqual(buildChromeArgs(base));});++ it("runs an opted-in WebGPU composition on SwiftShader's Vulkan with GPU compositing on", () => {+ const args = buildChromeArgs(+ { ...base, platform: "linux", requiresWebGpu: true },+ { browserGpuMode: "software", allowSoftwareWebGpu: true },+ );+ expect(args).toContain("--enable-features=CanvasDrawElement,Vulkan");+ expect(args).not.toContain("--enable-features=CanvasDrawElement");+ expect(args).toContain("--use-vulkan=swiftshader");+ expect(args).toContain("--enable-unsafe-webgpu");+ expect(args).not.toContain("--disable-gpu-compositing");+ });++ it.each([+ ["without the opt-in", true, { browserGpuMode: "software" }],+ [+ "for a composition without WebGPU",+ false,+ { browserGpuMode: "software", allowSoftwareWebGpu: true },+ ],+ ["on a GPU", true, { browserGpuMode: "hardware", allowSoftwareWebGpu: true }],+ ["under --disable-gpu", true, { disableGpu: true, allowSoftwareWebGpu: true }],+ ] as const)("leaves the launch as it was %s", (_, requiresWebGpu, config) => {+ expect(usesSoftwareWebGpu(requiresWebGpu, config)).toBe(false);+ expect(buildChromeArgs({ ...base, platform: "linux", requiresWebGpu }, config)).toEqual(+ buildChromeArgs(+ { ...base, platform: "linux", requiresWebGpu },+ { ...config, allowSoftwareWebGpu: false },+ ),+ );+ });});describe("compositionRequiresWebGpu", () => {@@ -255,7 +287,7 @@ describe("assertWebGpuAdapterAvailable", () => {describe("in-page adapter check (real callback, stubbed navigator.gpu)", () => {const runInPage = {- evaluate: (fn: (t: number) => unknown, t: number) => fn(t),+ evaluate: (fn: (...args: unknown[]) => unknown, ...args: unknown[]) => fn(...args),} as unknown as Page;const stubAdapter = (adapter: unknown) =>vi.stubGlobal("navigator", { gpu: { requestAdapter: async () => adapter } });@@ -273,6 +305,14 @@ describe("assertWebGpuAdapterAvailable", () => {);});+ it("accepts a software fallback adapter only on a launch that runs software WebGPU", async () => {+ stubAdapter({ info: { isFallbackAdapter: true } });+ await expect(assertWebGpuAdapterAvailable(runInPage, true, true)).resolves.toBeUndefined();+ await expect(assertWebGpuAdapterAvailable(runInPage, true, false)).rejects.toBeInstanceOf(+ WebGpuUnavailableError,+ );+ });+it("refuses a host with no navigator.gpu", async () => {vi.stubGlobal("navigator", {});await expect(assertWebGpuAdapterAvailable(runInPage, true)).rejects.toBeInstanceOf(diff --git a/packages/engine/src/services/browserManager.ts b/packages/engine/src/services/browserManager.tsindex 6eda90a05b..5719f473c3 100644--- a/packages/engine/src/services/browserManager.ts+++ b/packages/engine/src/services/browserManager.ts@@ -872,15 +872,33 @@ export interface BuildChromeArgsOptions {const CANVAS_DRAW_ELEMENT_FEATURE_FLAG = "--enable-features=CanvasDrawElement";const WEBGPU_FLAG = "--enable-unsafe-webgpu";-export function buildChromeArgs(- options: BuildChromeArgsOptions,- config?: Partial<Pick<EngineConfig, "browserGpuMode" | "disableGpu" | "chromePath">>,-): string[] {+type GpuConfig = Partial<+ Pick<EngineConfig, "browserGpuMode" | "disableGpu" | "chromePath" | "allowSoftwareWebGpu">+>;++/**+ * Whether this launch renders WebGPU on SwiftShader: the composition needs it, the render opted in, no GPU is in use.+ * SwiftShader draws canvas WebGPU only via Vulkan with GPU compositing on, and not at all under --disable-gpu.+ */+export function usesSoftwareWebGpu(+ requiresWebGpu: boolean | undefined,+ config?: GpuConfig,+): boolean {+ return (+ !!requiresWebGpu &&+ !!config?.allowSoftwareWebGpu &&+ !(config.disableGpu ?? DEFAULT_CONFIG.disableGpu) &&+ (config.browserGpuMode ?? DEFAULT_CONFIG.browserGpuMode) === "software"+ );+}++export function buildChromeArgs(options: BuildChromeArgsOptions, config?: GpuConfig): string[] {const platform = options.platform ?? process.platform;const gpuDisabled = config?.disableGpu ?? DEFAULT_CONFIG.disableGpu;const browserGpuMode = gpuDisabled? "software": (config?.browserGpuMode ?? DEFAULT_CONFIG.browserGpuMode);+ const softwareWebGpu = usesSoftwareWebGpu(options.requiresWebGpu, config);// Chrome flags tuned for headless rendering performance. The set below is a// fairly standard "headless-for-capture" configuration — similar profiles// appear in Puppeteer's defaults, Playwright, Remotion, and Chrome's own@@ -889,7 +907,9 @@ export function buildChromeArgs("--no-sandbox","--disable-setuid-sandbox","--disable-dev-shm-usage",- CANVAS_DRAW_ELEMENT_FEATURE_FLAG,+ softwareWebGpu+ ? `${CANVAS_DRAW_ELEMENT_FEATURE_FLAG},Vulkan`+ : CANVAS_DRAW_ELEMENT_FEATURE_FLAG,"--enable-webgl","--ignore-gpu-blocklist",...getBrowserGpuArgs(browserGpuMode, platform),@@ -934,6 +954,7 @@ export function buildChromeArgs(if (browserGpuMode !== "software" || options.requiresWebGpu) {chromeArgs.push(WEBGPU_FLAG);}+ if (softwareWebGpu) chromeArgs.push("--use-vulkan=swiftshader");// SwiftShader's GPU compositor can retain a transformed layer for several// sequential frames after a GSAP yoyo/reversal, and it re-presents stale@@ -956,7 +977,7 @@ export function buildChromeArgs(//// Remove this workaround once the pinned chrome-headless-shell includes// https://issues.chromium.org/issues/535256667.- if (browserGpuMode === "software") {+ if (browserGpuMode === "software" && !softwareWebGpu) {chromeArgs.push("--disable-gpu-compositing");}@@ -995,8 +1016,9 @@ export class WebGpuUnavailableError extends Error {constructor() {super("This composition declares data-requires-webgpu, but no hardware WebGPU adapter could be " +- "obtained on this browser launch (a software fallback adapter such as swiftshader reports " +- "one but cannot render it). Run on a host with a GPU, or remove data-requires-webgpu.",+ "obtained on this browser launch. Run on a host with a GPU, set " ++ "PRODUCER_ALLOW_SOFTWARE_WEBGPU=true to render it on SwiftShader (slow), or remove " ++ "data-requires-webgpu.",);this.name = "WebGpuUnavailableError";}@@ -1005,29 +1027,35 @@ export class WebGpuUnavailableError extends Error {const WEBGPU_ADAPTER_PROBE_TIMEOUT_MS = 10_000;/**- * Confirms the already-navigated page can obtain a hardware WebGPU adapter; no-op otherwise.+ * Confirms the already-navigated page can obtain a hardware WebGPU adapter, or any adapter when the launch runs+ * software WebGPU ({@link usesSoftwareWebGpu}); no-op otherwise.* `page` must be on a secure-context origin, or navigator.gpu always reads absent.*/export async function assertWebGpuAdapterAvailable(page: Page,requiresWebGpu: boolean,+ softwareWebGpu = false,): Promise<void> {if (!requiresWebGpu) return;// requestAdapter() has no native timeout; a broken driver can hang it// indefinitely. Race it in-page so a stuck adapter reads as "unavailable"// instead of hanging the caller.- const hasUsableAdapter = await page.evaluate(async (timeoutMs) => {- if (typeof navigator === "undefined" || !navigator.gpu) return false;- try {- const adapter = await Promise.race([- navigator.gpu.requestAdapter(),- new Promise<null>((resolve) => setTimeout(() => resolve(null), timeoutMs)),- ]);- return !!adapter && !adapter.info.isFallbackAdapter;- } catch {- return false;- }- }, WEBGPU_ADAPTER_PROBE_TIMEOUT_MS);+ const hasUsableAdapter = await page.evaluate(+ async (timeoutMs, acceptFallback) => {+ if (typeof navigator === "undefined" || !navigator.gpu) return false;+ try {+ const adapter = await Promise.race([+ navigator.gpu.requestAdapter(),+ new Promise<null>((resolve) => setTimeout(() => resolve(null), timeoutMs)),+ ]);+ return !!adapter && (acceptFallback || !adapter.info.isFallbackAdapter);+ } catch {+ return false;+ }+ },+ WEBGPU_ADAPTER_PROBE_TIMEOUT_MS,+ softwareWebGpu,+ );if (!hasUsableAdapter) throw new WebGpuUnavailableError();}diff --git a/packages/engine/src/services/frameCapture-softwareWebGpu.test.ts b/packages/engine/src/services/frameCapture-softwareWebGpu.test.tsnew file mode 100644index 0000000000..b155f16e8f--- /dev/null+++ b/packages/engine/src/services/frameCapture-softwareWebGpu.test.ts@@ -0,0 +1,73 @@+import { afterEach, describe, expect, it, vi } from "vitest";+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { createCaptureSession, initializeSession } from "./frameCapture.js";+import { assertWebGpuAdapterAvailable } from "./browserManager.js";++const launches: string[][] = [];+let resolvedMode: "software" | "hardware" = "software";++vi.mock("./browserManager.js", async (importOriginal) => ({+ ...(await importOriginal<typeof import("./browserManager.js")>()),+ resolveBrowserGpuMode: async () => resolvedMode,+ assertWebGpuAdapterAvailable: vi.fn(async () => {+ throw new Error("adapter checked");+ }),+ acquireBrowser: async (args: string[]) => {+ launches.push(args);+ return {+ captureMode: "screenshot",+ browser: {+ version: async () => "Chrome/150.0.0.0",+ newPage: async () => ({+ on: () => {},+ goto: async () => null,+ createCDPSession: async () => ({ on: () => {}, send: async () => ({}) }),+ evaluateOnNewDocument: async () => {},+ setViewport: async () => {},+ evaluate: async () => undefined,+ }),+ },+ };+ },+}));++const dirs: string[] = [];+afterEach(() => {+ launches.length = 0;+ vi.mocked(assertWebGpuAdapterAvailable).mockClear();+ for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });+});++/** The args a WebGPU capture session launches with, after its adapter check has run. */+async function webGpuSession(allowSoftwareWebGpu: boolean) {+ const dir = mkdtempSync(join(tmpdir(), "hf-software-webgpu-"));+ dirs.push(dir);+ const session = await createCaptureSession(+ "http://localhost:0",+ dir,+ { width: 1920, height: 1080, fps: { num: 30, den: 1 }, format: "jpeg", requiresWebGpu: true },+ null,+ { forceScreenshot: true, browserGpuMode: "auto", allowSoftwareWebGpu },+ );+ await expect(initializeSession(session)).rejects.toThrow("adapter checked");+ return launches.at(-1) ?? [];+}++describe("software WebGPU in a capture session", () => {+ it("launches on SwiftShader's Vulkan and accepts its adapter when opted in on a software host", async () => {+ resolvedMode = "software";+ expect(await webGpuSession(true)).toContain("--use-vulkan=swiftshader");+ expect(assertWebGpuAdapterAvailable).toHaveBeenLastCalledWith(expect.anything(), true, true);+ });++ it.each([+ ["without the opt-in", false, "software"],+ ["on a GPU", true, "hardware"],+ ] as const)("refuses a fallback adapter %s", async (_, allowSoftwareWebGpu, mode) => {+ resolvedMode = mode;+ expect(await webGpuSession(allowSoftwareWebGpu)).not.toContain("--use-vulkan=swiftshader");+ expect(assertWebGpuAdapterAvailable).toHaveBeenLastCalledWith(expect.anything(), true, false);+ });+});diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.tsindex 42a40db396..c87c29933e 100644--- a/packages/engine/src/services/frameCapture.ts+++ b/packages/engine/src/services/frameCapture.ts@@ -51,6 +51,7 @@ import {resolveHeadlessShellPath,compositionRequiresWebGpu,assertWebGpuAdapterAvailable,+ usesSoftwareWebGpu,type BrowserLease,type CaptureMode,} from "./browserManager.js";@@ -114,6 +115,8 @@ export interface CaptureSession {outputDir: string;/** The composition served at `serverUrl` declares `data-requires-webgpu`. */requiresWebGpu?: boolean;+ /** The browser was launched to run that WebGPU on SwiftShader ({@link usesSoftwareWebGpu}). */+ softwareWebGpu?: boolean;onBeforeCapture: BeforeCaptureHook | null;isInitialized: boolean;/**@@ -1305,9 +1308,10 @@ export async function createCaptureSession(.then((res) => res.text()).then(compositionRequiresWebGpu).catch(() => false));+ const gpuConfig = { ...config, browserGpuMode: resolvedGpuMode };const chromeArgs = buildChromeArgs({ width: options.width, height: options.height, captureMode: preMode, requiresWebGpu },- { ...config, browserGpuMode: resolvedGpuMode },+ gpuConfig,);const browserLease = await acquireBrowser(chromeArgs, config);@@ -1320,6 +1324,7 @@ export async function createCaptureSession(config,useDrawElement,requiresWebGpu,+ softwareWebGpu: usesSoftwareWebGpu(requiresWebGpu, gpuConfig),});}@@ -1332,6 +1337,7 @@ interface CaptureSessionConstructionInput {config?: Partial<EngineConfig>;useDrawElement: boolean;requiresWebGpu?: boolean;+ softwareWebGpu?: boolean;}async function constructCaptureSessionWithRollback(@@ -1386,6 +1392,7 @@ async function constructCaptureSession(config,useDrawElement,requiresWebGpu,+ softwareWebGpu,onPageCreated,} = input;const { browser, captureMode } = browserLease;@@ -1511,6 +1518,7 @@ async function constructCaptureSession(outputDir,onBeforeCapture,requiresWebGpu,+ softwareWebGpu,isInitialized: false,browserConsoleBuffer: [],scriptLoadFailures: [],@@ -2391,7 +2399,11 @@ export async function initializeSession(session: CaptureSession): Promise<void>);throw error;}- await assertWebGpuAdapterAvailable(page, session.requiresWebGpu ?? false);+ await assertWebGpuAdapterAvailable(+ page,+ session.requiresWebGpu ?? false,+ session.softwareWebGpu,+ );};if (session.captureMode === "screenshot") {diff --git a/skills-manifest.json b/skills-manifest.jsonindex 581b3c40e8..ccf1735284 100644--- a/skills-manifest.json+++ b/skills-manifest.json@@ -22,7 +22,7 @@"files": 29},"hyperframes-animation": {- "hash": "179715a416750b26",+ "hash": "bcee69f2229cda71","files": 122},"hyperframes-audio": {diff --git a/skills/hyperframes-animation/adapters/typegpu.md b/skills/hyperframes-animation/adapters/typegpu.mdindex 675d1bd753..666cb93d63 100644--- a/skills/hyperframes-animation/adapters/typegpu.md+++ b/skills/hyperframes-animation/adapters/typegpu.md@@ -17,7 +17,7 @@ The render engine auto-passes `--enable-unsafe-webgpu` and `--enable-features=Ca- Render from HyperFrames time, not `performance.now()`.- Listen for the `hf-seek` event and re-render at exactly that time.- Guard against environments where WebGPU is unavailable — the adapter does not check for you.-- If the composition cannot render without WebGPU, add `data-requires-webgpu` to its composition root. Local capture commands then report an actionable error instead of capturing a no-GPU fallback screen when auto-detection selects software rendering.+- If the composition cannot render without WebGPU, add `data-requires-webgpu` to its composition root. Local capture commands then report an actionable error instead of capturing a no-GPU fallback screen when auto-detection selects software rendering. To render it on a host without a GPU, set `PRODUCER_ALLOW_SOFTWARE_WEBGPU=true`: it renders on SwiftShader, several times slower.- After submitting GPU work, register queue completion synchronously with `e.detail.waitUntil(device.queue.onSubmittedWorkDone())`. HyperFrames awaits registered work before screenshots and frame capture.The adapter sets `window.__hfTypegpuTime` and dispatches an `hf-seek` event with `{ time, waitUntil }` on each seek. While Studio is paused, HyperFrames may dispatch the same time again to keep the WebGPU swapchain presented. Re-render that exact time; do not advance simulation state.