Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
feat(core,engine): track accelerated canvases before any page script runs (#5970)
Vance Ingalls0865b6f36d42
diff --git a/packages/core/src/compiler/htmlDocument.test.ts b/packages/core/src/compiler/htmlDocument.test.tsindex 3c0c1a7331..466d993e02 100644--- a/packages/core/src/compiler/htmlDocument.test.ts+++ b/packages/core/src/compiler/htmlDocument.test.ts@@ -8,6 +8,7 @@ import {injectScriptsIntoHtml,injectTagsAtHeadStart,insertBeforeCloseTag,+ insertRuntimeTag,hasCompositionOutsideTemplates,isFullHtmlDocument,parseHTMLContent,@@ -454,3 +455,29 @@ describe("hasCompositionOutsideTemplates", () => {expect(hasCompositionOutsideTemplates(html)).toBe(expected);});});++describe("insertRuntimeTag", () => {+ const runtime = '<script data-hf-runtime src="/rt.js"></script>';+ const page = (attrs: string) =>+ `<html><head><script>window.author=1</script></head><body><div id="root" ${attrs}></div></body></html>`;++ it("puts the canvas tracker ahead of author head scripts for a page that opts into blur", () => {+ const out = insertRuntimeTag(page('data-motion-blur="on"'), runtime);+ const tracker = out.indexOf("data-hf-accel-tracker");+ expect(tracker).toBeGreaterThan(-1);+ expect(tracker).toBeLessThan(out.indexOf("window.author"));+ expect(out.indexOf("window.author")).toBeLessThan(out.indexOf(runtime));+ });++ it("adds the tracker once, however many times the runtime is placed", () => {+ const once = insertRuntimeTag(page('data-motion-blur="on"'), runtime);+ const twice = insertRuntimeTag(once, runtime);+ expect(twice.split("data-hf-accel-tracker").length).toBe(2);+ });++ it("leaves pages that did not opt in alone", () => {+ for (const attrs of ["", 'data-motion-blur="off"', 'data-motion-blur-shutter="on"']) {+ expect(insertRuntimeTag(page(attrs), runtime)).not.toContain("data-hf-accel-tracker");+ }+ });+});diff --git a/packages/core/src/compiler/htmlDocument.ts b/packages/core/src/compiler/htmlDocument.tsindex a1861a7d28..d070905820 100644--- a/packages/core/src/compiler/htmlDocument.ts+++ b/packages/core/src/compiler/htmlDocument.ts@@ -1,3 +1,4 @@+import { acceleratedCanvasTrackerSource } from "../pageMotionBlur.js";import { parseHTML } from "linkedom";export const RUNTIME_BOOTSTRAP_ATTR = "data-hyperframes-preview-runtime";@@ -347,7 +348,7 @@ export function insertBeforeCloseTag(return insertBeforeDocumentTag(html, `</${name}`, markup);}-export function insertRuntimeTag(html: string, tag: string): string {+function insertTagInHead(html: string, tag: string): string {const withHead = insertBeforeCloseTag(html, "head", `${tag}\n`);if (withHead !== null) return withHead;const htmlOpenMatch = html.match(/<html\b[^>]*>/i);@@ -363,6 +364,16 @@ export function insertRuntimeTag(html: string, tag: string): string {return tag + html;}+const ACCEL_TRACKER_ATTR = "data-hf-accel-tracker";+const OPTS_INTO_MOTION_BLUR = /\sdata-motion-blur\s*=\s*["']?on[\s"'>]/i;++export function insertRuntimeTag(html: string, tag: string): string {+ const withRuntime = insertTagInHead(html, tag);+ if (!OPTS_INTO_MOTION_BLUR.test(html) || html.includes(ACCEL_TRACKER_ATTR)) return withRuntime;+ const tracker = `<script ${ACCEL_TRACKER_ATTR}>${acceleratedCanvasTrackerSource()}</script>`;+ return injectTagsAtHeadStart(withRuntime, tracker);+}+/*** Insert raw tag markup at the very start of `<head>`, ahead of every author* script (inline or external). Falls back to just before `<body>`, then to thediff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 75ead8efa5..3cee99b92f 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -437,4 +437,8 @@ export {type HfVfxParam,type HfVfxParamValues,} from "./vfx.js";-export { PAGE_DEVICE_SCALE_GLOBAL, PAGE_MOTION_BLUR_OFF_GLOBAL } from "./pageMotionBlur.js";+export {+ PAGE_DEVICE_SCALE_GLOBAL,+ PAGE_MOTION_BLUR_OFF_GLOBAL,+ acceleratedCanvasTrackerSource,+} from "./pageMotionBlur.js";diff --git a/packages/core/src/pageMotionBlur.test.ts b/packages/core/src/pageMotionBlur.test.tsnew file mode 100644index 0000000000..9c7e1e62cf--- /dev/null+++ b/packages/core/src/pageMotionBlur.test.ts@@ -0,0 +1,59 @@+// @vitest-environment jsdom+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import {+ acceleratedCanvasTrackerSource,+ installAcceleratedCanvasTracker,+ isAcceleratedCanvas,+} from "./pageMotionBlur";++type Proto = Record<string, unknown>;+const proto = HTMLCanvasElement.prototype as unknown as Proto;+const saved: Proto = {};++beforeEach(() => {+ for (const name of ["getContext", "transferControlToOffscreen"]) saved[name] = proto[name];+ proto.getContext = vi.fn(() => ({}));+ proto.transferControlToOffscreen = vi.fn(() => ({}));+});++afterEach(() => {+ for (const name of Object.keys(saved)) proto[name] = saved[name];+ Reflect.deleteProperty(window, "__hfAcceleratedCanvases");+});++describe("accelerated canvas tracker", () => {+ it("remembers WebGL and WebGPU canvases, not 2D ones", () => {+ installAcceleratedCanvasTracker();+ const [gl, gpu, flat] = [1, 2, 3].map(() => document.createElement("canvas"));+ gl?.getContext("webgl2");+ gpu?.getContext("webgpu");+ flat?.getContext("2d");+ expect([gl, gpu, flat].map((c) => c !== undefined && isAcceleratedCanvas(c))).toEqual([+ true,+ true,+ false,+ ]);+ });++ it("treats a canvas handed to an OffscreenCanvas as accelerated", () => {+ installAcceleratedCanvasTracker();+ const canvas = document.createElement("canvas");+ canvas.transferControlToOffscreen();+ expect(isAcceleratedCanvas(canvas)).toBe(true);+ });++ it("installs once however many times it runs", () => {+ const original = proto.getContext as ReturnType<typeof vi.fn>;+ installAcceleratedCanvasTracker();+ installAcceleratedCanvasTracker();+ document.createElement("canvas").getContext("2d");+ expect(original).toHaveBeenCalledTimes(1);+ });++ it("works from its source text, as the engine and the preview inline it", () => {+ (0, eval)(acceleratedCanvasTrackerSource());+ const canvas = document.createElement("canvas");+ canvas.getContext("webgl");+ expect(isAcceleratedCanvas(canvas)).toBe(true);+ });+});diff --git a/packages/core/src/pageMotionBlur.ts b/packages/core/src/pageMotionBlur.tsindex 4b422e1cef..9169613f1e 100644--- a/packages/core/src/pageMotionBlur.ts+++ b/packages/core/src/pageMotionBlur.ts@@ -12,3 +12,48 @@ export function supportsCaptureMotionBlur(win: Window = window): boolean {const canvas = w.HTMLCanvasElement?.prototype;return !!ctx && !!canvas && "drawElementImage" in ctx && "requestPaint" in canvas;}++const ACCELERATED_CANVASES_GLOBAL = "__hfAcceleratedCanvases";++/**+ * Remember canvases html-in-canvas cannot capture: WebGL, WebGPU, or handed to an OffscreenCanvas.+ * Self-contained: the engine and the preview run its source ahead of every author script.+ */+export function installAcceleratedCanvasTracker(): void {+ const key = "__hfAcceleratedCanvases";+ const known: unknown = Reflect.get(window, key);+ const seen = known instanceof WeakSet ? known : new WeakSet<object>();+ if (seen !== known) Object.defineProperty(window, key, { value: seen });+ const accelerated = ["webgl", "webgl2", "experimental-webgl", "webgpu"];+ const wrap = (+ name: string,+ onCall: (canvas: object, args: unknown[], result: unknown) => void,+ ) => {+ const proto = HTMLCanvasElement.prototype;+ const original: unknown = Reflect.get(proto, name);+ if (typeof original !== "function" || "__hfAccelTracked" in original) return;+ const tracked = function (this: object, ...args: unknown[]): unknown {+ const result: unknown = Reflect.apply(original, this, args);+ onCall(this, args, result);+ return result;+ };+ Object.defineProperty(tracked, "__hfAccelTracked", { value: true });+ Object.defineProperty(proto, name, { value: tracked, writable: true, configurable: true });+ };+ wrap("getContext", (canvas, args, ctx) => {+ if (ctx && accelerated.includes(String(args[0]))) seen.add(canvas);+ });+ wrap("transferControlToOffscreen", (canvas) => {+ seen.add(canvas);+ });+}++export function isAcceleratedCanvas(canvas: HTMLCanvasElement): boolean {+ const seen: unknown = Reflect.get(window, ACCELERATED_CANVASES_GLOBAL);+ return seen instanceof WeakSet && seen.has(canvas);+}++/** The tracker as a script body, for `evaluateOnNewDocument` and inline `<script>` injection. */+export function acceleratedCanvasTrackerSource(): string {+ return `(${installAcceleratedCanvasTracker.toString()})();`;+}diff --git a/packages/core/src/runtime/captureMotionBlur/index.test.ts b/packages/core/src/runtime/captureMotionBlur/index.test.tsindex 13590878cf..3cc52b744b 100644--- a/packages/core/src/runtime/captureMotionBlur/index.test.ts+++ b/packages/core/src/runtime/captureMotionBlur/index.test.ts@@ -17,6 +17,7 @@ type Win = Window & {reason: string | null;shutterDeg: number;moverNames: string[];+ failedFrames: number;};};__hf_page_composite_pending?: boolean;@@ -204,6 +205,22 @@ describe("opt-in and off switches", () => {expect(win.__hfMotionBlur?.report().reason).toMatch(/WebGL/);});+ it("stays off for a canvas handed to an OffscreenCanvas, whose context is out of reach", () => {+ installHtmlInCanvas();+ (HTMLCanvasElement.prototype as unknown as Record<string, unknown>).transferControlToOffscreen =+ () => ({});+ trackAcceleratedCanvases();+ const root = makeRoot();+ const canvas = document.createElement("canvas");+ root.append(canvas);+ canvas.transferControlToOffscreen();+ initCaptureMotionBlur(host(root));+ expect(win.__hfMotionBlur?.report()).toMatchObject({ state: "off" });+ expect(win.__hfMotionBlur?.report().reason).toMatch(/WebGL/);+ delete (HTMLCanvasElement.prototype as unknown as Record<string, unknown>)+ .transferControlToOffscreen;+ });+it("turns off instead of wrapping when a WebGL canvas appears after init", () => {installHtmlInCanvas();trackAcceleratedCanvases();@@ -672,6 +689,30 @@ describe("painting", () => {expect(win.__hfMotionBlur?.report()).toMatchObject({ state: "off" });});+ it("counts every frame that fell back to plain DOM, though the failure streak resets", async () => {+ vi.useFakeTimers();+ installHtmlInCanvas();+ const { root, h } = movingHost();+ initCaptureMotionBlur(h);+ const frame = async (t: number, fails: boolean): Promise<void> => {+ captureContext()?.drawElementImage.mockImplementation(() => {+ if (fails) throw new Error("No cached paint record");+ });+ const done = paintCaptureMotionBlur(t, { engineMode: true });+ releaseCaptureMotionBlurFrame();+ captureCanvas(root).dispatchEvent(new Event("paint"));+ captureCanvas(root).dispatchEvent(new Event("paint"));+ await done;+ };+ await frame(0.1, true);+ await frame(0.2, false);+ void paintCaptureMotionBlur(0.3, { engineMode: true });+ releaseCaptureMotionBlurFrame();+ await vi.advanceTimersByTimeAsync(2000);+ expect(win.__hfMotionBlur?.report()).toMatchObject({ failures: 0, failedFrames: 2 });+ expect(Object.keys(win.__hfMotionBlur?.report() ?? {})).toContain("failedFrames");+ });+it("keeps blurring in a render after repeated capture failures, each failed frame plain DOM", async () => {installHtmlInCanvas();const { root, h } = movingHost();@@ -689,7 +730,11 @@ describe("painting", () => {await engineFrame(i);expect(root.parentElement).toBe(document.body);}- expect(win.__hfMotionBlur?.report()).toMatchObject({ state: "on", failures: 5 });+ expect(win.__hfMotionBlur?.report()).toMatchObject({+ state: "on",+ failures: 5,+ failedFrames: 5,+ });captureContext()?.drawElementImage.mockReset();const done = paintCaptureMotionBlur(0.5, { engineMode: true });expect(captureCanvas(root).tagName).toBe("CANVAS");@@ -697,7 +742,12 @@ describe("painting", () => {captureCanvas(root).dispatchEvent(new Event("paint"));captureCanvas(root).dispatchEvent(new Event("paint"));await done;- expect(win.__hfMotionBlur?.report()).toMatchObject({ state: "on", failures: 0, frames: 6 });+ expect(win.__hfMotionBlur?.report()).toMatchObject({+ state: "on",+ failures: 0,+ failedFrames: 5,+ frames: 6,+ });expect(contexts[1]?.drawImage.mock.calls.length).toBeGreaterThan(1);});});diff --git a/packages/core/src/runtime/captureMotionBlur/index.ts b/packages/core/src/runtime/captureMotionBlur/index.tsindex 1261dc81b8..fbda3c223f 100644--- a/packages/core/src/runtime/captureMotionBlur/index.ts+++ b/packages/core/src/runtime/captureMotionBlur/index.ts@@ -14,6 +14,8 @@import {PAGE_DEVICE_SCALE_GLOBAL,PAGE_MOTION_BLUR_OFF_GLOBAL,+ installAcceleratedCanvasTracker,+ isAcceleratedCanvas,supportsCaptureMotionBlur,} from "../../pageMotionBlur";import { swallow } from "../diagnostics";@@ -105,29 +107,8 @@ function readConfig(root: HTMLElement): Config | null {};}-const ACCELERATED_CONTEXTS = new Set(["webgl", "webgl2", "experimental-webgl", "webgpu"]);-const acceleratedCanvases = new WeakSet<HTMLCanvasElement>();--/**- * Remember canvases given a WebGL / WebGPU context. Must run before composition scripts: once- * one has presented, html-in-canvas keeps capturing that frame however often it redraws.- */-export function trackAcceleratedCanvases(): void {- const proto = HTMLCanvasElement.prototype;- const original: unknown = proto.getContext;- if (typeof original !== "function" || "__hfAccelTracked" in original) return;- const tracked = function (this: HTMLCanvasElement, ...args: unknown[]): unknown {- const ctx: unknown = Reflect.apply(original, this, args);- if (ctx && ACCELERATED_CONTEXTS.has(String(args[0]))) acceleratedCanvases.add(this);- return ctx;- };- Object.defineProperty(tracked, "__hfAccelTracked", { value: true });- Object.defineProperty(proto, "getContext", {- value: tracked,- writable: true,- configurable: true,- });-}+/** Must run before composition scripts: see `installAcceleratedCanvasTracker`. */+export const trackAcceleratedCanvases = installAcceleratedCanvasTracker;/** What in the composition the capture cannot take, checked again per frame (scripts add canvases). */function captureBlocker(root: HTMLElement): string | null {@@ -135,7 +116,7 @@ function captureBlocker(root: HTMLElement): string | null {if (root.querySelector("canvas[layoutsubtree]"))return "the composition already uses html-in-canvas";for (const canvas of root.querySelectorAll("canvas")) {- if (acceleratedCanvases.has(canvas))+ if (isAcceleratedCanvas(canvas))return "the composition draws a WebGL/WebGPU canvas, which html-in-canvas captures frozen";}return null;diff --git a/packages/engine/src/services/pageMotionBlur.test.ts b/packages/engine/src/services/pageMotionBlur.test.tsindex cde3d5ab92..87e0ccaeb9 100644--- a/packages/engine/src/services/pageMotionBlur.test.ts+++ b/packages/engine/src/services/pageMotionBlur.test.ts@@ -1,5 +1,9 @@import { describe, expect, it, vi } from "vitest";-import { PAGE_DEVICE_SCALE_GLOBAL, PAGE_MOTION_BLUR_OFF_GLOBAL } from "@hyperframes/core";+import {+ PAGE_DEVICE_SCALE_GLOBAL,+ PAGE_MOTION_BLUR_OFF_GLOBAL,+ acceleratedCanvasTrackerSource,+} from "@hyperframes/core";import { installPageMotionBlur } from "./pageMotionBlur";const fakePage = () => ({ evaluateOnNewDocument: vi.fn(async () => ({ identifier: "1" })) });@@ -8,7 +12,7 @@ describe("installPageMotionBlur", () => {it("tells the page to stand down when the engine blurs", async () => {const page = fakePage();await installPageMotionBlur(page, { motionBlur: { samplesPerFrame: 16 } });- expect(page.evaluateOnNewDocument).toHaveBeenCalledTimes(1);+ expect(page.evaluateOnNewDocument).toHaveBeenCalledTimes(2);expect(page.evaluateOnNewDocument).toHaveBeenCalledWith(expect.any(Function),PAGE_MOTION_BLUR_OFF_GLOBAL,@@ -36,9 +40,9 @@ describe("installPageMotionBlur", () => {);});- it("leaves the page alone otherwise", async () => {+ it("installs the canvas tracker before any page script, and nothing else by default", async () => {const page = fakePage();await installPageMotionBlur(page, {});- expect(page.evaluateOnNewDocument).not.toHaveBeenCalled();+ expect(page.evaluateOnNewDocument.mock.calls).toEqual([[acceleratedCanvasTrackerSource()]]);});});diff --git a/packages/engine/src/services/pageMotionBlur.ts b/packages/engine/src/services/pageMotionBlur.tsindex e161d43fe1..ef15a7452a 100644--- a/packages/engine/src/services/pageMotionBlur.ts+++ b/packages/engine/src/services/pageMotionBlur.ts@@ -4,7 +4,11 @@* when the render must match a Studio preview that could not blur (`pageMotionBlur: "off"`).*/-import { PAGE_DEVICE_SCALE_GLOBAL, PAGE_MOTION_BLUR_OFF_GLOBAL } from "@hyperframes/core";+import {+ PAGE_DEVICE_SCALE_GLOBAL,+ PAGE_MOTION_BLUR_OFF_GLOBAL,+ acceleratedCanvasTrackerSource,+} from "@hyperframes/core";import type { Page } from "puppeteer-core";import type { CaptureOptions } from "../types.js";@@ -12,6 +16,7 @@ export async function installPageMotionBlur(page: Pick<Page, "evaluateOnNewDocument">,options: Pick<CaptureOptions, "motionBlur" | "pageMotionBlur" | "deviceScaleFactor">,): Promise<void> {+ await page.evaluateOnNewDocument(acceleratedCanvasTrackerSource());const scale = options.deviceScaleFactor ?? 1;if (scale > 1) {await page.evaluateOnNewDocument(