Skip to content

Write HTML. Render video. Built for agents.

feat(core,engine): track accelerated canvases before any page script runs (#5970)

Vance Ingalls0865b6f36d42
diff --git a/packages/core/src/compiler/htmlDocument.test.ts b/packages/core/src/compiler/htmlDocument.test.ts
index 3c0c1a7331..466d993e02 100644
--- a/packages/core/src/compiler/htmlDocument.test.ts
+++ b/packages/core/src/compiler/htmlDocument.test.ts
@@ -8,6 +8,7 @@ import {
injectScriptsIntoHtml,
injectTagsAtHeadStart,
insertBeforeCloseTag,
+ insertRuntimeTag,
hasCompositionOutsideTemplates,
isFullHtmlDocument,
parseHTMLContent,
@@ -454,3 +455,29 @@ describe("hasCompositionOutsideTemplates", () => {
expect(hasCompositionOutsideTemplates(html)).toBe(expected);
});
});
+
+describe("insertRuntimeTag", () => {
+ const runtime = '<script data-hf-runtime src="/rt.js"></script>';
+ const page = (attrs: string) =>
+ `<html><head><script>window.author=1</script></head><body><div id="root" ${attrs}></div></body></html>`;
+
+ it("puts the canvas tracker ahead of author head scripts for a page that opts into blur", () => {
+ const out = insertRuntimeTag(page('data-motion-blur="on"'), runtime);
+ const tracker = out.indexOf("data-hf-accel-tracker");
+ expect(tracker).toBeGreaterThan(-1);
+ expect(tracker).toBeLessThan(out.indexOf("window.author"));
+ expect(out.indexOf("window.author")).toBeLessThan(out.indexOf(runtime));
+ });
+
+ it("adds the tracker once, however many times the runtime is placed", () => {
+ const once = insertRuntimeTag(page('data-motion-blur="on"'), runtime);
+ const twice = insertRuntimeTag(once, runtime);
+ expect(twice.split("data-hf-accel-tracker").length).toBe(2);
+ });
+
+ it("leaves pages that did not opt in alone", () => {
+ for (const attrs of ["", 'data-motion-blur="off"', 'data-motion-blur-shutter="on"']) {
+ expect(insertRuntimeTag(page(attrs), runtime)).not.toContain("data-hf-accel-tracker");
+ }
+ });
+});
diff --git a/packages/core/src/compiler/htmlDocument.ts b/packages/core/src/compiler/htmlDocument.ts
index a1861a7d28..d070905820 100644
--- a/packages/core/src/compiler/htmlDocument.ts
+++ b/packages/core/src/compiler/htmlDocument.ts
@@ -1,3 +1,4 @@
+import { acceleratedCanvasTrackerSource } from "../pageMotionBlur.js";
import { parseHTML } from "linkedom";
export const RUNTIME_BOOTSTRAP_ATTR = "data-hyperframes-preview-runtime";
@@ -347,7 +348,7 @@ export function insertBeforeCloseTag(
return insertBeforeDocumentTag(html, `</${name}`, markup);
}
-export function insertRuntimeTag(html: string, tag: string): string {
+function insertTagInHead(html: string, tag: string): string {
const withHead = insertBeforeCloseTag(html, "head", `${tag}\n`);
if (withHead !== null) return withHead;
const htmlOpenMatch = html.match(/<html\b[^>]*>/i);
@@ -363,6 +364,16 @@ export function insertRuntimeTag(html: string, tag: string): string {
return tag + html;
}
+const ACCEL_TRACKER_ATTR = "data-hf-accel-tracker";
+const OPTS_INTO_MOTION_BLUR = /\sdata-motion-blur\s*=\s*["']?on[\s"'>]/i;
+
+export function insertRuntimeTag(html: string, tag: string): string {
+ const withRuntime = insertTagInHead(html, tag);
+ if (!OPTS_INTO_MOTION_BLUR.test(html) || html.includes(ACCEL_TRACKER_ATTR)) return withRuntime;
+ const tracker = `<script ${ACCEL_TRACKER_ATTR}>${acceleratedCanvasTrackerSource()}</script>`;
+ return injectTagsAtHeadStart(withRuntime, tracker);
+}
+
/**
* Insert raw tag markup at the very start of `<head>`, ahead of every author
* script (inline or external). Falls back to just before `<body>`, then to the
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 75ead8efa5..3cee99b92f 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -437,4 +437,8 @@ export {
type HfVfxParam,
type HfVfxParamValues,
} from "./vfx.js";
-export { PAGE_DEVICE_SCALE_GLOBAL, PAGE_MOTION_BLUR_OFF_GLOBAL } from "./pageMotionBlur.js";
+export {
+ PAGE_DEVICE_SCALE_GLOBAL,
+ PAGE_MOTION_BLUR_OFF_GLOBAL,
+ acceleratedCanvasTrackerSource,
+} from "./pageMotionBlur.js";
diff --git a/packages/core/src/pageMotionBlur.test.ts b/packages/core/src/pageMotionBlur.test.ts
new file mode 100644
index 0000000000..9c7e1e62cf
--- /dev/null
+++ b/packages/core/src/pageMotionBlur.test.ts
@@ -0,0 +1,59 @@
+// @vitest-environment jsdom
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ acceleratedCanvasTrackerSource,
+ installAcceleratedCanvasTracker,
+ isAcceleratedCanvas,
+} from "./pageMotionBlur";
+
+type Proto = Record<string, unknown>;
+const proto = HTMLCanvasElement.prototype as unknown as Proto;
+const saved: Proto = {};
+
+beforeEach(() => {
+ for (const name of ["getContext", "transferControlToOffscreen"]) saved[name] = proto[name];
+ proto.getContext = vi.fn(() => ({}));
+ proto.transferControlToOffscreen = vi.fn(() => ({}));
+});
+
+afterEach(() => {
+ for (const name of Object.keys(saved)) proto[name] = saved[name];
+ Reflect.deleteProperty(window, "__hfAcceleratedCanvases");
+});
+
+describe("accelerated canvas tracker", () => {
+ it("remembers WebGL and WebGPU canvases, not 2D ones", () => {
+ installAcceleratedCanvasTracker();
+ const [gl, gpu, flat] = [1, 2, 3].map(() => document.createElement("canvas"));
+ gl?.getContext("webgl2");
+ gpu?.getContext("webgpu");
+ flat?.getContext("2d");
+ expect([gl, gpu, flat].map((c) => c !== undefined && isAcceleratedCanvas(c))).toEqual([
+ true,
+ true,
+ false,
+ ]);
+ });
+
+ it("treats a canvas handed to an OffscreenCanvas as accelerated", () => {
+ installAcceleratedCanvasTracker();
+ const canvas = document.createElement("canvas");
+ canvas.transferControlToOffscreen();
+ expect(isAcceleratedCanvas(canvas)).toBe(true);
+ });
+
+ it("installs once however many times it runs", () => {
+ const original = proto.getContext as ReturnType<typeof vi.fn>;
+ installAcceleratedCanvasTracker();
+ installAcceleratedCanvasTracker();
+ document.createElement("canvas").getContext("2d");
+ expect(original).toHaveBeenCalledTimes(1);
+ });
+
+ it("works from its source text, as the engine and the preview inline it", () => {
+ (0, eval)(acceleratedCanvasTrackerSource());
+ const canvas = document.createElement("canvas");
+ canvas.getContext("webgl");
+ expect(isAcceleratedCanvas(canvas)).toBe(true);
+ });
+});
diff --git a/packages/core/src/pageMotionBlur.ts b/packages/core/src/pageMotionBlur.ts
index 4b422e1cef..9169613f1e 100644
--- a/packages/core/src/pageMotionBlur.ts
+++ b/packages/core/src/pageMotionBlur.ts
@@ -12,3 +12,48 @@ export function supportsCaptureMotionBlur(win: Window = window): boolean {
const canvas = w.HTMLCanvasElement?.prototype;
return !!ctx && !!canvas && "drawElementImage" in ctx && "requestPaint" in canvas;
}
+
+const ACCELERATED_CANVASES_GLOBAL = "__hfAcceleratedCanvases";
+
+/**
+ * Remember canvases html-in-canvas cannot capture: WebGL, WebGPU, or handed to an OffscreenCanvas.
+ * Self-contained: the engine and the preview run its source ahead of every author script.
+ */
+export function installAcceleratedCanvasTracker(): void {
+ const key = "__hfAcceleratedCanvases";
+ const known: unknown = Reflect.get(window, key);
+ const seen = known instanceof WeakSet ? known : new WeakSet<object>();
+ if (seen !== known) Object.defineProperty(window, key, { value: seen });
+ const accelerated = ["webgl", "webgl2", "experimental-webgl", "webgpu"];
+ const wrap = (
+ name: string,
+ onCall: (canvas: object, args: unknown[], result: unknown) => void,
+ ) => {
+ const proto = HTMLCanvasElement.prototype;
+ const original: unknown = Reflect.get(proto, name);
+ if (typeof original !== "function" || "__hfAccelTracked" in original) return;
+ const tracked = function (this: object, ...args: unknown[]): unknown {
+ const result: unknown = Reflect.apply(original, this, args);
+ onCall(this, args, result);
+ return result;
+ };
+ Object.defineProperty(tracked, "__hfAccelTracked", { value: true });
+ Object.defineProperty(proto, name, { value: tracked, writable: true, configurable: true });
+ };
+ wrap("getContext", (canvas, args, ctx) => {
+ if (ctx && accelerated.includes(String(args[0]))) seen.add(canvas);
+ });
+ wrap("transferControlToOffscreen", (canvas) => {
+ seen.add(canvas);
+ });
+}
+
+export function isAcceleratedCanvas(canvas: HTMLCanvasElement): boolean {
+ const seen: unknown = Reflect.get(window, ACCELERATED_CANVASES_GLOBAL);
+ return seen instanceof WeakSet && seen.has(canvas);
+}
+
+/** The tracker as a script body, for `evaluateOnNewDocument` and inline `<script>` injection. */
+export function acceleratedCanvasTrackerSource(): string {
+ return `(${installAcceleratedCanvasTracker.toString()})();`;
+}
diff --git a/packages/core/src/runtime/captureMotionBlur/index.test.ts b/packages/core/src/runtime/captureMotionBlur/index.test.ts
index 13590878cf..3cc52b744b 100644
--- a/packages/core/src/runtime/captureMotionBlur/index.test.ts
+++ b/packages/core/src/runtime/captureMotionBlur/index.test.ts
@@ -17,6 +17,7 @@ type Win = Window & {
reason: string | null;
shutterDeg: number;
moverNames: string[];
+ failedFrames: number;
};
};
__hf_page_composite_pending?: boolean;
@@ -204,6 +205,22 @@ describe("opt-in and off switches", () => {
expect(win.__hfMotionBlur?.report().reason).toMatch(/WebGL/);
});
+ it("stays off for a canvas handed to an OffscreenCanvas, whose context is out of reach", () => {
+ installHtmlInCanvas();
+ (HTMLCanvasElement.prototype as unknown as Record<string, unknown>).transferControlToOffscreen =
+ () => ({});
+ trackAcceleratedCanvases();
+ const root = makeRoot();
+ const canvas = document.createElement("canvas");
+ root.append(canvas);
+ canvas.transferControlToOffscreen();
+ initCaptureMotionBlur(host(root));
+ expect(win.__hfMotionBlur?.report()).toMatchObject({ state: "off" });
+ expect(win.__hfMotionBlur?.report().reason).toMatch(/WebGL/);
+ delete (HTMLCanvasElement.prototype as unknown as Record<string, unknown>)
+ .transferControlToOffscreen;
+ });
+
it("turns off instead of wrapping when a WebGL canvas appears after init", () => {
installHtmlInCanvas();
trackAcceleratedCanvases();
@@ -672,6 +689,30 @@ describe("painting", () => {
expect(win.__hfMotionBlur?.report()).toMatchObject({ state: "off" });
});
+ it("counts every frame that fell back to plain DOM, though the failure streak resets", async () => {
+ vi.useFakeTimers();
+ installHtmlInCanvas();
+ const { root, h } = movingHost();
+ initCaptureMotionBlur(h);
+ const frame = async (t: number, fails: boolean): Promise<void> => {
+ captureContext()?.drawElementImage.mockImplementation(() => {
+ if (fails) throw new Error("No cached paint record");
+ });
+ const done = paintCaptureMotionBlur(t, { engineMode: true });
+ releaseCaptureMotionBlurFrame();
+ captureCanvas(root).dispatchEvent(new Event("paint"));
+ captureCanvas(root).dispatchEvent(new Event("paint"));
+ await done;
+ };
+ await frame(0.1, true);
+ await frame(0.2, false);
+ void paintCaptureMotionBlur(0.3, { engineMode: true });
+ releaseCaptureMotionBlurFrame();
+ await vi.advanceTimersByTimeAsync(2000);
+ expect(win.__hfMotionBlur?.report()).toMatchObject({ failures: 0, failedFrames: 2 });
+ expect(Object.keys(win.__hfMotionBlur?.report() ?? {})).toContain("failedFrames");
+ });
+
it("keeps blurring in a render after repeated capture failures, each failed frame plain DOM", async () => {
installHtmlInCanvas();
const { root, h } = movingHost();
@@ -689,7 +730,11 @@ describe("painting", () => {
await engineFrame(i);
expect(root.parentElement).toBe(document.body);
}
- expect(win.__hfMotionBlur?.report()).toMatchObject({ state: "on", failures: 5 });
+ expect(win.__hfMotionBlur?.report()).toMatchObject({
+ state: "on",
+ failures: 5,
+ failedFrames: 5,
+ });
captureContext()?.drawElementImage.mockReset();
const done = paintCaptureMotionBlur(0.5, { engineMode: true });
expect(captureCanvas(root).tagName).toBe("CANVAS");
@@ -697,7 +742,12 @@ describe("painting", () => {
captureCanvas(root).dispatchEvent(new Event("paint"));
captureCanvas(root).dispatchEvent(new Event("paint"));
await done;
- expect(win.__hfMotionBlur?.report()).toMatchObject({ state: "on", failures: 0, frames: 6 });
+ expect(win.__hfMotionBlur?.report()).toMatchObject({
+ state: "on",
+ failures: 0,
+ failedFrames: 5,
+ frames: 6,
+ });
expect(contexts[1]?.drawImage.mock.calls.length).toBeGreaterThan(1);
});
});
diff --git a/packages/core/src/runtime/captureMotionBlur/index.ts b/packages/core/src/runtime/captureMotionBlur/index.ts
index 1261dc81b8..fbda3c223f 100644
--- a/packages/core/src/runtime/captureMotionBlur/index.ts
+++ b/packages/core/src/runtime/captureMotionBlur/index.ts
@@ -14,6 +14,8 @@
import {
PAGE_DEVICE_SCALE_GLOBAL,
PAGE_MOTION_BLUR_OFF_GLOBAL,
+ installAcceleratedCanvasTracker,
+ isAcceleratedCanvas,
supportsCaptureMotionBlur,
} from "../../pageMotionBlur";
import { swallow } from "../diagnostics";
@@ -105,29 +107,8 @@ function readConfig(root: HTMLElement): Config | null {
};
}
-const ACCELERATED_CONTEXTS = new Set(["webgl", "webgl2", "experimental-webgl", "webgpu"]);
-const acceleratedCanvases = new WeakSet<HTMLCanvasElement>();
-
-/**
- * Remember canvases given a WebGL / WebGPU context. Must run before composition scripts: once
- * one has presented, html-in-canvas keeps capturing that frame however often it redraws.
- */
-export function trackAcceleratedCanvases(): void {
- const proto = HTMLCanvasElement.prototype;
- const original: unknown = proto.getContext;
- if (typeof original !== "function" || "__hfAccelTracked" in original) return;
- const tracked = function (this: HTMLCanvasElement, ...args: unknown[]): unknown {
- const ctx: unknown = Reflect.apply(original, this, args);
- if (ctx && ACCELERATED_CONTEXTS.has(String(args[0]))) acceleratedCanvases.add(this);
- return ctx;
- };
- Object.defineProperty(tracked, "__hfAccelTracked", { value: true });
- Object.defineProperty(proto, "getContext", {
- value: tracked,
- writable: true,
- configurable: true,
- });
-}
+/** Must run before composition scripts: see `installAcceleratedCanvasTracker`. */
+export const trackAcceleratedCanvases = installAcceleratedCanvasTracker;
/** What in the composition the capture cannot take, checked again per frame (scripts add canvases). */
function captureBlocker(root: HTMLElement): string | null {
@@ -135,7 +116,7 @@ function captureBlocker(root: HTMLElement): string | null {
if (root.querySelector("canvas[layoutsubtree]"))
return "the composition already uses html-in-canvas";
for (const canvas of root.querySelectorAll("canvas")) {
- if (acceleratedCanvases.has(canvas))
+ if (isAcceleratedCanvas(canvas))
return "the composition draws a WebGL/WebGPU canvas, which html-in-canvas captures frozen";
}
return null;
diff --git a/packages/engine/src/services/pageMotionBlur.test.ts b/packages/engine/src/services/pageMotionBlur.test.ts
index cde3d5ab92..87e0ccaeb9 100644
--- a/packages/engine/src/services/pageMotionBlur.test.ts
+++ b/packages/engine/src/services/pageMotionBlur.test.ts
@@ -1,5 +1,9 @@
import { describe, expect, it, vi } from "vitest";
-import { PAGE_DEVICE_SCALE_GLOBAL, PAGE_MOTION_BLUR_OFF_GLOBAL } from "@hyperframes/core";
+import {
+ PAGE_DEVICE_SCALE_GLOBAL,
+ PAGE_MOTION_BLUR_OFF_GLOBAL,
+ acceleratedCanvasTrackerSource,
+} from "@hyperframes/core";
import { installPageMotionBlur } from "./pageMotionBlur";
const fakePage = () => ({ evaluateOnNewDocument: vi.fn(async () => ({ identifier: "1" })) });
@@ -8,7 +12,7 @@ describe("installPageMotionBlur", () => {
it("tells the page to stand down when the engine blurs", async () => {
const page = fakePage();
await installPageMotionBlur(page, { motionBlur: { samplesPerFrame: 16 } });
- expect(page.evaluateOnNewDocument).toHaveBeenCalledTimes(1);
+ expect(page.evaluateOnNewDocument).toHaveBeenCalledTimes(2);
expect(page.evaluateOnNewDocument).toHaveBeenCalledWith(
expect.any(Function),
PAGE_MOTION_BLUR_OFF_GLOBAL,
@@ -36,9 +40,9 @@ describe("installPageMotionBlur", () => {
);
});
- it("leaves the page alone otherwise", async () => {
+ it("installs the canvas tracker before any page script, and nothing else by default", async () => {
const page = fakePage();
await installPageMotionBlur(page, {});
- expect(page.evaluateOnNewDocument).not.toHaveBeenCalled();
+ expect(page.evaluateOnNewDocument.mock.calls).toEqual([[acceleratedCanvasTrackerSource()]]);
});
});
diff --git a/packages/engine/src/services/pageMotionBlur.ts b/packages/engine/src/services/pageMotionBlur.ts
index e161d43fe1..ef15a7452a 100644
--- a/packages/engine/src/services/pageMotionBlur.ts
+++ b/packages/engine/src/services/pageMotionBlur.ts
@@ -4,7 +4,11 @@
* when the render must match a Studio preview that could not blur (`pageMotionBlur: "off"`).
*/
-import { PAGE_DEVICE_SCALE_GLOBAL, PAGE_MOTION_BLUR_OFF_GLOBAL } from "@hyperframes/core";
+import {
+ PAGE_DEVICE_SCALE_GLOBAL,
+ PAGE_MOTION_BLUR_OFF_GLOBAL,
+ acceleratedCanvasTrackerSource,
+} from "@hyperframes/core";
import type { Page } from "puppeteer-core";
import type { CaptureOptions } from "../types.js";
@@ -12,6 +16,7 @@ export async function installPageMotionBlur(
page: Pick<Page, "evaluateOnNewDocument">,
options: Pick<CaptureOptions, "motionBlur" | "pageMotionBlur" | "deviceScaleFactor">,
): Promise<void> {
+ await page.evaluateOnNewDocument(acceleratedCanvasTrackerSource());
const scale = options.deviceScaleFactor ?? 1;
if (scale > 1) {
await page.evaluateOnNewDocument(