Skip to content

Write HTML. Render video. Built for agents.

feat(core): capture motion blur runtime lifecycle (#5967)

Vance Ingalls0f1d6908fdfb
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 50c3c5bcdf..51514f023e 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -437,3 +437,4 @@ export {
type HfVfxParam,
type HfVfxParamValues,
} from "./vfx.js";
+export { PAGE_MOTION_BLUR_OFF_GLOBAL } from "./pageMotionBlur.js";
diff --git a/packages/core/src/pageMotionBlur.ts b/packages/core/src/pageMotionBlur.ts
new file mode 100644
index 0000000000..d540d046cf
--- /dev/null
+++ b/packages/core/src/pageMotionBlur.ts
@@ -0,0 +1,12 @@
+// Set by the engine: its own sub-frame blur runs, or the Studio preview could not blur.
+export const PAGE_MOTION_BLUR_OFF_GLOBAL = "__HF_PAGE_MOTION_BLUR_OFF__";
+
+export function supportsCaptureMotionBlur(win: Window = window): boolean {
+ const w = win as Window & {
+ CanvasRenderingContext2D?: { prototype: object };
+ HTMLCanvasElement?: { prototype: object };
+ };
+ const ctx = w.CanvasRenderingContext2D?.prototype;
+ const canvas = w.HTMLCanvasElement?.prototype;
+ return !!ctx && !!canvas && "drawElementImage" in ctx && "requestPaint" in canvas;
+}
diff --git a/packages/core/src/runtime/captureMotionBlur/index.test.ts b/packages/core/src/runtime/captureMotionBlur/index.test.ts
new file mode 100644
index 0000000000..ccd7a8c55d
--- /dev/null
+++ b/packages/core/src/runtime/captureMotionBlur/index.test.ts
@@ -0,0 +1,311 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ type CaptureMotionBlurHost,
+ initCaptureMotionBlur,
+ paintCaptureMotionBlur,
+ releaseCaptureMotionBlurFrame,
+ resetCaptureMotionBlurForTests,
+ trackAcceleratedCanvases,
+} from ".";
+
+type Win = Window & {
+ __HF_PAGE_MOTION_BLUR_OFF__?: unknown;
+ __hfMotionBlur?: {
+ report: () => {
+ state: string;
+ reason: string | null;
+ shutterDeg: number;
+ moverNames: string[];
+ };
+ };
+ __hf_page_composite_pending?: boolean;
+ __hf_page_composite_resolve?: () => boolean;
+};
+const win = window as Win;
+
+// Every 2D context (page canvas or offscreen) is this recording fake.
+function fakeContext() {
+ return {
+ reset: vi.fn(),
+ drawElementImage: vi.fn(),
+ drawImage: vi.fn(),
+ clearRect: vi.fn(),
+ setTransform: vi.fn(),
+ globalAlpha: 1,
+ globalCompositeOperation: "source-over",
+ };
+}
+let contexts: Array<ReturnType<typeof fakeContext>> = [];
+
+function installHtmlInCanvas(): void {
+ vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockImplementation(() => {
+ const ctx = fakeContext();
+ contexts.push(ctx);
+ return ctx as never;
+ });
+ vi.stubGlobal(
+ "CanvasRenderingContext2D",
+ class {
+ drawElementImage(): void {}
+ },
+ );
+ (HTMLCanvasElement.prototype as unknown as Record<string, unknown>).requestPaint = () => {};
+ vi.stubGlobal(
+ "OffscreenCanvas",
+ class {
+ width: number;
+ height: number;
+ constructor(w: number, h: number) {
+ this.width = w;
+ this.height = h;
+ }
+ getContext() {
+ return fakeContext();
+ }
+ },
+ );
+}
+
+function makeRoot(attrs: Record<string, string> = { "data-motion-blur": "on" }): HTMLElement {
+ const root = document.createElement("div");
+ root.setAttribute("data-composition-id", "main");
+ root.setAttribute("data-width", "640");
+ root.setAttribute("data-height", "360");
+ for (const [k, v] of Object.entries(attrs)) root.setAttribute(k, v);
+ document.body.append(root);
+ return root;
+}
+
+function host(
+ root: HTMLElement,
+ seek = vi.fn(),
+): CaptureMotionBlurHost & { seek: ReturnType<typeof vi.fn> } {
+ return { root, fps: 30, durationSeconds: () => 1, timeline: () => "tl", seek };
+}
+
+const captureCanvas = (root: HTMLElement) => root.parentElement as HTMLCanvasElement;
+
+beforeEach(() => {
+ contexts = [];
+ vi.spyOn(console, "info").mockImplementation(() => {});
+ vi.spyOn(console, "warn").mockImplementation(() => {});
+});
+
+afterEach(() => {
+ resetCaptureMotionBlurForTests();
+ vi.restoreAllMocks();
+ vi.unstubAllGlobals();
+ vi.useRealTimers();
+ delete (HTMLCanvasElement.prototype as unknown as Record<string, unknown>).requestPaint;
+ delete win.__HF_PAGE_MOTION_BLUR_OFF__;
+ delete win.__hfMotionBlur;
+ delete win.__hf_page_composite_pending;
+ delete win.__hf_page_composite_resolve;
+ document.body.innerHTML = "";
+});
+
+/** A root with one 10×10 element that slides 300 px over the 1 s composition (10 px/frame). */
+function movingHost(attrs?: Record<string, string>) {
+ const root = makeRoot(attrs);
+ const el = document.createElement("div");
+ el.id = "slider";
+ root.append(el);
+ for (const [k, v] of Object.entries({
+ offsetWidth: 10,
+ offsetHeight: 10,
+ offsetLeft: 0,
+ offsetTop: 0,
+ offsetParent: root,
+ })) {
+ Object.defineProperty(el, k, { value: v, configurable: true });
+ }
+ const x = () => Number(/,\s*([-\d.]+),\s*0\)$/.exec(el.style.transform)?.[1] ?? 0);
+ el.getBoundingClientRect = () =>
+ ({ left: x(), top: 0, right: x() + 10, bottom: 10, width: 10, height: 10 }) as DOMRect;
+ const seek = vi.fn((t: number) => {
+ el.style.transform = `matrix(1, 0, 0, 1, ${300 * t}, 0)`;
+ });
+ return { root, el, h: host(root, seek) };
+}
+
+const PLAY = { playing: true };
+
+describe("opt-in and off switches", () => {
+ it("does nothing for a root that has not opted in", () => {
+ installHtmlInCanvas();
+ const root = makeRoot({});
+ initCaptureMotionBlur(host(root));
+ expect(root.parentElement).toBe(document.body);
+ expect(paintCaptureMotionBlur(0)).toBeNull();
+ expect(win.__hfMotionBlur).toBeUndefined();
+ });
+
+ it("stays off without html-in-canvas and says why", () => {
+ const root = makeRoot();
+ initCaptureMotionBlur(host(root));
+ expect(root.parentElement).toBe(document.body);
+ expect(win.__hfMotionBlur?.report()).toMatchObject({
+ state: "off",
+ reason: expect.stringContaining("html-in-canvas"),
+ });
+ });
+
+ it("stays off while the engine's own sub-frame blur is on (never both)", () => {
+ installHtmlInCanvas();
+ win.__HF_PAGE_MOTION_BLUR_OFF__ = "engine-motion-blur";
+ const { root, h } = movingHost();
+ initCaptureMotionBlur(h);
+ expect(paintCaptureMotionBlur(0.5, PLAY)).toBeNull();
+ expect(root.parentElement).toBe(document.body);
+ expect(win.__hfMotionBlur?.report().reason).toContain("engine");
+ });
+
+ it("stays off for a WebGL canvas, which html-in-canvas captures frozen, but not for a 2D one", () => {
+ installHtmlInCanvas();
+ trackAcceleratedCanvases();
+ const root = makeRoot();
+ const flat = document.createElement("canvas");
+ root.append(flat);
+ flat.getContext("2d");
+ initCaptureMotionBlur(host(root));
+ expect(win.__hfMotionBlur?.report().state).toBe("on");
+
+ resetCaptureMotionBlurForTests();
+ const gl = document.createElement("canvas");
+ root.append(gl);
+ gl.getContext("webgl");
+ initCaptureMotionBlur(host(root));
+ expect(win.__hfMotionBlur?.report().reason).toMatch(/WebGL/);
+ });
+
+ it("turns off instead of wrapping when a WebGL canvas appears after init", () => {
+ installHtmlInCanvas();
+ trackAcceleratedCanvases();
+ const { root, h } = movingHost();
+ initCaptureMotionBlur(h);
+ const late = document.createElement("canvas");
+ root.append(late);
+ late.getContext("webgl2");
+ void paintCaptureMotionBlur(0.5, { engineMode: true });
+ expect(root.parentElement).toBe(document.body);
+ expect(win.__hfMotionBlur?.report().reason).toMatch(/WebGL/);
+ });
+});
+
+describe("wrapping", () => {
+ it("wraps the root itself in a capture canvas, with an output canvas after it", () => {
+ installHtmlInCanvas();
+ const { root, h } = movingHost();
+ initCaptureMotionBlur(h);
+ expect(root.parentElement).toBe(document.body); // not until it plays
+ void paintCaptureMotionBlur(0.5, PLAY);
+ const src = captureCanvas(root);
+ expect(src.tagName).toBe("CANVAS");
+ expect(src.hasAttribute("layoutsubtree")).toBe(true);
+ expect(src.hasAttribute("data-hf-ignore")).toBe(true);
+ expect([src.width, src.height]).toEqual([640, 360]);
+ const out = src.nextElementSibling as HTMLElement;
+ expect(out.getAttribute("data-hf-motion-blur")).toBe("output");
+ // The output sits over the composition: clicks must reach what is under it.
+ expect(out.style.pointerEvents).toBe("none");
+ });
+
+ it("turns off instead of wrapping when a capture canvas appears after init", () => {
+ installHtmlInCanvas();
+ const { root, h } = movingHost();
+ initCaptureMotionBlur(h);
+ const late = document.createElement("canvas");
+ late.setAttribute("layoutsubtree", "");
+ root.append(late);
+ void paintCaptureMotionBlur(0.5, PLAY);
+ expect(root.parentElement).toBe(document.body);
+ expect(win.__hfMotionBlur?.report().state).toBe("off");
+ });
+});
+
+describe("paint timeouts", () => {
+ const HIDE = "data-hf-motion-blur-hide";
+ const info = () => vi.mocked(console.info);
+ const engineFrame = (t: number): Promise<void> | null => {
+ const done = paintCaptureMotionBlur(t, { engineMode: true });
+ releaseCaptureMotionBlurFrame();
+ return done;
+ };
+ const plainDom = (root: HTMLElement): void => {
+ expect(root.parentElement).toBe(document.body);
+ expect(document.querySelectorAll("canvas")).toHaveLength(0);
+ };
+
+ it("renders a released frame whose paint never comes plain, counts it, and leaves nothing for the engine to finish (first frame, output never drawn)", async () => {
+ vi.useFakeTimers();
+ installHtmlInCanvas();
+ const { root, el, h } = movingHost();
+ initCaptureMotionBlur(h);
+ const done = engineFrame(0.5);
+ expect(captureCanvas(root).tagName).toBe("CANVAS");
+ await vi.advanceTimersByTimeAsync(2001);
+ await done;
+ plainDom(root);
+ expect(el.hasAttribute(HIDE)).toBe(false);
+ expect(win.__hfMotionBlur?.report()).toMatchObject({
+ state: "on",
+ failures: 0,
+ failedFrames: 1,
+ });
+ expect(info()).toHaveBeenCalledWith(
+ expect.stringMatching(/capture failed \(1 so far\).*paint/),
+ );
+ win.__hf_page_composite_resolve?.(); // the engine's forced composite arrives late
+ plainDom(root);
+ expect(contexts[0]?.drawElementImage).not.toHaveBeenCalled();
+ expect(win.__hf_page_composite_pending).toBe(false);
+ void paintCaptureMotionBlur(0.6, { engineMode: true }); // the next frame wraps again
+ expect(captureCanvas(root).tagName).toBe("CANVAS");
+ });
+
+ it("does the same for a frame after one that painted, whose output already holds an earlier frame", async () => {
+ vi.useFakeTimers();
+ installHtmlInCanvas();
+ const { root, el, h } = movingHost();
+ initCaptureMotionBlur(h);
+ const first = engineFrame(0.5);
+ captureCanvas(root).dispatchEvent(new Event("paint"));
+ captureCanvas(root).dispatchEvent(new Event("paint"));
+ await first;
+ expect(win.__hfMotionBlur?.report()).toMatchObject({ failedFrames: 0, frames: 1 });
+ const captures = contexts[0]?.drawElementImage.mock.calls.length;
+ const second = engineFrame(0.6);
+ await vi.advanceTimersByTimeAsync(2001);
+ await second;
+ plainDom(root);
+ expect(el.hasAttribute(HIDE)).toBe(false);
+ expect(win.__hfMotionBlur?.report()).toMatchObject({ failures: 0, failedFrames: 1 });
+ win.__hf_page_composite_resolve?.();
+ plainDom(root);
+ expect(contexts[0]?.drawElementImage.mock.calls.length).toBe(captures);
+ });
+
+ it("keeps blurring through preview paint timeouts, each frame plain, and resumes when painting returns", async () => {
+ vi.useFakeTimers();
+ installHtmlInCanvas();
+ const { root, h } = movingHost();
+ initCaptureMotionBlur(h);
+ for (let i = 0; i < 4; i++) {
+ void paintCaptureMotionBlur(i / 30, PLAY);
+ expect(captureCanvas(root).tagName).toBe("CANVAS");
+ await vi.advanceTimersByTimeAsync(2000);
+ plainDom(root);
+ }
+ expect(win.__hfMotionBlur?.report()).toMatchObject({
+ state: "on",
+ failures: 0,
+ failedFrames: 4,
+ });
+ expect(info()).toHaveBeenCalledTimes(1);
+ const done = paintCaptureMotionBlur(0.5, PLAY);
+ captureCanvas(root).dispatchEvent(new Event("paint"));
+ captureCanvas(root).dispatchEvent(new Event("paint"));
+ await done;
+ expect(contexts[0]?.drawElementImage).toHaveBeenCalled();
+ });
+});
diff --git a/packages/core/src/runtime/captureMotionBlur/index.ts b/packages/core/src/runtime/captureMotionBlur/index.ts
new file mode 100644
index 0000000000..9324a8f22c
--- /dev/null
+++ b/packages/core/src/runtime/captureMotionBlur/index.ts
@@ -0,0 +1,605 @@
+/**
+ * Capture motion blur, in preview and render. Root attributes: data-motion-blur="on"|"off",
+ * -shutter (0–720°, default 180), -samples (2–64, default 16), -events="on" (run timeline
+ * callbacks at sub-frame samples). PAGE_MOTION_BLUR_OFF_GLOBAL (engine blur, or a preview that
+ * could not blur) turns all off.
+ *
+ * Per frame: read movers' poses across the shutter (seeks, no paint), capture the root and crop
+ * the movers, hide them and capture the background, then redraw each along its path.
+ * The root is wrapped, not its children (author selectors keep matching), and only while playing
+ * or rendering with something moving: paused, it is plain DOM so Studio can select inside it.
+ * Needs html-in-canvas (`drawElementImage`); without it nothing changes.
+ */
+
+import { PAGE_MOTION_BLUR_OFF_GLOBAL, supportsCaptureMotionBlur } from "../../pageMotionBlur";
+import { swallow } from "../diagnostics";
+import { createCompositor, type MoverFrame } from "./composite";
+import { discoverMovers } from "./discover";
+import { type Affine, createPoseReader, IDENTITY, tapsFromReads } from "./pose";
+
+export interface CaptureMotionBlurHost {
+ root: HTMLElement;
+ fps: number;
+ durationSeconds: () => number;
+ /** The bound root timeline; movers are re-scanned when it changes. */
+ timeline: () => unknown;
+ /** Seek the timeline and adapters (not media) for a pose read. */
+ seek: (t: number, suppressEvents: boolean) => void;
+}
+
+interface Config {
+ shutterDeg: number;
+ samples: number;
+ events: boolean;
+}
+
+interface Report {
+ state: "on" | "off";
+ reason: string | null;
+ movers: number;
+ /** The blurred elements, as `tag#id.class`, for diagnosing what was (not) picked. */
+ moverNames: string[];
+ /** One-time cost of finding the movers, in ms. */
+ scanMs: number;
+ shutterDeg: number;
+ frames: number;
+ meanMs: number;
+ failures: number;
+ failedFrames: number;
+}
+
+type DrawElementContext = CanvasRenderingContext2D & {
+ drawElementImage: (el: Element, x: number, y: number) => unknown;
+};
+type PaintCanvas = HTMLCanvasElement & { requestPaint: () => void };
+type CompositeWindow = Window & {
+ __hfMotionBlur?: { report: () => Report };
+ __hf_page_composite_pending?: boolean;
+ __hf_page_composite_resolve?: () => boolean;
+};
+
+/** Marks a mover hidden for the background capture; set only between a frame's two paints. */
+const HIDE_ATTR = "data-hf-motion-blur-hide";
+
+interface Job {
+ seq: number;
+ els: HTMLElement[];
+ frames: MoverFrame[];
+ /** Which movers were cropped (null until the first capture of this frame). */
+ kept: boolean[] | null;
+ /** Engine frames wait for a release: video frames are injected after the seek. */
+ armed: boolean;
+ /** The paint wait starts on arming, so slow video injection cannot time a frame out unseen. */
+ startTimer: () => void;
+ ms: number;
+ settle: () => void;
+}
+
+/** Pose reads per frame; the taps between them are interpolated. A render reads more. */
+const PREVIEW_READS = 4;
+const RENDER_READS = 8;
+/** Same ceiling as vfx: a real paint is two animation frames. */
+const PAINT_TIMEOUT_MS = 2000;
+/** Consecutive capture failures before the blur turns itself off for the session. */
+const MAX_FAILURES = 3;
+const LOG_EVERY = 25;
+
+const clamp = (x: number, lo: number, hi: number): number => Math.min(hi, Math.max(lo, x));
+
+function readConfig(root: HTMLElement): Config | null {
+ if (root.getAttribute("data-motion-blur") !== "on") return null;
+ const num = (name: string, fallback: number): number => {
+ const v = Number(root.getAttribute(name));
+ return root.hasAttribute(name) && Number.isFinite(v) ? v : fallback;
+ };
+ return {
+ shutterDeg: clamp(num("data-motion-blur-shutter", 180), 0, 720),
+ samples: Math.round(clamp(num("data-motion-blur-samples", 16), 2, 64)),
+ events: root.getAttribute("data-motion-blur-events") === "on",
+ };
+}
+
+const ACCELERATED_CONTEXTS = new Set(["webgl", "webgl2", "experimental-webgl", "webgpu"]);
+const acceleratedCanvases = new WeakSet<HTMLCanvasElement>();
+
+/**
+ * Remember canvases given a WebGL / WebGPU context. Must run before composition scripts: once
+ * one has presented, html-in-canvas keeps capturing that frame however often it redraws.
+ */
+export function trackAcceleratedCanvases(): void {
+ const proto = HTMLCanvasElement.prototype;
+ const original: unknown = proto.getContext;
+ if (typeof original !== "function" || "__hfAccelTracked" in original) return;
+ const tracked = function (this: HTMLCanvasElement, ...args: unknown[]): unknown {
+ const ctx: unknown = Reflect.apply(original, this, args);
+ if (ctx && ACCELERATED_CONTEXTS.has(String(args[0]))) acceleratedCanvases.add(this);
+ return ctx;
+ };
+ Object.defineProperty(tracked, "__hfAccelTracked", { value: true });
+ Object.defineProperty(proto, "getContext", {
+ value: tracked,
+ writable: true,
+ configurable: true,
+ });
+}
+
+/** What in the composition the capture cannot take, checked again per frame (scripts add canvases). */
+function captureBlocker(root: HTMLElement): string | null {
+ // A capture canvas inside another one blocks the renderer's main thread.
+ if (root.querySelector("canvas[layoutsubtree]"))
+ return "the composition already uses html-in-canvas";
+ for (const canvas of root.querySelectorAll("canvas")) {
+ if (acceleratedCanvases.has(canvas))
+ return "the composition draws a WebGL/WebGPU canvas, which html-in-canvas captures frozen";
+ }
+ return null;
+}
+
+function unsupportedReason(root: HTMLElement): string | null {
+ const off = (window as unknown as Record<string, unknown>)[PAGE_MOTION_BLUR_OFF_GLOBAL];
+ if (off) return `the renderer turned it off (${String(off)})`;
+ if (!supportsCaptureMotionBlur()) {
+ return "html-in-canvas (drawElementImage) is unavailable in this browser";
+ }
+ return captureBlocker(root);
+}
+
+function describe(el: Element): string {
+ const cls =
+ typeof el.className === "string" && el.className
+ ? `.${el.className.trim().split(/\s+/).join(".")}`
+ : "";
+ return `${el.tagName.toLowerCase()}${el.id ? `#${el.id}` : ""}${cls}`;
+}
+
+function rootSize(root: HTMLElement): { width: number; height: number } {
+ const w = Number(root.getAttribute("data-width")) || root.offsetWidth;
+ const h = Number(root.getAttribute("data-height")) || root.offsetHeight;
+ return { width: Math.max(1, Math.round(w)), height: Math.max(1, Math.round(h)) };
+}
+
+function makeCanvas(width: number, height: number, role: string): HTMLCanvasElement {
+ const canvas = document.createElement("canvas");
+ canvas.width = width;
+ canvas.height = height;
+ canvas.setAttribute("data-hf-ignore", "");
+ canvas.setAttribute("data-hf-motion-blur", role);
+ // ponytail: assumes the root sits at its parent's origin (true for compositions); place the
+ // canvases at the root's offset if a layout ever puts it elsewhere.
+ canvas.style.cssText = `position:absolute;left:0;top:0;width:${width}px;height:${height}px;`;
+ return canvas;
+}
+
+function createInstance(host: CaptureMotionBlurHost, config: Config) {
+ const { root } = host;
+ const { width, height } = rootSize(root);
+ const src = makeCanvas(width, height, "capture") as PaintCanvas;
+ const out = makeCanvas(width, height, "output");
+ out.style.pointerEvents = "none";
+ src.setAttribute("layoutsubtree", "");
+
+ const captureContext = src.getContext("2d") as DrawElementContext | null;
+ const outputContext = out.getContext("2d");
+ if (!captureContext || !outputContext) throw new Error("capture motion blur: no 2D context");
+ const sctx: DrawElementContext = captureContext;
+ const octx: CanvasRenderingContext2D = outputContext;
+ const compositor = createCompositor(width, height);
+ // Hides movers for the background capture without touching author or GSAP opacity.
+ const hideRule = document.createElement("style");
+ hideRule.setAttribute("data-hf-ignore", "");
+ hideRule.textContent = `[${HIDE_ATTR}] { opacity: 0 !important; }`;
+
+ let movers: HTMLElement[] = [];
+ let reader: ReturnType<typeof createPoseReader> | null = null;
+ let pending: Job | null = null;
+ // While a frame is in flight (two paints), newer requests collapse into the latest one.
+ let queued: { t: number; engineMode: boolean; release: () => void } | null = null;
+ let seq = 0;
+ let failures = 0;
+ let failedFrames = 0;
+ let disabled: string | null = null;
+ const timings: number[] = [];
+ let sampleMs = 0;
+ let scanMs = 0;
+ let scannedTimeline: unknown = undefined;
+ let wrapped = false;
+ // The output still shows the last frame of the previous wrap until this one draws.
+ let staleOutput = false;
+ // Once the engine drives frames, paused-style requests come from the transport, not a user.
+ let rendering = false;
+
+ function wrap(): void {
+ if (wrapped) return;
+ root.before(src);
+ src.append(root);
+ src.after(out);
+ document.head.append(hideRule);
+ wrapped = true;
+ staleOutput = true;
+ }
+
+ /** Back to plain DOM: drop any frame in flight, put the root where it was. */
+ function unwrap(): void {
+ queued?.release();
+ queued = null;
+ if (pending) {
+ reveal(pending);
+ pending.settle();
+ pending = null;
+ }
+ if (!wrapped) return;
+ wrapped = false;
+ hideRule.remove();
+ src.before(root);
+ src.remove();
+ out.remove();
+ }
+
+ const seek = (t: number): void => host.seek(t, !config.events);
+
+ function discover(restoreTo: number): void {
+ scannedTimeline = host.timeline();
+ const s0 = performance.now();
+ const found = discoverMovers({
+ scene: root,
+ durationSeconds: host.durationSeconds(),
+ fps: host.fps,
+ seek,
+ });
+ seek(restoreTo);
+ movers = found.movers;
+ reader = createPoseReader(root, found.dynamic);
+ scanMs = Math.round(performance.now() - s0);
+ }
+
+ function frames(t: number, reads: number): { els: HTMLElement[]; frames: MoverFrame[] } {
+ const r = reader;
+ if (!r) return { els: [], frames: [] };
+ const visible = movers.filter(
+ (el) =>
+ typeof el.checkVisibility !== "function" ||
+ el.checkVisibility({ visibilityProperty: true }),
+ );
+ const shutter = config.shutterDeg / 360 / host.fps;
+ if (visible.length === 0 || shutter <= 0) return { els: [], frames: [] };
+ const start = t - shutter / 2;
+ const poses = visible.map((): Affine[] => []);
+ for (let j = 0; j < reads; j++) {
+ seek(Math.max(0, start + (j / (reads - 1)) * shutter));
+ r.reset();
+ visible.forEach((el, i) => poses[i]?.push(r.pose(el)));
+ }
+ seek(t);
+ r.reset();
+ const result = visible.map((el, i) => ({
+ w: el.offsetWidth,
+ h: el.offsetHeight,
+ now: r.pose(el),
+ taps: tapsFromReads(poses[i] ?? [IDENTITY], config.samples),
+ }));
+ r.reset();
+ return { els: visible, frames: result };
+ }
+
+ const capture = (): void => {
+ sctx.reset();
+ sctx.drawElementImage(root, 0, 0);
+ };
+ const reveal = (job: Job): void => {
+ for (const el of job.els) el.removeAttribute(HIDE_ATTR);
+ };
+
+ /**
+ * One step of the current frame, run on each paint (or by the engine's composite hook):
+ * first capture the movers and crop their pixels, then hide them and, on the next paint,
+ * capture the background and draw it with the trails. A frame with nothing moving is done
+ * after the first capture. `finalStep` forces the frame to finish now (no further paint will
+ * come), drawing trails over the sharp frame if the background was never captured.
+ */
+ function step(finalStep = false): void {
+ const job = pending;
+ if (!job || disabled || !job.armed) return;
+ const t0 = performance.now();
+ let done = true;
+ try {
+ done = captureAndDraw(job, finalStep);
+ failures = 0;
+ } catch (err) {
+ failures++;
+ swallow("runtime.captureMotionBlur.compose", err);
+ if (failures >= MAX_FAILURES) disable(`capture failed ${failures} times in a row`);
+ }
+ job.ms += performance.now() - t0;
+ if (!done && !disabled) {
+ src.requestPaint();
+ return;
+ }
+ timings.push(sampleMs + job.ms);
+ if (timings.length > 600) timings.shift();
+ finishJob(job);
+ }
+
+ /** This paint's half of the frame; false when the background capture is still to come. */
+ function captureAndDraw(job: Job, finalStep: boolean): boolean {
+ capture();
+ try {
+ if (!job.kept) {
+ job.kept = compositor.crop(src, job.frames);
+ if (staleOutput) {
+ // First paint since wrapping: show this frame sharp until its trails are drawn.
+ octx.setTransform(1, 0, 0, 1, 0, 0);
+ octx.globalAlpha = 1;
+ octx.globalCompositeOperation = "copy";
+ octx.drawImage(src, 0, 0);
+ staleOutput = false;
+ }
+ if (job.kept.some(Boolean) && !finalStep) {
+ job.els.forEach((el, i) => job.kept?.[i] && el.setAttribute(HIDE_ATTR, ""));
+ return false;
+ }
+ }
+ compositor.draw(src, octx, job.frames, job.kept);
+ return true;
+ } finally {
+ sctx.reset(); // the capture canvas sits under the output: keep its own bitmap empty
+ }
+ }
+
+ /** End a frame (drawn or given up on) and start the newest frame requested meanwhile. */
+ function finishJob(job: Job): void {
+ reveal(job);
+ if (pending === job) pending = null;
+ job.settle();
+ const next = queued;
+ queued = null;
+ if (next && !disabled) void start(next.t, next.engineMode).then(next.release);
+ else next?.release();
+ }
+ const onPaint = (): void => step();
+
+ function countFailedFrame(why: string): void {
+ failedFrames++;
+ if (failedFrames % LOG_EVERY === 1)
+ console.info(`[motion-blur] capture failed (${failedFrames} so far): ${why}`);
+ }
+
+ // A tab that stops painting while the clock runs is not a broken capture: this frame is plain
+ // DOM (root back in the page, movers revealed), the next tries again, and timeouts never count
+ // toward giving up.
+ function onPaintTimeout(job: Job): void {
+ countFailedFrame(`no paint within ${PAINT_TIMEOUT_MS} ms`);
+ unwrap();
+ finishJob(job);
+ }
+
+ src.addEventListener("paint", onPaint);
+
+ function disable(reason: string): void {
+ if (disabled) return;
+ disabled = reason;
+ console.warn(`[motion-blur] off: ${reason}`);
+ teardown();
+ }
+
+ function teardown(): void {
+ src.removeEventListener("paint", onPaint);
+ unwrap();
+ }
+
+ /**
+ * Request one blurred frame at `t`; resolves once it is drawn (or given up on). A frame still
+ * in flight (between its two paints) is not abandoned: the request waits as the newest pending
+ * one. An engine frame replaces it, since the engine waits on exactly this frame.
+ */
+ function paint(t: number, engineMode: boolean): Promise<void> {
+ if (disabled) return Promise.resolve();
+ if (engineMode) rendering = true;
+ if (pending && !engineMode) {
+ const previous = queued;
+ return new Promise<void>((resolve) => {
+ queued = {
+ t,
+ engineMode,
+ release: () => {
+ previous?.release();
+ resolve();
+ },
+ };
+ });
+ }
+ return start(t, engineMode);
+ }
+
+ function start(t: number, engineMode: boolean): Promise<void> {
+ try {
+ if (!reader || host.timeline() !== scannedTimeline) discover(t);
+ if (movers.length === 0) {
+ unwrap();
+ return Promise.resolve();
+ }
+ // Checked again here, not only at init: a block's script can add its own capture canvas
+ // (wrapping one inside ours hangs the renderer) or a WebGL canvas later.
+ const blocked = captureBlocker(root);
+ if (blocked) {
+ disable(blocked);
+ return Promise.resolve();
+ }
+ wrap();
+ const s0 = performance.now();
+ const next = frames(t, engineMode ? RENDER_READS : PREVIEW_READS);
+ sampleMs = performance.now() - s0;
+ if (pending) finishJob(pending);
+ return new Promise<void>((resolve) => {
+ const mine = ++seq;
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const startTimer = (): void => {
+ timer = setTimeout(() => {
+ if (pending?.seq === mine) onPaintTimeout(pending);
+ else resolve();
+ }, PAINT_TIMEOUT_MS);
+ };
+ pending = {
+ seq: mine,
+ els: next.els,
+ frames: next.frames,
+ kept: null,
+ armed: !engineMode,
+ startTimer,
+ ms: 0,
+ settle: () => {
+ clearTimeout(timer);
+ resolve();
+ },
+ };
+ if (engineMode) return;
+ startTimer();
+ src.requestPaint();
+ });
+ } catch (err) {
+ swallow("runtime.captureMotionBlur.paint", err);
+ disable(err instanceof Error ? err.message : String(err));
+ return Promise.resolve();
+ }
+ }
+
+ function report(): Report {
+ const mean = timings.length ? timings.reduce((s, x) => s + x, 0) / timings.length : 0;
+ return {
+ state: disabled ? "off" : "on",
+ reason: disabled,
+ movers: movers.length,
+ moverNames: movers.map(describe),
+ scanMs,
+ shutterDeg: config.shutterDeg,
+ frames: timings.length,
+ meanMs: Math.round(mean * 10) / 10,
+ failures,
+ failedFrames,
+ };
+ }
+
+ return {
+ root,
+ paint,
+ /** Paused in preview: show plain DOM so it can be selected. A render never rests. */
+ rest: (): boolean => {
+ if (rendering) return false;
+ unwrap();
+ return true;
+ },
+ arm: () => {
+ if (!pending || pending.armed) return;
+ pending.armed = true;
+ pending.startTimer();
+ src.requestPaint();
+ },
+ /** Finish the current frame now: the engine's forced paint just ran. */
+ finish: () => {
+ if (pending) pending.armed = true;
+ step(true);
+ },
+ teardown,
+ report,
+ busy: (): boolean => pending !== null,
+ rediscover: () => {
+ reader = null;
+ },
+ };
+}
+
+type Instance = ReturnType<typeof createInstance>;
+let instance: Instance | null = null;
+let priorResolver: (() => boolean) | null = null;
+let ourResolver: (() => boolean) | null = null;
+
+/**
+ * Read the root's opt-in and wrap (or unwrap) it. Called wherever the runtime (re)initialises
+ * scene DOM; a repeat call on the same root only schedules a fresh mover scan.
+ */
+export function initCaptureMotionBlur(host: CaptureMotionBlurHost): void {
+ const config = readConfig(host.root);
+ const w = window as CompositeWindow;
+ if (instance && (instance.root !== host.root || !config)) {
+ instance.teardown();
+ instance = null;
+ delete w.__hfMotionBlur;
+ }
+ if (!config) return;
+ if (instance) {
+ instance.rediscover();
+ return;
+ }
+ const reason = unsupportedReason(host.root);
+ if (reason) {
+ console.info(`[motion-blur] off: ${reason}`);
+ w.__hfMotionBlur = {
+ report: () => ({
+ state: "off",
+ reason,
+ movers: 0,
+ moverNames: [],
+ scanMs: 0,
+ shutterDeg: config.shutterDeg,
+ frames: 0,
+ meanMs: 0,
+ failures: 0,
+ failedFrames: 0,
+ }),
+ };
+ return;
+ }
+ instance = createInstance(host, config);
+ const current = instance;
+ w.__hfMotionBlur = { report: () => current.report() };
+}
+
+/**
+ * Draw the blurred frame for `t` after a seek or a playback tick. In preview only playback blurs:
+ * a paused request (no `playing`) unwraps the composition instead and returns null. In engine
+ * mode it also claims the engine's
+ * page-composite step, composing with whoever already holds it, so a render frame is complete
+ * before the screenshot even if no paint event fired. Returns the frame's completion.
+ */
+export function paintCaptureMotionBlur(
+ t: number,
+ options?: { engineMode?: boolean; playing?: boolean },
+): Promise<void> | null {
+ if (!instance) return null;
+ const engineMode = options?.engineMode === true;
+ if (!engineMode && !options?.playing && instance.rest()) return null;
+ const done = instance.paint(t, engineMode);
+ // Nothing in flight (nothing moves, or blur turned off): leave the engine's composite step idle.
+ if (engineMode && instance.busy()) armPageComposite();
+ return done;
+}
+
+/** Start capturing a deferred engine frame: the page (injected video frames included) is final. */
+export function releaseCaptureMotionBlurFrame(): void {
+ instance?.arm();
+}
+
+function armPageComposite(): void {
+ const w = window as CompositeWindow;
+ const current = w.__hf_page_composite_resolve;
+ if (current !== ourResolver) {
+ priorResolver = typeof current === "function" ? current : null;
+ ourResolver = () => {
+ if (priorResolver) priorResolver();
+ // The engine's forced paint has just run, so the paint records are valid here even if
+ // no paint event reached us; compose() is a no-op when the event already drew the frame.
+ instance?.finish();
+ w.__hf_page_composite_pending = false;
+ return true;
+ };
+ w.__hf_page_composite_resolve = ourResolver;
+ }
+ w.__hf_page_composite_pending = true;
+}
+
+/** Test-only: drop module state between cases. */
+export function resetCaptureMotionBlurForTests(): void {
+ instance?.teardown();
+ instance = null;
+ priorResolver = null;
+ ourResolver = null;
+}
diff --git a/packages/core/src/runtime/entry.test.ts b/packages/core/src/runtime/entry.test.ts
index d54f98505f..5f64f36305 100644
--- a/packages/core/src/runtime/entry.test.ts
+++ b/packages/core/src/runtime/entry.test.ts
@@ -96,6 +96,23 @@ const neverDecodes = (clip: HTMLElement) => {
describe("runtime entry", () => {
afterEach(resetRuntimeGlobals);
+ it("starts a held capture-motion-blur frame when capture waits on the seek, after video injection", async () => {
+ const release = vi.fn();
+ vi.doMock("./captureMotionBlur", async (original) => ({
+ ...(await original<typeof import("./captureMotionBlur")>()),
+ releaseCaptureMotionBlurFrame: release,
+ }));
+ try {
+ mountRoot().appendChild(document.createElement("div"));
+ await evaluateRuntime();
+ expect(release).not.toHaveBeenCalled();
+ await window.__hfWaitForSeekCompletion?.();
+ expect(release).toHaveBeenCalledTimes(1);
+ } finally {
+ vi.doUnmock("./captureMotionBlur");
+ }
+ });
+
it("drives an async frame source without a GSAP timeline and waits before capture", async () => {
const root = mountRoot();
root.setAttribute("data-duration", "10");
diff --git a/packages/core/src/runtime/entry.ts b/packages/core/src/runtime/entry.ts
index 6f34c13baf..236222cfb7 100644
--- a/packages/core/src/runtime/entry.ts
+++ b/packages/core/src/runtime/entry.ts
@@ -17,6 +17,7 @@ import { runScriptsAfterFonts } from "./afterFonts";
import { AFTER_FONTS_SCRIPTS } from "../compiler/scriptRuns";
import { hasFrameSources, registerFrameSource } from "./frameSources";
import { createFilmBridge } from "./filmBridge";
+import { trackAcceleratedCanvases } from "./captureMotionBlur";
type HyperframeWindow = Window & {
__hyperframeRuntimeBootstrapped?: boolean;
@@ -43,6 +44,7 @@ type HyperframeWindow = Window & {
installAuthoredOpacityCapture();
installAuthoredMediaCapture();
installFlatGsapTransforms();
+trackAcceleratedCanvases();
window.__hfHasFrameSources = hasFrameSources;
hideTimedClipsUntilFirstPass();
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index ac9ca35880..7661f27b8e 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -125,6 +125,11 @@ import { applyVariableBindings, unproxiedMediaSrc } from "./applyVariableBinding
import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading";
import { COLOR_GRADING_AUTHORED_OPACITY_ATTR } from "../colorGrading";
import { initVfx, paintVfx } from "./vfx";
+import {
+ initCaptureMotionBlur,
+ paintCaptureMotionBlur,
+ releaseCaptureMotionBlurFrame,
+} from "./captureMotionBlur";
import { TransportClock } from "./clock";
import { WebAudioTransport } from "./webAudioTransport";
import {
@@ -3659,6 +3664,24 @@ export function initSandboxRuntimeModular(): void {
// Passes that must see scene DOM, which arrives after init: when scenes mount, or when one is
// swapped. Resolves when caption overrides have landed.
+ // Capture motion blur (data-motion-blur="on" on the root). Its pose reads seek the timeline
+ // and adapters only: media and transport bookkeeping stay on the frame's seek.
+ function initMotionBlur(): void {
+ const root = resolveRootCompositionElement();
+ if (!root) return;
+ initCaptureMotionBlur({
+ root,
+ fps: state.canonicalFps,
+ durationSeconds: () =>
+ getSafeTimelineDurationSeconds(state.capturedTimeline, 0) ||
+ Number(root.getAttribute("data-duration")) ||
+ 0,
+ timeline: () => state.capturedTimeline,
+ seek: (t, suppressEvents) => {
+ seekRootChildrenAndAdapters(state.capturedTimeline, t, { suppressEvents });
+ },
+ });
+ }
const settleSceneDom = (): Promise<void> => {
bindMediaMetadataListeners();
installAssetFailureDiagnostics();
@@ -3667,6 +3690,7 @@ export function initSandboxRuntimeModular(): void {
applyVariableBindings(document);
// An unregistered vfx chain paints nothing and logs nothing, so re-scan the new DOM.
initVfx(document.body, state.canonicalFps);
+ initMotionBlur();
return captionsApplied;
};
if (!externalCompositionsReady) {
@@ -4012,6 +4036,7 @@ export function initSandboxRuntimeModular(): void {
}
};
initVfx(document.body, state.canonicalFps);
+ initMotionBlur();
applyFetchedCaptionOverrides(captionOverrides, captionHosts, rewindCaptionTimelines);
releaseDetachedMedia();
childrenBound = false;
@@ -4064,6 +4089,7 @@ export function initSandboxRuntimeModular(): void {
});
// Per-pixel effect chains: compile once here, repaint on every seek below.
initVfx(document.body, state.canonicalFps);
+ initMotionBlur();
const applyPlaybackRate = (nextRate: number) => {
const parsed = Number(nextRate);
@@ -4120,6 +4146,8 @@ export function initSandboxRuntimeModular(): void {
}
colorGrading.redraw();
paintVfx(state.currentTime);
+ const blurred = paintCaptureMotionBlur(state.currentTime);
+ if (blurred) registerSeekCompletion(blurred);
postState(true);
};
@@ -4172,6 +4200,8 @@ export function initSandboxRuntimeModular(): void {
runAdapters("pause");
syncMediaForCurrentState();
colorGrading.redraw();
+ // Paused: back to plain DOM so Studio can select inside the composition.
+ void paintCaptureMotionBlur(state.currentTime);
postState(true);
},
seek: (timeSeconds, options) => {
@@ -4232,6 +4262,8 @@ export function initSandboxRuntimeModular(): void {
syncMediaForCurrentState();
colorGrading.redraw();
paintVfx(state.currentTime, { engineMode: true });
+ const blurred = paintCaptureMotionBlur(state.currentTime, { engineMode: true });
+ if (blurred) registerSeekCompletion(blurred);
postState(true);
},
getTime: () => heldSeek?.time ?? clock.now(),
@@ -4340,9 +4372,15 @@ export function initSandboxRuntimeModular(): void {
window.__hfTypegpuTime = t;
forceDispatchSeekEvent(t);
};
- window.__hfWaitForSeekCompletion = waitForSeekCompletion;
+ // Engine and CLI capture wait here once the page is final (the engine's injected video frames
+ // land after renderSeek), so a blurred engine frame is captured now, not at the seek.
+ const waitForFinalFrame: typeof waitForSeekCompletion = (options) => {
+ releaseCaptureMotionBlurFrame();
+ return waitForSeekCompletion(options);
+ };
+ window.__hfWaitForSeekCompletion = waitForFinalFrame;
runtimeCleanupCallbacks.push(() => {
- if (window.__hfWaitForSeekCompletion === waitForSeekCompletion) {
+ if (window.__hfWaitForSeekCompletion === waitForFinalFrame) {
delete window.__hfWaitForSeekCompletion;
}
});
@@ -5159,6 +5197,8 @@ export function initSandboxRuntimeModular(): void {
state.capturedTimeline !== lastTransportSeekTimeline
) {
seekTimelineAndAdapters(t, { revision: timingRevision });
+ // Playback: draw and move on; nothing waits on this frame's blur. Paused, it unwraps.
+ void paintCaptureMotionBlur(t, { playing: isPlaying });
if (!isPlaying) pausedSeekDeferredByManualGesture = false;
}
if (isPlaying) {
@@ -5181,6 +5221,7 @@ export function initSandboxRuntimeModular(): void {
}
runAdapters("pause");
syncMediaForCurrentState(timingRevision);
+ void paintCaptureMotionBlur(state.currentTime);
postState(true);
return;
}