Skip to content

Write HTML. Render video. Built for agents.

feat(core): capture motion blur pose, discovery and compositor (#5875)

Vance Ingalls8df2d8d95dec
diff --git a/packages/core/src/runtime/captureMotionBlur/composite.test.ts b/packages/core/src/runtime/captureMotionBlur/composite.test.ts
new file mode 100644
index 0000000000..3fd0033873
--- /dev/null
+++ b/packages/core/src/runtime/captureMotionBlur/composite.test.ts
@@ -0,0 +1,80 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { createCompositor, type MoverFrame } from "./composite";
+import { IDENTITY, translate } from "./pose";
+
+function fakeContext() {
+ return {
+ drawImage: vi.fn(),
+ clearRect: vi.fn(),
+ setTransform: vi.fn(),
+ globalAlpha: 1,
+ globalCompositeOperation: "source-over",
+ };
+}
+let offscreen: Array<ReturnType<typeof fakeContext>> = [];
+
+beforeEach(() => {
+ offscreen = [];
+ vi.stubGlobal(
+ "OffscreenCanvas",
+ class {
+ constructor(
+ public width: number,
+ public height: number,
+ ) {}
+ getContext() {
+ const ctx = fakeContext();
+ offscreen.push(ctx);
+ return ctx;
+ }
+ },
+ );
+});
+afterEach(() => vi.unstubAllGlobals());
+
+const capture = {} as CanvasImageSource;
+const background = {} as CanvasImageSource;
+const slide = (from: number, to: number, n = 16) =>
+ Array.from({ length: n }, (_, k) => translate(IDENTITY, from + ((to - from) * (k + 0.5)) / n, 0));
+const mover = (w: number, h: number, taps: ReturnType<typeof slide>): MoverFrame => ({
+ w,
+ h,
+ now: translate(IDENTITY, 50, 0),
+ taps,
+});
+
+describe("createCompositor", () => {
+ it("keeps no crop for a mover that does not travel a pixel, and draws only the background", () => {
+ const c = createCompositor(100, 100);
+ const movers = [mover(10, 10, slide(0, 0.5))];
+ const kept = c.crop(capture, movers);
+ expect(kept).toEqual([false]);
+ const out = fakeContext();
+ expect(c.draw(background, out as never, movers, kept)).toEqual({ blurred: 0, taps: 0 });
+ expect(out.drawImage).toHaveBeenCalledTimes(1);
+ expect(out.drawImage.mock.calls[0]?.[0]).toBe(background);
+ });
+
+ it("crops a moving mover once and lays one tap per draw over the background", () => {
+ const c = createCompositor(1000, 100);
+ const movers = [mover(10, 10, slide(0, 100))];
+ const kept = c.crop(capture, movers);
+ expect(kept).toEqual([true]);
+ const crop = offscreen.at(-1);
+ expect(crop?.drawImage).toHaveBeenCalledWith(capture, 0, 0);
+ const out = fakeContext();
+ const stats = c.draw(background, out as never, movers, kept);
+ const accum = offscreen[0];
+ expect(stats.blurred).toBe(1);
+ expect(accum?.drawImage).toHaveBeenCalledTimes(stats.taps);
+ expect(accum?.globalAlpha).toBeCloseTo(1 / stats.taps, 9);
+ // Background first, then the trail over it.
+ expect(out.drawImage.mock.calls[0]).toEqual([background, 0, 0]);
+ expect(out.drawImage).toHaveBeenCalledTimes(2);
+ });
+
+ it("leaves a mover sharp when its box is too big to copy every frame", () => {
+ const c = createCompositor(100, 100);
+ expect(c.crop(capture, [mover(1000, 1000, slide(0, 100))])).toEqual([false]);
+ });
+});
diff --git a/packages/core/src/runtime/captureMotionBlur/composite.ts b/packages/core/src/runtime/captureMotionBlur/composite.ts
new file mode 100644
index 0000000000..81532fad77
--- /dev/null
+++ b/packages/core/src/runtime/captureMotionBlur/composite.ts
@@ -0,0 +1,135 @@
+/**
+ * The per-frame draw: a background, then each mover's own pixels redrawn along its path.
+ *
+ * Each mover is cropped from a capture of the scene and drawn at every tap pose into a float
+ * buffer, additive at weight 1/n: the shutter integral. Its coverage fades at the path ends, so
+ * the trail needs a background without the mover (a second capture, movers hidden); falling back
+ * to the sharp frame leaves thin lines looking unblurred.
+ *
+ * Image-space approximation: a mover's box contents move with it; ink outside it is not redrawn.
+ */
+
+import { type Affine, invert, sweep, thinTaps } from "./pose";
+
+export interface MoverFrame {
+ w: number;
+ h: number;
+ /** The pose the capture shows (frame time). */
+ now: Affine;
+ /** Poses across the shutter. */
+ taps: Affine[];
+}
+
+interface CompositeStats {
+ blurred: number;
+ taps: number;
+}
+
+type Context2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+interface Layer {
+ canvas: OffscreenCanvas;
+ ctx: OffscreenCanvasRenderingContext2D;
+}
+
+// Float16 keeps n additive 1/n-weighted taps exact; 8-bit loses up to ~n/2 levels per channel.
+function makeLayer(w: number, h: number): Layer {
+ const canvas = new OffscreenCanvas(w, h);
+ let ctx: OffscreenCanvasRenderingContext2D | null = null;
+ try {
+ ctx = canvas.getContext("2d", { colorType: "float16" } as CanvasRenderingContext2DSettings);
+ } catch {
+ ctx = null;
+ }
+ ctx = ctx ?? canvas.getContext("2d");
+ if (!ctx) throw new Error("capture motion blur: no 2D context for the accumulation buffer");
+ return { canvas, ctx };
+}
+
+/** Whether a mover travels far enough within this shutter to draw a trail at all. */
+function moves(mover: MoverFrame, frameW: number, frameH: number): boolean {
+ const { travel, rect } = sweep(mover.taps, mover.w, mover.h, frameW, frameH);
+ return rect !== null && travel >= 1;
+}
+
+export function createCompositor(width: number, height: number) {
+ const accum = makeLayer(width, height);
+ const crops: Layer[] = [];
+ // A mover box this many times the frame is too big to copy every frame; it is left sharp.
+ const maxCropArea = width * height * 4;
+
+ const cropLayer = (i: number, w: number, h: number): Layer => {
+ let layer = crops[i];
+ if (!layer || layer.canvas.width < w || layer.canvas.height < h) {
+ layer = makeLayer(
+ Math.max(w, layer?.canvas.width ?? 1),
+ Math.max(h, layer?.canvas.height ?? 1),
+ );
+ crops[i] = layer;
+ }
+ return layer;
+ };
+
+ /** Copy each mover's frame-time pixels out of `src`; false marks a mover to leave sharp. */
+ function crop(src: CanvasImageSource, movers: MoverFrame[]): boolean[] {
+ return movers.map((mover, i) => {
+ const { w, h } = mover;
+ const back = invert(mover.now);
+ if (!back || w * h > maxCropArea || !moves(mover, width, height)) return false;
+ const c = cropLayer(i, w, h).ctx;
+ c.setTransform(1, 0, 0, 1, 0, 0);
+ c.globalCompositeOperation = "source-over";
+ c.globalAlpha = 1;
+ c.clearRect(0, 0, w, h);
+ c.setTransform(back.a, back.b, back.c, back.d, back.e, back.f);
+ c.drawImage(src, 0, 0);
+ return true;
+ });
+ }
+
+ function trail(out: Context2D, mover: MoverFrame, layer: Layer): number {
+ const { w, h } = mover;
+ const { travel, rect } = sweep(mover.taps, w, h, width, height);
+ if (!rect) return 0;
+ const taps = thinTaps(mover.taps, travel);
+ const a = accum.ctx;
+ a.setTransform(1, 0, 0, 1, 0, 0);
+ a.globalCompositeOperation = "source-over";
+ a.globalAlpha = 1;
+ a.clearRect(rect.x, rect.y, rect.w, rect.h);
+ a.globalCompositeOperation = "lighter";
+ a.globalAlpha = 1 / taps.length;
+ for (const m of taps) {
+ a.setTransform(m.a, m.b, m.c, m.d, m.e, m.f);
+ a.drawImage(layer.canvas, 0, 0, w, h, 0, 0, w, h);
+ }
+ out.drawImage(accum.canvas, rect.x, rect.y, rect.w, rect.h, rect.x, rect.y, rect.w, rect.h);
+ return taps.length;
+ }
+
+ /** `background` into `out`, then the trail of every mover `crop` kept. */
+ function draw(
+ background: CanvasImageSource,
+ out: Context2D,
+ movers: MoverFrame[],
+ kept: boolean[],
+ ): CompositeStats {
+ out.setTransform(1, 0, 0, 1, 0, 0);
+ out.globalAlpha = 1;
+ out.globalCompositeOperation = "copy";
+ out.drawImage(background, 0, 0);
+ out.globalCompositeOperation = "source-over";
+ const stats: CompositeStats = { blurred: 0, taps: 0 };
+ movers.forEach((mover, i) => {
+ const layer = crops[i];
+ if (!kept[i] || !layer) return;
+ const n = trail(out, mover, layer);
+ if (n > 0) {
+ stats.blurred++;
+ stats.taps += n;
+ }
+ });
+ return stats;
+ }
+
+ return { crop, draw };
+}
diff --git a/packages/core/src/runtime/captureMotionBlur/discover.test.ts b/packages/core/src/runtime/captureMotionBlur/discover.test.ts
new file mode 100644
index 0000000000..0f0aac55a3
--- /dev/null
+++ b/packages/core/src/runtime/captureMotionBlur/discover.test.ts
@@ -0,0 +1,89 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { discoverMovers } from "./discover";
+
+// jsdom lays nothing out: each element's rect follows the x its fake timeline writes.
+function element(parent: Element, id: string): HTMLElement {
+ const el = document.createElement("div");
+ el.id = id;
+ parent.append(el);
+ el.getBoundingClientRect = () => {
+ const x = parseFloat(el.style.transform.replace("translateX(", "")) || 0;
+ return { left: x, top: 0, right: x + 10, bottom: 10, width: 10, height: 10 } as DOMRect;
+ };
+ return el;
+}
+
+function setup() {
+ const scene = document.createElement("div");
+ const fast = element(scene, "fast");
+ const fastChild = element(fast, "fast-child");
+ const slow = element(scene, "slow");
+ const still = element(scene, "still");
+ const off = element(scene, "off");
+ off.setAttribute("data-motion-blur", "off");
+ // 1 s at 30 fps: `fast` and `off` cross 300 px (10 px/frame), `slow` 15 px (0.5 px/frame).
+ const seek = (t: number): void => {
+ fast.style.transform = `translateX(${300 * t}px)`;
+ fastChild.style.transform = `translateX(${200 * t}px)`;
+ slow.style.transform = `translateX(${15 * t}px)`;
+ off.style.transform = `translateX(${300 * t}px)`;
+ };
+ seek(0);
+ return { scene, fast, fastChild, slow, still, off, seek };
+}
+
+describe("discoverMovers", () => {
+ afterEach(() => vi.restoreAllMocks());
+
+ it("keeps fast, outermost movers that are not opted out", () => {
+ const { scene, fast, seek } = setup();
+ const { movers } = discoverMovers({ scene, durationSeconds: 1, fps: 30, seek });
+ expect(movers).toEqual([fast]);
+ });
+
+ it("marks every changing element dynamic, slow and opted-out ones included", () => {
+ const { scene, fast, fastChild, slow, still, off, seek } = setup();
+ const { dynamic } = discoverMovers({ scene, durationSeconds: 1, fps: 30, seek });
+ expect(
+ dynamic.has(fast) && dynamic.has(fastChild) && dynamic.has(slow) && dynamic.has(off),
+ ).toBe(true);
+ expect(dynamic.has(still)).toBe(false);
+ });
+
+ it("ignores a CSS animation of offset-distance, which the pose has no term for", () => {
+ const scene = document.createElement("div");
+ const box = document.createElement("div");
+ scene.append(box);
+ let x = 0; // moves with no inline style change: only getAnimations can nominate it
+ box.getBoundingClientRect = () =>
+ ({ left: x, right: x + 10, top: 0, bottom: 10, width: 10 }) as DOMRect;
+ const scan = (prop: string) => {
+ const effect = { target: box, getKeyframes: () => [{ [prop]: "0" }] };
+ Object.defineProperty(document, "getAnimations", {
+ value: () => [{ effect }],
+ configurable: true,
+ });
+ const seek = (t: number): void => void (x = 300 * t);
+ return discoverMovers({ scene, durationSeconds: 1, fps: 30, seek }).movers;
+ };
+ expect(scan("offsetDistance")).toEqual([]);
+ expect(scan("translate")).toEqual([box]);
+ Reflect.deleteProperty(document, "getAnimations");
+ });
+
+ it("drops a mover in a 3D context, or on an offset-path the pose cannot follow", () => {
+ const { scene, fast, seek } = setup();
+ const real = window.getComputedStyle;
+ let own: object = {};
+ vi.spyOn(window, "getComputedStyle").mockImplementation((el: Element) =>
+ el === fast
+ ? ({ transform: "none", transformStyle: "flat", perspective: "none", ...own } as never)
+ : real(el),
+ );
+ for (own of [
+ { transform: "matrix3d(1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1)" },
+ { offsetPath: "path('M 0 0')" },
+ ])
+ expect(discoverMovers({ scene, durationSeconds: 1, fps: 30, seek }).movers).toEqual([]);
+ });
+});
diff --git a/packages/core/src/runtime/captureMotionBlur/discover.ts b/packages/core/src/runtime/captureMotionBlur/discover.ts
new file mode 100644
index 0000000000..18ac0cc84f
--- /dev/null
+++ b/packages/core/src/runtime/captureMotionBlur/discover.ts
@@ -0,0 +1,124 @@
+/**
+ * Which elements to blur: the ones that move fast enough for a smear to show.
+ *
+ * One scan: candidates are elements whose inline style changes or that a CSS / WAAPI animation
+ * moves; they must reach MIN_SPEED px/frame outside any 3D context (the redraw is 2D), and only
+ * the outermost of nested candidates is kept.
+ */
+
+import { isHtmlElement } from "../domRealm";
+
+/** Under this peak speed (px per frame) a smear is invisible, so the element stays sharp. */
+const MIN_SPEED = 2;
+
+/** Scan resolution: at most this many timeline samples, however long the composition. */
+const SCAN_STEPS = 240;
+
+const ANIMATED_POSE_PROPS = new Set<string>([
+ "transform",
+ "translate",
+ "rotate",
+ "scale",
+ "left",
+ "top",
+ "right",
+ "bottom",
+]);
+
+export interface DiscoverOptions {
+ scene: HTMLElement;
+ durationSeconds: number;
+ fps: number;
+ /** Seek the composition (timeline + adapters) to a time. */
+ seek: (t: number) => void;
+}
+
+export interface Discovery {
+ movers: HTMLElement[];
+ /** Every element whose pose changes at all; their transforms are never cached. */
+ dynamic: Set<Element>;
+}
+
+// The whole inline style in one read; opacity or colour tweens only cost a rect check.
+const styleKey = (el: HTMLElement): string => el.getAttribute("style") ?? "";
+
+function optedOut(el: Element, scene: Element): boolean {
+ for (let n: Element | null = el; n && n !== scene; n = n.parentElement) {
+ if (n.getAttribute("data-motion-blur") === "off") return true;
+ }
+ return false;
+}
+
+function in3d(el: Element, scene: Element): boolean {
+ for (let n: Element | null = el; n && n !== scene; n = n.parentElement) {
+ const cs = getComputedStyle(n);
+ if (
+ cs.transform.startsWith("matrix3d") ||
+ cs.transformStyle === "preserve-3d" ||
+ cs.perspective !== "none" ||
+ (cs.offsetPath ?? "none") !== "none"
+ ) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/** Elements a CSS animation or WAAPI animation moves (no inline style to diff). */
+function pageAnimated(scene: Element): Set<HTMLElement> {
+ const out = new Set<HTMLElement>();
+ if (typeof document.getAnimations !== "function") return out;
+ for (const animation of document.getAnimations()) {
+ const effect = animation.effect as KeyframeEffect | null;
+ const target = effect?.target;
+ if (!isHtmlElement(target) || !scene.contains(target)) continue;
+ const frames = typeof effect?.getKeyframes === "function" ? effect.getKeyframes() : [];
+ if (frames.some((frame) => Object.keys(frame).some((k) => ANIMATED_POSE_PROPS.has(k))))
+ out.add(target);
+ }
+ return out;
+}
+
+/** How far a box's farthest-moving edge travelled between two reads; 0 when either is empty. */
+function edgeShift(a: DOMRect | undefined, b: DOMRect | undefined): number {
+ if (!a || !b || !a.width || !b.width) return 0;
+ return Math.max(
+ Math.abs(a.left - b.left),
+ Math.abs(a.top - b.top),
+ Math.abs(a.right - b.right),
+ Math.abs(a.bottom - b.bottom),
+ );
+}
+
+export function discoverMovers({ scene, durationSeconds, fps, seek }: DiscoverOptions): Discovery {
+ const all = [...scene.querySelectorAll("*")].filter(isHtmlElement);
+ const dynamic = new Set<Element>(pageAnimated(scene));
+ const first = all.map(styleKey);
+ const frames = Math.max(1, Math.round(durationSeconds * fps));
+ // One pass, about every other frame on a typical composition. A whip shorter than one step
+ // can be under-measured; ponytail: raise SCAN_STEPS if a fast move goes unblurred.
+ const steps = Math.min(frames, SCAN_STEPS);
+ const framesPerStep = frames / steps;
+ const fast = new Set<HTMLElement>();
+ const threeD = new Set<HTMLElement>();
+ let prev: Array<DOMRect | undefined> | null = null;
+
+ for (let i = 0; i <= steps; i++) {
+ seek((i / steps) * durationSeconds);
+ // Rects only for elements whose style has changed; the first read flushes layout once.
+ const rects: Array<DOMRect | undefined> = [];
+ all.forEach((el, j) => {
+ if (!dynamic.has(el) && styleKey(el) === first[j]) return;
+ dynamic.add(el);
+ rects[j] = el.getBoundingClientRect();
+ if (edgeShift(prev?.[j], rects[j]) / framesPerStep >= MIN_SPEED) fast.add(el);
+ });
+ // Style reads up the chain are the costly part; every 10th step is enough to see a 3D turn.
+ if (i % 10 === 0 || i === steps) for (const el of fast) if (in3d(el, scene)) threeD.add(el);
+ prev = rects;
+ }
+
+ const candidates = all.filter((el) => fast.has(el) && !threeD.has(el) && !optedOut(el, scene));
+ const movers = candidates.filter((el) => !candidates.some((o) => o !== el && o.contains(el)));
+ return { movers, dynamic };
+}
diff --git a/packages/core/src/runtime/captureMotionBlur/pose.test.ts b/packages/core/src/runtime/captureMotionBlur/pose.test.ts
new file mode 100644
index 0000000000..42c8967108
--- /dev/null
+++ b/packages/core/src/runtime/captureMotionBlur/pose.test.ts
@@ -0,0 +1,230 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+ type Affine,
+ IDENTITY,
+ corners,
+ createPoseReader,
+ individualIs3d,
+ individualTransform,
+ interpolate,
+ invert,
+ multiply,
+ parseTransform,
+ sweep,
+ tapsFromReads,
+ thinTaps,
+ translate,
+} from "./pose";
+
+const rot = (deg: number, e = 0, f = 0): Affine => {
+ const r = (deg * Math.PI) / 180;
+ return { a: Math.cos(r), b: Math.sin(r), c: -Math.sin(r), d: Math.cos(r), e, f };
+};
+const scaleOf = (m: Affine): number => Math.hypot(m.a, m.b);
+const angleOf = (m: Affine): number => (Math.atan2(m.b, m.a) * 180) / Math.PI;
+
+describe("pose math", () => {
+ it("interpolates a rotation without shrinking it", () => {
+ // Entry-wise lerp halfway through a 90° turn gives scale cos(45°) ≈ 0.707.
+ const mid = interpolate(rot(0), rot(90), 0.5);
+ expect(scaleOf(mid)).toBeCloseTo(1, 6);
+ expect(angleOf(mid)).toBeCloseTo(45, 6);
+ });
+
+ it("rotates the short way round", () => {
+ expect(angleOf(interpolate(rot(350), rot(10), 0.5))).toBeCloseTo(0, 6);
+ });
+
+ it("interpolates translation and scale linearly", () => {
+ const p: Affine = { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 };
+ const q: Affine = { a: 3, b: 0, c: 0, d: 2, e: 100, f: -40 };
+ expect(interpolate(p, q, 0.25)).toMatchObject({ a: 1.5, d: 1.25, e: 25, f: -10 });
+ });
+
+ it("inverts an affine", () => {
+ const m = multiply(rot(30, 12, -7), { a: 2, b: 0, c: 0.5, d: 3, e: 4, f: 5 });
+ const back = invert(m);
+ expect(back).not.toBeNull();
+ const id = multiply(m, back as Affine);
+ for (const [k, v] of Object.entries(IDENTITY)) expect(id[k as keyof Affine]).toBeCloseTo(v, 9);
+ });
+
+ it("refuses to invert a collapsed box", () => {
+ expect(invert({ a: 0, b: 0, c: 0, d: 1, e: 0, f: 0 })).toBeNull();
+ });
+
+ it("parses computed 2D and 3D transforms", () => {
+ const values = (css: string) => Object.values(parseTransform(css) ?? {});
+ expect(values("matrix(1, 2, 3, 4, 5, 6)")).toEqual([1, 2, 3, 4, 5, 6]);
+ expect(values("matrix3d(1,0,0,0, 0,1,0,0, 0,0,1,0, 7,8,9,1)")).toEqual([1, 0, 0, 1, 7, 8]);
+ expect(parseTransform("none")).toBeNull();
+ expect(parseTransform("matrix(1, 2)")).toBeNull();
+ });
+});
+
+describe("taps", () => {
+ it("spreads taps across the shutter, centred in each slice", () => {
+ const reads = [translate(IDENTITY, 0, 0), translate(IDENTITY, 100, 0)];
+ const xs = tapsFromReads(reads, 4).map((m) => m.e);
+ expect(xs.map((x) => Math.round(x * 100) / 100)).toEqual([12.5, 37.5, 62.5, 87.5]);
+ });
+
+ it("measures the fastest corner's path and the swept rect", () => {
+ const taps = [
+ translate(IDENTITY, 0, 0),
+ translate(IDENTITY, 30, 0),
+ translate(IDENTITY, 60, 0),
+ ];
+ const { travel, rect } = sweep(taps, 10, 10, 1920, 1080);
+ expect(travel).toBeCloseTo(60, 6);
+ expect(rect).toEqual({ x: 0, y: 0, w: 72, h: 12 });
+ });
+
+ it("reports an off-frame sweep as no rect", () => {
+ const taps = [translate(IDENTITY, -500, -500), translate(IDENTITY, -400, -500)];
+ expect(sweep(taps, 10, 10, 1920, 1080).rect).toBeNull();
+ });
+
+ it("thins taps to about one per 1.5 px, never below two", () => {
+ const taps = tapsFromReads([IDENTITY, translate(IDENTITY, 300, 0)], 32);
+ expect(thinTaps(taps, 3)).toHaveLength(2);
+ expect(thinTaps(taps, 15)).toHaveLength(10);
+ expect(thinTaps(taps, 300)).toHaveLength(32);
+ });
+
+ it("places a rotated box's corners", () => {
+ const [, tr] = corners(rot(90), 10, 0);
+ expect(tr?.[0]).toBeCloseTo(0, 9);
+ expect(tr?.[1]).toBeCloseTo(10, 9);
+ });
+});
+
+const extra = new Map<Element, Partial<CSSStyleDeclaration>>();
+
+describe("createPoseReader", () => {
+ afterEach(() => {
+ vi.restoreAllMocks();
+ extra.clear();
+ });
+
+ function place(el: HTMLElement, left: number, top: number, parent: Element | null): void {
+ Object.defineProperty(el, "offsetLeft", { value: left, configurable: true });
+ Object.defineProperty(el, "offsetTop", { value: top, configurable: true });
+ Object.defineProperty(el, "offsetParent", { value: parent, configurable: true });
+ }
+
+ function tree() {
+ const scene = document.createElement("div");
+ const group = document.createElement("div");
+ const mover = document.createElement("div");
+ scene.append(group);
+ group.append(mover);
+ place(group, 100, 50, scene);
+ place(mover, 10, 20, group);
+ const transforms = new Map<Element, string>([
+ [group, "matrix(2, 0, 0, 2, 0, 0)"],
+ [mover, "none"],
+ ]);
+ vi.spyOn(window, "getComputedStyle").mockImplementation(
+ (el: Element) =>
+ ({
+ transform: transforms.get(el) ?? "none",
+ transformOrigin: "0px 0px",
+ ...extra.get(el),
+ }) as CSSStyleDeclaration,
+ );
+ return { scene, group, mover, transforms };
+ }
+
+ it("composes layout offsets with every transform up the chain", () => {
+ const { scene, mover } = tree();
+ const reader = createPoseReader(scene, new Set([mover]));
+ // group at (100,50) scaled 2×, mover at (10,20) inside it → (120, 90), scale 2.
+ expect(reader.pose(mover)).toEqual({ a: 2, b: 0, c: 0, d: 2, e: 120, f: 90 });
+ });
+
+ it("re-reads a dynamic node's transform after reset, but caches a static one", () => {
+ const { scene, group, mover, transforms } = tree();
+ const reader = createPoseReader(scene, new Set([mover]));
+ reader.pose(mover);
+ transforms.set(mover, "matrix(1, 0, 0, 1, 5, 0)");
+ transforms.set(group, "matrix(1, 0, 0, 1, 0, 0)");
+ reader.reset();
+ // The mover's new 5 px shift lands (in the still-cached 2× group), the group's change does not.
+ expect(reader.pose(mover)).toMatchObject({ a: 2, e: 130, f: 90 });
+ });
+
+ it("puts them into a pose, before the transform property and about its origin", () => {
+ const { scene, group, mover } = tree();
+ extra.set(group, { scale: "2" });
+ extra.set(mover, { translate: "-50% -50%" });
+ Object.defineProperty(mover, "offsetWidth", { value: 20, configurable: true });
+ Object.defineProperty(mover, "offsetHeight", { value: 40, configurable: true });
+ const reader = createPoseReader(scene, new Set());
+ // group: layout (100,50), scale 2 from the origin, then its transform scale 2 → 4×.
+ // mover: layout (10,20) in the group, shifted (-10,-20): at the group's origin.
+ expect(reader.pose(mover)).toEqual({ a: 4, b: 0, c: 0, d: 4, e: 100, f: 50 });
+ });
+
+ it.each([
+ ["translate", "120px 0px", "matrix(1, 0, 0, 1, 120, 0)"],
+ ["translate", "180px 0px", "matrix(1, 0, 0, 1, 180, 0)"],
+ ["rotate", "90deg", "matrix(0, 1, -1, 0, 0, 0)"],
+ ["scale", "2 3", "matrix(2, 0, 0, 3, 0, 0)"],
+ ])("poses `%s: %s` like the equivalent matrix transform", (prop, value, matrix) => {
+ const { scene, mover, transforms } = tree();
+ const reader = createPoseReader(scene, new Set([mover]));
+ transforms.set(mover, matrix);
+ const control = reader.pose(mover);
+ transforms.set(mover, "none");
+ extra.set(mover, { [prop]: value });
+ reader.reset();
+ const pose = reader.pose(mover);
+ for (const k of ["a", "b", "c", "d", "e", "f"] as const)
+ expect(pose[k]).toBeCloseTo(control[k], 9);
+ });
+
+ it("measures from the offsetParent's border and subtracts scroll", () => {
+ const { scene, group, mover } = tree();
+ Object.defineProperty(group, "clientLeft", { value: 5, configurable: true });
+ Object.defineProperty(group, "clientTop", { value: 3, configurable: true });
+ Object.defineProperty(group, "scrollTop", { value: 7, configurable: true });
+ const reader = createPoseReader(scene, new Set());
+ // group's 2× transform scales mover's in-group offset (10 + 5, 20 + 3 - 7) → (30, 32).
+ expect(reader.pose(mover)).toMatchObject({ e: 130, f: 82 });
+ });
+});
+
+describe("individual transform properties", () => {
+ it("centres a box with translate percentages of its own size", () => {
+ expect(individualTransform({ translate: "-50% -50%" }, 100, 60)).toEqual({
+ ...IDENTITY,
+ e: -50,
+ f: -30,
+ });
+ expect(individualTransform({ translate: "calc(50% + 10px) 5px" }, 100, 60)).toMatchObject({
+ e: 60,
+ f: 5,
+ });
+ });
+
+ it("applies translate, then rotate, then scale", () => {
+ const m = individualTransform({ translate: "10px 0px", rotate: "90deg", scale: "2" }, 10, 10);
+ // A point (1, 0): scaled to (2, 0), rotated to (0, 2), shifted to (10, 2).
+ expect(m && [m.a + m.e, m.b + m.f]).toEqual([expect.closeTo(10, 9), expect.closeTo(2, 9)]);
+ });
+
+ it("is null when none is set", () => {
+ expect(individualTransform({ translate: "none", rotate: "none", scale: "none" }, 1, 1)).toBe(
+ null,
+ );
+ expect(individualTransform({}, 1, 1)).toBe(null);
+ });
+
+ it("flags a z translate or a rotate axis as 3D", () => {
+ expect(individualIs3d({ translate: "10px 20px 5px" })).toBe(true);
+ expect(individualIs3d({ rotate: "x 30deg" })).toBe(true);
+ expect(individualIs3d({ rotate: "1 1 0 57.2958deg" })).toBe(true);
+ expect(individualIs3d({ translate: "10px 20px", rotate: "30deg", scale: "2" })).toBe(false);
+ });
+});
diff --git a/packages/core/src/runtime/captureMotionBlur/pose.ts b/packages/core/src/runtime/captureMotionBlur/pose.ts
new file mode 100644
index 0000000000..8495676791
--- /dev/null
+++ b/packages/core/src/runtime/captureMotionBlur/pose.ts
@@ -0,0 +1,337 @@
+/**
+ * Pose math for capture motion blur: where an element's box sits in composition space at a
+ * given timeline time, and the in-between poses the shutter sweeps through.
+ *
+ * A pose is the full 2D affine from the element's own border box to the composition root:
+ * layout offsets plus every CSS transform up the ancestor chain. Reading it costs style and
+ * layout reads only; no paint.
+ */
+
+export interface Affine {
+ a: number;
+ b: number;
+ c: number;
+ d: number;
+ e: number;
+ f: number;
+}
+
+export const IDENTITY: Affine = { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 };
+
+export function multiply(m: Affine, n: Affine): Affine {
+ return {
+ a: m.a * n.a + m.c * n.b,
+ b: m.b * n.a + m.d * n.b,
+ c: m.a * n.c + m.c * n.d,
+ d: m.b * n.c + m.d * n.d,
+ e: m.a * n.e + m.c * n.f + m.e,
+ f: m.b * n.e + m.d * n.f + m.f,
+ };
+}
+
+export function translate(m: Affine, x: number, y: number): Affine {
+ return multiply(m, { a: 1, b: 0, c: 0, d: 1, e: x, f: y });
+}
+
+export function invert(m: Affine): Affine | null {
+ const det = m.a * m.d - m.b * m.c;
+ if (!Number.isFinite(det) || Math.abs(det) < 1e-12) return null;
+ return {
+ a: m.d / det,
+ b: -m.b / det,
+ c: -m.c / det,
+ d: m.a / det,
+ e: (m.c * m.f - m.d * m.e) / det,
+ f: (m.b * m.e - m.a * m.f) / det,
+ };
+}
+
+/** The four corners of a w×h box under `m`. */
+export function corners(m: Affine, w: number, h: number): Array<[number, number]> {
+ return [
+ [m.e, m.f],
+ [m.a * w + m.e, m.b * w + m.f],
+ [m.c * h + m.e, m.d * h + m.f],
+ [m.a * w + m.c * h + m.e, m.b * w + m.d * h + m.f],
+ ];
+}
+
+/**
+ * Interpolate two poses by translation, rotation, scale and shear rather than matrix entries:
+ * entry-wise lerp of a rotating box shrinks it mid-segment (halfway through a 90° turn it is
+ * scaled by cos 45°). Rotation takes the short way round.
+ */
+export function interpolate(p: Affine, q: Affine, f: number): Affine {
+ const dp = decompose(p);
+ const dq = decompose(q);
+ let turn = dq.angle - dp.angle;
+ if (turn > Math.PI) turn -= 2 * Math.PI;
+ else if (turn < -Math.PI) turn += 2 * Math.PI;
+ const lerp = (x: number, y: number): number => x + (y - x) * f;
+ return compose({
+ angle: dp.angle + turn * f,
+ sx: lerp(dp.sx, dq.sx),
+ sy: lerp(dp.sy, dq.sy),
+ shear: lerp(dp.shear, dq.shear),
+ e: lerp(p.e, q.e),
+ f: lerp(p.f, q.f),
+ });
+}
+
+interface Decomposed {
+ angle: number;
+ sx: number;
+ sy: number;
+ shear: number;
+ e: number;
+ f: number;
+}
+
+// QR split of the linear part: first column = sx·(cos, sin), second = shear·(cos, sin) + sy·(−sin, cos).
+function decompose(m: Affine): Decomposed {
+ const sx = Math.hypot(m.a, m.b);
+ if (sx < 1e-12) return { angle: 0, sx: 0, sy: m.d, shear: m.c, e: m.e, f: m.f };
+ return {
+ angle: Math.atan2(m.b, m.a),
+ sx,
+ sy: (m.a * m.d - m.b * m.c) / sx,
+ shear: (m.a * m.c + m.b * m.d) / sx,
+ e: m.e,
+ f: m.f,
+ };
+}
+
+function compose(d: Decomposed): Affine {
+ const cos = Math.cos(d.angle);
+ const sin = Math.sin(d.angle);
+ return {
+ a: d.sx * cos,
+ b: d.sx * sin,
+ c: d.shear * cos - d.sy * sin,
+ d: d.shear * sin + d.sy * cos,
+ e: d.e,
+ f: d.f,
+ };
+}
+
+/**
+ * `samples` taps spread evenly across the shutter, interpolated from `reads` poses read at
+ * evenly spaced times from the start to the end of the shutter (endpoints included).
+ */
+export function tapsFromReads(reads: Affine[], samples: number): Affine[] {
+ if (reads.length === 0) return [];
+ if (reads.length === 1) return Array.from({ length: samples }, () => reads[0] as Affine);
+ const last = reads.length - 1;
+ return Array.from({ length: samples }, (_, k) => {
+ const x = ((k + 0.5) / samples) * last;
+ const j = Math.min(last - 1, Math.floor(x));
+ return interpolate(reads[j] as Affine, reads[j + 1] as Affine, x - j);
+ });
+}
+
+export interface Sweep {
+ /** Path length of the box's fastest corner across the taps, in pixels. */
+ travel: number;
+ /** Pixel rect covering every tap, clipped to the frame; null when off-frame. */
+ rect: { x: number; y: number; w: number; h: number } | null;
+}
+
+export function sweep(taps: Affine[], w: number, h: number, frameW: number, frameH: number): Sweep {
+ let x0 = Infinity;
+ let y0 = Infinity;
+ let x1 = -Infinity;
+ let y1 = -Infinity;
+ let travel = 0;
+ let prev: Array<[number, number]> | null = null;
+ for (const m of taps) {
+ const cs = corners(m, w, h);
+ for (const [x, y] of cs) {
+ x0 = Math.min(x0, x);
+ y0 = Math.min(y0, y);
+ x1 = Math.max(x1, x);
+ y1 = Math.max(y1, y);
+ }
+ if (prev) {
+ let step = 0;
+ for (let j = 0; j < 4; j++) {
+ const [px, py] = prev[j] as [number, number];
+ const [cx, cy] = cs[j] as [number, number];
+ step = Math.max(step, Math.hypot(cx - px, cy - py));
+ }
+ travel += step;
+ }
+ prev = cs;
+ }
+ // 2 px of slack for bilinear filtering at the edges.
+ const rx0 = Math.max(0, Math.floor(x0) - 2);
+ const ry0 = Math.max(0, Math.floor(y0) - 2);
+ const rx1 = Math.min(frameW, Math.ceil(x1) + 2);
+ const ry1 = Math.min(frameH, Math.ceil(y1) + 2);
+ const rect = rx1 > rx0 && ry1 > ry0 ? { x: rx0, y: ry0, w: rx1 - rx0, h: ry1 - ry0 } : null;
+ return { travel, rect };
+}
+
+/** About one tap per `spacing` px of travel (no visible steps), at least 2, at most all. */
+export function thinTaps(taps: Affine[], travel: number, spacing = 1.5): Affine[] {
+ const n = Math.min(taps.length, Math.max(2, Math.ceil(travel / spacing)));
+ if (n >= taps.length) return taps;
+ return Array.from({ length: n }, (_, i) => {
+ const k = Math.round(((i + 0.5) / n) * taps.length - 0.5);
+ return taps[k] as Affine;
+ });
+}
+
+/**
+ * Reads poses relative to `scene`, memoised for one timeline time: movers share ancestors, so
+ * each node's matrix is computed once from its parent's. Call `reset()` after every seek.
+ *
+ * `dynamic` names the nodes whose transform can change over time; everyone else's own
+ * transform is read once and cached for the reader's lifetime.
+ */
+export function createPoseReader(scene: HTMLElement, dynamic: ReadonlySet<Element>) {
+ const staticOwn = new Map<Element, Affine | null>();
+ let positions = new Map<Element, [number, number]>();
+ let matrices = new Map<Element, Affine>();
+
+ const ownTransform = (n: HTMLElement): Affine | null => {
+ const cached = !dynamic.has(n) && staticOwn.has(n);
+ if (cached) return staticOwn.get(n) ?? null;
+ const cs = getComputedStyle(n);
+ const t = multiplyAll([
+ individualTransform(cs, n.offsetWidth, n.offsetHeight),
+ parseTransform(cs.transform),
+ ]);
+ let own: Affine | null = null;
+ if (t) {
+ const [ox = 0, oy = 0] = cs.transformOrigin.split(" ").map((v) => parseFloat(v) || 0);
+ own = translate(multiply(translate(IDENTITY, ox, oy), t), -ox, -oy);
+ }
+ if (!dynamic.has(n)) staticOwn.set(n, own);
+ return own;
+ };
+
+ // Untransformed border-box position relative to the scene, along the offsetParent chain.
+ // offsetLeft/Top measure from the offsetParent's padding edge, so its border is added back.
+ const layoutPos = (n: Element | null): [number, number] => {
+ if (!n || n === scene || !isOffsetElement(n)) return [0, 0];
+ let p = positions.get(n);
+ if (!p) {
+ const op = n.offsetParent;
+ const [px, py] = layoutPos(op);
+ p = [px + n.offsetLeft + (op?.clientLeft ?? 0), py + n.offsetTop + (op?.clientTop ?? 0)];
+ positions.set(n, p);
+ }
+ return p;
+ };
+
+ const matrixOf = (n: Element | null): Affine => {
+ if (!n || n === scene || !isOffsetElement(n)) return IDENTITY;
+ let m = matrices.get(n);
+ if (!m) {
+ const parent = n.parentElement;
+ const [x, y] = layoutPos(n);
+ const [px, py] = layoutPos(parent);
+ const [sx, sy] = scrollOf(parent);
+ m = translate(matrixOf(parent), x - px - sx, y - py - sy);
+ const own = ownTransform(n);
+ if (own) m = multiply(m, own);
+ matrices.set(n, m);
+ }
+ return m;
+ };
+
+ return {
+ pose: matrixOf as (el: HTMLElement) => Affine,
+ reset(): void {
+ positions = new Map();
+ matrices = new Map();
+ },
+ };
+}
+
+function scrollOf(el: Element | null): [number, number] {
+ return el ? [el.scrollLeft, el.scrollTop] : [0, 0];
+}
+
+function multiplyAll(ms: Array<Affine | null>): Affine | null {
+ let out: Affine | null = null;
+ for (const m of ms) if (m) out = out ? multiply(out, m) : m;
+ return out;
+}
+
+type IndividualProps = Pick<CSSStyleDeclaration, "translate" | "rotate" | "scale">;
+
+// Splits at spaces outside parentheses (`calc(50% + 10px) 5%` is two tokens).
+const tokens = (value: string): string[] => value.trim().split(/\s+(?![^(]*\))/);
+
+// A computed length or percentage (`calc(50% + 10px)` included) in px against `basis`.
+function lengthPx(token: string, basis: number): number {
+ let sum = 0;
+ for (const m of token.matchAll(/([+-]?\s*[\d.]+(?:e[+-]?\d+)?)(px|%)?/g)) {
+ const v = Number(m[1]?.replace(/\s/g, ""));
+ sum += m[2] === "%" ? (v / 100) * basis : v;
+ }
+ return sum;
+}
+
+const ANGLE_UNIT: Record<string, number> = {
+ deg: Math.PI / 180,
+ rad: 1,
+ grad: Math.PI / 200,
+ turn: 2 * Math.PI,
+};
+
+/**
+ * The `translate`, `rotate` and `scale` properties as one 2D affine, in the CSS order (translate,
+ * then rotate, then scale; `transform` comes after). Null when none is set.
+ */
+export function individualTransform(
+ cs: Partial<IndividualProps>,
+ width: number,
+ height: number,
+): Affine | null {
+ const t = cs.translate ?? "none";
+ const r = cs.rotate ?? "none";
+ const s = cs.scale ?? "none";
+ const parts: Array<Affine | null> = [];
+ if (t !== "none") {
+ const [x = "0", y = "0"] = tokens(t);
+ parts.push({ ...IDENTITY, e: lengthPx(x, width), f: lengthPx(y, height) });
+ }
+ const turn = /^(-?[\d.]+)(deg|rad|grad|turn)$/.exec(r);
+ if (turn) {
+ const a = Number(turn[1]) * (ANGLE_UNIT[turn[2] ?? "deg"] ?? 0);
+ parts.push({ a: Math.cos(a), b: Math.sin(a), c: -Math.sin(a), d: Math.cos(a), e: 0, f: 0 });
+ }
+ if (s !== "none") {
+ const [sx = 1, sy = sx] = tokens(s).map(Number);
+ parts.push({ ...IDENTITY, a: sx, d: sy });
+ }
+ return multiplyAll(parts);
+}
+
+/** Whether `translate` has a z part or `rotate` an axis: a 3D move the 2D pose cannot track. */
+export function individualIs3d(cs: Partial<IndividualProps>): boolean {
+ const t = cs.translate ?? "none";
+ const r = cs.rotate ?? "none";
+ return (t !== "none" && tokens(t).length > 2) || !/^(none|-?[\d.]+[a-z]+)$/.test(r);
+}
+
+/**
+ * A computed `transform` (always `none`, `matrix(...)` or `matrix3d(...)` once resolved) as its
+ * 2D affine part; null for `none` or anything unparsable.
+ */
+export function parseTransform(value: string): Affine | null {
+ const match = /^matrix(3d)?\(([^)]*)\)$/.exec(value.trim());
+ if (!match) return null;
+ const v = (match[2] ?? "").split(",").map(Number);
+ if (v.some((x) => !Number.isFinite(x))) return null;
+ const pick = match[1] ? [0, 1, 4, 5, 12, 13] : [0, 1, 2, 3, 4, 5];
+ if (v.length !== (match[1] ? 16 : 6)) return null;
+ const [a = 1, b = 0, c = 0, d = 1, e = 0, f = 0] = pick.map((i) => v[i] ?? 0);
+ return { a, b, c, d, e, f };
+}
+
+function isOffsetElement(n: Element): n is HTMLElement {
+ return typeof (n as HTMLElement).offsetLeft === "number";
+}