Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(core): playback housekeeping runs on a timer, and rebinds stop re-sampling silent media (#4595)
Miguel Ángel4ed269ad1e3d
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.tsindex 3f01cdf406..2571701f00 100644--- a/packages/core/src/runtime/init.ts+++ b/packages/core/src/runtime/init.ts@@ -103,6 +103,7 @@ import { swallow } from "./diagnostics";import {CHANGE_DRIVEN_SERVICE_MIN_INTERVAL_MS,MEDIA_BIND_INTERVAL_FRAMES,+ PLAYING_POLL_INTERVAL_MS,TIMELINE_POST_INTERVAL_FRAMES,shouldAttemptPeriodicTimelineBind,} from "./timelineRebindPolicy";@@ -1217,7 +1218,8 @@ export function initSandboxRuntimeModular(): void {childList: true,subtree: true,attributes: true,- attributeFilter: DURATION_FLOOR_INPUT_ATTRIBUTES,+ // Sizing inputs too: postTimeline applies them, and it runs when this revision moves.+ attributeFilter: [...DURATION_FLOOR_INPUT_ATTRIBUTES, "data-width", "data-height"],});// Media duration changes are published as events, not DOM mutations, and// they do not bubble — so listen in the capture phase, which reaches every@@ -3007,6 +3009,7 @@ export function initSandboxRuntimeModular(): void {let assetsReadyStarted = false;let assetsSettled = false;let liveRootDurationOverrideSeconds = 0;+ let lastPostedManifest = "";const computeClipTreeSignature = (): string => {let sig = "";for (const el of document.querySelectorAll("[data-start]")) {@@ -3032,6 +3035,7 @@ export function initSandboxRuntimeModular(): void {canonicalFps: state.canonicalFps,});window.__clipManifest = payload;+ lastPostedManifest = JSON.stringify(payload);const currentSignature = computeClipTreeSignature();if (clipTreeSignature !== currentSignature) {@@ -3065,6 +3069,14 @@ export function initSandboxRuntimeModular(): void {scheduleRootStageLayoutDiagnostics();};+ // What no observer sees (a label, a track or an inline z-index edited on a playing clip) still reaches Studio.+ const postTimelineIfManifestChanged = () => {+ const manifest = JSON.stringify(+ collectRuntimeTimelinePayload({ canonicalFps: state.canonicalFps }),+ );+ if (manifest !== lastPostedManifest) postTimeline();+ };+const finitePositiveDuration = (value: number | null | undefined): number =>typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;@@ -3829,6 +3841,9 @@ export function initSandboxRuntimeModular(): void {/** A composition change has been seen and not yet carried to consumers. */let compositionChangePending = false;let lastChangeDrivenServiceAtMs = Number.NEGATIVE_INFINITY;+ let lastPlayingPollAtMs = Number.NEGATIVE_INFINITY;+ let playingPollWitness = "";+ let lastTickPlaying = false;const seekRuntimeTimeline = (timeline: RuntimeTimelineLike,@@ -4195,30 +4210,32 @@ export function initSandboxRuntimeModular(): void {try {transportTickCount += 1;- // The three periodic jobs below fire on a frame counter, which is only a- // proxy for "the document may have changed since last time". The counter- // stops advancing while the loop is parked, so on THAT path the question- // is asked directly: the composition timing revision moves on exactly the- // inputs these three derive from.- //- // Only on that path. While the clock is playing the counter advances at- // frame rate and is already a good cadence, and raising it is a real- // regression: postTimeline walks the whole document, so a composition- // that mutates the DOM every frame would turn one post per 20 frames into- // one per frame. And the change-driven path is rate-limited to the same- // posts-per-second the counter yields at 60 Hz, so no consumer ever sees- // a faster cadence than the frame counter alone produced.+ // The jobs below run when the composition revision moves (rate-limited). What no+ // observer sees is polled: on the frame counter paused, on a timer playing.const timingRevision = readCompositionTimingRevision();if (timingRevision !== lastSeenTimingRevision) {lastSeenTimingRevision = timingRevision;compositionChangePending = true;}const nowMs = Date.now();+ const playing = clock.isPlaying();+ const playingPollDue = playing && nowMs - lastPlayingPollAtMs >= PLAYING_POLL_INTERVAL_MS;+ const stoppedPlaying = lastTickPlaying && !playing;+ lastTickPlaying = playing;+ if (playingPollDue) {+ lastPlayingPollAtMs = nowMs;+ const witness = readParkedPollWitness();+ if (witness !== playingPollWitness) {+ playingPollWitness = witness;+ compositionChangePending = true;+ }+ }const changeDrivenService =compositionChangePending &&- !clock.isPlaying() &&nowMs - lastChangeDrivenServiceAtMs >= CHANGE_DRIVEN_SERVICE_MIN_INTERVAL_MS;if (changeDrivenService) lastChangeDrivenServiceAtMs = nowMs;+ const pausedCounterDue = (interval: number) =>+ !playing && transportTickCount % interval === 0;// Slower operations: timeline binding (~every 60 frames / ~1s at 60fps).// `compositionChanged` goes IN to the policy, never around it: the policy@@ -4227,10 +4244,11 @@ export function initSandboxRuntimeModular(): void {if (shouldAttemptPeriodicTimelineBind({tick: transportTickCount,- isPlaying: clock.isPlaying(),+ isPlaying: playing,hasCapturedTimeline: state.capturedTimeline != null,currentTimeSeconds: clock.now(),compositionChanged: changeDrivenService,+ playingPollDue,})) {const prevTimeline = state.capturedTimeline;@@ -4246,7 +4264,7 @@ export function initSandboxRuntimeModular(): void {postTimeline();}}- if (changeDrivenService || transportTickCount % TIMELINE_POST_INTERVAL_FRAMES === 0) {+ if (changeDrivenService || pausedCounterDue(TIMELINE_POST_INTERVAL_FRAMES)) {// The manifest is what carries a composition change to its consumers,// so posting it is what discharges the pending flag — whichever path// got here. Clearing it when the change is merely SEEN would drop it@@ -4259,8 +4277,10 @@ export function initSandboxRuntimeModular(): void {// change happens to wake the loop again.postTimeline();compositionChangePending = false;+ } else if (playingPollDue || stoppedPlaying) {+ postTimelineIfManifestChanged();}- if (changeDrivenService || transportTickCount % MEDIA_BIND_INTERVAL_FRAMES === 0) {+ if (changeDrivenService || pausedCounterDue(MEDIA_BIND_INTERVAL_FRAMES)) {bindMediaMetadataListeners();}diff --git a/packages/core/src/runtime/mediaVolumeEnvelope.test.ts b/packages/core/src/runtime/mediaVolumeEnvelope.test.tsindex da3d9650a2..53093e4c52 100644--- a/packages/core/src/runtime/mediaVolumeEnvelope.test.ts+++ b/packages/core/src/runtime/mediaVolumeEnvelope.test.ts@@ -1,5 +1,6 @@/** @vitest-environment jsdom */-import { describe, expect, it } from "vitest";+import gsap from "gsap";+import { describe, expect, it, vi } from "vitest";import {interpolateVolumeGain,probeAndCacheElementVolume,@@ -140,6 +141,120 @@ describe("probeAndCacheElementVolume", () => {expect(cache.has(audio)).toBe(false);});+ it("does not seek to probe an element whose volume no tween names", () => {+ const audio = document.createElement("audio");+ audio.dataset.start = "0";+ audio.dataset.duration = "25";+ document.body.append(audio);+ const title = document.createElement("h1");++ let seekCount = 0;+ const timeline = {+ totalTime(next?: number) {+ if (next !== undefined) seekCount += 1;+ return 0;+ },+ getChildren: () => [+ { targets: () => [title], vars: { volume: 0, opacity: 1 } },+ { targets: () => [audio], vars: { playbackRate: 2 } },+ ],+ };+ const cache = new WeakMap<HTMLMediaElement, { time: number; volume: number }[]>();++ // What a rebind does for every bound media element: 25 s at 60 samples/s on main.+ probeAndCacheElementVolume(audio, timeline, 25, cache);++ expect(seekCount).toBe(0);+ expect(cache.has(audio)).toBe(false);+ });++ it("samples a gain tweened through a plain-object proxy that writes the element's volume", () => {+ const audio = document.createElement("audio");+ audio.dataset.start = "0";+ audio.dataset.duration = "1";+ audio.dataset.volume = "0";+ document.body.append(audio);+ const proxy = {+ get gain() {+ return audio.volume;+ },+ set gain(value: number) {+ audio.volume = value;+ },+ };+ const timeline = gsap.timeline({ paused: true });+ timeline.fromTo(proxy, { gain: 0 }, { gain: 0.75, duration: 1, ease: "none" }, 0);+ const cache = new WeakMap<HTMLMediaElement, { time: number; volume: number }[]>();++ probeAndCacheElementVolume(audio, timeline, 1, cache);++ const envelope = cache.get(audio);+ expect(envelope).toBeDefined();+ expect(interpolateVolumeGain(envelope!, 0.5)).toBeCloseTo(0.375, 3);+ });++ it.each([+ ["a {} spacer", (tl: gsap.core.Timeline) => tl.to({}, { duration: 25 })],+ ["a tl.call", (tl: gsap.core.Timeline) => tl.call(() => {}, [], 3)],+ ["a plain counter", (tl: gsap.core.Timeline) => tl.to({ n: 0 }, { n: 100, duration: 5 })],+ ])("does not seek for %s", (_name, add) => {+ const audio = document.createElement("audio");+ audio.dataset.start = "0";+ audio.dataset.duration = "25";+ document.body.append(audio);+ const timeline = gsap.timeline({ paused: true });+ add(timeline);+ const seek = vi.spyOn(timeline, "totalTime");++ probeAndCacheElementVolume(audio, timeline, 25, new WeakMap());++ expect(seek.mock.calls.filter((args) => args.length > 0)).toHaveLength(0);+ });++ it("samples a class instance whose volume setter writes the element", () => {+ const audio = document.createElement("audio");+ audio.dataset.start = "0";+ audio.dataset.duration = "1";+ audio.dataset.volume = "0";+ document.body.append(audio);+ class Fader {+ get volume() {+ return audio.volume;+ }+ set volume(value: number) {+ audio.volume = value;+ }+ }+ const timeline = gsap.timeline({ paused: true });+ timeline.fromTo(new Fader(), { volume: 0 }, { volume: 0.5, duration: 1, ease: "none" }, 0);+ const cache = new WeakMap<HTMLMediaElement, { time: number; volume: number }[]>();++ probeAndCacheElementVolume(audio, timeline, 1, cache);++ expect(interpolateVolumeGain(cache.get(audio) ?? [], 0.5)).toBeCloseTo(0.25, 3);+ });++ it("still samples a keyframed volume tween on the element", () => {+ const audio = document.createElement("audio");+ audio.dataset.volume = "1";+ document.body.append(audio);++ const timeline = {+ totalTime(next?: number) {+ if (next !== undefined) audio.volume = next >= 1 ? 0 : 1;+ return 0;+ },+ getChildren: () => [{ targets: () => [audio], vars: { keyframes: [{ volume: 0 }] } }],+ };+ const cache = new WeakMap<HTMLMediaElement, { time: number; volume: number }[]>();++ probeAndCacheElementVolume(audio, timeline, 2, cache);++ expect(cache.get(audio)).toEqual(+ expect.arrayContaining([expect.objectContaining({ volume: 0 })]),+ );+ });+it("restores the timeline playhead after sampling volume automation", () => {const audio = document.createElement("audio");audio.dataset.volume = "1";diff --git a/packages/core/src/runtime/mediaVolumeEnvelope.ts b/packages/core/src/runtime/mediaVolumeEnvelope.tsindex a0eb9cc1dd..a4d8bbeea2 100644--- a/packages/core/src/runtime/mediaVolumeEnvelope.ts+++ b/packages/core/src/runtime/mediaVolumeEnvelope.ts@@ -202,7 +202,9 @@ function probeKeyframesInWindow(return hasAutomation ? keyframes : null;}-export type RuntimeTimelineRef = Partial<Pick<RuntimeTimelineLike, "totalTime" | "seek">>;+export type RuntimeTimelineRef = Partial<+ Pick<RuntimeTimelineLike, "totalTime" | "seek" | "getChildren">+>;export interface VolumeProbeOptions {/**@@ -216,6 +218,46 @@ export interface VolumeProbeOptions {allowLiveTimelineSeek?: boolean;}+function namesKey(vars: unknown, matches: (key: string) => boolean, depth = 0): boolean {+ if (depth > 3 || vars === null || typeof vars !== "object") return false;+ if (Array.isArray(vars)) return vars.some((item) => namesKey(item, matches, depth + 1));+ if (Object.getPrototypeOf(vars) !== Object.prototype) return false;+ return Object.entries(vars).some(+ ([key, value]) => matches(key) || namesKey(value, matches, depth + 1),+ );+}++const isDomNode = (target: unknown): boolean =>+ typeof (target as { nodeType?: unknown } | null)?.nodeType === "number";++function hasSetter(target: unknown, key: string): boolean {+ for (let o = Object(target); o; o = Object.getPrototypeOf(o)) {+ const descriptor = Object.getOwnPropertyDescriptor(o, key);+ if (descriptor) return typeof descriptor.set === "function";+ }+ return false;+}++/**+ * Whether seeking `timeline` can move `el.volume`: a tween on the element that names `volume`, or a tween+ * on a non-DOM object whose tweened property is a setter (a gain proxy). Spacers, calls and counters skip.+ */+function timelineCanMoveVolume(timeline: RuntimeTimelineRef, el: HTMLMediaElement): boolean {+ if (typeof timeline.getChildren !== "function") return true;+ try {+ return timeline.getChildren(true, true, false).some((tween) => {+ const targets: unknown[] = tween.targets?.() ?? [];+ return targets.some((target) =>+ isDomNode(target)+ ? target === el && namesKey(tween.vars, (key) => key === "volume")+ : namesKey(tween.vars, (key) => hasSetter(target, key)),+ );+ });+ } catch {+ return true;+ }+}+/*** Probe a media element and, if volume automation is detected, store a* NORMALISED envelope in `cache`. Safe to call with a null timeline — returns@@ -238,6 +280,8 @@ export function probeAndCacheElementVolume(if (!timeline) return;if (!isMediaElement(mediaEl)) return;if (compositionDuration <= 0) return;+ // Sampling is ~60 whole-timeline seeks per second of clip, paid again on every rebind.+ if (!timelineCanMoveVolume(timeline, mediaEl)) return;const seekFn = (t: number) => {try {diff --git a/packages/core/src/runtime/timelineRebindPolicy.test.ts b/packages/core/src/runtime/timelineRebindPolicy.test.tsindex 2b54b9a63c..ea30fcf0fc 100644--- a/packages/core/src/runtime/timelineRebindPolicy.test.ts+++ b/packages/core/src/runtime/timelineRebindPolicy.test.ts@@ -32,6 +32,7 @@ describe("shouldAttemptPeriodicTimelineBind", () => {isPlaying: true,hasCapturedTimeline: true,currentTimeSeconds: PLAY_REBIND_HOLD_SECONDS - 0.001,+ playingPollDue: true,}),).toBe(false);expect(@@ -40,6 +41,7 @@ describe("shouldAttemptPeriodicTimelineBind", () => {isPlaying: true,hasCapturedTimeline: true,currentTimeSeconds: PLAY_REBIND_HOLD_SECONDS,+ playingPollDue: true,}),).toBe(true);});@@ -51,7 +53,21 @@ describe("shouldAttemptPeriodicTimelineBind", () => {isPlaying: true,hasCapturedTimeline: false,currentTimeSeconds: 0,+ playingPollDue: true,}),).toBe(true);});++ it("checks on the poll timer, not the frame counter, while playing", () => {+ const playing = {+ tick: TIMELINE_REBIND_INTERVAL_FRAMES,+ isPlaying: true,+ hasCapturedTimeline: true,+ currentTimeSeconds: PLAY_REBIND_HOLD_SECONDS,+ };+ expect(shouldAttemptPeriodicTimelineBind(playing)).toBe(false);+ expect(shouldAttemptPeriodicTimelineBind({ ...playing, tick: 7, playingPollDue: true })).toBe(+ true,+ );+ });});diff --git a/packages/core/src/runtime/timelineRebindPolicy.ts b/packages/core/src/runtime/timelineRebindPolicy.tsindex 8753f7270e..b4d4ff1417 100644--- a/packages/core/src/runtime/timelineRebindPolicy.ts+++ b/packages/core/src/runtime/timelineRebindPolicy.ts@@ -10,6 +10,7 @@ export const MEDIA_BIND_INTERVAL_FRAMES = 30;* it: `TIMELINE_POST_INTERVAL_FRAMES` frames at a 60 Hz display.*/export const CHANGE_DRIVEN_SERVICE_MIN_INTERVAL_MS = (1000 * TIMELINE_POST_INTERVAL_FRAMES) / 60;+export const PLAYING_POLL_INTERVAL_MS = (1000 * TIMELINE_REBIND_INTERVAL_FRAMES) / 60;export function shouldAttemptPeriodicTimelineBind(input: {tick: number;@@ -24,6 +25,7 @@ export function shouldAttemptPeriodicTimelineBind(input: {* async rebind cannot race the first two seconds of playback.*/compositionChanged?: boolean;+ playingPollDue?: boolean;}): boolean {// The hold is the outer rule and applies to every trigger.if (@@ -34,6 +36,7 @@ export function shouldAttemptPeriodicTimelineBind(input: {return false;}if (input.compositionChanged === true) return true;+ if (input.isPlaying) return input.playingPollDue === true;return (Number.isInteger(input.tick) &&input.tick > 0 &&diff --git a/packages/core/src/runtime/transportPark.test.ts b/packages/core/src/runtime/transportPark.test.tsindex 027dd6a2a1..fa99cd78d4 100644--- a/packages/core/src/runtime/transportPark.test.ts+++ b/packages/core/src/runtime/transportPark.test.ts@@ -123,6 +123,12 @@ describe("parked transport loop", () => {return frames;};+ /** One frame at 120 Hz: the clock and the animation frame advance together. */+ const frame120Hz = () => {+ vi.advanceTimersByTime(8);+ raf.step(8);+ };+/*** Park the transport AND drain init's own one-shot timers, several of which* post state and would otherwise be mistaken for a heartbeat. Leaves exactly@@ -467,7 +473,7 @@ describe("parked transport loop", () => {expect(posted.filter((m) => m["type"] === "state")).toHaveLength(0);});- it("keeps the manifest on its frame cadence while playing, whatever the DOM does", () => {+ it("rate-limits the manifest while playing, whatever the DOM does", () => {mount();initSandboxRuntimeModular();quiesce();@@ -492,6 +498,85 @@ describe("parked transport loop", () => {expect(posts).toBeLessThanOrEqual(2);});+ it("does not re-post the manifest while playing a film that is not changing", () => {+ mount();+ initSandboxRuntimeModular();+ quiesce();+ window.__player!.play();+ // Past the rebind policy's play hold, at 120 Hz.+ for (let i = 0; i < 300; i += 1) frame120Hz();+ const before = posted.filter((m) => m["type"] === "timeline").length;+ for (let i = 0; i < 250; i += 1) frame120Hz();+ expect(posted.filter((m) => m["type"] === "timeline").length - before).toBe(0);+ });++ it("posts a playing clip's new track, label and z-index on the next poll", async () => {+ mount(+ '<div id="clip" data-start="0" data-duration="5" data-track-index="0" data-timeline-label="Before" style="z-index: 1"></div>',+ );+ initSandboxRuntimeModular();+ quiesce();+ window.__player!.play();+ for (let i = 0; i < 300; i += 1) frame120Hz();+ const clip = document.getElementById("clip")!;+ clip.setAttribute("data-track-index", "7");+ clip.setAttribute("data-timeline-label", "After");+ clip.style.zIndex = "9";+ await flushObservers();+ for (let i = 0; i < 130; i += 1) frame120Hz();+ const clips = (posted.filter((m) => m["type"] === "timeline").at(-1)?.["clips"] ?? []) as Array<+ Record<string, unknown>+ >;+ const clip_ = clips.find((c) => c["id"] === "clip");+ expect([clip_?.["track"], clip_?.["timelineLabel"], clip_?.["zIndex"]]).toEqual([+ 7,+ "After",+ 9,+ ]);+ });++ it("posts a label edited while playing and paused before the next poll", async () => {+ mount('<div id="clip" data-start="0" data-duration="5" data-timeline-label="Before"></div>');+ initSandboxRuntimeModular();+ quiesce();+ window.__player!.play();+ for (let i = 0; i < 300; i += 1) frame120Hz();+ document.getElementById("clip")!.setAttribute("data-timeline-label", "After");+ await flushObservers();+ for (let i = 0; i < 12; i += 1) frame120Hz();+ window.__player!.pause();+ for (let i = 0; i < 300; i += 1) frame120Hz();+ const clips = (posted.filter((m) => m["type"] === "timeline").at(-1)?.["clips"] ?? []) as Array<+ Record<string, unknown>+ >;+ expect(clips.find((c) => c["id"] === "clip")?.["timelineLabel"]).toBe("After");+ });++ it("applies a data-width change while playing", async () => {+ mount();+ initSandboxRuntimeModular();+ quiesce();+ window.__player!.play();+ for (let i = 0; i < 300; i += 1) frame120Hz();+ const root = document.getElementById("root")!;+ root.setAttribute("data-width", "640");+ await flushObservers();+ for (let i = 0; i < 60; i += 1) frame120Hz();+ expect(root.style.width).toBe("640px");+ });++ it("applies a data-height change while parked", async () => {+ mount();+ initSandboxRuntimeModular();+ quiesce();+ const root = document.getElementById("root")!;+ root.setAttribute("data-height", "360");+ await flushObservers();+ vi.advanceTimersByTime(PARK_HEARTBEAT_MS);+ settle();+ expect(root.style.height).toBe("360px");+ });+it("does not park when the manifest post throws with a change still pending", async () => {mount();initSandboxRuntimeModular();