Skip to content

Write HTML. Render video. Built for agents.

perf(core): playback housekeeping runs on a timer, and rebinds stop re-sampling silent media (#4595)

Miguel Ángel4ed269ad1e3d
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index 3f01cdf406..2571701f00 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -103,6 +103,7 @@ import { swallow } from "./diagnostics";
import {
CHANGE_DRIVEN_SERVICE_MIN_INTERVAL_MS,
MEDIA_BIND_INTERVAL_FRAMES,
+ PLAYING_POLL_INTERVAL_MS,
TIMELINE_POST_INTERVAL_FRAMES,
shouldAttemptPeriodicTimelineBind,
} from "./timelineRebindPolicy";
@@ -1217,7 +1218,8 @@ export function initSandboxRuntimeModular(): void {
childList: true,
subtree: true,
attributes: true,
- attributeFilter: DURATION_FLOOR_INPUT_ATTRIBUTES,
+ // Sizing inputs too: postTimeline applies them, and it runs when this revision moves.
+ attributeFilter: [...DURATION_FLOOR_INPUT_ATTRIBUTES, "data-width", "data-height"],
});
// Media duration changes are published as events, not DOM mutations, and
// they do not bubble — so listen in the capture phase, which reaches every
@@ -3007,6 +3009,7 @@ export function initSandboxRuntimeModular(): void {
let assetsReadyStarted = false;
let assetsSettled = false;
let liveRootDurationOverrideSeconds = 0;
+ let lastPostedManifest = "";
const computeClipTreeSignature = (): string => {
let sig = "";
for (const el of document.querySelectorAll("[data-start]")) {
@@ -3032,6 +3035,7 @@ export function initSandboxRuntimeModular(): void {
canonicalFps: state.canonicalFps,
});
window.__clipManifest = payload;
+ lastPostedManifest = JSON.stringify(payload);
const currentSignature = computeClipTreeSignature();
if (clipTreeSignature !== currentSignature) {
@@ -3065,6 +3069,14 @@ export function initSandboxRuntimeModular(): void {
scheduleRootStageLayoutDiagnostics();
};
+ // What no observer sees (a label, a track or an inline z-index edited on a playing clip) still reaches Studio.
+ const postTimelineIfManifestChanged = () => {
+ const manifest = JSON.stringify(
+ collectRuntimeTimelinePayload({ canonicalFps: state.canonicalFps }),
+ );
+ if (manifest !== lastPostedManifest) postTimeline();
+ };
+
const finitePositiveDuration = (value: number | null | undefined): number =>
typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
@@ -3829,6 +3841,9 @@ export function initSandboxRuntimeModular(): void {
/** A composition change has been seen and not yet carried to consumers. */
let compositionChangePending = false;
let lastChangeDrivenServiceAtMs = Number.NEGATIVE_INFINITY;
+ let lastPlayingPollAtMs = Number.NEGATIVE_INFINITY;
+ let playingPollWitness = "";
+ let lastTickPlaying = false;
const seekRuntimeTimeline = (
timeline: RuntimeTimelineLike,
@@ -4195,30 +4210,32 @@ export function initSandboxRuntimeModular(): void {
try {
transportTickCount += 1;
- // The three periodic jobs below fire on a frame counter, which is only a
- // proxy for "the document may have changed since last time". The counter
- // stops advancing while the loop is parked, so on THAT path the question
- // is asked directly: the composition timing revision moves on exactly the
- // inputs these three derive from.
- //
- // Only on that path. While the clock is playing the counter advances at
- // frame rate and is already a good cadence, and raising it is a real
- // regression: postTimeline walks the whole document, so a composition
- // that mutates the DOM every frame would turn one post per 20 frames into
- // one per frame. And the change-driven path is rate-limited to the same
- // posts-per-second the counter yields at 60 Hz, so no consumer ever sees
- // a faster cadence than the frame counter alone produced.
+ // The jobs below run when the composition revision moves (rate-limited). What no
+ // observer sees is polled: on the frame counter paused, on a timer playing.
const timingRevision = readCompositionTimingRevision();
if (timingRevision !== lastSeenTimingRevision) {
lastSeenTimingRevision = timingRevision;
compositionChangePending = true;
}
const nowMs = Date.now();
+ const playing = clock.isPlaying();
+ const playingPollDue = playing && nowMs - lastPlayingPollAtMs >= PLAYING_POLL_INTERVAL_MS;
+ const stoppedPlaying = lastTickPlaying && !playing;
+ lastTickPlaying = playing;
+ if (playingPollDue) {
+ lastPlayingPollAtMs = nowMs;
+ const witness = readParkedPollWitness();
+ if (witness !== playingPollWitness) {
+ playingPollWitness = witness;
+ compositionChangePending = true;
+ }
+ }
const changeDrivenService =
compositionChangePending &&
- !clock.isPlaying() &&
nowMs - lastChangeDrivenServiceAtMs >= CHANGE_DRIVEN_SERVICE_MIN_INTERVAL_MS;
if (changeDrivenService) lastChangeDrivenServiceAtMs = nowMs;
+ const pausedCounterDue = (interval: number) =>
+ !playing && transportTickCount % interval === 0;
// Slower operations: timeline binding (~every 60 frames / ~1s at 60fps).
// `compositionChanged` goes IN to the policy, never around it: the policy
@@ -4227,10 +4244,11 @@ export function initSandboxRuntimeModular(): void {
if (
shouldAttemptPeriodicTimelineBind({
tick: transportTickCount,
- isPlaying: clock.isPlaying(),
+ isPlaying: playing,
hasCapturedTimeline: state.capturedTimeline != null,
currentTimeSeconds: clock.now(),
compositionChanged: changeDrivenService,
+ playingPollDue,
})
) {
const prevTimeline = state.capturedTimeline;
@@ -4246,7 +4264,7 @@ export function initSandboxRuntimeModular(): void {
postTimeline();
}
}
- if (changeDrivenService || transportTickCount % TIMELINE_POST_INTERVAL_FRAMES === 0) {
+ if (changeDrivenService || pausedCounterDue(TIMELINE_POST_INTERVAL_FRAMES)) {
// The manifest is what carries a composition change to its consumers,
// so posting it is what discharges the pending flag — whichever path
// got here. Clearing it when the change is merely SEEN would drop it
@@ -4259,8 +4277,10 @@ export function initSandboxRuntimeModular(): void {
// change happens to wake the loop again.
postTimeline();
compositionChangePending = false;
+ } else if (playingPollDue || stoppedPlaying) {
+ postTimelineIfManifestChanged();
}
- if (changeDrivenService || transportTickCount % MEDIA_BIND_INTERVAL_FRAMES === 0) {
+ if (changeDrivenService || pausedCounterDue(MEDIA_BIND_INTERVAL_FRAMES)) {
bindMediaMetadataListeners();
}
diff --git a/packages/core/src/runtime/mediaVolumeEnvelope.test.ts b/packages/core/src/runtime/mediaVolumeEnvelope.test.ts
index da3d9650a2..53093e4c52 100644
--- a/packages/core/src/runtime/mediaVolumeEnvelope.test.ts
+++ b/packages/core/src/runtime/mediaVolumeEnvelope.test.ts
@@ -1,5 +1,6 @@
/** @vitest-environment jsdom */
-import { describe, expect, it } from "vitest";
+import gsap from "gsap";
+import { describe, expect, it, vi } from "vitest";
import {
interpolateVolumeGain,
probeAndCacheElementVolume,
@@ -140,6 +141,120 @@ describe("probeAndCacheElementVolume", () => {
expect(cache.has(audio)).toBe(false);
});
+ it("does not seek to probe an element whose volume no tween names", () => {
+ const audio = document.createElement("audio");
+ audio.dataset.start = "0";
+ audio.dataset.duration = "25";
+ document.body.append(audio);
+ const title = document.createElement("h1");
+
+ let seekCount = 0;
+ const timeline = {
+ totalTime(next?: number) {
+ if (next !== undefined) seekCount += 1;
+ return 0;
+ },
+ getChildren: () => [
+ { targets: () => [title], vars: { volume: 0, opacity: 1 } },
+ { targets: () => [audio], vars: { playbackRate: 2 } },
+ ],
+ };
+ const cache = new WeakMap<HTMLMediaElement, { time: number; volume: number }[]>();
+
+ // What a rebind does for every bound media element: 25 s at 60 samples/s on main.
+ probeAndCacheElementVolume(audio, timeline, 25, cache);
+
+ expect(seekCount).toBe(0);
+ expect(cache.has(audio)).toBe(false);
+ });
+
+ it("samples a gain tweened through a plain-object proxy that writes the element's volume", () => {
+ const audio = document.createElement("audio");
+ audio.dataset.start = "0";
+ audio.dataset.duration = "1";
+ audio.dataset.volume = "0";
+ document.body.append(audio);
+ const proxy = {
+ get gain() {
+ return audio.volume;
+ },
+ set gain(value: number) {
+ audio.volume = value;
+ },
+ };
+ const timeline = gsap.timeline({ paused: true });
+ timeline.fromTo(proxy, { gain: 0 }, { gain: 0.75, duration: 1, ease: "none" }, 0);
+ const cache = new WeakMap<HTMLMediaElement, { time: number; volume: number }[]>();
+
+ probeAndCacheElementVolume(audio, timeline, 1, cache);
+
+ const envelope = cache.get(audio);
+ expect(envelope).toBeDefined();
+ expect(interpolateVolumeGain(envelope!, 0.5)).toBeCloseTo(0.375, 3);
+ });
+
+ it.each([
+ ["a {} spacer", (tl: gsap.core.Timeline) => tl.to({}, { duration: 25 })],
+ ["a tl.call", (tl: gsap.core.Timeline) => tl.call(() => {}, [], 3)],
+ ["a plain counter", (tl: gsap.core.Timeline) => tl.to({ n: 0 }, { n: 100, duration: 5 })],
+ ])("does not seek for %s", (_name, add) => {
+ const audio = document.createElement("audio");
+ audio.dataset.start = "0";
+ audio.dataset.duration = "25";
+ document.body.append(audio);
+ const timeline = gsap.timeline({ paused: true });
+ add(timeline);
+ const seek = vi.spyOn(timeline, "totalTime");
+
+ probeAndCacheElementVolume(audio, timeline, 25, new WeakMap());
+
+ expect(seek.mock.calls.filter((args) => args.length > 0)).toHaveLength(0);
+ });
+
+ it("samples a class instance whose volume setter writes the element", () => {
+ const audio = document.createElement("audio");
+ audio.dataset.start = "0";
+ audio.dataset.duration = "1";
+ audio.dataset.volume = "0";
+ document.body.append(audio);
+ class Fader {
+ get volume() {
+ return audio.volume;
+ }
+ set volume(value: number) {
+ audio.volume = value;
+ }
+ }
+ const timeline = gsap.timeline({ paused: true });
+ timeline.fromTo(new Fader(), { volume: 0 }, { volume: 0.5, duration: 1, ease: "none" }, 0);
+ const cache = new WeakMap<HTMLMediaElement, { time: number; volume: number }[]>();
+
+ probeAndCacheElementVolume(audio, timeline, 1, cache);
+
+ expect(interpolateVolumeGain(cache.get(audio) ?? [], 0.5)).toBeCloseTo(0.25, 3);
+ });
+
+ it("still samples a keyframed volume tween on the element", () => {
+ const audio = document.createElement("audio");
+ audio.dataset.volume = "1";
+ document.body.append(audio);
+
+ const timeline = {
+ totalTime(next?: number) {
+ if (next !== undefined) audio.volume = next >= 1 ? 0 : 1;
+ return 0;
+ },
+ getChildren: () => [{ targets: () => [audio], vars: { keyframes: [{ volume: 0 }] } }],
+ };
+ const cache = new WeakMap<HTMLMediaElement, { time: number; volume: number }[]>();
+
+ probeAndCacheElementVolume(audio, timeline, 2, cache);
+
+ expect(cache.get(audio)).toEqual(
+ expect.arrayContaining([expect.objectContaining({ volume: 0 })]),
+ );
+ });
+
it("restores the timeline playhead after sampling volume automation", () => {
const audio = document.createElement("audio");
audio.dataset.volume = "1";
diff --git a/packages/core/src/runtime/mediaVolumeEnvelope.ts b/packages/core/src/runtime/mediaVolumeEnvelope.ts
index a0eb9cc1dd..a4d8bbeea2 100644
--- a/packages/core/src/runtime/mediaVolumeEnvelope.ts
+++ b/packages/core/src/runtime/mediaVolumeEnvelope.ts
@@ -202,7 +202,9 @@ function probeKeyframesInWindow(
return hasAutomation ? keyframes : null;
}
-export type RuntimeTimelineRef = Partial<Pick<RuntimeTimelineLike, "totalTime" | "seek">>;
+export type RuntimeTimelineRef = Partial<
+ Pick<RuntimeTimelineLike, "totalTime" | "seek" | "getChildren">
+>;
export interface VolumeProbeOptions {
/**
@@ -216,6 +218,46 @@ export interface VolumeProbeOptions {
allowLiveTimelineSeek?: boolean;
}
+function namesKey(vars: unknown, matches: (key: string) => boolean, depth = 0): boolean {
+ if (depth > 3 || vars === null || typeof vars !== "object") return false;
+ if (Array.isArray(vars)) return vars.some((item) => namesKey(item, matches, depth + 1));
+ if (Object.getPrototypeOf(vars) !== Object.prototype) return false;
+ return Object.entries(vars).some(
+ ([key, value]) => matches(key) || namesKey(value, matches, depth + 1),
+ );
+}
+
+const isDomNode = (target: unknown): boolean =>
+ typeof (target as { nodeType?: unknown } | null)?.nodeType === "number";
+
+function hasSetter(target: unknown, key: string): boolean {
+ for (let o = Object(target); o; o = Object.getPrototypeOf(o)) {
+ const descriptor = Object.getOwnPropertyDescriptor(o, key);
+ if (descriptor) return typeof descriptor.set === "function";
+ }
+ return false;
+}
+
+/**
+ * Whether seeking `timeline` can move `el.volume`: a tween on the element that names `volume`, or a tween
+ * on a non-DOM object whose tweened property is a setter (a gain proxy). Spacers, calls and counters skip.
+ */
+function timelineCanMoveVolume(timeline: RuntimeTimelineRef, el: HTMLMediaElement): boolean {
+ if (typeof timeline.getChildren !== "function") return true;
+ try {
+ return timeline.getChildren(true, true, false).some((tween) => {
+ const targets: unknown[] = tween.targets?.() ?? [];
+ return targets.some((target) =>
+ isDomNode(target)
+ ? target === el && namesKey(tween.vars, (key) => key === "volume")
+ : namesKey(tween.vars, (key) => hasSetter(target, key)),
+ );
+ });
+ } catch {
+ return true;
+ }
+}
+
/**
* Probe a media element and, if volume automation is detected, store a
* NORMALISED envelope in `cache`. Safe to call with a null timeline — returns
@@ -238,6 +280,8 @@ export function probeAndCacheElementVolume(
if (!timeline) return;
if (!isMediaElement(mediaEl)) return;
if (compositionDuration <= 0) return;
+ // Sampling is ~60 whole-timeline seeks per second of clip, paid again on every rebind.
+ if (!timelineCanMoveVolume(timeline, mediaEl)) return;
const seekFn = (t: number) => {
try {
diff --git a/packages/core/src/runtime/timelineRebindPolicy.test.ts b/packages/core/src/runtime/timelineRebindPolicy.test.ts
index 2b54b9a63c..ea30fcf0fc 100644
--- a/packages/core/src/runtime/timelineRebindPolicy.test.ts
+++ b/packages/core/src/runtime/timelineRebindPolicy.test.ts
@@ -32,6 +32,7 @@ describe("shouldAttemptPeriodicTimelineBind", () => {
isPlaying: true,
hasCapturedTimeline: true,
currentTimeSeconds: PLAY_REBIND_HOLD_SECONDS - 0.001,
+ playingPollDue: true,
}),
).toBe(false);
expect(
@@ -40,6 +41,7 @@ describe("shouldAttemptPeriodicTimelineBind", () => {
isPlaying: true,
hasCapturedTimeline: true,
currentTimeSeconds: PLAY_REBIND_HOLD_SECONDS,
+ playingPollDue: true,
}),
).toBe(true);
});
@@ -51,7 +53,21 @@ describe("shouldAttemptPeriodicTimelineBind", () => {
isPlaying: true,
hasCapturedTimeline: false,
currentTimeSeconds: 0,
+ playingPollDue: true,
}),
).toBe(true);
});
+
+ it("checks on the poll timer, not the frame counter, while playing", () => {
+ const playing = {
+ tick: TIMELINE_REBIND_INTERVAL_FRAMES,
+ isPlaying: true,
+ hasCapturedTimeline: true,
+ currentTimeSeconds: PLAY_REBIND_HOLD_SECONDS,
+ };
+ expect(shouldAttemptPeriodicTimelineBind(playing)).toBe(false);
+ expect(shouldAttemptPeriodicTimelineBind({ ...playing, tick: 7, playingPollDue: true })).toBe(
+ true,
+ );
+ });
});
diff --git a/packages/core/src/runtime/timelineRebindPolicy.ts b/packages/core/src/runtime/timelineRebindPolicy.ts
index 8753f7270e..b4d4ff1417 100644
--- a/packages/core/src/runtime/timelineRebindPolicy.ts
+++ b/packages/core/src/runtime/timelineRebindPolicy.ts
@@ -10,6 +10,7 @@ export const MEDIA_BIND_INTERVAL_FRAMES = 30;
* it: `TIMELINE_POST_INTERVAL_FRAMES` frames at a 60 Hz display.
*/
export const CHANGE_DRIVEN_SERVICE_MIN_INTERVAL_MS = (1000 * TIMELINE_POST_INTERVAL_FRAMES) / 60;
+export const PLAYING_POLL_INTERVAL_MS = (1000 * TIMELINE_REBIND_INTERVAL_FRAMES) / 60;
export function shouldAttemptPeriodicTimelineBind(input: {
tick: number;
@@ -24,6 +25,7 @@ export function shouldAttemptPeriodicTimelineBind(input: {
* async rebind cannot race the first two seconds of playback.
*/
compositionChanged?: boolean;
+ playingPollDue?: boolean;
}): boolean {
// The hold is the outer rule and applies to every trigger.
if (
@@ -34,6 +36,7 @@ export function shouldAttemptPeriodicTimelineBind(input: {
return false;
}
if (input.compositionChanged === true) return true;
+ if (input.isPlaying) return input.playingPollDue === true;
return (
Number.isInteger(input.tick) &&
input.tick > 0 &&
diff --git a/packages/core/src/runtime/transportPark.test.ts b/packages/core/src/runtime/transportPark.test.ts
index 027dd6a2a1..fa99cd78d4 100644
--- a/packages/core/src/runtime/transportPark.test.ts
+++ b/packages/core/src/runtime/transportPark.test.ts
@@ -123,6 +123,12 @@ describe("parked transport loop", () => {
return frames;
};
+ /** One frame at 120 Hz: the clock and the animation frame advance together. */
+ const frame120Hz = () => {
+ vi.advanceTimersByTime(8);
+ raf.step(8);
+ };
+
/**
* Park the transport AND drain init's own one-shot timers, several of which
* post state and would otherwise be mistaken for a heartbeat. Leaves exactly
@@ -467,7 +473,7 @@ describe("parked transport loop", () => {
expect(posted.filter((m) => m["type"] === "state")).toHaveLength(0);
});
- it("keeps the manifest on its frame cadence while playing, whatever the DOM does", () => {
+ it("rate-limits the manifest while playing, whatever the DOM does", () => {
mount();
initSandboxRuntimeModular();
quiesce();
@@ -492,6 +498,85 @@ describe("parked transport loop", () => {
expect(posts).toBeLessThanOrEqual(2);
});
+ it("does not re-post the manifest while playing a film that is not changing", () => {
+ mount();
+ initSandboxRuntimeModular();
+ quiesce();
+ window.__player!.play();
+ // Past the rebind policy's play hold, at 120 Hz.
+ for (let i = 0; i < 300; i += 1) frame120Hz();
+ const before = posted.filter((m) => m["type"] === "timeline").length;
+ for (let i = 0; i < 250; i += 1) frame120Hz();
+ expect(posted.filter((m) => m["type"] === "timeline").length - before).toBe(0);
+ });
+
+ it("posts a playing clip's new track, label and z-index on the next poll", async () => {
+ mount(
+ '<div id="clip" data-start="0" data-duration="5" data-track-index="0" data-timeline-label="Before" style="z-index: 1"></div>',
+ );
+ initSandboxRuntimeModular();
+ quiesce();
+ window.__player!.play();
+ for (let i = 0; i < 300; i += 1) frame120Hz();
+ const clip = document.getElementById("clip")!;
+ clip.setAttribute("data-track-index", "7");
+ clip.setAttribute("data-timeline-label", "After");
+ clip.style.zIndex = "9";
+ await flushObservers();
+ for (let i = 0; i < 130; i += 1) frame120Hz();
+ const clips = (posted.filter((m) => m["type"] === "timeline").at(-1)?.["clips"] ?? []) as Array<
+ Record<string, unknown>
+ >;
+ const clip_ = clips.find((c) => c["id"] === "clip");
+ expect([clip_?.["track"], clip_?.["timelineLabel"], clip_?.["zIndex"]]).toEqual([
+ 7,
+ "After",
+ 9,
+ ]);
+ });
+
+ it("posts a label edited while playing and paused before the next poll", async () => {
+ mount('<div id="clip" data-start="0" data-duration="5" data-timeline-label="Before"></div>');
+ initSandboxRuntimeModular();
+ quiesce();
+ window.__player!.play();
+ for (let i = 0; i < 300; i += 1) frame120Hz();
+ document.getElementById("clip")!.setAttribute("data-timeline-label", "After");
+ await flushObservers();
+ for (let i = 0; i < 12; i += 1) frame120Hz();
+ window.__player!.pause();
+ for (let i = 0; i < 300; i += 1) frame120Hz();
+ const clips = (posted.filter((m) => m["type"] === "timeline").at(-1)?.["clips"] ?? []) as Array<
+ Record<string, unknown>
+ >;
+ expect(clips.find((c) => c["id"] === "clip")?.["timelineLabel"]).toBe("After");
+ });
+
+ it("applies a data-width change while playing", async () => {
+ mount();
+ initSandboxRuntimeModular();
+ quiesce();
+ window.__player!.play();
+ for (let i = 0; i < 300; i += 1) frame120Hz();
+ const root = document.getElementById("root")!;
+ root.setAttribute("data-width", "640");
+ await flushObservers();
+ for (let i = 0; i < 60; i += 1) frame120Hz();
+ expect(root.style.width).toBe("640px");
+ });
+
+ it("applies a data-height change while parked", async () => {
+ mount();
+ initSandboxRuntimeModular();
+ quiesce();
+ const root = document.getElementById("root")!;
+ root.setAttribute("data-height", "360");
+ await flushObservers();
+ vi.advanceTimersByTime(PARK_HEARTBEAT_MS);
+ settle();
+ expect(root.style.height).toBe("360px");
+ });
+
it("does not park when the manifest post throws with a change still pending", async () => {
mount();
initSandboxRuntimeModular();