Skip to content

Write HTML. Render video. Built for agents.

perf(core): playback and jumps do only the work on screen (#5754)

Miguel Ángel959dd3e584cc
diff --git a/packages/core/src/runtime/adapters/seek-dispatch.ts b/packages/core/src/runtime/adapters/seek-dispatch.ts
index 8bce055eef..2173e8b110 100644
--- a/packages/core/src/runtime/adapters/seek-dispatch.ts
+++ b/packages/core/src/runtime/adapters/seek-dispatch.ts
@@ -102,6 +102,21 @@ export function isSeekCompletionBarrierActive(): boolean {
return _activeCompletionBarriers > 0;
}
+/** Settles with every pending completion, without consuming a failure the barrier reports. */
+export async function seekCompletionsSettled(): Promise<void> {
+ for (
+ let pending = [..._pendingCompletions];
+ pending.length > 0;
+ pending = [..._pendingCompletions]
+ ) {
+ await Promise.all(pending);
+ }
+}
+
+export function hasPendingSeekCompletion(): boolean {
+ return _pendingCompletions.size > 0;
+}
+
/** `videoFrames: false` skips only the video-frame holds: a geometry read needs every other completion. */
export async function waitForSeekCompletion(options?: { videoFrames?: boolean }): Promise<void> {
const awaited = () =>
diff --git a/packages/core/src/runtime/init.playbackScope.test.ts b/packages/core/src/runtime/init.playbackScope.test.ts
new file mode 100644
index 0000000000..bb6d17bb98
--- /dev/null
+++ b/packages/core/src/runtime/init.playbackScope.test.ts
@@ -0,0 +1,256 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { initSandboxRuntimeModular } from "./init";
+import * as timeline from "./timeline";
+import type { RuntimeTimelineLike } from "./types";
+
+vi.mock("../compositionReadiness", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("../compositionReadiness")>();
+ return {
+ ...actual,
+ settleCompositionReadiness: vi.fn(),
+ settleFirstFrameCompositionReadiness: vi.fn(),
+ };
+});
+vi.mock("./timeline", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("./timeline")>();
+ return { ...actual, isRuntimeElementVisibleAt: vi.fn(actual.isRuntimeElementVisibleAt) };
+});
+
+// Pins what a playing frame touches: clips on screen and clips whose edge it crossed, never the whole film.
+
+type CountingTimeline = RuntimeTimelineLike & { seeks: number; childScans: number };
+
+function countingTimeline(duration: number): CountingTimeline {
+ const state = { time: 0, paused: true };
+ const tl: CountingTimeline = {
+ seeks: 0,
+ childScans: 0,
+ play: () => {
+ state.paused = false;
+ },
+ pause: () => {
+ state.paused = true;
+ },
+ seek: (time?: number) => {
+ if (time !== undefined) state.time = time;
+ return state.time;
+ },
+ totalTime: (time?: number) => {
+ if (time !== undefined) {
+ tl.seeks += 1;
+ state.time = time;
+ }
+ return state.time;
+ },
+ time: () => state.time,
+ duration: () => duration,
+ add: () => {},
+ paused: (value?: boolean) => {
+ if (typeof value === "boolean") state.paused = value;
+ return state.paused;
+ },
+ timeScale: () => {},
+ set: () => {},
+ getChildren: () => {
+ tl.childScans += 1;
+ return [];
+ },
+ };
+ return tl;
+}
+
+function createManualRaf() {
+ let now = 0;
+ let nextId = 0;
+ const callbacks = new Map<number, FrameRequestCallback>();
+ return {
+ requestAnimationFrame: (callback: FrameRequestCallback) => {
+ nextId += 1;
+ callbacks.set(nextId, callback);
+ return nextId;
+ },
+ cancelAnimationFrame: (id: number) => {
+ callbacks.delete(id);
+ },
+ pending: () => callbacks.size,
+ step: (milliseconds: number) => {
+ now += milliseconds;
+ const batch = Array.from(callbacks.values());
+ callbacks.clear();
+ for (const callback of batch) callback(now);
+ },
+ };
+}
+
+describe("a playing frame's scope", () => {
+ const originalRaf = window.requestAnimationFrame;
+ const originalCancelRaf = window.cancelAnimationFrame;
+ let raf: ReturnType<typeof createManualRaf>;
+ let posted: Array<Record<string, unknown>>;
+ let nowMs: number;
+
+ beforeEach(() => {
+ vi.useFakeTimers({
+ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"],
+ });
+ nowMs = 1000;
+ vi.spyOn(performance, "now").mockImplementation(() => nowMs);
+ (globalThis as typeof globalThis & { CSS?: { escape?: (v: string) => string } }).CSS ??= {};
+ globalThis.CSS.escape ??= (value: string) => value;
+ raf = createManualRaf();
+ window.requestAnimationFrame = raf.requestAnimationFrame as typeof window.requestAnimationFrame;
+ window.cancelAnimationFrame = raf.cancelAnimationFrame as typeof window.cancelAnimationFrame;
+ posted = [];
+ vi.spyOn(window.parent, "postMessage").mockImplementation((message: unknown) => {
+ posted.push(message as Record<string, unknown>);
+ });
+ });
+
+ afterEach(() => {
+ window.__hfRuntimeTeardown?.();
+ document.body.innerHTML = "";
+ window.__timelines = {} as Record<string, RuntimeTimelineLike>;
+ delete window.__player;
+ delete window.__playerReady;
+ vi.restoreAllMocks();
+ vi.useRealTimers();
+ window.requestAnimationFrame = originalRaf;
+ window.cancelAnimationFrame = originalCancelRaf;
+ });
+
+ /** Advances clock and frame together, `ms` at a time, for `count` frames. */
+ const frames = (count: number, ms = 50) => {
+ for (let i = 0; i < count; i += 1) {
+ nowMs += ms;
+ vi.advanceTimersByTime(ms);
+ raf.step(ms);
+ }
+ };
+
+ /** Clips a..f at 2 s apart; c and d sit inside host h, which shows from 4 s to 9 s. */
+ const CLIPS: Record<string, [number, number]> = {
+ a: [0, 3],
+ b: [2, 6],
+ h: [4, 5],
+ c: [4.5, 2],
+ d: [7, 5],
+ e: [10, 4],
+ f: [16, 3],
+ };
+ const mountFilm = (registry: Record<string, unknown> = {}) => {
+ const clip = (id: string, inner = "") =>
+ `<div id="${id}" data-start="${CLIPS[id]![0]}" data-duration="${CLIPS[id]![1]}">${inner}</div>`;
+ document.body.innerHTML =
+ `<div id="root" data-composition-id="main" data-root="true" data-start="0" data-duration="20">` +
+ clip("a") +
+ clip("b") +
+ clip("h", clip("c") + clip("d")) +
+ clip("e") +
+ clip("f") +
+ `</div>`;
+ window.__timelines = { main: countingTimeline(20), ...registry } as Record<
+ string,
+ RuntimeTimelineLike
+ >;
+ initSandboxRuntimeModular();
+ frames(10, 1);
+ };
+ const shownAt = (id: string, t: number): boolean => {
+ const within = ([start, duration]: [number, number]) => t >= start && t < start + duration;
+ return within(CLIPS[id]!) && (id !== "c" && id !== "d" ? true : within(CLIPS.h!));
+ };
+
+ it("shows each clip exactly when it and its host are in window, deciding only the clips whose edge it crossed", () => {
+ mountFilm();
+ const decide = vi.mocked(timeline.isRuntimeElementVisibleAt);
+ window.__player!.play();
+ frames(2);
+ let quietFrames = 0;
+ for (let i = 0; i < 300; i += 1) {
+ const before = window.__player!.getTime();
+ decide.mockClear();
+ frames(1);
+ const after = window.__player!.getTime();
+ for (const id of Object.keys(CLIPS)) {
+ const expected = shownAt(id, after) ? "visible" : "hidden";
+ expect(document.getElementById(id)!.style.visibility, `${id} at ${after}`).toBe(expected);
+ }
+ const edgeCrossed = Object.values(CLIPS).some(([start, duration]) =>
+ [start, start + duration].some((edge) =>
+ [0, 2, 4].some((lead) =>
+ [edge - lead, edge + lead].some((x) => x >= before && x <= after),
+ ),
+ ),
+ );
+ if (!edgeCrossed) {
+ quietFrames += 1;
+ expect(decide, `frame ${before} -> ${after}`).not.toHaveBeenCalled();
+ }
+ }
+ expect(quietFrames).toBeGreaterThan(20);
+ });
+
+ it("shows a clip on the next frame when its timing moves it onto the playhead mid-play", async () => {
+ mountFilm();
+ window.__player!.play();
+ while (window.__player!.getTime() < 5) frames(1);
+ const moved = document.getElementById("e")!;
+ const stretched = document.getElementById("a")!;
+ expect([moved.style.visibility, stretched.style.visibility]).toEqual(["hidden", "hidden"]);
+ moved.setAttribute("data-start", "4");
+ stretched.setAttribute("data-duration", "19");
+ await Promise.resolve();
+ frames(1);
+ expect([moved.style.visibility, stretched.style.visibility]).toEqual(["visible", "visible"]);
+ });
+
+ it("leaves a scene that ended alone while the film plays on, after resting it at its end once", () => {
+ const scene = countingTimeline(8);
+ document.body.innerHTML =
+ `<div id="root" data-composition-id="main" data-root="true" data-start="0" data-duration="20">` +
+ `<div id="host" data-composition-id="scene" data-start="1" data-duration="2"></div></div>`;
+ window.__timelines = { main: countingTimeline(20), scene } as Record<
+ string,
+ RuntimeTimelineLike
+ >;
+ initSandboxRuntimeModular();
+ frames(10, 1);
+ window.__player!.play();
+ frames(80);
+ expect(window.__player!.getTime()).toBeGreaterThan(3.5);
+ scene.seeks = 0;
+ frames(40);
+ expect(scene.seeks).toBe(0);
+ expect(scene.time()).toBe(2);
+ });
+
+ it("checks the timeline registry once a second while playing, not every frame", () => {
+ const scene = countingTimeline(2);
+ mountFilm({ scene });
+ window.__player!.play();
+ frames(2);
+ scene.childScans = 0;
+ frames(40);
+ expect(scene.childScans).toBeLessThanOrEqual(3);
+ });
+
+ it("holds clip-list posts while playing and sends one when playback stops", () => {
+ mountFilm();
+ window.__player!.play();
+ frames(4);
+ posted.length = 0;
+ document.getElementById("e")!.setAttribute("data-duration", "5");
+ frames(40);
+ expect(posted.filter((m) => m["type"] === "timeline")).toHaveLength(0);
+ window.__player!.pause();
+ frames(4);
+ expect(posted.filter((m) => m["type"] === "timeline").length).toBeGreaterThanOrEqual(1);
+ });
+
+ it("plays past a registry entry that is not a timeline", () => {
+ mountFilm({ stray: 42 });
+ window.__player!.play();
+ expect(() => frames(40)).not.toThrow();
+ expect(window.__player!.getTime()).toBeGreaterThan(1.5);
+ });
+});
diff --git a/packages/core/src/runtime/init.seekOrder.test.ts b/packages/core/src/runtime/init.seekOrder.test.ts
new file mode 100644
index 0000000000..6a30f1d43e
--- /dev/null
+++ b/packages/core/src/runtime/init.seekOrder.test.ts
@@ -0,0 +1,90 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { registerSeekCompletion } from "./adapters/seek-dispatch";
+import { initSandboxRuntimeModular } from "./init";
+import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
+
+it("asks the on-screen video for the jump's frame before rendering the page there", () => {
+ resetRuntimeFixtureDom();
+ vi.spyOn(window, "requestAnimationFrame").mockImplementation(() => 0);
+ document.body.innerHTML =
+ '<div data-composition-id="main" data-root="true" data-duration="60"><video id="clip" data-start="30" data-duration="10" src="/movie.mp4"></video></div>';
+ const video = document.querySelector("video")!;
+ Object.defineProperty(video, "duration", { value: 10, configurable: true });
+ Object.defineProperty(video, "readyState", { value: 1, configurable: true });
+ Object.defineProperty(video, "paused", { value: true, configurable: true });
+ video.pause = vi.fn();
+ video.load = vi.fn();
+ const order: string[] = [];
+ vi.spyOn(video, "currentTime", "set").mockImplementation(() => order.push("video"));
+ const main = createMockTimeline(60);
+ const seekMain = main.totalTime;
+ main.totalTime = (time?: number) => {
+ if (time !== undefined) order.push("page");
+ return seekMain(time);
+ };
+ window.__timelines = { main };
+ initSandboxRuntimeModular();
+ order.length = 0;
+ window.__player!.seek(35);
+ expect(order[0]).toBe("video");
+ expect(order).toContain("page");
+});
+
+function mountTimingFilm() {
+ resetRuntimeFixtureDom();
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval"] });
+ const frames: FrameRequestCallback[] = [];
+ vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => frames.push(callback));
+ const step = (count = 1) => {
+ for (let i = 0; i < count; i += 1) {
+ for (const callback of frames.splice(0)) callback(performance.now());
+ vi.advanceTimersByTime(20);
+ }
+ };
+ const posts: string[] = [];
+ vi.spyOn(window.parent, "postMessage").mockImplementation((message: unknown) => {
+ const type = (message as { type?: string }).type;
+ if (type === "timeline") posts.push(type);
+ });
+ document.body.innerHTML =
+ '<div data-composition-id="main" data-root="true" data-duration="60"><div id="clip" data-start="30" data-duration="10"></div></div>';
+ window.__timelines = { main: createMockTimeline(60) };
+ initSandboxRuntimeModular();
+ step(40);
+ let land = () => {};
+ registerSeekCompletion(new Promise<void>((resolve) => (land = resolve)));
+ posts.length = 0;
+ const editTiming = async () => {
+ document.getElementById("clip")!.setAttribute("data-duration", "12");
+ await Promise.resolve();
+ step(40);
+ };
+ return { step, posts, land: () => land(), editTiming };
+}
+
+it("posts the clip list once a jump has landed, not while it lands", async () => {
+ const film = mountTimingFilm();
+ window.__player!.seek(35);
+ await film.editTiming();
+ expect(film.posts).toHaveLength(0);
+ film.land();
+ await vi.waitFor(() => {
+ film.step();
+ expect(film.posts).toHaveLength(1);
+ });
+ vi.useRealTimers();
+});
+
+it("posts at once when nothing was jumped to, so readers at load see the clip list", async () => {
+ const film = mountTimingFilm();
+ await film.editTiming();
+ expect(film.posts.length).toBeGreaterThanOrEqual(1);
+ film.land();
+ vi.useRealTimers();
+});
+
+afterEach(() => {
+ window.__hfRuntimeTeardown?.();
+ vi.restoreAllMocks();
+ resetRuntimeFixtureDom();
+});
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index 40a4fe48b9..d21a57c485 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -44,7 +44,9 @@ import {
} from "./adapters/video-texture-compat";
import {
forceDispatchSeekEvent,
+ hasPendingSeekCompletion,
registerSeekCompletion,
+ seekCompletionsSettled,
waitForSeekCompletion,
} from "./adapters/seek-dispatch";
import { sourceTimeAt } from "../speedRamp";
@@ -80,6 +82,7 @@ import { createRuntimeState } from "./state";
import {
collectRuntimeTimelinePayload,
isRuntimeElementVisibleAt,
+ runtimeElementWindow,
parseAuthoredTrack,
} from "./timeline";
import { findRootCompositionElement, parseLayoutDimension } from "./compositionDimension";
@@ -1320,6 +1323,7 @@ export function initSandboxRuntimeModular(): void {
let durationFloorsCache: DurationFloors | null = null;
let durationFloorsRevision = -1;
let timelineRegistrySignature: string | null = null;
+ let registrySignatureReadAtMs = Number.NEGATIVE_INFINITY;
let compositionTimingObserver: MutationObserver | null = null;
// Bumped whenever anything above can have moved a clip's window. Two caches
@@ -1431,11 +1435,16 @@ export function initSandboxRuntimeModular(): void {
}
// `window.__timelines` is a plain object: no MutationObserver and no event
// can see a composition script registering into it or lengthening what it
- // already registered, so its shape is compared on every read.
- const signature = readTimelineRegistrySignature();
- if (signature !== timelineRegistrySignature) {
- timelineRegistrySignature = signature;
- invalidateCompositionTimingCaches();
+ // already registered, so its shape is polled like everything else no
+ // observer sees: on every read paused, on the playing poll timer playing.
+ const now = Date.now();
+ if (!clock.isPlaying() || now - registrySignatureReadAtMs >= PLAYING_POLL_INTERVAL_MS) {
+ registrySignatureReadAtMs = now;
+ const signature = readTimelineRegistrySignature();
+ if (signature !== timelineRegistrySignature) {
+ timelineRegistrySignature = signature;
+ invalidateCompositionTimingCaches();
+ }
}
return compositionTimingRevision;
};
@@ -2888,37 +2897,77 @@ export function initSandboxRuntimeModular(): void {
dueSoon(el, visibleAt, loopStartSeconds, RELEASE_LOOKAHEAD_SECONDS)));
const visibilityDecided = new WeakMap<Element, boolean>();
- const mediaWindows = new WeakMap<object, Map<Element, { start: number; end: number }>>();
+ // Every time at which an element's visibility, upcoming or preload decision can flip: its own window
+ // edges, the look-ahead and release edges around them, and the window edges of each timed ancestor.
+ const decisionEdges = new Map<Element, number[] | null>();
+ let decisionEdgesRevision = -1;
const VISIBILITY_REACH_SECONDS = RELEASE_LOOKAHEAD_SECONDS + LOOKAHEAD_SECONDS + 1;
const PLAYED_STEP_MAX_SECONDS = 1;
+ const decisionEdgesOf = (
+ node: Element,
+ rootComp: HTMLElement,
+ windowOf: (el: Element) => { start: number; end: number } | null,
+ ): number[] | null => {
+ const own = windowOf(node);
+ if (!own || !Number.isFinite(own.start)) return null;
+ const edges: number[] = [];
+ for (const ahead of [0, LOOKAHEAD_SECONDS, RELEASE_LOOKAHEAD_SECONDS])
+ edges.push(own.start - ahead, own.end - ahead, own.end + ahead);
+ for (let el = node.parentElement; el && el !== rootComp; el = el.parentElement) {
+ if (!el.hasAttribute("data-start")) continue;
+ const span = windowOf(el);
+ if (!span || !Number.isFinite(span.start)) return null;
+ edges.push(span.start, span.end);
+ }
+ return edges.filter((edge) => Number.isFinite(edge));
+ };
const unchangedSincePlayedFrom = (
from: number,
to: number,
rootComp: HTMLElement | null,
+ timingRevision: number,
): ((node: Element) => boolean) | null => {
if (!rootComp || !(Math.abs(to - from) <= PLAYED_STEP_MAX_SECONDS)) return null;
- const index = resolveMediaClipIndex();
- let windows = mediaWindows.get(index);
- if (!windows) {
- windows = new Map(index.clips.map((clip) => [clip.el, clip]));
- mediaWindows.set(index, windows);
+ if (decisionEdgesRevision !== timingRevision) {
+ decisionEdges.clear();
+ decisionEdgesRevision = timingRevision;
+ return null;
}
+ const windowOptions = {
+ timelineRegistry: window.__timelines ?? {},
+ resolver: timingResolverFor(true),
+ };
+ const windowOf = (el: Element) => runtimeElementWindow(el, windowOptions);
const lo = Math.min(from, to);
const hi = Math.max(from, to);
- const near = (start: number, end: number, a: number, b: number) =>
- start - VISIBILITY_REACH_SECONDS <= b && end + VISIBILITY_REACH_SECONDS >= a;
return (node) => {
- if (!isMediaElement(node) || node.parentElement !== rootComp) return false;
- if (visibilityDecided.get(node) !== false || mediaNearPlayhead.get(node) === true)
+ const decided = visibilityDecided.get(node);
+ if (
+ decided === undefined ||
+ (isMediaElement(node) && (armedBy.has(node) || armedFrom.has(node)))
+ )
return false;
- if (armedBy.has(node) || armedFrom.has(node)) return false;
- const span = windows.get(node);
- if (!span || !Number.isFinite(span.start)) return false;
- const end = Number.isFinite(span.end) ? span.end : Infinity;
- if (near(span.start, end, lo, hi)) return false;
- return (
- loopStartSeconds === null || !near(span.start, end, loopStartSeconds, loopStartSeconds)
- );
+ // A script that changed a clip's visibility gets it decided again, as a full pass would.
+ if (
+ !node.hasAttribute(RUNTIME_STAMPED_ATTR) &&
+ (node as HTMLElement).style.visibility !== (decided ? "visible" : "hidden")
+ )
+ return false;
+ let edges = decisionEdges.get(node);
+ if (edges === undefined) {
+ edges = decisionEdgesOf(node, rootComp, windowOf);
+ decisionEdges.set(node, edges);
+ }
+ if (!edges) return false;
+ for (const edge of edges) {
+ if (edge >= lo && edge <= hi) return false;
+ if (
+ loopStartSeconds !== null &&
+ Math.abs(edge - loopStartSeconds) <= VISIBILITY_REACH_SECONDS
+ )
+ return false;
+ }
+ return true;
};
};
@@ -2932,9 +2981,9 @@ export function initSandboxRuntimeModular(): void {
let decidedTimedClip = false;
const visibleAt = timedVisibilityAt(timingRevision);
const unchanged =
- playedFromSeconds === undefined
+ playedFromSeconds === undefined || timingRevision === undefined
? null
- : unchangedSincePlayedFrom(playedFromSeconds, currentTime, rootComp);
+ : unchangedSincePlayedFrom(playedFromSeconds, currentTime, rootComp, timingRevision);
for (const rawNode of visibilityNodes) {
if (!isStylableElement(rawNode)) continue;
@@ -3101,8 +3150,7 @@ export function initSandboxRuntimeModular(): void {
resolveHostSlots: resolveMediaHostSlots,
});
- const resolveMediaClipIndex = (): MediaClipIndex => {
- const revision = readCompositionTimingRevision();
+ const resolveMediaClipIndex = (revision = readCompositionTimingRevision()): MediaClipIndex => {
if (mediaClipIndex && mediaClipIndex.revision === revision) return mediaClipIndex;
const clips = buildRuntimeMediaCache().mediaClips;
mediaClipIndex = {
@@ -3303,7 +3351,7 @@ export function initSandboxRuntimeModular(): void {
: 0;
const mediaClips = withTimingResolver(() => {
if (!indexed) return buildRuntimeMediaCache().mediaClips;
- const index = resolveMediaClipIndex();
+ const index = resolveMediaClipIndex(timingRevision);
// Windows come from the index, but every field handed to `syncRuntimeMedia`
// is re-read here: `el.duration` is reset by any `el.load()` the retry path
// made last pass, and a cached copy of it would be exactly the stale
@@ -3378,6 +3426,7 @@ export function initSandboxRuntimeModular(): void {
// this the one place a parked transport needs to hear about, instead of a
// wake call bolted onto each mutator (and forgotten on the next one).
if (force) wakeTransport();
+ if (force && timelinePostHeld && !clock.isPlaying()) postTimeline();
const frame = Math.max(0, Math.round((state.currentTime || 0) * state.canonicalFps));
const now = Date.now();
const shouldPost =
@@ -3419,6 +3468,24 @@ export function initSandboxRuntimeModular(): void {
}
return sig;
};
+ // Studio re-reads the whole manifest on every post, so playback holds them until it stops.
+ let timelinePostHeld = false;
+ // A jump paints its frame first; the posts its scene mounts cause become one after.
+ let jumpLanding = false;
+ let timelinePostAfterSeek = false;
+ const postTimelineAfterSeek = () => {
+ if (timelinePostAfterSeek) return;
+ timelinePostAfterSeek = true;
+ void seekCompletionsSettled().then(() =>
+ window.requestAnimationFrame(() =>
+ setTimeout(() => {
+ timelinePostAfterSeek = false;
+ jumpLanding = false;
+ if (!state.tornDown) postTimeline();
+ }),
+ ),
+ );
+ };
const postTimeline = () => {
sanitizeCompositionDurationAttributes();
applyCompositionSizing();
@@ -3432,6 +3499,11 @@ export function initSandboxRuntimeModular(): void {
}
}
bindRootTimelineIfAvailable();
+ timelinePostHeld = clock.isPlaying();
+ if (timelinePostHeld) return;
+ if (jumpLanding && !renderCaptureSeekStarted && hasPendingSeekCompletion())
+ return postTimelineAfterSeek();
+ jumpLanding = false;
const payload = collectRuntimeTimelinePayload({
canonicalFps: state.canonicalFps,
});
@@ -4024,15 +4096,18 @@ export function initSandboxRuntimeModular(): void {
state.currentTime = clock.now();
state.isPlaying = false;
state.mediaForceSyncNextTick = true;
+ const keepPlaying = options?.keepPlaying && wasPlaying;
+ jumpLanding = !keepPlaying;
+ // Decoding starts before the page renders, so the two overlap.
+ if (!keepPlaying) syncMediaForCurrentState();
const tl = state.capturedTimeline;
pauseTimelineIfPossible(tl);
const pageAnimations = seekTimelineAndAdapters(state.currentTime);
runAdapters("pause", 0, pageAnimations);
- if (options?.keepPlaying && wasPlaying) {
+ if (keepPlaying) {
transport.play();
return;
}
- syncMediaForCurrentState();
colorGrading.redraw();
paintVfx(state.currentTime);
postState(true);
@@ -4450,38 +4525,85 @@ export function initSandboxRuntimeModular(): void {
standaloneHosts.set(compositionId, node ?? { missingAt: compositionTimingRevision });
return node;
};
+ type SceneClock = {
+ start: number;
+ hiddenFrom: number;
+ playbackStart: number;
+ rate: ReturnType<typeof readElementRateSpec>;
+ timelineDuration: number | null;
+ };
+ let sceneClocks = new WeakMap<object, SceneClock | null>();
+ let sceneClocksRevision = -1;
+ const sceneClockOf = (compositionId: string, timeline: RuntimeTimelineLike) => {
+ if (sceneClocks.has(timeline)) return sceneClocks.get(timeline)!;
+ const node = standaloneHostOf(compositionId);
+ const start = node ? resolveStartForElement(node, 0) : Number.NaN;
+ const clock: SceneClock | null =
+ node && Number.isFinite(start)
+ ? {
+ start,
+ // Rest only where the visibility pass hides the host.
+ hiddenFrom:
+ node.hasAttribute("data-start") && !isDiscoveryStamped(node)
+ ? start + (resolveDurationForElement(node) ?? Infinity)
+ : Infinity,
+ playbackStart: readElementPlaybackStart(node),
+ rate: readElementRateSpec(node),
+ timelineDuration: getTimelineDurationSeconds(timeline),
+ }
+ : null;
+ sceneClocks.set(timeline, clock);
+ return clock;
+ };
+ const restingAt = new WeakMap<object, number>();
+ const restsAt = (timeline: RuntimeTimelineLike, time: number): boolean => {
+ if (restingAt.get(timeline) !== time) return false;
+ try {
+ return (
+ (timeline.paused as () => unknown)() === true && Number(timeline.totalTime?.()) === time
+ );
+ } catch {
+ return false;
+ }
+ };
const seekStandaloneRegisteredTimelines = (
timeSeconds: number,
+ revision: number,
options?: RuntimeSeekOptions & { activateChildren?: boolean },
) => {
+ if (sceneClocksRevision !== revision || renderCaptureSeekStarted) {
+ sceneClocks = new WeakMap();
+ sceneClocksRevision = revision;
+ }
const timelines = (window.__timelines ?? {}) as Record<string, RuntimeTimelineLike | undefined>;
const rootCompositionId =
resolveRootCompositionElement()?.getAttribute("data-composition-id") ?? null;
for (const [compositionId, timeline] of Object.entries(timelines)) {
- if (!timeline || compositionId === rootCompositionId) continue;
- const node = standaloneHostOf(compositionId);
- if (!node) continue;
- const start = resolveStartForElement(node, 0);
- if (!Number.isFinite(start)) continue;
- // Rest only where the visibility pass hides the host and no rearmed root propagation moves it.
- const hiddenFrom =
- !options?.activateChildren && node.hasAttribute("data-start") && !isDiscoveryStamped(node)
- ? start + (resolveDurationForElement(node) ?? Infinity)
- : Infinity;
- const timelineDuration = getTimelineDurationSeconds(timeline);
+ if (!timeline || typeof timeline !== "object" || compositionId === rootCompositionId)
+ continue;
+ const scene = sceneClockOf(compositionId, timeline);
+ if (!scene) continue;
const sourceTime =
- readElementPlaybackStart(node) +
+ scene.playbackStart +
sourceTimeAt(
- readElementRateSpec(node),
- Math.max(0, Math.min(timeSeconds, hiddenFrom) - start),
+ scene.rate,
+ // A rearmed child follows root propagation, so it rests nowhere the root would not.
+ Math.max(
+ 0,
+ (options?.activateChildren ? timeSeconds : Math.min(timeSeconds, scene.hiddenFrom)) -
+ scene.start,
+ ),
);
const localTime = Math.max(
0,
- timelineDuration != null && timelineDuration > 0
- ? Math.min(timelineDuration, sourceTime)
+ scene.timelineDuration != null && scene.timelineDuration > 0
+ ? Math.min(scene.timelineDuration, sourceTime)
: sourceTime,
);
+ // A scene before or after its clip rests where the last frame left it: no redraw.
+ if (!renderCaptureSeekStarted && restsAt(timeline, localTime)) continue;
seekRuntimeTimeline(timeline, localTime, "runtime.init.transport.childTimeline", options);
+ restingAt.set(timeline, localTime);
}
};
@@ -4578,7 +4700,7 @@ export function initSandboxRuntimeModular(): void {
*/
function seekTimelineAndAdapters(
t: number,
- opts?: { activateChildren?: boolean; suppressEvents?: boolean },
+ opts?: { activateChildren?: boolean; suppressEvents?: boolean; revision?: number },
): () => Animation[] {
const tl = state.capturedTimeline;
// Critical for a sub-composition whose data-start is at or near 0: it is added
@@ -4598,10 +4720,10 @@ export function initSandboxRuntimeModular(): void {
function seekRootChildrenAndAdapters(
tl: RuntimeTimelineLike | null,
t: number,
- opts?: { activateChildren?: boolean; suppressEvents?: boolean },
+ opts?: { activateChildren?: boolean; suppressEvents?: boolean; revision?: number },
): () => Animation[] {
const suppressEvents = opts?.suppressEvents === true;
- const revision = readCompositionTimingRevision();
+ const revision = opts?.revision ?? readCompositionTimingRevision();
if (revision !== tweensRenderedThrough.revision)
tweensRenderedThrough = { revision, time: Number.NEGATIVE_INFINITY };
const jumped = !(Math.abs(t - lastTransportSeekTime) <= 1.5 / state.canonicalFps);
@@ -4656,7 +4778,7 @@ export function initSandboxRuntimeModular(): void {
// safe because nothing between frames READS a sibling's paused() — grepped over
// the deterministic adapters, syncTimedElementVisibility, the hf-timelines-built
// handler and __hfReseekGpu. It only ever moved the state the seek left behind.
- seekStandaloneRegisteredTimelines(t, opts);
+ seekStandaloneRegisteredTimelines(t, revision, opts);
hideClipsAgain?.();
tweensRenderedThrough.time = Math.max(tweensRenderedThrough.time, t);
const pageAnimations = pageAnimationsForOnePass();
@@ -4978,7 +5100,7 @@ export function initSandboxRuntimeModular(): void {
// change happens to wake the loop again.
postTimeline();
compositionChangePending = false;
- } else if (playingPollDue || stoppedPlaying) {
+ } else if (stoppedPlaying) {
postTimelineIfManifestChanged();
}
if (changeDrivenService || pausedCounterDue(MEDIA_BIND_INTERVAL_FRAMES)) {
@@ -5018,7 +5140,7 @@ export function initSandboxRuntimeModular(): void {
t !== lastTransportSeekTime ||
state.capturedTimeline !== lastTransportSeekTimeline
) {
- seekTimelineAndAdapters(t);
+ seekTimelineAndAdapters(t, { revision: timingRevision });
if (!isPlaying) pausedSeekDeferredByManualGesture = false;
}
if (isPlaying) {
diff --git a/packages/core/src/runtime/previewRasterHints.test.ts b/packages/core/src/runtime/previewRasterHints.test.ts
index 513a0d2623..be59388671 100644
--- a/packages/core/src/runtime/previewRasterHints.test.ts
+++ b/packages/core/src/runtime/previewRasterHints.test.ts
@@ -1,6 +1,7 @@
import { afterEach, describe, expect, it, vi } from "vitest";
-import { PREVIEW_RASTER_ATTR } from "../studioPreviewMark";
+import { PREVIEW_RASTER_ATTR, STUDIO_PREVIEW_UPCOMING_ATTR } from "../studioPreviewMark";
import { setPreviewRasterScale } from "./previewRasterHints";
+import { CLIP_HIDDEN_ATTR } from "./timedClipHide";
function layer(style: string, parent: Element = document.body): HTMLElement {
const element = document.createElement("div");
@@ -158,6 +159,19 @@ describe("setPreviewRasterScale", () => {
expect(marked(words)).toBe(true);
});
+ it("ignores the clip attributes the runtime toggles while playing", async () => {
+ const scene = layer(POSITIONED);
+ for (let i = 0; i < 20; i += 1) layer("position: absolute", scene);
+ setPreviewRasterScale(0.5);
+ const reads = vi.spyOn(window, "getComputedStyle");
+ scene.toggleAttribute(STUDIO_PREVIEW_UPCOMING_ATTR, true);
+ scene.toggleAttribute(CLIP_HIDDEN_ATTR, true);
+ await Promise.resolve();
+ expect(reads).not.toHaveBeenCalled();
+ expect(marked(scene)).toBe(true);
+ reads.mockRestore();
+ });
+
it("re-checks the page when a linked stylesheet loads", async () => {
const sheet = document.createElement("style");
document.head.append(sheet);
@@ -176,6 +190,39 @@ describe("setPreviewRasterScale", () => {
link.remove();
});
+ it("decides every mark in a changed subtree before writing any, so the page restyles once", async () => {
+ const sheet = document.createElement("style");
+ sheet.textContent = "[data-flat] > div { position: static !important }";
+ document.head.append(sheet);
+ const panel = layer("");
+ const rows = Array.from({ length: 10 }, () => layer(POSITIONED, panel));
+ setPreviewRasterScale(0.5);
+ expect(rows.every(marked)).toBe(true);
+ const order: string[] = [];
+ const realRead = window.getComputedStyle.bind(window);
+ const reads = vi.spyOn(window, "getComputedStyle").mockImplementation((el, pseudo) => {
+ order.push("read");
+ return realRead(el, pseudo);
+ });
+ const realToggle = Element.prototype.toggleAttribute;
+ const writes = vi.spyOn(Element.prototype, "toggleAttribute").mockImplementation(function (
+ this: Element,
+ name: string,
+ force?: boolean,
+ ) {
+ if (name === PREVIEW_RASTER_ATTR) order.push("write");
+ return realToggle.call(this, name, force);
+ });
+ panel.setAttribute("data-flat", "");
+ await Promise.resolve();
+ reads.mockRestore();
+ writes.mockRestore();
+ expect(rows.some(marked)).toBe(false);
+ expect(order.filter((step) => step === "write")).toHaveLength(10);
+ expect(order.lastIndexOf("read")).toBeLessThan(order.indexOf("write"));
+ sheet.remove();
+ });
+
it("checks each element once for a batch of attribute changes inside one subtree", async () => {
const group = layer(POSITIONED);
const words = Array.from({ length: 20 }, () => layer("position: absolute", group));
diff --git a/packages/core/src/runtime/previewRasterHints.ts b/packages/core/src/runtime/previewRasterHints.ts
index 36406b7a58..0277b6ab18 100644
--- a/packages/core/src/runtime/previewRasterHints.ts
+++ b/packages/core/src/runtime/previewRasterHints.ts
@@ -1,5 +1,6 @@
-import { PREVIEW_RASTER_ATTR } from "../studioPreviewMark";
+import { PREVIEW_ONLY_ATTRS, PREVIEW_RASTER_ATTR } from "../studioPreviewMark";
import { isElementNode } from "./domRealm";
+import { CLIP_HIDDEN_ATTR } from "./timedClipHide";
// Chromium rasters a `will-change: transform` layer at no less than device scale, so a preview shown
// small rasters each one at full size. Below 1x the hint is dropped wherever nothing depends on the
@@ -7,6 +8,8 @@ import { isElementNode } from "./domRealm";
const TRANSFORM_HINTS = new Set(["transform", "translate", "rotate", "scale"]);
const MARKED = `[${PREVIEW_RASTER_ATTR}]`;
const DROP_HINTS = `${MARKED} { will-change: auto !important; }`;
+// Toggled by the runtime as clips come and go; its own rules on them only hide, never restack.
+const RUNTIME_PLAYBACK_ATTRS = new Set<string>([...PREVIEW_ONLY_ATTRS, CLIP_HIDDEN_ATTR]);
let sheet: HTMLStyleElement | null = null;
let observer: MutationObserver | null = null;
@@ -58,13 +61,16 @@ function fits(element: Element, readOwnHints: boolean): boolean {
return drop;
}
-function check(element: Element, readOwnHints: boolean): void {
- const drop = fits(element, readOwnHints);
+function writeMark(element: Element, drop: boolean): void {
if (drop !== element.hasAttribute(PREVIEW_RASTER_ATTR)) {
element.toggleAttribute(PREVIEW_RASTER_ATTR, drop);
}
}
+function check(element: Element, readOwnHints: boolean): void {
+ writeMark(element, fits(element, readOwnHints));
+}
+
function unmarkHolders(element: Element): void {
let holder = element.parentElement?.closest(MARKED);
while (holder) {
@@ -76,12 +82,17 @@ function unmarkHolders(element: Element): void {
}
function followTree(root: Element, batch: FollowBatch): void {
+ const decidedBeforeAnyWrite: Array<[Element, boolean]> = [];
for (const element of [root, ...root.querySelectorAll("*")]) {
if (batch.checked.has(element)) continue;
batch.checked.add(element);
- check(element, element === root || batch.changed.has(element));
- unmarkHolders(element);
+ decidedBeforeAnyWrite.push([
+ element,
+ fits(element, element === root || batch.changed.has(element)),
+ ]);
}
+ for (const [element, drop] of decidedBeforeAnyWrite) writeMark(element, drop);
+ for (const [element] of decidedBeforeAnyWrite) unmarkHolders(element);
}
type FollowBatch = { changed: Set<Element>; checked: Set<Element> };
@@ -111,7 +122,8 @@ function markLayers(): void {
}
function followAttribute(record: MutationRecord, batch: FollowBatch): void {
- if (!isElementNode(record.target) || record.attributeName === PREVIEW_RASTER_ATTR) return;
+ if (!isElementNode(record.target) || RUNTIME_PLAYBACK_ATTRS.has(record.attributeName ?? ""))
+ return;
// Any attribute can match a selector; inline style is the per-frame case, so it is cheaper.
if (record.attributeName === "style") followStyle(record.target);
else followTree(record.target, batch);
@@ -129,7 +141,7 @@ function ownAttributeChanges(records: MutationRecord[]): Set<Element> {
const changed = new Set<Element>();
for (const record of records) {
const name = record.attributeName;
- if (name === null || name === "style" || name === PREVIEW_RASTER_ATTR) continue;
+ if (name === null || name === "style" || RUNTIME_PLAYBACK_ATTRS.has(name)) continue;
if (isElementNode(record.target)) changed.add(record.target);
}
return changed;
diff --git a/packages/core/src/runtime/timeline.ts b/packages/core/src/runtime/timeline.ts
index 83e28000b0..21805116b7 100644
--- a/packages/core/src/runtime/timeline.ts
+++ b/packages/core/src/runtime/timeline.ts
@@ -31,20 +31,16 @@ import { isElementNode, isMediaElement } from "./domRealm";
import { LOOP_INFLATED_TIMELINE_SECONDS } from "./compositionLength";
import { cutToHostSlots } from "../mediaTiming";
-export function isRuntimeElementVisibleAt(
+export function runtimeElementWindow(
rawNode: Element,
options: {
- currentTime: number;
- compositionDuration: number;
- canonicalFps: number;
- exportRenderSeek: boolean;
timelineRegistry: Record<string, RuntimeTimelineLike | undefined>;
resolver: ReturnType<typeof createRuntimeStartTimeResolver>;
},
-): boolean {
+): { start: number; end: number } | null {
const tag = rawNode.tagName.toLowerCase();
if (tag === "script" || tag === "style" || tag === "link" || tag === "meta") {
- return false;
+ return null;
}
const isMedia = tag === "video" || tag === "audio";
@@ -70,12 +66,29 @@ export function isRuntimeElementVisibleAt(
duration = liveDuration;
}
}
- const computedEnd =
- duration != null && duration > 0 ? start + duration : Number.POSITIVE_INFINITY;
+ return {
+ start,
+ end: duration != null && duration > 0 ? start + duration : Number.POSITIVE_INFINITY,
+ };
+}
+
+export function isRuntimeElementVisibleAt(
+ rawNode: Element,
+ options: {
+ currentTime: number;
+ compositionDuration: number;
+ canonicalFps: number;
+ exportRenderSeek: boolean;
+ timelineRegistry: Record<string, RuntimeTimelineLike | undefined>;
+ resolver: ReturnType<typeof createRuntimeStartTimeResolver>;
+ },
+): boolean {
+ const span = runtimeElementWindow(rawNode, options);
+ if (!span) return false;
// Export seeks snap to frame boundaries; interactive visibility uses authored seconds.
const clipWindow = options.exportRenderSeek
- ? exportClipWindow(start, computedEnd, options.canonicalFps)
- : { start, end: computedEnd };
+ ? exportClipWindow(span.start, span.end, options.canonicalFps)
+ : span;
return isClipVisibleAt(
options.currentTime,
clipWindow.start,
diff --git a/packages/core/src/runtime/transportPark.test.ts b/packages/core/src/runtime/transportPark.test.ts
index 0906841ef8..447020089c 100644
--- a/packages/core/src/runtime/transportPark.test.ts
+++ b/packages/core/src/runtime/transportPark.test.ts
@@ -735,7 +735,7 @@ describe("parked transport loop", () => {
expect(posted.filter((m) => m["type"] === "timeline").length - before).toBe(0);
});
- it("posts a playing clip's new track, label and z-index on the next poll", async () => {
+ it("posts a playing clip's new track, label and z-index once playback stops", async () => {
mount(
'<div id="clip" data-start="0" data-duration="5" data-track-index="0" data-timeline-label="Before" style="z-index: 1"></div>',
);
@@ -748,7 +748,11 @@ describe("parked transport loop", () => {
clip.setAttribute("data-timeline-label", "After");
clip.style.zIndex = "9";
await flushObservers();
+ posted.length = 0;
for (let i = 0; i < 130; i += 1) frame120Hz();
+ expect(posted.filter((m) => m["type"] === "timeline")).toHaveLength(0);
+ window.__player!.pause();
+ for (let i = 0; i < 10; i += 1) frame120Hz();
const clips = (posted.filter((m) => m["type"] === "timeline").at(-1)?.["clips"] ?? []) as Array<
Record<string, unknown>
>;
diff --git a/packages/studio/src/player/hooks/previewMessageRouter.test.ts b/packages/studio/src/player/hooks/previewMessageRouter.test.ts
new file mode 100644
index 0000000000..d5d7492822
--- /dev/null
+++ b/packages/studio/src/player/hooks/previewMessageRouter.test.ts
@@ -0,0 +1,23 @@
+import { describe, expect, it, vi } from "vitest";
+import { createPreviewMessageHandler } from "./previewMessageRouter";
+
+function route(isPlaying: boolean) {
+ const enrich = vi.fn();
+ const handle = createPreviewMessageHandler({
+ iframeRef: { current: null },
+ processTimelineMessageRef: { current: vi.fn() },
+ enrichMissingCompositionsRef: { current: enrich },
+ lastTimelineMessageRef: { current: 0 },
+ getAdapter: () => null,
+ syncTimelineElements: vi.fn(),
+ });
+ handle(new MessageEvent("message", { data: { source: "hf-preview", type: "state", isPlaying } }));
+ return enrich;
+}
+
+describe("preview state messages", () => {
+ it("re-reads the preview for missing compositions only while it is stopped", () => {
+ expect(route(true)).not.toHaveBeenCalled();
+ expect(route(false)).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/packages/studio/src/player/hooks/previewMessageRouter.ts b/packages/studio/src/player/hooks/previewMessageRouter.ts
index 91eee73719..cee225f88f 100644
--- a/packages/studio/src/player/hooks/previewMessageRouter.ts
+++ b/packages/studio/src/player/hooks/previewMessageRouter.ts
@@ -29,6 +29,7 @@ interface PreviewMessage {
source?: string;
type?: string;
clips?: unknown;
+ isPlaying?: unknown;
}
export interface PreviewMessageRouterDeps {
@@ -68,6 +69,7 @@ function acceptedPreviewMessage(
function handleStateMessage(
deps: PreviewMessageRouterDeps,
iframe: HTMLIFrameElement | null,
+ playingHoldsClipListPosts: boolean,
): void {
try {
if (usePlayerStore.getState().elements.length === 0) {
@@ -76,7 +78,7 @@ function handleStateMessage(
deps.processTimelineMessageRef.current(manifest);
}
}
- if (Date.now() - deps.lastTimelineMessageRef.current > 500) {
+ if (!playingHoldsClipListPosts && Date.now() - deps.lastTimelineMessageRef.current > 500) {
deps.enrichMissingCompositionsRef.current();
}
} catch {}
@@ -112,7 +114,7 @@ export function createPreviewMessageHandler(
const iframe = deps.iframeRef.current;
const data = acceptedPreviewMessage(e, iframe);
if (!data) return;
- if (data.type === "state") return handleStateMessage(deps, iframe);
+ if (data.type === "state") return handleStateMessage(deps, iframe, data.isPlaying === true);
if (data.type === "timeline" && Array.isArray(data.clips)) {
handleTimelineMessage(deps, iframe, data as unknown as ClipManifestMessage);
}