Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(core): playback and jumps do only the work on screen (#5754)
Miguel Ángel959dd3e584cc
diff --git a/packages/core/src/runtime/adapters/seek-dispatch.ts b/packages/core/src/runtime/adapters/seek-dispatch.tsindex 8bce055eef..2173e8b110 100644--- a/packages/core/src/runtime/adapters/seek-dispatch.ts+++ b/packages/core/src/runtime/adapters/seek-dispatch.ts@@ -102,6 +102,21 @@ export function isSeekCompletionBarrierActive(): boolean {return _activeCompletionBarriers > 0;}+/** Settles with every pending completion, without consuming a failure the barrier reports. */+export async function seekCompletionsSettled(): Promise<void> {+ for (+ let pending = [..._pendingCompletions];+ pending.length > 0;+ pending = [..._pendingCompletions]+ ) {+ await Promise.all(pending);+ }+}++export function hasPendingSeekCompletion(): boolean {+ return _pendingCompletions.size > 0;+}+/** `videoFrames: false` skips only the video-frame holds: a geometry read needs every other completion. */export async function waitForSeekCompletion(options?: { videoFrames?: boolean }): Promise<void> {const awaited = () =>diff --git a/packages/core/src/runtime/init.playbackScope.test.ts b/packages/core/src/runtime/init.playbackScope.test.tsnew file mode 100644index 0000000000..bb6d17bb98--- /dev/null+++ b/packages/core/src/runtime/init.playbackScope.test.ts@@ -0,0 +1,256 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { initSandboxRuntimeModular } from "./init";+import * as timeline from "./timeline";+import type { RuntimeTimelineLike } from "./types";++vi.mock("../compositionReadiness", async (importOriginal) => {+ const actual = await importOriginal<typeof import("../compositionReadiness")>();+ return {+ ...actual,+ settleCompositionReadiness: vi.fn(),+ settleFirstFrameCompositionReadiness: vi.fn(),+ };+});+vi.mock("./timeline", async (importOriginal) => {+ const actual = await importOriginal<typeof import("./timeline")>();+ return { ...actual, isRuntimeElementVisibleAt: vi.fn(actual.isRuntimeElementVisibleAt) };+});++// Pins what a playing frame touches: clips on screen and clips whose edge it crossed, never the whole film.++type CountingTimeline = RuntimeTimelineLike & { seeks: number; childScans: number };++function countingTimeline(duration: number): CountingTimeline {+ const state = { time: 0, paused: true };+ const tl: CountingTimeline = {+ seeks: 0,+ childScans: 0,+ play: () => {+ state.paused = false;+ },+ pause: () => {+ state.paused = true;+ },+ seek: (time?: number) => {+ if (time !== undefined) state.time = time;+ return state.time;+ },+ totalTime: (time?: number) => {+ if (time !== undefined) {+ tl.seeks += 1;+ state.time = time;+ }+ return state.time;+ },+ time: () => state.time,+ duration: () => duration,+ add: () => {},+ paused: (value?: boolean) => {+ if (typeof value === "boolean") state.paused = value;+ return state.paused;+ },+ timeScale: () => {},+ set: () => {},+ getChildren: () => {+ tl.childScans += 1;+ return [];+ },+ };+ return tl;+}++function createManualRaf() {+ let now = 0;+ let nextId = 0;+ const callbacks = new Map<number, FrameRequestCallback>();+ return {+ requestAnimationFrame: (callback: FrameRequestCallback) => {+ nextId += 1;+ callbacks.set(nextId, callback);+ return nextId;+ },+ cancelAnimationFrame: (id: number) => {+ callbacks.delete(id);+ },+ pending: () => callbacks.size,+ step: (milliseconds: number) => {+ now += milliseconds;+ const batch = Array.from(callbacks.values());+ callbacks.clear();+ for (const callback of batch) callback(now);+ },+ };+}++describe("a playing frame's scope", () => {+ const originalRaf = window.requestAnimationFrame;+ const originalCancelRaf = window.cancelAnimationFrame;+ let raf: ReturnType<typeof createManualRaf>;+ let posted: Array<Record<string, unknown>>;+ let nowMs: number;++ beforeEach(() => {+ vi.useFakeTimers({+ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"],+ });+ nowMs = 1000;+ vi.spyOn(performance, "now").mockImplementation(() => nowMs);+ (globalThis as typeof globalThis & { CSS?: { escape?: (v: string) => string } }).CSS ??= {};+ globalThis.CSS.escape ??= (value: string) => value;+ raf = createManualRaf();+ window.requestAnimationFrame = raf.requestAnimationFrame as typeof window.requestAnimationFrame;+ window.cancelAnimationFrame = raf.cancelAnimationFrame as typeof window.cancelAnimationFrame;+ posted = [];+ vi.spyOn(window.parent, "postMessage").mockImplementation((message: unknown) => {+ posted.push(message as Record<string, unknown>);+ });+ });++ afterEach(() => {+ window.__hfRuntimeTeardown?.();+ document.body.innerHTML = "";+ window.__timelines = {} as Record<string, RuntimeTimelineLike>;+ delete window.__player;+ delete window.__playerReady;+ vi.restoreAllMocks();+ vi.useRealTimers();+ window.requestAnimationFrame = originalRaf;+ window.cancelAnimationFrame = originalCancelRaf;+ });++ /** Advances clock and frame together, `ms` at a time, for `count` frames. */+ const frames = (count: number, ms = 50) => {+ for (let i = 0; i < count; i += 1) {+ nowMs += ms;+ vi.advanceTimersByTime(ms);+ raf.step(ms);+ }+ };++ /** Clips a..f at 2 s apart; c and d sit inside host h, which shows from 4 s to 9 s. */+ const CLIPS: Record<string, [number, number]> = {+ a: [0, 3],+ b: [2, 6],+ h: [4, 5],+ c: [4.5, 2],+ d: [7, 5],+ e: [10, 4],+ f: [16, 3],+ };+ const mountFilm = (registry: Record<string, unknown> = {}) => {+ const clip = (id: string, inner = "") =>+ `<div id="${id}" data-start="${CLIPS[id]![0]}" data-duration="${CLIPS[id]![1]}">${inner}</div>`;+ document.body.innerHTML =+ `<div id="root" data-composition-id="main" data-root="true" data-start="0" data-duration="20">` ++ clip("a") ++ clip("b") ++ clip("h", clip("c") + clip("d")) ++ clip("e") ++ clip("f") ++ `</div>`;+ window.__timelines = { main: countingTimeline(20), ...registry } as Record<+ string,+ RuntimeTimelineLike+ >;+ initSandboxRuntimeModular();+ frames(10, 1);+ };+ const shownAt = (id: string, t: number): boolean => {+ const within = ([start, duration]: [number, number]) => t >= start && t < start + duration;+ return within(CLIPS[id]!) && (id !== "c" && id !== "d" ? true : within(CLIPS.h!));+ };++ it("shows each clip exactly when it and its host are in window, deciding only the clips whose edge it crossed", () => {+ mountFilm();+ const decide = vi.mocked(timeline.isRuntimeElementVisibleAt);+ window.__player!.play();+ frames(2);+ let quietFrames = 0;+ for (let i = 0; i < 300; i += 1) {+ const before = window.__player!.getTime();+ decide.mockClear();+ frames(1);+ const after = window.__player!.getTime();+ for (const id of Object.keys(CLIPS)) {+ const expected = shownAt(id, after) ? "visible" : "hidden";+ expect(document.getElementById(id)!.style.visibility, `${id} at ${after}`).toBe(expected);+ }+ const edgeCrossed = Object.values(CLIPS).some(([start, duration]) =>+ [start, start + duration].some((edge) =>+ [0, 2, 4].some((lead) =>+ [edge - lead, edge + lead].some((x) => x >= before && x <= after),+ ),+ ),+ );+ if (!edgeCrossed) {+ quietFrames += 1;+ expect(decide, `frame ${before} -> ${after}`).not.toHaveBeenCalled();+ }+ }+ expect(quietFrames).toBeGreaterThan(20);+ });++ it("shows a clip on the next frame when its timing moves it onto the playhead mid-play", async () => {+ mountFilm();+ window.__player!.play();+ while (window.__player!.getTime() < 5) frames(1);+ const moved = document.getElementById("e")!;+ const stretched = document.getElementById("a")!;+ expect([moved.style.visibility, stretched.style.visibility]).toEqual(["hidden", "hidden"]);+ moved.setAttribute("data-start", "4");+ stretched.setAttribute("data-duration", "19");+ await Promise.resolve();+ frames(1);+ expect([moved.style.visibility, stretched.style.visibility]).toEqual(["visible", "visible"]);+ });++ it("leaves a scene that ended alone while the film plays on, after resting it at its end once", () => {+ const scene = countingTimeline(8);+ document.body.innerHTML =+ `<div id="root" data-composition-id="main" data-root="true" data-start="0" data-duration="20">` ++ `<div id="host" data-composition-id="scene" data-start="1" data-duration="2"></div></div>`;+ window.__timelines = { main: countingTimeline(20), scene } as Record<+ string,+ RuntimeTimelineLike+ >;+ initSandboxRuntimeModular();+ frames(10, 1);+ window.__player!.play();+ frames(80);+ expect(window.__player!.getTime()).toBeGreaterThan(3.5);+ scene.seeks = 0;+ frames(40);+ expect(scene.seeks).toBe(0);+ expect(scene.time()).toBe(2);+ });++ it("checks the timeline registry once a second while playing, not every frame", () => {+ const scene = countingTimeline(2);+ mountFilm({ scene });+ window.__player!.play();+ frames(2);+ scene.childScans = 0;+ frames(40);+ expect(scene.childScans).toBeLessThanOrEqual(3);+ });++ it("holds clip-list posts while playing and sends one when playback stops", () => {+ mountFilm();+ window.__player!.play();+ frames(4);+ posted.length = 0;+ document.getElementById("e")!.setAttribute("data-duration", "5");+ frames(40);+ expect(posted.filter((m) => m["type"] === "timeline")).toHaveLength(0);+ window.__player!.pause();+ frames(4);+ expect(posted.filter((m) => m["type"] === "timeline").length).toBeGreaterThanOrEqual(1);+ });++ it("plays past a registry entry that is not a timeline", () => {+ mountFilm({ stray: 42 });+ window.__player!.play();+ expect(() => frames(40)).not.toThrow();+ expect(window.__player!.getTime()).toBeGreaterThan(1.5);+ });+});diff --git a/packages/core/src/runtime/init.seekOrder.test.ts b/packages/core/src/runtime/init.seekOrder.test.tsnew file mode 100644index 0000000000..6a30f1d43e--- /dev/null+++ b/packages/core/src/runtime/init.seekOrder.test.ts@@ -0,0 +1,90 @@+import { afterEach, expect, it, vi } from "vitest";+import { registerSeekCompletion } from "./adapters/seek-dispatch";+import { initSandboxRuntimeModular } from "./init";+import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";++it("asks the on-screen video for the jump's frame before rendering the page there", () => {+ resetRuntimeFixtureDom();+ vi.spyOn(window, "requestAnimationFrame").mockImplementation(() => 0);+ document.body.innerHTML =+ '<div data-composition-id="main" data-root="true" data-duration="60"><video id="clip" data-start="30" data-duration="10" src="/movie.mp4"></video></div>';+ const video = document.querySelector("video")!;+ Object.defineProperty(video, "duration", { value: 10, configurable: true });+ Object.defineProperty(video, "readyState", { value: 1, configurable: true });+ Object.defineProperty(video, "paused", { value: true, configurable: true });+ video.pause = vi.fn();+ video.load = vi.fn();+ const order: string[] = [];+ vi.spyOn(video, "currentTime", "set").mockImplementation(() => order.push("video"));+ const main = createMockTimeline(60);+ const seekMain = main.totalTime;+ main.totalTime = (time?: number) => {+ if (time !== undefined) order.push("page");+ return seekMain(time);+ };+ window.__timelines = { main };+ initSandboxRuntimeModular();+ order.length = 0;+ window.__player!.seek(35);+ expect(order[0]).toBe("video");+ expect(order).toContain("page");+});++function mountTimingFilm() {+ resetRuntimeFixtureDom();+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval"] });+ const frames: FrameRequestCallback[] = [];+ vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => frames.push(callback));+ const step = (count = 1) => {+ for (let i = 0; i < count; i += 1) {+ for (const callback of frames.splice(0)) callback(performance.now());+ vi.advanceTimersByTime(20);+ }+ };+ const posts: string[] = [];+ vi.spyOn(window.parent, "postMessage").mockImplementation((message: unknown) => {+ const type = (message as { type?: string }).type;+ if (type === "timeline") posts.push(type);+ });+ document.body.innerHTML =+ '<div data-composition-id="main" data-root="true" data-duration="60"><div id="clip" data-start="30" data-duration="10"></div></div>';+ window.__timelines = { main: createMockTimeline(60) };+ initSandboxRuntimeModular();+ step(40);+ let land = () => {};+ registerSeekCompletion(new Promise<void>((resolve) => (land = resolve)));+ posts.length = 0;+ const editTiming = async () => {+ document.getElementById("clip")!.setAttribute("data-duration", "12");+ await Promise.resolve();+ step(40);+ };+ return { step, posts, land: () => land(), editTiming };+}++it("posts the clip list once a jump has landed, not while it lands", async () => {+ const film = mountTimingFilm();+ window.__player!.seek(35);+ await film.editTiming();+ expect(film.posts).toHaveLength(0);+ film.land();+ await vi.waitFor(() => {+ film.step();+ expect(film.posts).toHaveLength(1);+ });+ vi.useRealTimers();+});++it("posts at once when nothing was jumped to, so readers at load see the clip list", async () => {+ const film = mountTimingFilm();+ await film.editTiming();+ expect(film.posts.length).toBeGreaterThanOrEqual(1);+ film.land();+ vi.useRealTimers();+});++afterEach(() => {+ window.__hfRuntimeTeardown?.();+ vi.restoreAllMocks();+ resetRuntimeFixtureDom();+});diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.tsindex 40a4fe48b9..d21a57c485 100644--- a/packages/core/src/runtime/init.ts+++ b/packages/core/src/runtime/init.ts@@ -44,7 +44,9 @@ import {} from "./adapters/video-texture-compat";import {forceDispatchSeekEvent,+ hasPendingSeekCompletion,registerSeekCompletion,+ seekCompletionsSettled,waitForSeekCompletion,} from "./adapters/seek-dispatch";import { sourceTimeAt } from "../speedRamp";@@ -80,6 +82,7 @@ import { createRuntimeState } from "./state";import {collectRuntimeTimelinePayload,isRuntimeElementVisibleAt,+ runtimeElementWindow,parseAuthoredTrack,} from "./timeline";import { findRootCompositionElement, parseLayoutDimension } from "./compositionDimension";@@ -1320,6 +1323,7 @@ export function initSandboxRuntimeModular(): void {let durationFloorsCache: DurationFloors | null = null;let durationFloorsRevision = -1;let timelineRegistrySignature: string | null = null;+ let registrySignatureReadAtMs = Number.NEGATIVE_INFINITY;let compositionTimingObserver: MutationObserver | null = null;// Bumped whenever anything above can have moved a clip's window. Two caches@@ -1431,11 +1435,16 @@ export function initSandboxRuntimeModular(): void {}// `window.__timelines` is a plain object: no MutationObserver and no event// can see a composition script registering into it or lengthening what it- // already registered, so its shape is compared on every read.- const signature = readTimelineRegistrySignature();- if (signature !== timelineRegistrySignature) {- timelineRegistrySignature = signature;- invalidateCompositionTimingCaches();+ // already registered, so its shape is polled like everything else no+ // observer sees: on every read paused, on the playing poll timer playing.+ const now = Date.now();+ if (!clock.isPlaying() || now - registrySignatureReadAtMs >= PLAYING_POLL_INTERVAL_MS) {+ registrySignatureReadAtMs = now;+ const signature = readTimelineRegistrySignature();+ if (signature !== timelineRegistrySignature) {+ timelineRegistrySignature = signature;+ invalidateCompositionTimingCaches();+ }}return compositionTimingRevision;};@@ -2888,37 +2897,77 @@ export function initSandboxRuntimeModular(): void {dueSoon(el, visibleAt, loopStartSeconds, RELEASE_LOOKAHEAD_SECONDS)));const visibilityDecided = new WeakMap<Element, boolean>();- const mediaWindows = new WeakMap<object, Map<Element, { start: number; end: number }>>();+ // Every time at which an element's visibility, upcoming or preload decision can flip: its own window+ // edges, the look-ahead and release edges around them, and the window edges of each timed ancestor.+ const decisionEdges = new Map<Element, number[] | null>();+ let decisionEdgesRevision = -1;const VISIBILITY_REACH_SECONDS = RELEASE_LOOKAHEAD_SECONDS + LOOKAHEAD_SECONDS + 1;const PLAYED_STEP_MAX_SECONDS = 1;+ const decisionEdgesOf = (+ node: Element,+ rootComp: HTMLElement,+ windowOf: (el: Element) => { start: number; end: number } | null,+ ): number[] | null => {+ const own = windowOf(node);+ if (!own || !Number.isFinite(own.start)) return null;+ const edges: number[] = [];+ for (const ahead of [0, LOOKAHEAD_SECONDS, RELEASE_LOOKAHEAD_SECONDS])+ edges.push(own.start - ahead, own.end - ahead, own.end + ahead);+ for (let el = node.parentElement; el && el !== rootComp; el = el.parentElement) {+ if (!el.hasAttribute("data-start")) continue;+ const span = windowOf(el);+ if (!span || !Number.isFinite(span.start)) return null;+ edges.push(span.start, span.end);+ }+ return edges.filter((edge) => Number.isFinite(edge));+ };const unchangedSincePlayedFrom = (from: number,to: number,rootComp: HTMLElement | null,+ timingRevision: number,): ((node: Element) => boolean) | null => {if (!rootComp || !(Math.abs(to - from) <= PLAYED_STEP_MAX_SECONDS)) return null;- const index = resolveMediaClipIndex();- let windows = mediaWindows.get(index);- if (!windows) {- windows = new Map(index.clips.map((clip) => [clip.el, clip]));- mediaWindows.set(index, windows);+ if (decisionEdgesRevision !== timingRevision) {+ decisionEdges.clear();+ decisionEdgesRevision = timingRevision;+ return null;}+ const windowOptions = {+ timelineRegistry: window.__timelines ?? {},+ resolver: timingResolverFor(true),+ };+ const windowOf = (el: Element) => runtimeElementWindow(el, windowOptions);const lo = Math.min(from, to);const hi = Math.max(from, to);- const near = (start: number, end: number, a: number, b: number) =>- start - VISIBILITY_REACH_SECONDS <= b && end + VISIBILITY_REACH_SECONDS >= a;return (node) => {- if (!isMediaElement(node) || node.parentElement !== rootComp) return false;- if (visibilityDecided.get(node) !== false || mediaNearPlayhead.get(node) === true)+ const decided = visibilityDecided.get(node);+ if (+ decided === undefined ||+ (isMediaElement(node) && (armedBy.has(node) || armedFrom.has(node)))+ )return false;- if (armedBy.has(node) || armedFrom.has(node)) return false;- const span = windows.get(node);- if (!span || !Number.isFinite(span.start)) return false;- const end = Number.isFinite(span.end) ? span.end : Infinity;- if (near(span.start, end, lo, hi)) return false;- return (- loopStartSeconds === null || !near(span.start, end, loopStartSeconds, loopStartSeconds)- );+ // A script that changed a clip's visibility gets it decided again, as a full pass would.+ if (+ !node.hasAttribute(RUNTIME_STAMPED_ATTR) &&+ (node as HTMLElement).style.visibility !== (decided ? "visible" : "hidden")+ )+ return false;+ let edges = decisionEdges.get(node);+ if (edges === undefined) {+ edges = decisionEdgesOf(node, rootComp, windowOf);+ decisionEdges.set(node, edges);+ }+ if (!edges) return false;+ for (const edge of edges) {+ if (edge >= lo && edge <= hi) return false;+ if (+ loopStartSeconds !== null &&+ Math.abs(edge - loopStartSeconds) <= VISIBILITY_REACH_SECONDS+ )+ return false;+ }+ return true;};};@@ -2932,9 +2981,9 @@ export function initSandboxRuntimeModular(): void {let decidedTimedClip = false;const visibleAt = timedVisibilityAt(timingRevision);const unchanged =- playedFromSeconds === undefined+ playedFromSeconds === undefined || timingRevision === undefined? null- : unchangedSincePlayedFrom(playedFromSeconds, currentTime, rootComp);+ : unchangedSincePlayedFrom(playedFromSeconds, currentTime, rootComp, timingRevision);for (const rawNode of visibilityNodes) {if (!isStylableElement(rawNode)) continue;@@ -3101,8 +3150,7 @@ export function initSandboxRuntimeModular(): void {resolveHostSlots: resolveMediaHostSlots,});- const resolveMediaClipIndex = (): MediaClipIndex => {- const revision = readCompositionTimingRevision();+ const resolveMediaClipIndex = (revision = readCompositionTimingRevision()): MediaClipIndex => {if (mediaClipIndex && mediaClipIndex.revision === revision) return mediaClipIndex;const clips = buildRuntimeMediaCache().mediaClips;mediaClipIndex = {@@ -3303,7 +3351,7 @@ export function initSandboxRuntimeModular(): void {: 0;const mediaClips = withTimingResolver(() => {if (!indexed) return buildRuntimeMediaCache().mediaClips;- const index = resolveMediaClipIndex();+ const index = resolveMediaClipIndex(timingRevision);// Windows come from the index, but every field handed to `syncRuntimeMedia`// is re-read here: `el.duration` is reset by any `el.load()` the retry path// made last pass, and a cached copy of it would be exactly the stale@@ -3378,6 +3426,7 @@ export function initSandboxRuntimeModular(): void {// this the one place a parked transport needs to hear about, instead of a// wake call bolted onto each mutator (and forgotten on the next one).if (force) wakeTransport();+ if (force && timelinePostHeld && !clock.isPlaying()) postTimeline();const frame = Math.max(0, Math.round((state.currentTime || 0) * state.canonicalFps));const now = Date.now();const shouldPost =@@ -3419,6 +3468,24 @@ export function initSandboxRuntimeModular(): void {}return sig;};+ // Studio re-reads the whole manifest on every post, so playback holds them until it stops.+ let timelinePostHeld = false;+ // A jump paints its frame first; the posts its scene mounts cause become one after.+ let jumpLanding = false;+ let timelinePostAfterSeek = false;+ const postTimelineAfterSeek = () => {+ if (timelinePostAfterSeek) return;+ timelinePostAfterSeek = true;+ void seekCompletionsSettled().then(() =>+ window.requestAnimationFrame(() =>+ setTimeout(() => {+ timelinePostAfterSeek = false;+ jumpLanding = false;+ if (!state.tornDown) postTimeline();+ }),+ ),+ );+ };const postTimeline = () => {sanitizeCompositionDurationAttributes();applyCompositionSizing();@@ -3432,6 +3499,11 @@ export function initSandboxRuntimeModular(): void {}}bindRootTimelineIfAvailable();+ timelinePostHeld = clock.isPlaying();+ if (timelinePostHeld) return;+ if (jumpLanding && !renderCaptureSeekStarted && hasPendingSeekCompletion())+ return postTimelineAfterSeek();+ jumpLanding = false;const payload = collectRuntimeTimelinePayload({canonicalFps: state.canonicalFps,});@@ -4024,15 +4096,18 @@ export function initSandboxRuntimeModular(): void {state.currentTime = clock.now();state.isPlaying = false;state.mediaForceSyncNextTick = true;+ const keepPlaying = options?.keepPlaying && wasPlaying;+ jumpLanding = !keepPlaying;+ // Decoding starts before the page renders, so the two overlap.+ if (!keepPlaying) syncMediaForCurrentState();const tl = state.capturedTimeline;pauseTimelineIfPossible(tl);const pageAnimations = seekTimelineAndAdapters(state.currentTime);runAdapters("pause", 0, pageAnimations);- if (options?.keepPlaying && wasPlaying) {+ if (keepPlaying) {transport.play();return;}- syncMediaForCurrentState();colorGrading.redraw();paintVfx(state.currentTime);postState(true);@@ -4450,38 +4525,85 @@ export function initSandboxRuntimeModular(): void {standaloneHosts.set(compositionId, node ?? { missingAt: compositionTimingRevision });return node;};+ type SceneClock = {+ start: number;+ hiddenFrom: number;+ playbackStart: number;+ rate: ReturnType<typeof readElementRateSpec>;+ timelineDuration: number | null;+ };+ let sceneClocks = new WeakMap<object, SceneClock | null>();+ let sceneClocksRevision = -1;+ const sceneClockOf = (compositionId: string, timeline: RuntimeTimelineLike) => {+ if (sceneClocks.has(timeline)) return sceneClocks.get(timeline)!;+ const node = standaloneHostOf(compositionId);+ const start = node ? resolveStartForElement(node, 0) : Number.NaN;+ const clock: SceneClock | null =+ node && Number.isFinite(start)+ ? {+ start,+ // Rest only where the visibility pass hides the host.+ hiddenFrom:+ node.hasAttribute("data-start") && !isDiscoveryStamped(node)+ ? start + (resolveDurationForElement(node) ?? Infinity)+ : Infinity,+ playbackStart: readElementPlaybackStart(node),+ rate: readElementRateSpec(node),+ timelineDuration: getTimelineDurationSeconds(timeline),+ }+ : null;+ sceneClocks.set(timeline, clock);+ return clock;+ };+ const restingAt = new WeakMap<object, number>();+ const restsAt = (timeline: RuntimeTimelineLike, time: number): boolean => {+ if (restingAt.get(timeline) !== time) return false;+ try {+ return (+ (timeline.paused as () => unknown)() === true && Number(timeline.totalTime?.()) === time+ );+ } catch {+ return false;+ }+ };const seekStandaloneRegisteredTimelines = (timeSeconds: number,+ revision: number,options?: RuntimeSeekOptions & { activateChildren?: boolean },) => {+ if (sceneClocksRevision !== revision || renderCaptureSeekStarted) {+ sceneClocks = new WeakMap();+ sceneClocksRevision = revision;+ }const timelines = (window.__timelines ?? {}) as Record<string, RuntimeTimelineLike | undefined>;const rootCompositionId =resolveRootCompositionElement()?.getAttribute("data-composition-id") ?? null;for (const [compositionId, timeline] of Object.entries(timelines)) {- if (!timeline || compositionId === rootCompositionId) continue;- const node = standaloneHostOf(compositionId);- if (!node) continue;- const start = resolveStartForElement(node, 0);- if (!Number.isFinite(start)) continue;- // Rest only where the visibility pass hides the host and no rearmed root propagation moves it.- const hiddenFrom =- !options?.activateChildren && node.hasAttribute("data-start") && !isDiscoveryStamped(node)- ? start + (resolveDurationForElement(node) ?? Infinity)- : Infinity;- const timelineDuration = getTimelineDurationSeconds(timeline);+ if (!timeline || typeof timeline !== "object" || compositionId === rootCompositionId)+ continue;+ const scene = sceneClockOf(compositionId, timeline);+ if (!scene) continue;const sourceTime =- readElementPlaybackStart(node) ++ scene.playbackStart +sourceTimeAt(- readElementRateSpec(node),- Math.max(0, Math.min(timeSeconds, hiddenFrom) - start),+ scene.rate,+ // A rearmed child follows root propagation, so it rests nowhere the root would not.+ Math.max(+ 0,+ (options?.activateChildren ? timeSeconds : Math.min(timeSeconds, scene.hiddenFrom)) -+ scene.start,+ ),);const localTime = Math.max(0,- timelineDuration != null && timelineDuration > 0- ? Math.min(timelineDuration, sourceTime)+ scene.timelineDuration != null && scene.timelineDuration > 0+ ? Math.min(scene.timelineDuration, sourceTime): sourceTime,);+ // A scene before or after its clip rests where the last frame left it: no redraw.+ if (!renderCaptureSeekStarted && restsAt(timeline, localTime)) continue;seekRuntimeTimeline(timeline, localTime, "runtime.init.transport.childTimeline", options);+ restingAt.set(timeline, localTime);}};@@ -4578,7 +4700,7 @@ export function initSandboxRuntimeModular(): void {*/function seekTimelineAndAdapters(t: number,- opts?: { activateChildren?: boolean; suppressEvents?: boolean },+ opts?: { activateChildren?: boolean; suppressEvents?: boolean; revision?: number },): () => Animation[] {const tl = state.capturedTimeline;// Critical for a sub-composition whose data-start is at or near 0: it is added@@ -4598,10 +4720,10 @@ export function initSandboxRuntimeModular(): void {function seekRootChildrenAndAdapters(tl: RuntimeTimelineLike | null,t: number,- opts?: { activateChildren?: boolean; suppressEvents?: boolean },+ opts?: { activateChildren?: boolean; suppressEvents?: boolean; revision?: number },): () => Animation[] {const suppressEvents = opts?.suppressEvents === true;- const revision = readCompositionTimingRevision();+ const revision = opts?.revision ?? readCompositionTimingRevision();if (revision !== tweensRenderedThrough.revision)tweensRenderedThrough = { revision, time: Number.NEGATIVE_INFINITY };const jumped = !(Math.abs(t - lastTransportSeekTime) <= 1.5 / state.canonicalFps);@@ -4656,7 +4778,7 @@ export function initSandboxRuntimeModular(): void {// safe because nothing between frames READS a sibling's paused() — grepped over// the deterministic adapters, syncTimedElementVisibility, the hf-timelines-built// handler and __hfReseekGpu. It only ever moved the state the seek left behind.- seekStandaloneRegisteredTimelines(t, opts);+ seekStandaloneRegisteredTimelines(t, revision, opts);hideClipsAgain?.();tweensRenderedThrough.time = Math.max(tweensRenderedThrough.time, t);const pageAnimations = pageAnimationsForOnePass();@@ -4978,7 +5100,7 @@ export function initSandboxRuntimeModular(): void {// change happens to wake the loop again.postTimeline();compositionChangePending = false;- } else if (playingPollDue || stoppedPlaying) {+ } else if (stoppedPlaying) {postTimelineIfManifestChanged();}if (changeDrivenService || pausedCounterDue(MEDIA_BIND_INTERVAL_FRAMES)) {@@ -5018,7 +5140,7 @@ export function initSandboxRuntimeModular(): void {t !== lastTransportSeekTime ||state.capturedTimeline !== lastTransportSeekTimeline) {- seekTimelineAndAdapters(t);+ seekTimelineAndAdapters(t, { revision: timingRevision });if (!isPlaying) pausedSeekDeferredByManualGesture = false;}if (isPlaying) {diff --git a/packages/core/src/runtime/previewRasterHints.test.ts b/packages/core/src/runtime/previewRasterHints.test.tsindex 513a0d2623..be59388671 100644--- a/packages/core/src/runtime/previewRasterHints.test.ts+++ b/packages/core/src/runtime/previewRasterHints.test.ts@@ -1,6 +1,7 @@import { afterEach, describe, expect, it, vi } from "vitest";-import { PREVIEW_RASTER_ATTR } from "../studioPreviewMark";+import { PREVIEW_RASTER_ATTR, STUDIO_PREVIEW_UPCOMING_ATTR } from "../studioPreviewMark";import { setPreviewRasterScale } from "./previewRasterHints";+import { CLIP_HIDDEN_ATTR } from "./timedClipHide";function layer(style: string, parent: Element = document.body): HTMLElement {const element = document.createElement("div");@@ -158,6 +159,19 @@ describe("setPreviewRasterScale", () => {expect(marked(words)).toBe(true);});+ it("ignores the clip attributes the runtime toggles while playing", async () => {+ const scene = layer(POSITIONED);+ for (let i = 0; i < 20; i += 1) layer("position: absolute", scene);+ setPreviewRasterScale(0.5);+ const reads = vi.spyOn(window, "getComputedStyle");+ scene.toggleAttribute(STUDIO_PREVIEW_UPCOMING_ATTR, true);+ scene.toggleAttribute(CLIP_HIDDEN_ATTR, true);+ await Promise.resolve();+ expect(reads).not.toHaveBeenCalled();+ expect(marked(scene)).toBe(true);+ reads.mockRestore();+ });+it("re-checks the page when a linked stylesheet loads", async () => {const sheet = document.createElement("style");document.head.append(sheet);@@ -176,6 +190,39 @@ describe("setPreviewRasterScale", () => {link.remove();});+ it("decides every mark in a changed subtree before writing any, so the page restyles once", async () => {+ const sheet = document.createElement("style");+ sheet.textContent = "[data-flat] > div { position: static !important }";+ document.head.append(sheet);+ const panel = layer("");+ const rows = Array.from({ length: 10 }, () => layer(POSITIONED, panel));+ setPreviewRasterScale(0.5);+ expect(rows.every(marked)).toBe(true);+ const order: string[] = [];+ const realRead = window.getComputedStyle.bind(window);+ const reads = vi.spyOn(window, "getComputedStyle").mockImplementation((el, pseudo) => {+ order.push("read");+ return realRead(el, pseudo);+ });+ const realToggle = Element.prototype.toggleAttribute;+ const writes = vi.spyOn(Element.prototype, "toggleAttribute").mockImplementation(function (+ this: Element,+ name: string,+ force?: boolean,+ ) {+ if (name === PREVIEW_RASTER_ATTR) order.push("write");+ return realToggle.call(this, name, force);+ });+ panel.setAttribute("data-flat", "");+ await Promise.resolve();+ reads.mockRestore();+ writes.mockRestore();+ expect(rows.some(marked)).toBe(false);+ expect(order.filter((step) => step === "write")).toHaveLength(10);+ expect(order.lastIndexOf("read")).toBeLessThan(order.indexOf("write"));+ sheet.remove();+ });+it("checks each element once for a batch of attribute changes inside one subtree", async () => {const group = layer(POSITIONED);const words = Array.from({ length: 20 }, () => layer("position: absolute", group));diff --git a/packages/core/src/runtime/previewRasterHints.ts b/packages/core/src/runtime/previewRasterHints.tsindex 36406b7a58..0277b6ab18 100644--- a/packages/core/src/runtime/previewRasterHints.ts+++ b/packages/core/src/runtime/previewRasterHints.ts@@ -1,5 +1,6 @@-import { PREVIEW_RASTER_ATTR } from "../studioPreviewMark";+import { PREVIEW_ONLY_ATTRS, PREVIEW_RASTER_ATTR } from "../studioPreviewMark";import { isElementNode } from "./domRealm";+import { CLIP_HIDDEN_ATTR } from "./timedClipHide";// Chromium rasters a `will-change: transform` layer at no less than device scale, so a preview shown// small rasters each one at full size. Below 1x the hint is dropped wherever nothing depends on the@@ -7,6 +8,8 @@ import { isElementNode } from "./domRealm";const TRANSFORM_HINTS = new Set(["transform", "translate", "rotate", "scale"]);const MARKED = `[${PREVIEW_RASTER_ATTR}]`;const DROP_HINTS = `${MARKED} { will-change: auto !important; }`;+// Toggled by the runtime as clips come and go; its own rules on them only hide, never restack.+const RUNTIME_PLAYBACK_ATTRS = new Set<string>([...PREVIEW_ONLY_ATTRS, CLIP_HIDDEN_ATTR]);let sheet: HTMLStyleElement | null = null;let observer: MutationObserver | null = null;@@ -58,13 +61,16 @@ function fits(element: Element, readOwnHints: boolean): boolean {return drop;}-function check(element: Element, readOwnHints: boolean): void {- const drop = fits(element, readOwnHints);+function writeMark(element: Element, drop: boolean): void {if (drop !== element.hasAttribute(PREVIEW_RASTER_ATTR)) {element.toggleAttribute(PREVIEW_RASTER_ATTR, drop);}}+function check(element: Element, readOwnHints: boolean): void {+ writeMark(element, fits(element, readOwnHints));+}+function unmarkHolders(element: Element): void {let holder = element.parentElement?.closest(MARKED);while (holder) {@@ -76,12 +82,17 @@ function unmarkHolders(element: Element): void {}function followTree(root: Element, batch: FollowBatch): void {+ const decidedBeforeAnyWrite: Array<[Element, boolean]> = [];for (const element of [root, ...root.querySelectorAll("*")]) {if (batch.checked.has(element)) continue;batch.checked.add(element);- check(element, element === root || batch.changed.has(element));- unmarkHolders(element);+ decidedBeforeAnyWrite.push([+ element,+ fits(element, element === root || batch.changed.has(element)),+ ]);}+ for (const [element, drop] of decidedBeforeAnyWrite) writeMark(element, drop);+ for (const [element] of decidedBeforeAnyWrite) unmarkHolders(element);}type FollowBatch = { changed: Set<Element>; checked: Set<Element> };@@ -111,7 +122,8 @@ function markLayers(): void {}function followAttribute(record: MutationRecord, batch: FollowBatch): void {- if (!isElementNode(record.target) || record.attributeName === PREVIEW_RASTER_ATTR) return;+ if (!isElementNode(record.target) || RUNTIME_PLAYBACK_ATTRS.has(record.attributeName ?? ""))+ return;// Any attribute can match a selector; inline style is the per-frame case, so it is cheaper.if (record.attributeName === "style") followStyle(record.target);else followTree(record.target, batch);@@ -129,7 +141,7 @@ function ownAttributeChanges(records: MutationRecord[]): Set<Element> {const changed = new Set<Element>();for (const record of records) {const name = record.attributeName;- if (name === null || name === "style" || name === PREVIEW_RASTER_ATTR) continue;+ if (name === null || name === "style" || RUNTIME_PLAYBACK_ATTRS.has(name)) continue;if (isElementNode(record.target)) changed.add(record.target);}return changed;diff --git a/packages/core/src/runtime/timeline.ts b/packages/core/src/runtime/timeline.tsindex 83e28000b0..21805116b7 100644--- a/packages/core/src/runtime/timeline.ts+++ b/packages/core/src/runtime/timeline.ts@@ -31,20 +31,16 @@ import { isElementNode, isMediaElement } from "./domRealm";import { LOOP_INFLATED_TIMELINE_SECONDS } from "./compositionLength";import { cutToHostSlots } from "../mediaTiming";-export function isRuntimeElementVisibleAt(+export function runtimeElementWindow(rawNode: Element,options: {- currentTime: number;- compositionDuration: number;- canonicalFps: number;- exportRenderSeek: boolean;timelineRegistry: Record<string, RuntimeTimelineLike | undefined>;resolver: ReturnType<typeof createRuntimeStartTimeResolver>;},-): boolean {+): { start: number; end: number } | null {const tag = rawNode.tagName.toLowerCase();if (tag === "script" || tag === "style" || tag === "link" || tag === "meta") {- return false;+ return null;}const isMedia = tag === "video" || tag === "audio";@@ -70,12 +66,29 @@ export function isRuntimeElementVisibleAt(duration = liveDuration;}}- const computedEnd =- duration != null && duration > 0 ? start + duration : Number.POSITIVE_INFINITY;+ return {+ start,+ end: duration != null && duration > 0 ? start + duration : Number.POSITIVE_INFINITY,+ };+}++export function isRuntimeElementVisibleAt(+ rawNode: Element,+ options: {+ currentTime: number;+ compositionDuration: number;+ canonicalFps: number;+ exportRenderSeek: boolean;+ timelineRegistry: Record<string, RuntimeTimelineLike | undefined>;+ resolver: ReturnType<typeof createRuntimeStartTimeResolver>;+ },+): boolean {+ const span = runtimeElementWindow(rawNode, options);+ if (!span) return false;// Export seeks snap to frame boundaries; interactive visibility uses authored seconds.const clipWindow = options.exportRenderSeek- ? exportClipWindow(start, computedEnd, options.canonicalFps)- : { start, end: computedEnd };+ ? exportClipWindow(span.start, span.end, options.canonicalFps)+ : span;return isClipVisibleAt(options.currentTime,clipWindow.start,diff --git a/packages/core/src/runtime/transportPark.test.ts b/packages/core/src/runtime/transportPark.test.tsindex 0906841ef8..447020089c 100644--- a/packages/core/src/runtime/transportPark.test.ts+++ b/packages/core/src/runtime/transportPark.test.ts@@ -735,7 +735,7 @@ describe("parked transport loop", () => {expect(posted.filter((m) => m["type"] === "timeline").length - before).toBe(0);});- it("posts a playing clip's new track, label and z-index on the next poll", async () => {+ it("posts a playing clip's new track, label and z-index once playback stops", async () => {mount('<div id="clip" data-start="0" data-duration="5" data-track-index="0" data-timeline-label="Before" style="z-index: 1"></div>',);@@ -748,7 +748,11 @@ describe("parked transport loop", () => {clip.setAttribute("data-timeline-label", "After");clip.style.zIndex = "9";await flushObservers();+ posted.length = 0;for (let i = 0; i < 130; i += 1) frame120Hz();+ expect(posted.filter((m) => m["type"] === "timeline")).toHaveLength(0);+ window.__player!.pause();+ for (let i = 0; i < 10; i += 1) frame120Hz();const clips = (posted.filter((m) => m["type"] === "timeline").at(-1)?.["clips"] ?? []) as Array<Record<string, unknown>>;diff --git a/packages/studio/src/player/hooks/previewMessageRouter.test.ts b/packages/studio/src/player/hooks/previewMessageRouter.test.tsnew file mode 100644index 0000000000..d5d7492822--- /dev/null+++ b/packages/studio/src/player/hooks/previewMessageRouter.test.ts@@ -0,0 +1,23 @@+import { describe, expect, it, vi } from "vitest";+import { createPreviewMessageHandler } from "./previewMessageRouter";++function route(isPlaying: boolean) {+ const enrich = vi.fn();+ const handle = createPreviewMessageHandler({+ iframeRef: { current: null },+ processTimelineMessageRef: { current: vi.fn() },+ enrichMissingCompositionsRef: { current: enrich },+ lastTimelineMessageRef: { current: 0 },+ getAdapter: () => null,+ syncTimelineElements: vi.fn(),+ });+ handle(new MessageEvent("message", { data: { source: "hf-preview", type: "state", isPlaying } }));+ return enrich;+}++describe("preview state messages", () => {+ it("re-reads the preview for missing compositions only while it is stopped", () => {+ expect(route(true)).not.toHaveBeenCalled();+ expect(route(false)).toHaveBeenCalledTimes(1);+ });+});diff --git a/packages/studio/src/player/hooks/previewMessageRouter.ts b/packages/studio/src/player/hooks/previewMessageRouter.tsindex 91eee73719..cee225f88f 100644--- a/packages/studio/src/player/hooks/previewMessageRouter.ts+++ b/packages/studio/src/player/hooks/previewMessageRouter.ts@@ -29,6 +29,7 @@ interface PreviewMessage {source?: string;type?: string;clips?: unknown;+ isPlaying?: unknown;}export interface PreviewMessageRouterDeps {@@ -68,6 +69,7 @@ function acceptedPreviewMessage(function handleStateMessage(deps: PreviewMessageRouterDeps,iframe: HTMLIFrameElement | null,+ playingHoldsClipListPosts: boolean,): void {try {if (usePlayerStore.getState().elements.length === 0) {@@ -76,7 +78,7 @@ function handleStateMessage(deps.processTimelineMessageRef.current(manifest);}}- if (Date.now() - deps.lastTimelineMessageRef.current > 500) {+ if (!playingHoldsClipListPosts && Date.now() - deps.lastTimelineMessageRef.current > 500) {deps.enrichMissingCompositionsRef.current();}} catch {}@@ -112,7 +114,7 @@ export function createPreviewMessageHandler(const iframe = deps.iframeRef.current;const data = acceptedPreviewMessage(e, iframe);if (!data) return;- if (data.type === "state") return handleStateMessage(deps, iframe);+ if (data.type === "state") return handleStateMessage(deps, iframe, data.isPlaying === true);if (data.type === "timeline" && Array.isArray(data.clips)) {handleTimelineMessage(deps, iframe, data as unknown as ClipManifestMessage);}