Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(core): the preview loads only the media near the playhead (#4682)
Miguel Ángel5a77dbcea12b
diff --git a/packages/core/src/compositionReadiness.test.ts b/packages/core/src/compositionReadiness.test.tsindex d03b9f19a4..5172460bca 100644--- a/packages/core/src/compositionReadiness.test.ts+++ b/packages/core/src/compositionReadiness.test.ts@@ -8,6 +8,7 @@ import {settleFirstFrameCompositionReadiness,} from "./compositionReadiness.js";import { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";+import { STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js";import { isRuntimeElementVisibleAt } from "./runtime/timeline.js";function docWith(bodyHtml: string): Document {@@ -179,6 +180,23 @@ describe("mediaReadinessInput", () => {await expect(pending).resolves.toBeUndefined();});+ it("waits on a later clip in Studio's preview only once the runtime has set it to buffer", () => {+ const html =+ '<video id="deferred" data-start="30" data-duration="5" preload="none" src="a.mp4"></video>' ++ '<video id="untrimmed" data-start="30" preload="metadata" src="b.mp4"></video>' ++ '<video id="due" data-start="30" data-duration="5" preload="auto" src="c.mp4"></video>';+ const preview = docWith(html);+ preview.head+ .appendChild(preview.createElement("meta"))+ .setAttribute("name", STUDIO_PREVIEW_MARK_META);+ const plain = docWith(html);+ const ids = (doc: Document) =>+ scanPendingCompositionAssets(doc, { scope: "all" }).pendingMedia.map((el) => el.id);++ expect(ids(preview)).toEqual(["due"]);+ expect(ids(plain)).toEqual(["deferred", "untrimmed", "due"]);+ });+it("waits on a later scene's image in the full scan unless it is lazy", () => {const doc = docWith('<div data-start="30" data-duration="5"><img id="later" src="later.png">' +diff --git a/packages/core/src/compositionReadiness.ts b/packages/core/src/compositionReadiness.tsindex 3dfb9f97e9..49c69f1b17 100644--- a/packages/core/src/compositionReadiness.ts+++ b/packages/core/src/compositionReadiness.ts@@ -1,6 +1,7 @@import { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";import { isRuntimeElementVisibleAt } from "./runtime/timeline.js";import type { RuntimeTimelineLike } from "./runtime/types.js";+import { STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js";/** A composition is "ready" once every declared input settles, not just once* its duration is known. Each input returns null (nothing to wait on) or a@@ -103,9 +104,18 @@ export function scanPendingCompositionAssets(timelineRegistry: runtimeWindow?.__timelines,includeAuthoredTimingAttrs: true,});+ // Studio's preview buffers a clip off the first frame only once it comes due, as with a lazy image.+ const defersMedia = doc.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`) !== null;const pendingMedia = Array.from(doc.querySelectorAll("video, audio")).filter(isRealmHtmlMediaElement)- .filter((el) => shouldIncludeAsset(el, scope, resolver, runtimeWindow?.__timelines ?? {}))+ .filter((el) =>+ shouldIncludeAsset(+ el,+ defersMedia && el.preload !== "auto" ? FIRST_FRAME_READINESS_SCOPE : scope,+ resolver,+ runtimeWindow?.__timelines ?? {},+ ),+ ).filter((el) => el.readyState < HAVE_FUTURE_DATA);// A lazy image off the first frame loads only once shown, so no scope can wait on it.const pendingImages = Array.from(doc.querySelectorAll("img"))diff --git a/packages/core/src/runtime/entry.test.ts b/packages/core/src/runtime/entry.test.tsindex 726b4e82b9..abc146f400 100644--- a/packages/core/src/runtime/entry.test.ts+++ b/packages/core/src/runtime/entry.test.ts@@ -303,6 +303,331 @@ describe("runtime entry", () => {expect(document.querySelectorAll("style[data-hf-first-pass-hide]")).toHaveLength(0);});+ describe("media preload window", () => {+ const loads: HTMLMediaElement[] = [];+ const spyLoad = () => {+ loads.length = 0;+ vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(+ function (this: HTMLMediaElement) {+ loads.push(this);+ },+ );+ };+ const loadsOf = (...els: HTMLMediaElement[]) =>+ els.map((el) => loads.filter((loaded) => loaded === el).length);+ const videos = (root: HTMLElement, ...starts: string[]) =>+ starts.map((start) => timed(root, "video", start));+ const armed = (...els: HTMLMediaElement[]) => els.map((el) => el.preload === "auto");+ afterEach(() => vi.restoreAllMocks());++ it("loads a preview's media only on screen or due within the look-ahead, and frees what the playhead left", async () => {+ spyLoad();+ servePreview();+ const [now, soon, later] = videos(mountRoot(), "0", "1.5", "9");++ await evaluateRuntime();+ expect(armed(now, soon, later)).toEqual([true, true, false]);+ expect(later.preload).toBe("none");+ expect(loadsOf(now, soon, later)).toEqual([1, 1, 0]);++ now.setAttribute("src", "now.mp4");+ const srcWrites = new MutationObserver(() => {});+ srcWrites.observe(now, { attributeFilter: ["src"] });+ window.__player?.seek(7.6);+ expect(armed(now, soon, later)).toEqual([false, false, true]);+ // The two that left are reloaded empty with src dropped for the reload, ending any fetch in flight.+ expect([now.preload, soon.preload]).toEqual(["none", "none"]);+ expect(loadsOf(now, soon, later)).toEqual([2, 2, 1]);+ expect(srcWrites.takeRecords()).toHaveLength(2);+ expect(now.getAttribute("src")).toBe("now.mp4");+ });++ it("loads a clip with no authored length as a render does, so no reload drops its duration", async () => {+ spyLoad();+ servePreview();+ const [untrimmed, later] = videos(mountRoot(), "0", "8");+ untrimmed.removeAttribute("data-duration");+ later.removeAttribute("data-duration");+ Object.defineProperty(untrimmed, "readyState", { value: 1 });+ Object.defineProperty(untrimmed, "duration", { value: 2 });++ await evaluateRuntime();+ window.__player?.seek(3.6);+ window.__player?.seek(0.5);+ window.__player?.seek(7.5);+ expect(armed(untrimmed, later)).toEqual([true, true]);+ expect(loadsOf(untrimmed, later)).toEqual([1, 1]);+ });++ it("arms a clip a far jump lands on without reloading it under the seek", async () => {+ spyLoad();+ servePreview();+ const [now, later] = videos(mountRoot(), "0", "8");++ await evaluateRuntime();+ const beforeJump = loadsOf(later)[0];+ window.__player?.seek(8.4);+ expect(armed(now, later)).toEqual([false, true]);+ expect(loadsOf(later)).toEqual([beforeJump]);+ });++ // jsdom's media elements cannot play; the runtime only needs play() to settle.+ const stubPlayback = () => {+ vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(undefined);+ vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {});+ };+ const tracks = (root: HTMLElement) => {+ stubPlayback();+ const [now, next, afterNext, otherTrack] = videos(root, "0", "4", "7", "3");+ otherTrack.setAttribute("data-track-index", "2");+ return { now, next, afterNext, otherTrack };+ };++ it("arms the next clip on a clip's track when it starts playing, and nothing else", async () => {+ spyLoad();+ servePreview();+ const { next, afterNext, otherTrack } = tracks(mountRoot());++ await evaluateRuntime();+ window.__player?.play();+ expect(armed(next, afterNext, otherTrack)).toEqual([true, false, false]);+ expect(loadsOf(next, afterNext, otherTrack)).toEqual([1, 0, 0]);+ });++ it("arms nothing ahead of a paused playhead, and keeps what it armed through a pause", async () => {+ spyLoad();+ servePreview();+ const { next, afterNext } = tracks(mountRoot());++ await evaluateRuntime();+ window.__player?.seek(0.5);+ expect(armed(next)).toEqual([false]);+ window.__player?.play();+ window.__player?.pause();+ expect(armed(next, afterNext)).toEqual([true, false]);+ expect(loadsOf(next)).toEqual([1]);+ });++ it("holds a clip past the look-ahead edge a scrub crosses, and frees it once well clear", async () => {+ spyLoad();+ servePreview();+ const [, later] = videos(mountRoot(), "0", "5");++ await evaluateRuntime();+ window.__player?.seek(3.5);+ window.__player?.seek(2.5);+ window.__player?.seek(3.5);+ expect(armed(later)).toEqual([true]);+ expect(loadsOf(later)).toEqual([1]);+ window.__player?.seek(0.5);+ expect(armed(later)).toEqual([false]);+ });++ it("keeps the clip a step back across a cut returns to loaded", async () => {+ spyLoad();+ servePreview();+ const [outgoing, incoming] = videos(mountRoot(), "0", "2");++ await evaluateRuntime();+ window.__player?.seek(2.1);+ expect(armed(outgoing, incoming)).toEqual([true, true]);+ window.__player?.seek(1.9);+ expect(armed(outgoing)).toEqual([true]);+ expect(loadsOf(outgoing)).toEqual([1]);+ });++ it("arms a clip behind a playhead shuttling back before the playhead reaches it", async () => {+ spyLoad();+ servePreview();+ const [earlier, current] = videos(mountRoot(), "0", "2");+ current.setAttribute("data-duration", "6");++ await evaluateRuntime();+ window.__player?.seek(6.5);+ expect(armed(earlier)).toEqual([false]);+ let armedAt: number | undefined;+ for (let tenths = 45; tenths >= 20 && armedAt === undefined; tenths--) {+ window.__player?.seek(tenths / 10);+ if (armed(earlier)[0]) armedAt = tenths / 10;+ }+ // Within the 2 s look-ahead of its end, as a clip ahead is armed within 2 s of its start.+ expect(armedAt).toBe(4);+ });++ it("still frees a clip that ended well behind the playhead", async () => {+ spyLoad();+ servePreview();+ const [earlier, current] = videos(mountRoot(), "0", "2");+ current.setAttribute("data-duration", "6");++ await evaluateRuntime();+ window.__player?.seek(2.1);+ window.__player?.seek(5.9);+ expect(armed(earlier)).toEqual([true]);+ expect(loadsOf(earlier)).toEqual([1]);+ window.__player?.seek(6.1);+ expect(armed(earlier)).toEqual([false]);+ expect(earlier.preload).toBe("none");+ expect(loadsOf(earlier)).toEqual([2]);+ });++ it("drops an arm once the clip that set it leaves the screen, as a jump while playing does", async () => {+ spyLoad();+ servePreview();+ const { next } = tracks(mountRoot());+ window.__timelines = { main: pausedTimeline(20) };++ await evaluateRuntime();+ window.__player?.play();+ window.__player?.seek(10.5, { keepPlaying: true });+ expect(armed(next)).toEqual([false]);+ });++ it("stops holding a clip it armed once that clip has played", async () => {+ spyLoad();+ servePreview();+ const { next } = tracks(mountRoot());+ window.__timelines = { main: pausedTimeline(20) };++ await evaluateRuntime();+ window.__player?.play();+ window.__player?.seek(4.5, { keepPlaying: true });+ window.__player?.seek(10.5, { keepPlaying: true });+ expect(armed(next)).toEqual([false]);+ });++ it("arms a clip that starts exactly where the playing one ends", async () => {+ spyLoad();+ servePreview();+ stubPlayback();+ // 1.1 + 2.2 is 3.3000000000000003: back to back only within the boundary tolerance.+ const [first, second] = videos(mountRoot(), "1.1", "3.3");+ first.setAttribute("data-duration", "2.2");++ await evaluateRuntime();+ window.__player?.seek(1.2);+ window.__player?.play();+ expect(armed(second)).toEqual([true]);+ });++ it("arms the next clip only in the same host and on an authored track", async () => {+ spyLoad();+ servePreview();+ stubPlayback();+ const root = mountRoot();+ const [now] = videos(root, "0");+ const host = timed(root, "div", "0");+ host.setAttribute("data-composition-id", "inner");+ host.setAttribute("data-duration", "10");+ const [otherHost] = videos(host, "4");+ const [untracked, untrackedNext] = videos(root, "0", "4");+ for (const el of [untracked, untrackedNext]) el.removeAttribute("data-track-index");+ now.setAttribute("data-track-index", "1");++ await evaluateRuntime();+ window.__player?.play();+ expect(armed(otherHost, untrackedNext)).toEqual([false, false]);+ });++ it("arms a short nested clip whose start is already root time before it is due", async () => {+ spyLoad();+ servePreview();+ const host = timed(mountRoot(), "div", "5");+ host.setAttribute("data-composition-id", "inner");+ host.setAttribute("data-duration", "5");+ const [clip] = videos(host, "6");+ clip.setAttribute("data-hf-media-start-basis", "global");+ clip.setAttribute("data-duration", "0.4");++ await evaluateRuntime();+ window.__player?.seek(4.5);+ expect(armed(clip)).toEqual([true]);+ });++ it("keeps the clips due at the loop start loaded while the loop plays", async () => {+ spyLoad();+ servePreview();+ const { now, next, afterNext } = tracks(mountRoot());+ window.__timelines = { main: pausedTimeline(20) };++ await evaluateRuntime();+ window.__hf?.setLoopStart?.(0);+ window.__player?.play();+ window.__player?.seek(19.5, { keepPlaying: true });+ expect(armed(now, next, afterNext)).toEqual([true, true, false]);+ expect(loadsOf(now, next)).toEqual([1, 1]);+ window.__hf?.setLoopStart?.(null);+ window.__player?.seek(19.6, { keepPlaying: true });+ expect(armed(now)).toEqual([false]);+ });++ it("holds the loop start only while playing, and for a loop start that is a time", async () => {+ spyLoad();+ servePreview();+ const { now } = tracks(mountRoot());++ await evaluateRuntime();+ window.__hf?.setLoopStart?.(0);+ window.__player?.play();+ window.__player?.seek(9.5, { keepPlaying: true });+ window.__player?.pause();+ expect(armed(now)).toEqual([false]);+ window.__hf?.setLoopStart?.(-1);+ window.__player?.play();+ expect(armed(now)).toEqual([false]);+ });++ it("never arms a next clip with no authored length, so it is not reloaded", async () => {+ spyLoad();+ servePreview();+ const { next } = tracks(mountRoot());+ next.removeAttribute("data-duration");++ await evaluateRuntime();+ const atBind = loadsOf(next)[0];+ window.__player?.play();+ expect(loadsOf(next)).toEqual([atBind]);+ });++ it("keeps an armed next clip through a window pass that finds it outside the look-ahead", async () => {+ spyLoad();+ servePreview();+ const { next } = tracks(mountRoot());++ await evaluateRuntime();+ window.__player?.play();+ window.__player?.seek(0.5, { keepPlaying: true });+ expect(armed(next)).toEqual([true]);+ expect(loadsOf(next)).toEqual([1]);+ });++ it("gives each parsed clip after the first frame preload none before it can fetch", async () => {+ servePreview();+ const root = mountRoot();+ timed(root, "div", "0");+ await evaluateRuntime();+ const [first, later] = videos(root, "0", "5");+ const untrimmed = timed(root, "audio", "5");+ untrimmed.removeAttribute("data-duration");+ await Promise.resolve();+ expect([first.preload, later.preload, untrimmed.preload]).toEqual(["", "none", ""]);+ });++ it("keeps a render loading every media element at bind", async () => {+ spyLoad();+ const [now, soon, later] = videos(mountRoot(), "0", "1.5", "5");++ await evaluateRuntime();+ window.__player?.seek(3.6);+ expect(armed(now, soon, later)).toEqual([true, true, true]);+ expect(loadsOf(now, soon, later)).toEqual([1, 1, 1]);+ const parsedLater = timed(document.body, "video", "5");+ await Promise.resolve();+ expect(parsedLater.preload).not.toBe("none");+ expect((window as { __hfMediaDeferral?: unknown }).__hfMediaDeferral).toBeUndefined();+ });+ });+it("grades media inside a clip once the first pass shows the clip, with no seek", async () => {const getContext = vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null);const scene = timed(mountRoot(), "div", "0");diff --git a/packages/core/src/runtime/entry.ts b/packages/core/src/runtime/entry.tsindex cd8cf63196..e445e1c89d 100644--- a/packages/core/src/runtime/entry.ts+++ b/packages/core/src/runtime/entry.ts@@ -1,5 +1,6 @@import { initSandboxRuntimeModular, installAuthoredMediaCapture } from "./init";import { installAuthoredOpacityCapture } from "./colorGrading";+import { deferMediaUntilDue } from "./preloadMedia";import { hideTimedClipsUntilFirstPass } from "./timedClipHide";import { fitTextFontSize } from "../text/fitTextFontSize";import { pretext } from "../text/pretext";@@ -31,6 +32,7 @@ installAuthoredOpacityCapture();installAuthoredMediaCapture();hideTimedClipsUntilFirstPass();+deferMediaUntilDue();// Expose runtime helpers immediately so composition scripts can use them// before DOMContentLoaded (font sizing runs during script evaluation, anddiff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.tsindex 8e9e329eee..774fb9e579 100644--- a/packages/core/src/runtime/init.ts+++ b/packages/core/src/runtime/init.ts@@ -1,5 +1,5 @@// fallow-ignore-file code-duplication complexity-import { preloadMedia } from "./preloadMedia";+import { preloadMedia, releaseMedia, lengthIsAuthored, stopMediaDownload } from "./preloadMedia";import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge";import { instantTolerance } from "../clipFacts";import { isInClipWindow } from "./clipWindow";@@ -50,6 +50,7 @@ import {collectRuntimeTimelinePayload,isRuntimeElementVisibleAt,LOOP_INFLATED_TIMELINE_SECONDS,+ parseAuthoredTrack,} from "./timeline";import {findRootCompositionElement,@@ -2416,9 +2417,7 @@ export function initSandboxRuntimeModular(): void {if (mediaEl.isConnected) continue;unbindMedia(mediaEl);mediaEl.pause();- for (const source of mediaEl.querySelectorAll("source")) source.remove();- mediaEl.removeAttribute("src");- mediaEl.load();+ stopMediaDownload(mediaEl);}};@@ -2430,6 +2429,7 @@ export function initSandboxRuntimeModular(): void {const bindMediaMetadataListeners = () => {if (state.tornDown) return;const mediaEls = Array.from(document.querySelectorAll("video, audio")) as HTMLMediaElement[];+ const nearPlayheadPending: HTMLMediaElement[] = [];for (const mediaEl of mediaEls) {if (metadataBoundMedia.has(mediaEl)) continue;metadataBoundMedia.add(mediaEl);@@ -2464,7 +2464,9 @@ export function initSandboxRuntimeModular(): void {// mode, for <audio>, or when the codec map is absent.maybeProxyProactively(mediaEl);- preloadMedia(mediaEl);+ // Studio's preview loads a timed clip only near the playhead; the visibility pass decides.+ if (inPreloadWindow(mediaEl)) nearPlayheadPending.push(mediaEl);+ else preloadMedia(mediaEl);// Probe volume automation from the GSAP timeline — same approach as the// renderer (see discoverAudioVolumeAutomationFromTimeline / audioMixer).@@ -2473,6 +2475,8 @@ export function initSandboxRuntimeModular(): void {// fires after the timeline has been captured (every 30 transport ticks).probeAndCacheVolumeKeyframes(mediaEl);}+ if (nearPlayheadPending.length > 0)+ syncTimedElementVisibility(state.currentTime, nearPlayheadPending);};const probeAndCacheVolumeKeyframes = (mediaEl: HTMLMediaElement) => {@@ -2633,15 +2637,101 @@ export function initSandboxRuntimeModular(): void {const hiddenImagesSkipped = skipsHiddenImages();const LOOKAHEAD_SECONDS = 2;- // Unskipped while due in the look-ahead window, so a clip shorter than the window still loads first.+ const RELEASE_LOOKAHEAD_SECONDS = 4;+ // Unskipped while within the window on either side of the playhead, so a step or shuttle back+ // across a cut finds the clip it left still loaded; the edge checks catch a clip shorter than it.const dueSoon = (node: HTMLElement,visibleAt: ReturnType<typeof timedVisibilityAt>,t: number,+ ahead = LOOKAHEAD_SECONDS,) => {- const start = resolveStartForElement(node, Number.NaN);- return (start > t && start <= t + LOOKAHEAD_SECONDS) || visibleAt(node, t + LOOKAHEAD_SECONDS);+ const start = isMediaElement(node)+ ? resolveAbsoluteMediaStartSeconds(node)+ : resolveStartForElement(node, Number.NaN);+ const end = start + (resolveDurationForElement(node) ?? Number.NaN);+ return (+ (start > t && start <= t + ahead) ||+ (end <= t && end >= t - ahead) ||+ visibleAt(node, t + ahead)+ );+ };+ // Where Studio's loop wraps to (window.__hf.setLoopStart); null when it does not loop.+ let loopStartSeconds: number | null = null;+ // A clip with no authored length loads as in a render: a reload would drop the duration it is+ // timed by.+ const inPreloadWindow = (el: HTMLMediaElement) =>+ hiddenImagesSkipped && el.hasAttribute("data-start") && lengthIsAuthored(el);++ // Media on screen or due within the look-ahead loads; the rest stops holding a connection to+ // the origin that also serves Studio's thumbnails.+ const mediaNearPlayhead = new WeakMap<HTMLMediaElement, boolean>();+ // During playback a clip that starts arms the next clip on its track, which a cold 2 s look-ahead+ // cannot fetch in time on a slow link.+ const armedBy = new WeakMap<HTMLMediaElement, HTMLMediaElement>();+ const armedFrom = new WeakSet<HTMLMediaElement>();+ // An untracked clip (NaN) shares a track with nothing, as in the timeline payload.+ const hostOf = (el: Element) => el.parentElement?.closest("[data-composition-id]");+ const armNextOnTrack = (el: HTMLMediaElement) => {+ const track = parseAuthoredTrack(el, Number.NaN);+ const clips = buildRuntimeMediaCache(+ Array.from(metadataBoundMedia).filter(+ (m) =>+ m.isConnected && parseAuthoredTrack(m, Number.NaN) === track && hostOf(m) === hostOf(el),+ ),+ ).mediaClips;+ const end = clips.find((clip) => clip.el === el)?.end ?? Number.NaN;+ let next: (typeof clips)[number] | undefined;+ for (const clip of clips)+ if (+ clip.el !== el &&+ clip.start >= end - instantTolerance(end) &&+ !(next && next.start <= clip.start)+ )+ next = clip;+ if (!next || !inPreloadWindow(next.el)) return;+ armedBy.set(next.el, el);+ if (mediaNearPlayhead.get(next.el) !== true) {+ mediaNearPlayhead.set(next.el, true);+ preloadMedia(next.el);+ }+ };+ const preloadNearPlayhead = (el: HTMLMediaElement, visible: boolean, upcoming: boolean) => {+ const playing = clock.isPlaying();+ if (!(visible && playing)) armedFrom.delete(el);+ else if (!armedFrom.has(el)) {+ armedFrom.add(el);+ armNextOnTrack(el);+ }+ // An arm lasts while the clip that set it plays on screen, so a jump away drops it.+ const armer = armedBy.get(el);+ const near = visible || upcoming || (armer !== undefined && armedFrom.has(armer));+ const decided = mediaNearPlayhead.get(el);+ if (decided === near) return;+ mediaNearPlayhead.set(el, near);+ if (!near) {+ if (el.preload !== "none") {+ el.preload = "none";+ // Frees a clip the window armed; on one the parser started, it would only restart the fetch.+ if (decided) releaseMedia(el);+ }+ } else if (!visible || decided === undefined) {+ // Not a clip a jump lands on: the media sync arms that one, and load() would undo its seek.+ preloadMedia(el);+ }};+ // Held past the look-ahead so a scrub across its edge does not refetch, and while playing when+ // due at the loop start.+ const mediaStaysDue = (+ el: HTMLMediaElement,+ visibleAt: ReturnType<typeof timedVisibilityAt>,+ t: number,+ ) =>+ (mediaNearPlayhead.get(el) === true && dueSoon(el, visibleAt, t, RELEASE_LOOKAHEAD_SECONDS)) ||+ (loopStartSeconds !== null &&+ clock.isPlaying() &&+ (visibleAt(el, loopStartSeconds) ||+ dueSoon(el, visibleAt, loopStartSeconds, RELEASE_LOOKAHEAD_SECONDS)));const applyTimedElementVisibility = (currentTime: number,@@ -2679,10 +2769,15 @@ export function initSandboxRuntimeModular(): void {visibleAt(rawNode, currentTime) &&clipChainVisibleAt(rawNode.parentElement, currentTime, visibleAt, rootComp);rawNode.style.visibility = isVisibleNow ? "visible" : "hidden";- rawNode.toggleAttribute(- STUDIO_PREVIEW_UPCOMING_ATTR,- hiddenImagesSkipped && !isVisibleNow && dueSoon(rawNode, visibleAt, currentTime),- );+ const upcoming =+ hiddenImagesSkipped && !isVisibleNow && dueSoon(rawNode, visibleAt, currentTime);+ rawNode.toggleAttribute(STUDIO_PREVIEW_UPCOMING_ATTR, upcoming);+ if (isMediaElement(rawNode) && metadataBoundMedia.has(rawNode) && inPreloadWindow(rawNode))+ preloadNearPlayhead(+ rawNode,+ isVisibleNow,+ upcoming || mediaStaysDue(rawNode, visibleAt, currentTime),+ );if (!isMediaElement(rawNode) && !isImageElement(rawNode)) decidedTimedClip = true;if (isVideoElement(rawNode) || isImageElement(rawNode)) {colorGradingRuntime?.setSourceVisibility(rawNode, isVisibleNow);@@ -2711,8 +2806,8 @@ export function initSandboxRuntimeModular(): void {// Scope 2 of 3 (see `withTimingResolver`). One resolver for the whole// visibility pass: every node costs 2 x (1 + ancestor depth) resolves, and- // ancestor chains are shared between siblings. Nothing in this body calls- // `el.load()` or awaits, so no duration can change under the cache.+ // ancestor chains are shared between siblings. Its `el.load()` calls reach only clips timed by an+ // authored length, and it never awaits, so no duration the cache read can change.const syncTimedElementVisibility = (currentTime: number,visibilityNodes: Element[] = Array.from(document.querySelectorAll("[data-start]")),@@ -2896,6 +2991,9 @@ export function initSandboxRuntimeModular(): void {};window.__hf.leasePausedMedia = leasePausedMedia;window.__hf.releasePausedMedia = releasePausedMedia;+ window.__hf.setLoopStart = (seconds) => {+ loopStartSeconds = seconds != null && Number.isFinite(seconds) && seconds >= 0 ? seconds : null;+ };// A sub-composition is hidden after its host clip, so its animation counts only until then.const autoNestedHostEndSeconds = (child: RuntimeTimelineChildLike): number => {const hostId = autoNestedHostIds.get(child);diff --git a/packages/core/src/runtime/preloadMedia.test.ts b/packages/core/src/runtime/preloadMedia.test.tsindex 0756df285b..d892c1158e 100644--- a/packages/core/src/runtime/preloadMedia.test.ts+++ b/packages/core/src/runtime/preloadMedia.test.ts@@ -1,5 +1,5 @@import { describe, expect, it, vi } from "vitest";-import { preloadMedia } from "./preloadMedia";+import { preloadMedia, releaseMedia } from "./preloadMedia";function media(tagName = "VIDEO", networkState = 2, readyState = 0) {return { tagName, networkState, readyState, preload: "metadata", load: vi.fn() };@@ -34,3 +34,28 @@ describe("initial media preload", () => {expect(element.load).not.toHaveBeenCalled();});});++describe("releasing a clip's download", () => {+ it("reloads a src clip with its src dropped, then puts the src back", () => {+ const video = document.createElement("video");+ video.setAttribute("src", "a.mp4");+ const atLoad: (string | null)[] = [];+ video.load = () => atLoad.push(video.getAttribute("src"));++ releaseMedia(video);+ expect(atLoad).toEqual([null]);+ expect(video.getAttribute("src")).toBe("a.mp4");+ });++ it("leaves a clip on <source> children, their order and its markup untouched", () => {+ const video = document.createElement("video");+ video.innerHTML =+ '\n <track kind="captions">\n <source src="b.webm">\n <source src="b.mp4">\n';+ const markup = video.innerHTML;+ video.load = vi.fn();++ releaseMedia(video);+ expect(video.load).not.toHaveBeenCalled();+ expect(video.innerHTML).toBe(markup);+ });+});diff --git a/packages/core/src/runtime/preloadMedia.ts b/packages/core/src/runtime/preloadMedia.tsindex 82d2ab49e3..2d98cd8798 100644--- a/packages/core/src/runtime/preloadMedia.ts+++ b/packages/core/src/runtime/preloadMedia.ts@@ -1,3 +1,6 @@+import { parseStrictFiniteTimingNumber } from "./playbackRate";+import { skipsHiddenImages } from "./timedClipHide";+type PreloadableMedia = Pick<HTMLMediaElement,"tagName" | "preload" | "readyState" | "networkState" | "load"@@ -9,3 +12,47 @@ export function preloadMedia(media: PreloadableMedia): void {const videoAlreadyLoading = media.tagName === "VIDEO" && media.networkState === 2;if (media.readyState < 3 && !videoAlreadyLoading) media.load();}++/** Ends a fetch in flight for good, which preload none alone does not. */+export function stopMediaDownload(media: HTMLMediaElement): void {+ for (const source of media.querySelectorAll("source")) source.remove();+ media.removeAttribute("src");+ media.load();+}++export function releaseMedia(media: HTMLMediaElement): void {+ if (media.querySelector("source")) return;+ const src = media.getAttribute("src");+ stopMediaDownload(media);+ if (src !== null) media.setAttribute("src", src);+}++export function lengthIsAuthored(media: Element): boolean {+ return parseStrictFiniteTimingNumber(media.getAttribute("data-duration")) != null;+}++type DeferralWindow = Window & { __hfMediaDeferral?: MutationObserver };++/** Preview only: later clips parse at preload none, as Chromium ignores a none set mid-fetch.+ * One watcher per page: a runtime evaluated again replaces the last one's. */+export function deferMediaUntilDue(): void {+ const win = window as DeferralWindow;+ win.__hfMediaDeferral?.disconnect();+ win.__hfMediaDeferral = undefined;+ if (!skipsHiddenImages()) return;+ const defer = (el: Element) => {+ const start = parseStrictFiniteTimingNumber(el.getAttribute("data-start"));+ if (start != null && start > 0 && lengthIsAuthored(el))+ (el as HTMLMediaElement).preload = "none";+ };+ win.__hfMediaDeferral = new MutationObserver((records) => {+ for (const record of records)+ for (const node of record.addedNodes) {+ if (node.nodeType !== 1) continue;+ const el = node as Element;+ if (el.matches("video, audio")) defer(el);+ for (const media of el.querySelectorAll("video, audio")) defer(media);+ }+ });+ win.__hfMediaDeferral.observe(document.documentElement, { childList: true, subtree: true });+}diff --git a/packages/core/src/runtime/timeline.ts b/packages/core/src/runtime/timeline.tsindex 2ce1a957e1..4768776560 100644--- a/packages/core/src/runtime/timeline.ts+++ b/packages/core/src/runtime/timeline.ts@@ -143,7 +143,7 @@ function maxDefinedNumber(...values: Array<number | null>): number | null {* `parseInt(...) || fallback` silently replaced authored track 0 with the* synthetic fallback, so track-0 clips drifted to the bottom of the timeline.*/-function parseAuthoredTrack(el: Element, fallback: number): number {+export function parseAuthoredTrack(el: Element, fallback: number): number {const raw = el.getAttribute("data-track-index") ?? el.getAttribute("data-track");if (raw == null) return fallback;const parsed = Number.parseInt(raw, 10);diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.tsindex bca2aecff2..232cee4859 100644--- a/packages/core/src/runtime/window.d.ts+++ b/packages/core/src/runtime/window.d.ts@@ -68,6 +68,8 @@ declare global {* runtime's paused-side enforcement leaves it alone. Always release. */leasePausedMedia?: (el: HTMLMediaElement) => void;releasePausedMedia?: (el: HTMLMediaElement) => void;+ /** Where Studio's loop wraps to while it plays, or null: the clips due there stay loaded. */+ setLoopStart?: (seconds: number | null) => void;/** Read-only level taps for the Studio meters: nothing exists until* `start()`, and `stop()` removes every tap. Peaks are linear per channel. */audioMeter?: {diff --git a/packages/player/src/parent-media.test.ts b/packages/player/src/parent-media.test.tsindex b19fe48e60..3fb6eb1f07 100644--- a/packages/player/src/parent-media.test.ts+++ b/packages/player/src/parent-media.test.ts@@ -220,6 +220,21 @@ describe("ParentMediaManager following its clips", () => {const copy = "https://example.test/clip.mov?hf-proxy=h264";+ it("drops a clip's proxy when the runtime stops preloading the clip, and adopts it again when due", async () => {+ const mgr = makeManager();+ const clip = adoptedClip(mgr, "https://example.test/clip.mp4");+ const proxy = mgr.entries[0].el;++ clip.preload = "none";+ await flushObserver();+ expect(mgr.entries).toHaveLength(0);+ expect(proxy.getAttribute("src")).toBe("");++ clip.preload = "auto";+ await flushObserver();+ expect(mgr.entries.map((m) => m.el.src)).toEqual(["https://example.test/clip.mp4"]);+ });+it("keeps a re-pointed clip's proxy on its file while the composition owns playback", async () => {const mgr = makeManager();const clip = adoptedClip(mgr, "https://example.test/clip.mov");diff --git a/packages/player/src/parent-media.ts b/packages/player/src/parent-media.tsindex da4d7dc1d4..5d3a57ebf5 100644--- a/packages/player/src/parent-media.ts+++ b/packages/player/src/parent-media.ts@@ -380,8 +380,8 @@ export class ParentMediaManager {// fallow-ignore-next-line complexityprivate _adoptIframeMedia(iframeEl: HTMLMediaElement): void {- // Skip elements the preloader has demoted — the observer will re-trigger- // when the preload attribute is promoted to "auto".+ // Skip elements the preloader has demoted: the observer adopts one when its preload turns+ // "auto" and drops its proxy, which fetches the whole file, when it turns back.if (iframeEl.preload === "metadata" || iframeEl.preload === "none") return;const src = this._resolveIframeMediaSrc(iframeEl);@@ -465,10 +465,10 @@ export class ParentMediaManager {const target = m.target;if (isRealmHtmlMediaElement(target) &&- target.matches("audio[data-start], video[data-start]") &&- target.preload === "auto"+ target.matches("audio[data-start], video[data-start]")) {- this._adoptIframeMedia(target);+ if (target.preload === "auto") this._adoptIframeMedia(target);+ else this._detachIframeMedia(target);}continue;}diff --git a/packages/studio/src/player/hooks/useTimelinePlayer.seek.test.ts b/packages/studio/src/player/hooks/useTimelinePlayer.seek.test.tsindex 473065bc31..effd4eb106 100644--- a/packages/studio/src/player/hooks/useTimelinePlayer.seek.test.ts+++ b/packages/studio/src/player/hooks/useTimelinePlayer.seek.test.ts@@ -353,7 +353,11 @@ describe("useTimelinePlayer seek keepPlaying option (#834)", () => {describe("useTimelinePlayer RAF loop wrap-around", () => {type SeekCall = { time: number; options?: { keepPlaying?: boolean } };- function attachInstrumentedAdapter(api: ReturnType<typeof useTimelinePlayer>, duration = 30) {+ function attachInstrumentedAdapter(+ api: ReturnType<typeof useTimelinePlayer>,+ duration = 30,+ hooks: object = {},+ ) {let currentTime = 0;let playing = false;const seekCalls: SeekCall[] = [];@@ -377,6 +381,7 @@ describe("useTimelinePlayer RAF loop wrap-around", () => {};attachIframeWindow(api, {__player: adapter,+ __hf: hooks,postMessage: () => {},scrollTo: () => {},addEventListener: () => {},@@ -446,6 +451,29 @@ describe("useTimelinePlayer RAF loop wrap-around", () => {}});+ it("tells the preview where the loop wraps to while it plays, and that it stopped looping", () => {+ const raf = installRafCapture();+ try {+ const { api, root } = renderTimelinePlayerHarness();+ const setLoopStart = vi.fn();+ const { adapter } = attachInstrumentedAdapter(api, 30, { setLoopStart });+ act(() => {+ usePlayerStore.getState().setInPoint(2);+ usePlayerStore.getState().setOutPoint(5);+ api.play();+ });+ adapter.setTime(3);+ act(() => void raf.flushOne());+ expect(setLoopStart).toHaveBeenLastCalledWith(2);+ act(() => usePlayerStore.getState().setLoopEnabled(false));+ act(() => void raf.flushOne());+ expect(setLoopStart).toHaveBeenLastCalledWith(null);+ unmountWithAct(root);+ } finally {+ raf.restore();+ }+ });+it("does not seek and pauses cleanly when forward playback reaches the end without loop", () => {const raf = installRafCapture();try {diff --git a/packages/studio/src/player/hooks/useTimelinePlayer.ts b/packages/studio/src/player/hooks/useTimelinePlayer.tsindex b470feae2e..2612b594e6 100644--- a/packages/studio/src/player/hooks/useTimelinePlayer.ts+++ b/packages/studio/src/player/hooks/useTimelinePlayer.ts@@ -152,12 +152,18 @@ export function useTimelinePlayer({[],);+ const setLoopStart = useCallback((seconds: number | null) => {+ try {+ (iframeRef.current?.contentWindow as IframeWindow | null)?.__hf?.setLoopStart?.(seconds);+ } catch {}+ }, []);const { startRAFLoop, stopRAFLoop, stopReverseLoop } = useTimelinePlayerLoop({rafRef,reverseRafRef,getAdapter,setCurrentTime,setIsPlaying,+ setLoopStart,});const applyPlaybackRate = useCallback((rate: number) => {diff --git a/packages/studio/src/player/hooks/useTimelinePlayerLoop.ts b/packages/studio/src/player/hooks/useTimelinePlayerLoop.tsindex bda3d50601..306b326d38 100644--- a/packages/studio/src/player/hooks/useTimelinePlayerLoop.ts+++ b/packages/studio/src/player/hooks/useTimelinePlayerLoop.ts@@ -22,6 +22,7 @@ interface UseTimelinePlayerLoopParams {getAdapter: () => PlaybackAdapter | null;setCurrentTime: (v: number) => void;setIsPlaying: (v: boolean) => void;+ setLoopStart: (seconds: number | null) => void;}interface UseTimelinePlayerLoopResult {@@ -36,6 +37,7 @@ export function useTimelinePlayerLoop({getAdapter,setCurrentTime,setIsPlaying,+ setLoopStart,}: UseTimelinePlayerLoopParams): UseTimelinePlayerLoopResult {const stopReverseLoop = useCallback(() => {cancelAnimationFrame(reverseRafRef.current);@@ -55,8 +57,10 @@ export function useTimelinePlayerLoop({const rawLoopStart = inPoint !== null ? inPoint : 0;const loopEnd = rawLoopStart < rawLoopEnd ? rawLoopEnd : dur;const loopStart = rawLoopStart < rawLoopEnd ? rawLoopStart : 0;+ const looping = usePlayerStore.getState().loopEnabled && dur > 0;+ setLoopStart(looping ? loopStart : null);if (time >= loopEnd) {- if (usePlayerStore.getState().loopEnabled && dur > 0) {+ if (looping) {// keepPlaying skips the adapter's implicit pause; play() below is then a no-op.adapter.seek(loopStart, { keepPlaying: true });liveTime.notify(loopStart);@@ -75,7 +79,7 @@ export function useTimelinePlayerLoop({rafRef.current = requestAnimationFrame(tick);};rafRef.current = requestAnimationFrame(tick);- }, [rafRef, getAdapter, setCurrentTime, setIsPlaying]);+ }, [rafRef, getAdapter, setCurrentTime, setIsPlaying, setLoopStart]);const stopRAFLoop = useCallback(() => {cancelAnimationFrame(rafRef.current);diff --git a/packages/studio/src/player/lib/playbackTypes.ts b/packages/studio/src/player/lib/playbackTypes.tsindex 22747dd12c..707a3518b1 100644--- a/packages/studio/src/player/lib/playbackTypes.ts+++ b/packages/studio/src/player/lib/playbackTypes.ts@@ -69,5 +69,6 @@ export type IframeWindow = Window & {__hf?: {leasePausedMedia?: (el: HTMLMediaElement) => void;releasePausedMedia?: (el: HTMLMediaElement) => void;+ setLoopStart?: (seconds: number | null) => void;};};