Skip to content

Write HTML. Render video. Built for agents.

perf(core): the preview loads only the media near the playhead (#4682)

Miguel Ángel5a77dbcea12b
diff --git a/packages/core/src/compositionReadiness.test.ts b/packages/core/src/compositionReadiness.test.ts
index d03b9f19a4..5172460bca 100644
--- a/packages/core/src/compositionReadiness.test.ts
+++ b/packages/core/src/compositionReadiness.test.ts
@@ -8,6 +8,7 @@ import {
settleFirstFrameCompositionReadiness,
} from "./compositionReadiness.js";
import { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
+import { STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js";
import { isRuntimeElementVisibleAt } from "./runtime/timeline.js";
function docWith(bodyHtml: string): Document {
@@ -179,6 +180,23 @@ describe("mediaReadinessInput", () => {
await expect(pending).resolves.toBeUndefined();
});
+ it("waits on a later clip in Studio's preview only once the runtime has set it to buffer", () => {
+ const html =
+ '<video id="deferred" data-start="30" data-duration="5" preload="none" src="a.mp4"></video>' +
+ '<video id="untrimmed" data-start="30" preload="metadata" src="b.mp4"></video>' +
+ '<video id="due" data-start="30" data-duration="5" preload="auto" src="c.mp4"></video>';
+ const preview = docWith(html);
+ preview.head
+ .appendChild(preview.createElement("meta"))
+ .setAttribute("name", STUDIO_PREVIEW_MARK_META);
+ const plain = docWith(html);
+ const ids = (doc: Document) =>
+ scanPendingCompositionAssets(doc, { scope: "all" }).pendingMedia.map((el) => el.id);
+
+ expect(ids(preview)).toEqual(["due"]);
+ expect(ids(plain)).toEqual(["deferred", "untrimmed", "due"]);
+ });
+
it("waits on a later scene's image in the full scan unless it is lazy", () => {
const doc = docWith(
'<div data-start="30" data-duration="5"><img id="later" src="later.png">' +
diff --git a/packages/core/src/compositionReadiness.ts b/packages/core/src/compositionReadiness.ts
index 3dfb9f97e9..49c69f1b17 100644
--- a/packages/core/src/compositionReadiness.ts
+++ b/packages/core/src/compositionReadiness.ts
@@ -1,6 +1,7 @@
import { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
import { isRuntimeElementVisibleAt } from "./runtime/timeline.js";
import type { RuntimeTimelineLike } from "./runtime/types.js";
+import { STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js";
/** A composition is "ready" once every declared input settles, not just once
* its duration is known. Each input returns null (nothing to wait on) or a
@@ -103,9 +104,18 @@ export function scanPendingCompositionAssets(
timelineRegistry: runtimeWindow?.__timelines,
includeAuthoredTimingAttrs: true,
});
+ // Studio's preview buffers a clip off the first frame only once it comes due, as with a lazy image.
+ const defersMedia = doc.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`) !== null;
const pendingMedia = Array.from(doc.querySelectorAll("video, audio"))
.filter(isRealmHtmlMediaElement)
- .filter((el) => shouldIncludeAsset(el, scope, resolver, runtimeWindow?.__timelines ?? {}))
+ .filter((el) =>
+ shouldIncludeAsset(
+ el,
+ defersMedia && el.preload !== "auto" ? FIRST_FRAME_READINESS_SCOPE : scope,
+ resolver,
+ runtimeWindow?.__timelines ?? {},
+ ),
+ )
.filter((el) => el.readyState < HAVE_FUTURE_DATA);
// A lazy image off the first frame loads only once shown, so no scope can wait on it.
const pendingImages = Array.from(doc.querySelectorAll("img"))
diff --git a/packages/core/src/runtime/entry.test.ts b/packages/core/src/runtime/entry.test.ts
index 726b4e82b9..abc146f400 100644
--- a/packages/core/src/runtime/entry.test.ts
+++ b/packages/core/src/runtime/entry.test.ts
@@ -303,6 +303,331 @@ describe("runtime entry", () => {
expect(document.querySelectorAll("style[data-hf-first-pass-hide]")).toHaveLength(0);
});
+ describe("media preload window", () => {
+ const loads: HTMLMediaElement[] = [];
+ const spyLoad = () => {
+ loads.length = 0;
+ vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(
+ function (this: HTMLMediaElement) {
+ loads.push(this);
+ },
+ );
+ };
+ const loadsOf = (...els: HTMLMediaElement[]) =>
+ els.map((el) => loads.filter((loaded) => loaded === el).length);
+ const videos = (root: HTMLElement, ...starts: string[]) =>
+ starts.map((start) => timed(root, "video", start));
+ const armed = (...els: HTMLMediaElement[]) => els.map((el) => el.preload === "auto");
+ afterEach(() => vi.restoreAllMocks());
+
+ it("loads a preview's media only on screen or due within the look-ahead, and frees what the playhead left", async () => {
+ spyLoad();
+ servePreview();
+ const [now, soon, later] = videos(mountRoot(), "0", "1.5", "9");
+
+ await evaluateRuntime();
+ expect(armed(now, soon, later)).toEqual([true, true, false]);
+ expect(later.preload).toBe("none");
+ expect(loadsOf(now, soon, later)).toEqual([1, 1, 0]);
+
+ now.setAttribute("src", "now.mp4");
+ const srcWrites = new MutationObserver(() => {});
+ srcWrites.observe(now, { attributeFilter: ["src"] });
+ window.__player?.seek(7.6);
+ expect(armed(now, soon, later)).toEqual([false, false, true]);
+ // The two that left are reloaded empty with src dropped for the reload, ending any fetch in flight.
+ expect([now.preload, soon.preload]).toEqual(["none", "none"]);
+ expect(loadsOf(now, soon, later)).toEqual([2, 2, 1]);
+ expect(srcWrites.takeRecords()).toHaveLength(2);
+ expect(now.getAttribute("src")).toBe("now.mp4");
+ });
+
+ it("loads a clip with no authored length as a render does, so no reload drops its duration", async () => {
+ spyLoad();
+ servePreview();
+ const [untrimmed, later] = videos(mountRoot(), "0", "8");
+ untrimmed.removeAttribute("data-duration");
+ later.removeAttribute("data-duration");
+ Object.defineProperty(untrimmed, "readyState", { value: 1 });
+ Object.defineProperty(untrimmed, "duration", { value: 2 });
+
+ await evaluateRuntime();
+ window.__player?.seek(3.6);
+ window.__player?.seek(0.5);
+ window.__player?.seek(7.5);
+ expect(armed(untrimmed, later)).toEqual([true, true]);
+ expect(loadsOf(untrimmed, later)).toEqual([1, 1]);
+ });
+
+ it("arms a clip a far jump lands on without reloading it under the seek", async () => {
+ spyLoad();
+ servePreview();
+ const [now, later] = videos(mountRoot(), "0", "8");
+
+ await evaluateRuntime();
+ const beforeJump = loadsOf(later)[0];
+ window.__player?.seek(8.4);
+ expect(armed(now, later)).toEqual([false, true]);
+ expect(loadsOf(later)).toEqual([beforeJump]);
+ });
+
+ // jsdom's media elements cannot play; the runtime only needs play() to settle.
+ const stubPlayback = () => {
+ vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(undefined);
+ vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {});
+ };
+ const tracks = (root: HTMLElement) => {
+ stubPlayback();
+ const [now, next, afterNext, otherTrack] = videos(root, "0", "4", "7", "3");
+ otherTrack.setAttribute("data-track-index", "2");
+ return { now, next, afterNext, otherTrack };
+ };
+
+ it("arms the next clip on a clip's track when it starts playing, and nothing else", async () => {
+ spyLoad();
+ servePreview();
+ const { next, afterNext, otherTrack } = tracks(mountRoot());
+
+ await evaluateRuntime();
+ window.__player?.play();
+ expect(armed(next, afterNext, otherTrack)).toEqual([true, false, false]);
+ expect(loadsOf(next, afterNext, otherTrack)).toEqual([1, 0, 0]);
+ });
+
+ it("arms nothing ahead of a paused playhead, and keeps what it armed through a pause", async () => {
+ spyLoad();
+ servePreview();
+ const { next, afterNext } = tracks(mountRoot());
+
+ await evaluateRuntime();
+ window.__player?.seek(0.5);
+ expect(armed(next)).toEqual([false]);
+ window.__player?.play();
+ window.__player?.pause();
+ expect(armed(next, afterNext)).toEqual([true, false]);
+ expect(loadsOf(next)).toEqual([1]);
+ });
+
+ it("holds a clip past the look-ahead edge a scrub crosses, and frees it once well clear", async () => {
+ spyLoad();
+ servePreview();
+ const [, later] = videos(mountRoot(), "0", "5");
+
+ await evaluateRuntime();
+ window.__player?.seek(3.5);
+ window.__player?.seek(2.5);
+ window.__player?.seek(3.5);
+ expect(armed(later)).toEqual([true]);
+ expect(loadsOf(later)).toEqual([1]);
+ window.__player?.seek(0.5);
+ expect(armed(later)).toEqual([false]);
+ });
+
+ it("keeps the clip a step back across a cut returns to loaded", async () => {
+ spyLoad();
+ servePreview();
+ const [outgoing, incoming] = videos(mountRoot(), "0", "2");
+
+ await evaluateRuntime();
+ window.__player?.seek(2.1);
+ expect(armed(outgoing, incoming)).toEqual([true, true]);
+ window.__player?.seek(1.9);
+ expect(armed(outgoing)).toEqual([true]);
+ expect(loadsOf(outgoing)).toEqual([1]);
+ });
+
+ it("arms a clip behind a playhead shuttling back before the playhead reaches it", async () => {
+ spyLoad();
+ servePreview();
+ const [earlier, current] = videos(mountRoot(), "0", "2");
+ current.setAttribute("data-duration", "6");
+
+ await evaluateRuntime();
+ window.__player?.seek(6.5);
+ expect(armed(earlier)).toEqual([false]);
+ let armedAt: number | undefined;
+ for (let tenths = 45; tenths >= 20 && armedAt === undefined; tenths--) {
+ window.__player?.seek(tenths / 10);
+ if (armed(earlier)[0]) armedAt = tenths / 10;
+ }
+ // Within the 2 s look-ahead of its end, as a clip ahead is armed within 2 s of its start.
+ expect(armedAt).toBe(4);
+ });
+
+ it("still frees a clip that ended well behind the playhead", async () => {
+ spyLoad();
+ servePreview();
+ const [earlier, current] = videos(mountRoot(), "0", "2");
+ current.setAttribute("data-duration", "6");
+
+ await evaluateRuntime();
+ window.__player?.seek(2.1);
+ window.__player?.seek(5.9);
+ expect(armed(earlier)).toEqual([true]);
+ expect(loadsOf(earlier)).toEqual([1]);
+ window.__player?.seek(6.1);
+ expect(armed(earlier)).toEqual([false]);
+ expect(earlier.preload).toBe("none");
+ expect(loadsOf(earlier)).toEqual([2]);
+ });
+
+ it("drops an arm once the clip that set it leaves the screen, as a jump while playing does", async () => {
+ spyLoad();
+ servePreview();
+ const { next } = tracks(mountRoot());
+ window.__timelines = { main: pausedTimeline(20) };
+
+ await evaluateRuntime();
+ window.__player?.play();
+ window.__player?.seek(10.5, { keepPlaying: true });
+ expect(armed(next)).toEqual([false]);
+ });
+
+ it("stops holding a clip it armed once that clip has played", async () => {
+ spyLoad();
+ servePreview();
+ const { next } = tracks(mountRoot());
+ window.__timelines = { main: pausedTimeline(20) };
+
+ await evaluateRuntime();
+ window.__player?.play();
+ window.__player?.seek(4.5, { keepPlaying: true });
+ window.__player?.seek(10.5, { keepPlaying: true });
+ expect(armed(next)).toEqual([false]);
+ });
+
+ it("arms a clip that starts exactly where the playing one ends", async () => {
+ spyLoad();
+ servePreview();
+ stubPlayback();
+ // 1.1 + 2.2 is 3.3000000000000003: back to back only within the boundary tolerance.
+ const [first, second] = videos(mountRoot(), "1.1", "3.3");
+ first.setAttribute("data-duration", "2.2");
+
+ await evaluateRuntime();
+ window.__player?.seek(1.2);
+ window.__player?.play();
+ expect(armed(second)).toEqual([true]);
+ });
+
+ it("arms the next clip only in the same host and on an authored track", async () => {
+ spyLoad();
+ servePreview();
+ stubPlayback();
+ const root = mountRoot();
+ const [now] = videos(root, "0");
+ const host = timed(root, "div", "0");
+ host.setAttribute("data-composition-id", "inner");
+ host.setAttribute("data-duration", "10");
+ const [otherHost] = videos(host, "4");
+ const [untracked, untrackedNext] = videos(root, "0", "4");
+ for (const el of [untracked, untrackedNext]) el.removeAttribute("data-track-index");
+ now.setAttribute("data-track-index", "1");
+
+ await evaluateRuntime();
+ window.__player?.play();
+ expect(armed(otherHost, untrackedNext)).toEqual([false, false]);
+ });
+
+ it("arms a short nested clip whose start is already root time before it is due", async () => {
+ spyLoad();
+ servePreview();
+ const host = timed(mountRoot(), "div", "5");
+ host.setAttribute("data-composition-id", "inner");
+ host.setAttribute("data-duration", "5");
+ const [clip] = videos(host, "6");
+ clip.setAttribute("data-hf-media-start-basis", "global");
+ clip.setAttribute("data-duration", "0.4");
+
+ await evaluateRuntime();
+ window.__player?.seek(4.5);
+ expect(armed(clip)).toEqual([true]);
+ });
+
+ it("keeps the clips due at the loop start loaded while the loop plays", async () => {
+ spyLoad();
+ servePreview();
+ const { now, next, afterNext } = tracks(mountRoot());
+ window.__timelines = { main: pausedTimeline(20) };
+
+ await evaluateRuntime();
+ window.__hf?.setLoopStart?.(0);
+ window.__player?.play();
+ window.__player?.seek(19.5, { keepPlaying: true });
+ expect(armed(now, next, afterNext)).toEqual([true, true, false]);
+ expect(loadsOf(now, next)).toEqual([1, 1]);
+ window.__hf?.setLoopStart?.(null);
+ window.__player?.seek(19.6, { keepPlaying: true });
+ expect(armed(now)).toEqual([false]);
+ });
+
+ it("holds the loop start only while playing, and for a loop start that is a time", async () => {
+ spyLoad();
+ servePreview();
+ const { now } = tracks(mountRoot());
+
+ await evaluateRuntime();
+ window.__hf?.setLoopStart?.(0);
+ window.__player?.play();
+ window.__player?.seek(9.5, { keepPlaying: true });
+ window.__player?.pause();
+ expect(armed(now)).toEqual([false]);
+ window.__hf?.setLoopStart?.(-1);
+ window.__player?.play();
+ expect(armed(now)).toEqual([false]);
+ });
+
+ it("never arms a next clip with no authored length, so it is not reloaded", async () => {
+ spyLoad();
+ servePreview();
+ const { next } = tracks(mountRoot());
+ next.removeAttribute("data-duration");
+
+ await evaluateRuntime();
+ const atBind = loadsOf(next)[0];
+ window.__player?.play();
+ expect(loadsOf(next)).toEqual([atBind]);
+ });
+
+ it("keeps an armed next clip through a window pass that finds it outside the look-ahead", async () => {
+ spyLoad();
+ servePreview();
+ const { next } = tracks(mountRoot());
+
+ await evaluateRuntime();
+ window.__player?.play();
+ window.__player?.seek(0.5, { keepPlaying: true });
+ expect(armed(next)).toEqual([true]);
+ expect(loadsOf(next)).toEqual([1]);
+ });
+
+ it("gives each parsed clip after the first frame preload none before it can fetch", async () => {
+ servePreview();
+ const root = mountRoot();
+ timed(root, "div", "0");
+ await evaluateRuntime();
+ const [first, later] = videos(root, "0", "5");
+ const untrimmed = timed(root, "audio", "5");
+ untrimmed.removeAttribute("data-duration");
+ await Promise.resolve();
+ expect([first.preload, later.preload, untrimmed.preload]).toEqual(["", "none", ""]);
+ });
+
+ it("keeps a render loading every media element at bind", async () => {
+ spyLoad();
+ const [now, soon, later] = videos(mountRoot(), "0", "1.5", "5");
+
+ await evaluateRuntime();
+ window.__player?.seek(3.6);
+ expect(armed(now, soon, later)).toEqual([true, true, true]);
+ expect(loadsOf(now, soon, later)).toEqual([1, 1, 1]);
+ const parsedLater = timed(document.body, "video", "5");
+ await Promise.resolve();
+ expect(parsedLater.preload).not.toBe("none");
+ expect((window as { __hfMediaDeferral?: unknown }).__hfMediaDeferral).toBeUndefined();
+ });
+ });
+
it("grades media inside a clip once the first pass shows the clip, with no seek", async () => {
const getContext = vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null);
const scene = timed(mountRoot(), "div", "0");
diff --git a/packages/core/src/runtime/entry.ts b/packages/core/src/runtime/entry.ts
index cd8cf63196..e445e1c89d 100644
--- a/packages/core/src/runtime/entry.ts
+++ b/packages/core/src/runtime/entry.ts
@@ -1,5 +1,6 @@
import { initSandboxRuntimeModular, installAuthoredMediaCapture } from "./init";
import { installAuthoredOpacityCapture } from "./colorGrading";
+import { deferMediaUntilDue } from "./preloadMedia";
import { hideTimedClipsUntilFirstPass } from "./timedClipHide";
import { fitTextFontSize } from "../text/fitTextFontSize";
import { pretext } from "../text/pretext";
@@ -31,6 +32,7 @@ installAuthoredOpacityCapture();
installAuthoredMediaCapture();
hideTimedClipsUntilFirstPass();
+deferMediaUntilDue();
// Expose runtime helpers immediately so composition scripts can use them
// before DOMContentLoaded (font sizing runs during script evaluation, and
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index 8e9e329eee..774fb9e579 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -1,5 +1,5 @@
// fallow-ignore-file code-duplication complexity
-import { preloadMedia } from "./preloadMedia";
+import { preloadMedia, releaseMedia, lengthIsAuthored, stopMediaDownload } from "./preloadMedia";
import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge";
import { instantTolerance } from "../clipFacts";
import { isInClipWindow } from "./clipWindow";
@@ -50,6 +50,7 @@ import {
collectRuntimeTimelinePayload,
isRuntimeElementVisibleAt,
LOOP_INFLATED_TIMELINE_SECONDS,
+ parseAuthoredTrack,
} from "./timeline";
import {
findRootCompositionElement,
@@ -2416,9 +2417,7 @@ export function initSandboxRuntimeModular(): void {
if (mediaEl.isConnected) continue;
unbindMedia(mediaEl);
mediaEl.pause();
- for (const source of mediaEl.querySelectorAll("source")) source.remove();
- mediaEl.removeAttribute("src");
- mediaEl.load();
+ stopMediaDownload(mediaEl);
}
};
@@ -2430,6 +2429,7 @@ export function initSandboxRuntimeModular(): void {
const bindMediaMetadataListeners = () => {
if (state.tornDown) return;
const mediaEls = Array.from(document.querySelectorAll("video, audio")) as HTMLMediaElement[];
+ const nearPlayheadPending: HTMLMediaElement[] = [];
for (const mediaEl of mediaEls) {
if (metadataBoundMedia.has(mediaEl)) continue;
metadataBoundMedia.add(mediaEl);
@@ -2464,7 +2464,9 @@ export function initSandboxRuntimeModular(): void {
// mode, for <audio>, or when the codec map is absent.
maybeProxyProactively(mediaEl);
- preloadMedia(mediaEl);
+ // Studio's preview loads a timed clip only near the playhead; the visibility pass decides.
+ if (inPreloadWindow(mediaEl)) nearPlayheadPending.push(mediaEl);
+ else preloadMedia(mediaEl);
// Probe volume automation from the GSAP timeline — same approach as the
// renderer (see discoverAudioVolumeAutomationFromTimeline / audioMixer).
@@ -2473,6 +2475,8 @@ export function initSandboxRuntimeModular(): void {
// fires after the timeline has been captured (every 30 transport ticks).
probeAndCacheVolumeKeyframes(mediaEl);
}
+ if (nearPlayheadPending.length > 0)
+ syncTimedElementVisibility(state.currentTime, nearPlayheadPending);
};
const probeAndCacheVolumeKeyframes = (mediaEl: HTMLMediaElement) => {
@@ -2633,15 +2637,101 @@ export function initSandboxRuntimeModular(): void {
const hiddenImagesSkipped = skipsHiddenImages();
const LOOKAHEAD_SECONDS = 2;
- // Unskipped while due in the look-ahead window, so a clip shorter than the window still loads first.
+ const RELEASE_LOOKAHEAD_SECONDS = 4;
+ // Unskipped while within the window on either side of the playhead, so a step or shuttle back
+ // across a cut finds the clip it left still loaded; the edge checks catch a clip shorter than it.
const dueSoon = (
node: HTMLElement,
visibleAt: ReturnType<typeof timedVisibilityAt>,
t: number,
+ ahead = LOOKAHEAD_SECONDS,
) => {
- const start = resolveStartForElement(node, Number.NaN);
- return (start > t && start <= t + LOOKAHEAD_SECONDS) || visibleAt(node, t + LOOKAHEAD_SECONDS);
+ const start = isMediaElement(node)
+ ? resolveAbsoluteMediaStartSeconds(node)
+ : resolveStartForElement(node, Number.NaN);
+ const end = start + (resolveDurationForElement(node) ?? Number.NaN);
+ return (
+ (start > t && start <= t + ahead) ||
+ (end <= t && end >= t - ahead) ||
+ visibleAt(node, t + ahead)
+ );
+ };
+ // Where Studio's loop wraps to (window.__hf.setLoopStart); null when it does not loop.
+ let loopStartSeconds: number | null = null;
+ // A clip with no authored length loads as in a render: a reload would drop the duration it is
+ // timed by.
+ const inPreloadWindow = (el: HTMLMediaElement) =>
+ hiddenImagesSkipped && el.hasAttribute("data-start") && lengthIsAuthored(el);
+
+ // Media on screen or due within the look-ahead loads; the rest stops holding a connection to
+ // the origin that also serves Studio's thumbnails.
+ const mediaNearPlayhead = new WeakMap<HTMLMediaElement, boolean>();
+ // During playback a clip that starts arms the next clip on its track, which a cold 2 s look-ahead
+ // cannot fetch in time on a slow link.
+ const armedBy = new WeakMap<HTMLMediaElement, HTMLMediaElement>();
+ const armedFrom = new WeakSet<HTMLMediaElement>();
+ // An untracked clip (NaN) shares a track with nothing, as in the timeline payload.
+ const hostOf = (el: Element) => el.parentElement?.closest("[data-composition-id]");
+ const armNextOnTrack = (el: HTMLMediaElement) => {
+ const track = parseAuthoredTrack(el, Number.NaN);
+ const clips = buildRuntimeMediaCache(
+ Array.from(metadataBoundMedia).filter(
+ (m) =>
+ m.isConnected && parseAuthoredTrack(m, Number.NaN) === track && hostOf(m) === hostOf(el),
+ ),
+ ).mediaClips;
+ const end = clips.find((clip) => clip.el === el)?.end ?? Number.NaN;
+ let next: (typeof clips)[number] | undefined;
+ for (const clip of clips)
+ if (
+ clip.el !== el &&
+ clip.start >= end - instantTolerance(end) &&
+ !(next && next.start <= clip.start)
+ )
+ next = clip;
+ if (!next || !inPreloadWindow(next.el)) return;
+ armedBy.set(next.el, el);
+ if (mediaNearPlayhead.get(next.el) !== true) {
+ mediaNearPlayhead.set(next.el, true);
+ preloadMedia(next.el);
+ }
+ };
+ const preloadNearPlayhead = (el: HTMLMediaElement, visible: boolean, upcoming: boolean) => {
+ const playing = clock.isPlaying();
+ if (!(visible && playing)) armedFrom.delete(el);
+ else if (!armedFrom.has(el)) {
+ armedFrom.add(el);
+ armNextOnTrack(el);
+ }
+ // An arm lasts while the clip that set it plays on screen, so a jump away drops it.
+ const armer = armedBy.get(el);
+ const near = visible || upcoming || (armer !== undefined && armedFrom.has(armer));
+ const decided = mediaNearPlayhead.get(el);
+ if (decided === near) return;
+ mediaNearPlayhead.set(el, near);
+ if (!near) {
+ if (el.preload !== "none") {
+ el.preload = "none";
+ // Frees a clip the window armed; on one the parser started, it would only restart the fetch.
+ if (decided) releaseMedia(el);
+ }
+ } else if (!visible || decided === undefined) {
+ // Not a clip a jump lands on: the media sync arms that one, and load() would undo its seek.
+ preloadMedia(el);
+ }
};
+ // Held past the look-ahead so a scrub across its edge does not refetch, and while playing when
+ // due at the loop start.
+ const mediaStaysDue = (
+ el: HTMLMediaElement,
+ visibleAt: ReturnType<typeof timedVisibilityAt>,
+ t: number,
+ ) =>
+ (mediaNearPlayhead.get(el) === true && dueSoon(el, visibleAt, t, RELEASE_LOOKAHEAD_SECONDS)) ||
+ (loopStartSeconds !== null &&
+ clock.isPlaying() &&
+ (visibleAt(el, loopStartSeconds) ||
+ dueSoon(el, visibleAt, loopStartSeconds, RELEASE_LOOKAHEAD_SECONDS)));
const applyTimedElementVisibility = (
currentTime: number,
@@ -2679,10 +2769,15 @@ export function initSandboxRuntimeModular(): void {
visibleAt(rawNode, currentTime) &&
clipChainVisibleAt(rawNode.parentElement, currentTime, visibleAt, rootComp);
rawNode.style.visibility = isVisibleNow ? "visible" : "hidden";
- rawNode.toggleAttribute(
- STUDIO_PREVIEW_UPCOMING_ATTR,
- hiddenImagesSkipped && !isVisibleNow && dueSoon(rawNode, visibleAt, currentTime),
- );
+ const upcoming =
+ hiddenImagesSkipped && !isVisibleNow && dueSoon(rawNode, visibleAt, currentTime);
+ rawNode.toggleAttribute(STUDIO_PREVIEW_UPCOMING_ATTR, upcoming);
+ if (isMediaElement(rawNode) && metadataBoundMedia.has(rawNode) && inPreloadWindow(rawNode))
+ preloadNearPlayhead(
+ rawNode,
+ isVisibleNow,
+ upcoming || mediaStaysDue(rawNode, visibleAt, currentTime),
+ );
if (!isMediaElement(rawNode) && !isImageElement(rawNode)) decidedTimedClip = true;
if (isVideoElement(rawNode) || isImageElement(rawNode)) {
colorGradingRuntime?.setSourceVisibility(rawNode, isVisibleNow);
@@ -2711,8 +2806,8 @@ export function initSandboxRuntimeModular(): void {
// Scope 2 of 3 (see `withTimingResolver`). One resolver for the whole
// visibility pass: every node costs 2 x (1 + ancestor depth) resolves, and
- // ancestor chains are shared between siblings. Nothing in this body calls
- // `el.load()` or awaits, so no duration can change under the cache.
+ // ancestor chains are shared between siblings. Its `el.load()` calls reach only clips timed by an
+ // authored length, and it never awaits, so no duration the cache read can change.
const syncTimedElementVisibility = (
currentTime: number,
visibilityNodes: Element[] = Array.from(document.querySelectorAll("[data-start]")),
@@ -2896,6 +2991,9 @@ export function initSandboxRuntimeModular(): void {
};
window.__hf.leasePausedMedia = leasePausedMedia;
window.__hf.releasePausedMedia = releasePausedMedia;
+ window.__hf.setLoopStart = (seconds) => {
+ loopStartSeconds = seconds != null && Number.isFinite(seconds) && seconds >= 0 ? seconds : null;
+ };
// A sub-composition is hidden after its host clip, so its animation counts only until then.
const autoNestedHostEndSeconds = (child: RuntimeTimelineChildLike): number => {
const hostId = autoNestedHostIds.get(child);
diff --git a/packages/core/src/runtime/preloadMedia.test.ts b/packages/core/src/runtime/preloadMedia.test.ts
index 0756df285b..d892c1158e 100644
--- a/packages/core/src/runtime/preloadMedia.test.ts
+++ b/packages/core/src/runtime/preloadMedia.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest";
-import { preloadMedia } from "./preloadMedia";
+import { preloadMedia, releaseMedia } from "./preloadMedia";
function media(tagName = "VIDEO", networkState = 2, readyState = 0) {
return { tagName, networkState, readyState, preload: "metadata", load: vi.fn() };
@@ -34,3 +34,28 @@ describe("initial media preload", () => {
expect(element.load).not.toHaveBeenCalled();
});
});
+
+describe("releasing a clip's download", () => {
+ it("reloads a src clip with its src dropped, then puts the src back", () => {
+ const video = document.createElement("video");
+ video.setAttribute("src", "a.mp4");
+ const atLoad: (string | null)[] = [];
+ video.load = () => atLoad.push(video.getAttribute("src"));
+
+ releaseMedia(video);
+ expect(atLoad).toEqual([null]);
+ expect(video.getAttribute("src")).toBe("a.mp4");
+ });
+
+ it("leaves a clip on <source> children, their order and its markup untouched", () => {
+ const video = document.createElement("video");
+ video.innerHTML =
+ '\n <track kind="captions">\n <source src="b.webm">\n <source src="b.mp4">\n';
+ const markup = video.innerHTML;
+ video.load = vi.fn();
+
+ releaseMedia(video);
+ expect(video.load).not.toHaveBeenCalled();
+ expect(video.innerHTML).toBe(markup);
+ });
+});
diff --git a/packages/core/src/runtime/preloadMedia.ts b/packages/core/src/runtime/preloadMedia.ts
index 82d2ab49e3..2d98cd8798 100644
--- a/packages/core/src/runtime/preloadMedia.ts
+++ b/packages/core/src/runtime/preloadMedia.ts
@@ -1,3 +1,6 @@
+import { parseStrictFiniteTimingNumber } from "./playbackRate";
+import { skipsHiddenImages } from "./timedClipHide";
+
type PreloadableMedia = Pick<
HTMLMediaElement,
"tagName" | "preload" | "readyState" | "networkState" | "load"
@@ -9,3 +12,47 @@ export function preloadMedia(media: PreloadableMedia): void {
const videoAlreadyLoading = media.tagName === "VIDEO" && media.networkState === 2;
if (media.readyState < 3 && !videoAlreadyLoading) media.load();
}
+
+/** Ends a fetch in flight for good, which preload none alone does not. */
+export function stopMediaDownload(media: HTMLMediaElement): void {
+ for (const source of media.querySelectorAll("source")) source.remove();
+ media.removeAttribute("src");
+ media.load();
+}
+
+export function releaseMedia(media: HTMLMediaElement): void {
+ if (media.querySelector("source")) return;
+ const src = media.getAttribute("src");
+ stopMediaDownload(media);
+ if (src !== null) media.setAttribute("src", src);
+}
+
+export function lengthIsAuthored(media: Element): boolean {
+ return parseStrictFiniteTimingNumber(media.getAttribute("data-duration")) != null;
+}
+
+type DeferralWindow = Window & { __hfMediaDeferral?: MutationObserver };
+
+/** Preview only: later clips parse at preload none, as Chromium ignores a none set mid-fetch.
+ * One watcher per page: a runtime evaluated again replaces the last one's. */
+export function deferMediaUntilDue(): void {
+ const win = window as DeferralWindow;
+ win.__hfMediaDeferral?.disconnect();
+ win.__hfMediaDeferral = undefined;
+ if (!skipsHiddenImages()) return;
+ const defer = (el: Element) => {
+ const start = parseStrictFiniteTimingNumber(el.getAttribute("data-start"));
+ if (start != null && start > 0 && lengthIsAuthored(el))
+ (el as HTMLMediaElement).preload = "none";
+ };
+ win.__hfMediaDeferral = new MutationObserver((records) => {
+ for (const record of records)
+ for (const node of record.addedNodes) {
+ if (node.nodeType !== 1) continue;
+ const el = node as Element;
+ if (el.matches("video, audio")) defer(el);
+ for (const media of el.querySelectorAll("video, audio")) defer(media);
+ }
+ });
+ win.__hfMediaDeferral.observe(document.documentElement, { childList: true, subtree: true });
+}
diff --git a/packages/core/src/runtime/timeline.ts b/packages/core/src/runtime/timeline.ts
index 2ce1a957e1..4768776560 100644
--- a/packages/core/src/runtime/timeline.ts
+++ b/packages/core/src/runtime/timeline.ts
@@ -143,7 +143,7 @@ function maxDefinedNumber(...values: Array<number | null>): number | null {
* `parseInt(...) || fallback` silently replaced authored track 0 with the
* synthetic fallback, so track-0 clips drifted to the bottom of the timeline.
*/
-function parseAuthoredTrack(el: Element, fallback: number): number {
+export function parseAuthoredTrack(el: Element, fallback: number): number {
const raw = el.getAttribute("data-track-index") ?? el.getAttribute("data-track");
if (raw == null) return fallback;
const parsed = Number.parseInt(raw, 10);
diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts
index bca2aecff2..232cee4859 100644
--- a/packages/core/src/runtime/window.d.ts
+++ b/packages/core/src/runtime/window.d.ts
@@ -68,6 +68,8 @@ declare global {
* runtime's paused-side enforcement leaves it alone. Always release. */
leasePausedMedia?: (el: HTMLMediaElement) => void;
releasePausedMedia?: (el: HTMLMediaElement) => void;
+ /** Where Studio's loop wraps to while it plays, or null: the clips due there stay loaded. */
+ setLoopStart?: (seconds: number | null) => void;
/** Read-only level taps for the Studio meters: nothing exists until
* `start()`, and `stop()` removes every tap. Peaks are linear per channel. */
audioMeter?: {
diff --git a/packages/player/src/parent-media.test.ts b/packages/player/src/parent-media.test.ts
index b19fe48e60..3fb6eb1f07 100644
--- a/packages/player/src/parent-media.test.ts
+++ b/packages/player/src/parent-media.test.ts
@@ -220,6 +220,21 @@ describe("ParentMediaManager following its clips", () => {
const copy = "https://example.test/clip.mov?hf-proxy=h264";
+ it("drops a clip's proxy when the runtime stops preloading the clip, and adopts it again when due", async () => {
+ const mgr = makeManager();
+ const clip = adoptedClip(mgr, "https://example.test/clip.mp4");
+ const proxy = mgr.entries[0].el;
+
+ clip.preload = "none";
+ await flushObserver();
+ expect(mgr.entries).toHaveLength(0);
+ expect(proxy.getAttribute("src")).toBe("");
+
+ clip.preload = "auto";
+ await flushObserver();
+ expect(mgr.entries.map((m) => m.el.src)).toEqual(["https://example.test/clip.mp4"]);
+ });
+
it("keeps a re-pointed clip's proxy on its file while the composition owns playback", async () => {
const mgr = makeManager();
const clip = adoptedClip(mgr, "https://example.test/clip.mov");
diff --git a/packages/player/src/parent-media.ts b/packages/player/src/parent-media.ts
index da4d7dc1d4..5d3a57ebf5 100644
--- a/packages/player/src/parent-media.ts
+++ b/packages/player/src/parent-media.ts
@@ -380,8 +380,8 @@ export class ParentMediaManager {
// fallow-ignore-next-line complexity
private _adoptIframeMedia(iframeEl: HTMLMediaElement): void {
- // Skip elements the preloader has demoted — the observer will re-trigger
- // when the preload attribute is promoted to "auto".
+ // Skip elements the preloader has demoted: the observer adopts one when its preload turns
+ // "auto" and drops its proxy, which fetches the whole file, when it turns back.
if (iframeEl.preload === "metadata" || iframeEl.preload === "none") return;
const src = this._resolveIframeMediaSrc(iframeEl);
@@ -465,10 +465,10 @@ export class ParentMediaManager {
const target = m.target;
if (
isRealmHtmlMediaElement(target) &&
- target.matches("audio[data-start], video[data-start]") &&
- target.preload === "auto"
+ target.matches("audio[data-start], video[data-start]")
) {
- this._adoptIframeMedia(target);
+ if (target.preload === "auto") this._adoptIframeMedia(target);
+ else this._detachIframeMedia(target);
}
continue;
}
diff --git a/packages/studio/src/player/hooks/useTimelinePlayer.seek.test.ts b/packages/studio/src/player/hooks/useTimelinePlayer.seek.test.ts
index 473065bc31..effd4eb106 100644
--- a/packages/studio/src/player/hooks/useTimelinePlayer.seek.test.ts
+++ b/packages/studio/src/player/hooks/useTimelinePlayer.seek.test.ts
@@ -353,7 +353,11 @@ describe("useTimelinePlayer seek keepPlaying option (#834)", () => {
describe("useTimelinePlayer RAF loop wrap-around", () => {
type SeekCall = { time: number; options?: { keepPlaying?: boolean } };
- function attachInstrumentedAdapter(api: ReturnType<typeof useTimelinePlayer>, duration = 30) {
+ function attachInstrumentedAdapter(
+ api: ReturnType<typeof useTimelinePlayer>,
+ duration = 30,
+ hooks: object = {},
+ ) {
let currentTime = 0;
let playing = false;
const seekCalls: SeekCall[] = [];
@@ -377,6 +381,7 @@ describe("useTimelinePlayer RAF loop wrap-around", () => {
};
attachIframeWindow(api, {
__player: adapter,
+ __hf: hooks,
postMessage: () => {},
scrollTo: () => {},
addEventListener: () => {},
@@ -446,6 +451,29 @@ describe("useTimelinePlayer RAF loop wrap-around", () => {
}
});
+ it("tells the preview where the loop wraps to while it plays, and that it stopped looping", () => {
+ const raf = installRafCapture();
+ try {
+ const { api, root } = renderTimelinePlayerHarness();
+ const setLoopStart = vi.fn();
+ const { adapter } = attachInstrumentedAdapter(api, 30, { setLoopStart });
+ act(() => {
+ usePlayerStore.getState().setInPoint(2);
+ usePlayerStore.getState().setOutPoint(5);
+ api.play();
+ });
+ adapter.setTime(3);
+ act(() => void raf.flushOne());
+ expect(setLoopStart).toHaveBeenLastCalledWith(2);
+ act(() => usePlayerStore.getState().setLoopEnabled(false));
+ act(() => void raf.flushOne());
+ expect(setLoopStart).toHaveBeenLastCalledWith(null);
+ unmountWithAct(root);
+ } finally {
+ raf.restore();
+ }
+ });
+
it("does not seek and pauses cleanly when forward playback reaches the end without loop", () => {
const raf = installRafCapture();
try {
diff --git a/packages/studio/src/player/hooks/useTimelinePlayer.ts b/packages/studio/src/player/hooks/useTimelinePlayer.ts
index b470feae2e..2612b594e6 100644
--- a/packages/studio/src/player/hooks/useTimelinePlayer.ts
+++ b/packages/studio/src/player/hooks/useTimelinePlayer.ts
@@ -152,12 +152,18 @@ export function useTimelinePlayer({
[],
);
+ const setLoopStart = useCallback((seconds: number | null) => {
+ try {
+ (iframeRef.current?.contentWindow as IframeWindow | null)?.__hf?.setLoopStart?.(seconds);
+ } catch {}
+ }, []);
const { startRAFLoop, stopRAFLoop, stopReverseLoop } = useTimelinePlayerLoop({
rafRef,
reverseRafRef,
getAdapter,
setCurrentTime,
setIsPlaying,
+ setLoopStart,
});
const applyPlaybackRate = useCallback((rate: number) => {
diff --git a/packages/studio/src/player/hooks/useTimelinePlayerLoop.ts b/packages/studio/src/player/hooks/useTimelinePlayerLoop.ts
index bda3d50601..306b326d38 100644
--- a/packages/studio/src/player/hooks/useTimelinePlayerLoop.ts
+++ b/packages/studio/src/player/hooks/useTimelinePlayerLoop.ts
@@ -22,6 +22,7 @@ interface UseTimelinePlayerLoopParams {
getAdapter: () => PlaybackAdapter | null;
setCurrentTime: (v: number) => void;
setIsPlaying: (v: boolean) => void;
+ setLoopStart: (seconds: number | null) => void;
}
interface UseTimelinePlayerLoopResult {
@@ -36,6 +37,7 @@ export function useTimelinePlayerLoop({
getAdapter,
setCurrentTime,
setIsPlaying,
+ setLoopStart,
}: UseTimelinePlayerLoopParams): UseTimelinePlayerLoopResult {
const stopReverseLoop = useCallback(() => {
cancelAnimationFrame(reverseRafRef.current);
@@ -55,8 +57,10 @@ export function useTimelinePlayerLoop({
const rawLoopStart = inPoint !== null ? inPoint : 0;
const loopEnd = rawLoopStart < rawLoopEnd ? rawLoopEnd : dur;
const loopStart = rawLoopStart < rawLoopEnd ? rawLoopStart : 0;
+ const looping = usePlayerStore.getState().loopEnabled && dur > 0;
+ setLoopStart(looping ? loopStart : null);
if (time >= loopEnd) {
- if (usePlayerStore.getState().loopEnabled && dur > 0) {
+ if (looping) {
// keepPlaying skips the adapter's implicit pause; play() below is then a no-op.
adapter.seek(loopStart, { keepPlaying: true });
liveTime.notify(loopStart);
@@ -75,7 +79,7 @@ export function useTimelinePlayerLoop({
rafRef.current = requestAnimationFrame(tick);
};
rafRef.current = requestAnimationFrame(tick);
- }, [rafRef, getAdapter, setCurrentTime, setIsPlaying]);
+ }, [rafRef, getAdapter, setCurrentTime, setIsPlaying, setLoopStart]);
const stopRAFLoop = useCallback(() => {
cancelAnimationFrame(rafRef.current);
diff --git a/packages/studio/src/player/lib/playbackTypes.ts b/packages/studio/src/player/lib/playbackTypes.ts
index 22747dd12c..707a3518b1 100644
--- a/packages/studio/src/player/lib/playbackTypes.ts
+++ b/packages/studio/src/player/lib/playbackTypes.ts
@@ -69,5 +69,6 @@ export type IframeWindow = Window & {
__hf?: {
leasePausedMedia?: (el: HTMLMediaElement) => void;
releasePausedMedia?: (el: HTMLMediaElement) => void;
+ setLoopStart?: (seconds: number | null) => void;
};
};