Skip to content

Write HTML. Render video. Built for agents.

fix(runtime): prepare trimmed video frames before preview cuts (#5633)

Miguel Ángel8f8f91c9de53
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index 98311e4943..d114738235 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -53,6 +53,7 @@ import {
resolveNaturalMediaTimelineDuration,
resolveRuntimeMediaClipDuration,
syncRuntimeMedia,
+ VIDEO_PRESEEK_SECONDS,
type RuntimeMediaClip,
} from "./media";
import {
@@ -3041,6 +3042,17 @@ export function initSandboxRuntimeModular(): void {
for (const clip of clipsWithEndpointBetween(index.byEnd, (c) => c.end, lo, hi)) {
visiting.add(clip.el);
}
+ if (state.isPlaying && !window.__HF_EXPORT_RENDER_SEEK_CONFIG) {
+ // An endpoint-only pass reaches trimmed video too late: its first visible tick starts a seek.
+ for (const clip of clipsWithEndpointBetween(
+ index.byStart,
+ (c) => c.start,
+ toSeconds,
+ toSeconds + VIDEO_PRESEEK_SECONDS * state.playbackRate,
+ )) {
+ if (clip.el.tagName === "VIDEO") visiting.add(clip.el);
+ }
+ }
return [...visiting];
};
diff --git a/packages/core/src/runtime/init.videoPreseek.test.ts b/packages/core/src/runtime/init.videoPreseek.test.ts
new file mode 100644
index 0000000000..bf1e38fb5c
--- /dev/null
+++ b/packages/core/src/runtime/init.videoPreseek.test.ts
@@ -0,0 +1,60 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { initSandboxRuntimeModular } from "./init";
+import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
+
+it("prepares a hidden video when playback enters its lookahead, after metadata arrives", () => {
+ resetRuntimeFixtureDom();
+ let now = 0;
+ let id = 0;
+ const callbacks = new Map<number, FrameRequestCallback>();
+ vi.spyOn(performance, "now").mockImplementation(() => now);
+ vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => {
+ callbacks.set(++id, callback);
+ return id;
+ });
+ vi.spyOn(window, "cancelAnimationFrame").mockImplementation((key) => {
+ callbacks.delete(key);
+ });
+ const step = () => {
+ now += 100;
+ const pending = [...callbacks.values()];
+ callbacks.clear();
+ for (const callback of pending) callback(now);
+ };
+ document.body.innerHTML =
+ '<div data-composition-id="main" data-root="true" data-duration="6"><video id="incoming" data-start="4" data-duration="2" data-media-start="8" src="/movie.mp4"></video></div>';
+ const video = document.querySelector("video")!;
+ Object.defineProperty(video, "duration", { value: 20, configurable: true });
+ Object.defineProperty(video, "readyState", { value: 0, configurable: true });
+ Object.defineProperty(video, "paused", { value: true, writable: true, configurable: true });
+ const play = vi.fn(() => {
+ Object.assign(video, { paused: false });
+ return Promise.resolve();
+ });
+ video.play = play;
+ video.pause = vi.fn(() => {
+ Object.assign(video, { paused: true });
+ });
+ video.load = vi.fn();
+ const seek = vi.spyOn(video, "currentTime", "set");
+ window.__timelines = { main: createMockTimeline(6) };
+ initSandboxRuntimeModular();
+ window.__player!.play();
+ for (let i = 0; i < 28; i++) step();
+ expect(seek).not.toHaveBeenCalled();
+ Object.defineProperty(video, "readyState", { value: 4, configurable: true });
+ step();
+ expect(video.style.visibility).toBe("hidden");
+ expect(video.currentTime).toBe(8);
+ expect(play).not.toHaveBeenCalled();
+ for (let i = 0; i < 11; i++) step();
+ expect(video.style.visibility).toBe("visible");
+ expect(seek).toHaveBeenCalledTimes(1);
+ expect(play).toHaveBeenCalledTimes(1);
+});
+
+afterEach(() => {
+ window.__hfRuntimeTeardown?.();
+ vi.restoreAllMocks();
+ resetRuntimeFixtureDom();
+});
diff --git a/packages/core/src/runtime/media.preseek.test.ts b/packages/core/src/runtime/media.preseek.test.ts
new file mode 100644
index 0000000000..3b11d06803
--- /dev/null
+++ b/packages/core/src/runtime/media.preseek.test.ts
@@ -0,0 +1,101 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { syncRuntimeMedia, type RuntimeMediaClip } from "./media";
+
+function fixture(overrides: Partial<RuntimeMediaClip> = {}, tag = "video") {
+ const el = document.createElement(tag) as HTMLMediaElement;
+ document.body.appendChild(el);
+ Object.defineProperty(el, "readyState", { value: 4, configurable: true });
+ el.play = vi.fn(() => Promise.resolve());
+ el.pause = vi.fn();
+ const seek = vi.spyOn(el, "currentTime", "set");
+ const clip: RuntimeMediaClip = {
+ el,
+ start: 4,
+ end: 6,
+ duration: 2,
+ mediaStart: 8,
+ playbackRate: 1,
+ volume: null,
+ loop: false,
+ sourceDuration: 20,
+ ...overrides,
+ };
+ const tick = (timeSeconds: number, playing = true, playbackRate = 1) =>
+ syncRuntimeMedia({
+ clips: [clip],
+ timeSeconds,
+ playing,
+ playbackRate,
+ getCompositionDuration: () => 10,
+ });
+ return { el, clip, seek, tick };
+}
+
+afterEach(() => {
+ document.body.innerHTML = "";
+ delete window.__HF_EXPORT_RENDER_SEEK_CONFIG;
+ vi.restoreAllMocks();
+});
+
+describe("hidden incoming video preparation", () => {
+ it("lands the in-point before the cut without playing or seeking it again at activation", () => {
+ const { el, seek, tick } = fixture();
+ tick(0);
+ expect(seek).not.toHaveBeenCalled();
+ tick(2.5);
+ expect(el.currentTime).toBe(8);
+ expect(el.play).not.toHaveBeenCalled();
+ tick(3);
+ tick(4);
+ expect(seek).toHaveBeenCalledTimes(1);
+ expect(el.play).toHaveBeenCalledTimes(1);
+ });
+
+ it("retries after metadata arrives without interrupting an outstanding seek", () => {
+ const { el, seek, tick } = fixture();
+ Object.defineProperty(el, "readyState", { value: 0, configurable: true });
+ tick(2.5);
+ expect(seek).not.toHaveBeenCalled();
+ Object.defineProperty(el, "readyState", { value: 1, configurable: true });
+ Object.defineProperty(el, "seeking", { value: true, configurable: true });
+ tick(2.6);
+ expect(seek).not.toHaveBeenCalled();
+ Object.defineProperty(el, "seeking", { value: false, configurable: true });
+ tick(2.7);
+ expect(el.currentTime).toBe(8);
+ });
+
+ it.each([
+ { origin: 2, playbackRate: 2, expected: 12 },
+ { origin: 2, playbackRate: 2, loop: true, sourceDuration: 10, expected: 8 },
+ { origin: 2, playbackRate: 2, sourceDuration: 10, expected: 10 },
+ ])("prepares the activation source position for %j", ({ expected, ...fields }) => {
+ const { el, seek, tick } = fixture(fields);
+ tick(3);
+ expect(el.currentTime).toBe(expected);
+ tick(4);
+ expect(seek).toHaveBeenCalledTimes(1);
+ });
+
+ it("uses wall-time lookahead at a faster transport rate", () => {
+ const { el, tick } = fixture();
+ tick(1, true, 2);
+ expect(el.currentTime).toBe(8);
+ });
+
+ it.each(["audio", "paused", "past", "export", "injected", "playing", "disconnected"])(
+ "leaves %s media untouched",
+ (kind) => {
+ const { el, tick, seek } = fixture({}, kind === "audio" ? "audio" : "video");
+ if (kind === "export") window.__HF_EXPORT_RENDER_SEEK_CONFIG = {};
+ if (kind === "injected") {
+ el.id = "incoming";
+ el.insertAdjacentHTML("afterend", '<img id="__render_frame_incoming__">');
+ }
+ if (kind === "playing") Object.defineProperty(el, "paused", { value: false });
+ if (kind === "disconnected") el.remove();
+ tick(kind === "past" ? 7 : 3, kind !== "paused");
+ expect(seek).not.toHaveBeenCalled();
+ },
+ );
+});
diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts
index b8490149c0..a23338c319 100644
--- a/packages/core/src/runtime/media.ts
+++ b/packages/core/src/runtime/media.ts
@@ -263,6 +263,46 @@ export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {
}
export const MEDIA_HARD_SYNC_SECONDS = 0.5;
+export const VIDEO_PRESEEK_SECONDS = 1.5;
+
+function prepareUpcomingVideo(
+ clip: RuntimeMediaClip,
+ time: number,
+ playing: boolean,
+ rate: number,
+): void {
+ const { el } = clip;
+ const dueIn = clip.start - time;
+ if (
+ !playing ||
+ el.tagName !== "VIDEO" ||
+ dueIn <= 0 ||
+ dueIn > VIDEO_PRESEEK_SECONDS * rate ||
+ !el.paused ||
+ el.seeking ||
+ el.readyState < el.HAVE_METADATA ||
+ findInjectedRenderFrame(el) ||
+ ("__HF_EXPORT_RENDER_SEEK_CONFIG" in window && window.__HF_EXPORT_RENDER_SEEK_CONFIG)
+ )
+ return;
+ let target =
+ clip.mediaStart +
+ sourceTimeAt(clip.rate ?? clip.playbackRate, clip.start - (clip.origin ?? clip.start));
+ if (clip.sourceDuration != null) {
+ const loopLength = clip.sourceDuration - clip.mediaStart;
+ if (clip.loop && loopLength > 0 && target >= clip.sourceDuration)
+ target = clip.mediaStart + ((target - clip.mediaStart) % loopLength);
+ else if (!clip.loop) target = Math.min(target, clip.sourceDuration);
+ }
+ if (!Number.isFinite(target) || target < 0 || Math.abs(el.currentTime - target) <= 0.001) return;
+ try {
+ // A trimmed video exposed during its in-point seek has no current frame at the cut.
+ // Hidden preparation must not hold the visible-frame seek barrier.
+ el.currentTime = target;
+ } catch (err) {
+ swallow("runtime.media.preseek", err);
+ }
+}
/** Drift a playing audio element may carry before sync pulls it back onto the playhead. */
export const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;
@@ -657,5 +697,6 @@ export function syncRuntimeMedia(params: {
evictMediaSyncState(el);
if (remainsInsideAuthoredWindow) lastRelativeTime.set(el, relTime);
if (!el.paused) el.pause();
+ prepareUpcomingVideo(clip, params.timeSeconds, params.playing, params.playbackRate);
}
}