Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(runtime): prepare trimmed video frames before preview cuts (#5633)
Miguel Ángel8f8f91c9de53
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.tsindex 98311e4943..d114738235 100644--- a/packages/core/src/runtime/init.ts+++ b/packages/core/src/runtime/init.ts@@ -53,6 +53,7 @@ import {resolveNaturalMediaTimelineDuration,resolveRuntimeMediaClipDuration,syncRuntimeMedia,+ VIDEO_PRESEEK_SECONDS,type RuntimeMediaClip,} from "./media";import {@@ -3041,6 +3042,17 @@ export function initSandboxRuntimeModular(): void {for (const clip of clipsWithEndpointBetween(index.byEnd, (c) => c.end, lo, hi)) {visiting.add(clip.el);}+ if (state.isPlaying && !window.__HF_EXPORT_RENDER_SEEK_CONFIG) {+ // An endpoint-only pass reaches trimmed video too late: its first visible tick starts a seek.+ for (const clip of clipsWithEndpointBetween(+ index.byStart,+ (c) => c.start,+ toSeconds,+ toSeconds + VIDEO_PRESEEK_SECONDS * state.playbackRate,+ )) {+ if (clip.el.tagName === "VIDEO") visiting.add(clip.el);+ }+ }return [...visiting];};diff --git a/packages/core/src/runtime/init.videoPreseek.test.ts b/packages/core/src/runtime/init.videoPreseek.test.tsnew file mode 100644index 0000000000..bf1e38fb5c--- /dev/null+++ b/packages/core/src/runtime/init.videoPreseek.test.ts@@ -0,0 +1,60 @@+import { afterEach, expect, it, vi } from "vitest";+import { initSandboxRuntimeModular } from "./init";+import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";++it("prepares a hidden video when playback enters its lookahead, after metadata arrives", () => {+ resetRuntimeFixtureDom();+ let now = 0;+ let id = 0;+ const callbacks = new Map<number, FrameRequestCallback>();+ vi.spyOn(performance, "now").mockImplementation(() => now);+ vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => {+ callbacks.set(++id, callback);+ return id;+ });+ vi.spyOn(window, "cancelAnimationFrame").mockImplementation((key) => {+ callbacks.delete(key);+ });+ const step = () => {+ now += 100;+ const pending = [...callbacks.values()];+ callbacks.clear();+ for (const callback of pending) callback(now);+ };+ document.body.innerHTML =+ '<div data-composition-id="main" data-root="true" data-duration="6"><video id="incoming" data-start="4" data-duration="2" data-media-start="8" src="/movie.mp4"></video></div>';+ const video = document.querySelector("video")!;+ Object.defineProperty(video, "duration", { value: 20, configurable: true });+ Object.defineProperty(video, "readyState", { value: 0, configurable: true });+ Object.defineProperty(video, "paused", { value: true, writable: true, configurable: true });+ const play = vi.fn(() => {+ Object.assign(video, { paused: false });+ return Promise.resolve();+ });+ video.play = play;+ video.pause = vi.fn(() => {+ Object.assign(video, { paused: true });+ });+ video.load = vi.fn();+ const seek = vi.spyOn(video, "currentTime", "set");+ window.__timelines = { main: createMockTimeline(6) };+ initSandboxRuntimeModular();+ window.__player!.play();+ for (let i = 0; i < 28; i++) step();+ expect(seek).not.toHaveBeenCalled();+ Object.defineProperty(video, "readyState", { value: 4, configurable: true });+ step();+ expect(video.style.visibility).toBe("hidden");+ expect(video.currentTime).toBe(8);+ expect(play).not.toHaveBeenCalled();+ for (let i = 0; i < 11; i++) step();+ expect(video.style.visibility).toBe("visible");+ expect(seek).toHaveBeenCalledTimes(1);+ expect(play).toHaveBeenCalledTimes(1);+});++afterEach(() => {+ window.__hfRuntimeTeardown?.();+ vi.restoreAllMocks();+ resetRuntimeFixtureDom();+});diff --git a/packages/core/src/runtime/media.preseek.test.ts b/packages/core/src/runtime/media.preseek.test.tsnew file mode 100644index 0000000000..3b11d06803--- /dev/null+++ b/packages/core/src/runtime/media.preseek.test.ts@@ -0,0 +1,101 @@+import { afterEach, describe, expect, it, vi } from "vitest";+import { syncRuntimeMedia, type RuntimeMediaClip } from "./media";++function fixture(overrides: Partial<RuntimeMediaClip> = {}, tag = "video") {+ const el = document.createElement(tag) as HTMLMediaElement;+ document.body.appendChild(el);+ Object.defineProperty(el, "readyState", { value: 4, configurable: true });+ el.play = vi.fn(() => Promise.resolve());+ el.pause = vi.fn();+ const seek = vi.spyOn(el, "currentTime", "set");+ const clip: RuntimeMediaClip = {+ el,+ start: 4,+ end: 6,+ duration: 2,+ mediaStart: 8,+ playbackRate: 1,+ volume: null,+ loop: false,+ sourceDuration: 20,+ ...overrides,+ };+ const tick = (timeSeconds: number, playing = true, playbackRate = 1) =>+ syncRuntimeMedia({+ clips: [clip],+ timeSeconds,+ playing,+ playbackRate,+ getCompositionDuration: () => 10,+ });+ return { el, clip, seek, tick };+}++afterEach(() => {+ document.body.innerHTML = "";+ delete window.__HF_EXPORT_RENDER_SEEK_CONFIG;+ vi.restoreAllMocks();+});++describe("hidden incoming video preparation", () => {+ it("lands the in-point before the cut without playing or seeking it again at activation", () => {+ const { el, seek, tick } = fixture();+ tick(0);+ expect(seek).not.toHaveBeenCalled();+ tick(2.5);+ expect(el.currentTime).toBe(8);+ expect(el.play).not.toHaveBeenCalled();+ tick(3);+ tick(4);+ expect(seek).toHaveBeenCalledTimes(1);+ expect(el.play).toHaveBeenCalledTimes(1);+ });++ it("retries after metadata arrives without interrupting an outstanding seek", () => {+ const { el, seek, tick } = fixture();+ Object.defineProperty(el, "readyState", { value: 0, configurable: true });+ tick(2.5);+ expect(seek).not.toHaveBeenCalled();+ Object.defineProperty(el, "readyState", { value: 1, configurable: true });+ Object.defineProperty(el, "seeking", { value: true, configurable: true });+ tick(2.6);+ expect(seek).not.toHaveBeenCalled();+ Object.defineProperty(el, "seeking", { value: false, configurable: true });+ tick(2.7);+ expect(el.currentTime).toBe(8);+ });++ it.each([+ { origin: 2, playbackRate: 2, expected: 12 },+ { origin: 2, playbackRate: 2, loop: true, sourceDuration: 10, expected: 8 },+ { origin: 2, playbackRate: 2, sourceDuration: 10, expected: 10 },+ ])("prepares the activation source position for %j", ({ expected, ...fields }) => {+ const { el, seek, tick } = fixture(fields);+ tick(3);+ expect(el.currentTime).toBe(expected);+ tick(4);+ expect(seek).toHaveBeenCalledTimes(1);+ });++ it("uses wall-time lookahead at a faster transport rate", () => {+ const { el, tick } = fixture();+ tick(1, true, 2);+ expect(el.currentTime).toBe(8);+ });++ it.each(["audio", "paused", "past", "export", "injected", "playing", "disconnected"])(+ "leaves %s media untouched",+ (kind) => {+ const { el, tick, seek } = fixture({}, kind === "audio" ? "audio" : "video");+ if (kind === "export") window.__HF_EXPORT_RENDER_SEEK_CONFIG = {};+ if (kind === "injected") {+ el.id = "incoming";+ el.insertAdjacentHTML("afterend", '<img id="__render_frame_incoming__">');+ }+ if (kind === "playing") Object.defineProperty(el, "paused", { value: false });+ if (kind === "disconnected") el.remove();+ tick(kind === "past" ? 7 : 3, kind !== "paused");+ expect(seek).not.toHaveBeenCalled();+ },+ );+});diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.tsindex b8490149c0..a23338c319 100644--- a/packages/core/src/runtime/media.ts+++ b/packages/core/src/runtime/media.ts@@ -263,6 +263,46 @@ export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {}export const MEDIA_HARD_SYNC_SECONDS = 0.5;+export const VIDEO_PRESEEK_SECONDS = 1.5;++function prepareUpcomingVideo(+ clip: RuntimeMediaClip,+ time: number,+ playing: boolean,+ rate: number,+): void {+ const { el } = clip;+ const dueIn = clip.start - time;+ if (+ !playing ||+ el.tagName !== "VIDEO" ||+ dueIn <= 0 ||+ dueIn > VIDEO_PRESEEK_SECONDS * rate ||+ !el.paused ||+ el.seeking ||+ el.readyState < el.HAVE_METADATA ||+ findInjectedRenderFrame(el) ||+ ("__HF_EXPORT_RENDER_SEEK_CONFIG" in window && window.__HF_EXPORT_RENDER_SEEK_CONFIG)+ )+ return;+ let target =+ clip.mediaStart ++ sourceTimeAt(clip.rate ?? clip.playbackRate, clip.start - (clip.origin ?? clip.start));+ if (clip.sourceDuration != null) {+ const loopLength = clip.sourceDuration - clip.mediaStart;+ if (clip.loop && loopLength > 0 && target >= clip.sourceDuration)+ target = clip.mediaStart + ((target - clip.mediaStart) % loopLength);+ else if (!clip.loop) target = Math.min(target, clip.sourceDuration);+ }+ if (!Number.isFinite(target) || target < 0 || Math.abs(el.currentTime - target) <= 0.001) return;+ try {+ // A trimmed video exposed during its in-point seek has no current frame at the cut.+ // Hidden preparation must not hold the visible-frame seek barrier.+ el.currentTime = target;+ } catch (err) {+ swallow("runtime.media.preseek", err);+ }+}/** Drift a playing audio element may carry before sync pulls it back onto the playhead. */export const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;@@ -657,5 +697,6 @@ export function syncRuntimeMedia(params: {evictMediaSyncState(el);if (remainsInsideAuthoredWindow) lastRelativeTime.set(el, relTime);if (!el.paused) el.pause();+ prepareUpcomingVideo(clip, params.timeSeconds, params.playing, params.playbackRate);}}