Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(core): the playhead no longer jumps back when an audio clip starts (#4679)
Miguel Ángelfdbc5dcaf717
diff --git a/packages/core/src/runtime/audioClockSource.test.ts b/packages/core/src/runtime/audioClockSource.test.tsnew file mode 100644index 0000000000..3ff15d5092--- /dev/null+++ b/packages/core/src/runtime/audioClockSource.test.ts@@ -0,0 +1,215 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { initSandboxRuntimeModular } from "./init";+import type { RuntimeTimelineLike } from "./types";++type GraphNode = {+ kind: string;+ outputs: Set<GraphNode>;+ gain?: { value: number };+ connect(dest: GraphNode): GraphNode;+ disconnect(dest?: GraphNode): void;+};++function makeParam(value: number) {+ return {+ value,+ cancelScheduledValues() {},+ cancelAndHoldAtTime() {},+ setValueAtTime() {},+ linearRampToValueAtTime() {},+ setTargetAtTime() {},+ };+}++function makeNode(kind: string, extra: Partial<GraphNode> = {}): GraphNode {+ const node: GraphNode = {+ kind,+ outputs: new Set(),+ connect(dest) {+ node.outputs.add(dest);+ return dest;+ },+ disconnect(dest) {+ if (dest) node.outputs.delete(dest);+ else node.outputs.clear();+ },+ ...extra,+ };+ return node;+}++const sources = new Map<HTMLMediaElement, GraphNode>();++class GraphAudioContext {+ state = "running";+ currentTime = 0;+ destination = makeNode("destination");+ resume() {+ return Promise.resolve();+ }+ close() {+ return Promise.resolve();+ }+ createGain() {+ return makeNode("gain", { gain: makeParam(1) });+ }+ createMediaElementSource(el: HTMLMediaElement) {+ const node = makeNode("media-element");+ sources.set(el, node);+ return node;+ }+}++function createTimeline(duration: number): RuntimeTimelineLike {+ const s = { time: 0, paused: true };+ return {+ play: () => void (s.paused = false),+ pause: () => void (s.paused = true),+ seek: (t?: number) => (t === undefined ? s.time : (s.time = t)),+ totalTime: (t?: number) => (t === undefined ? s.time : (s.time = t)),+ time: () => s.time,+ duration: () => duration,+ add: () => {},+ paused: (v?: boolean) => (typeof v === "boolean" ? (s.paused = v) : s.paused),+ timeScale: () => {},+ set: () => {},+ getChildren: () => [],+ };+}++function mount(bodyHtml: string): void {+ document.body.innerHTML =+ `<div data-composition-id="main" data-root="true" data-start="0" data-duration="10"` ++ ` data-width="1920" data-height="1080">${bodyHtml}</div>`;+ for (const el of document.querySelectorAll("audio")) {+ el.load = () => {};+ el.play = vi.fn(() => Promise.resolve());+ el.pause = vi.fn();+ for (const [key, value] of Object.entries({ paused: true, readyState: 4, currentTime: 0 })) {+ Object.defineProperty(el, key, { value, writable: true, configurable: true });+ }+ }+ window.__timelines = { main: createTimeline(10) };+}++async function flush() {+ for (let i = 0; i < 4; i++) await Promise.resolve();+}++let frameCallbacks: FrameRequestCallback[] = [];+let nowMs = 0;+function stepFrames(count: number) {+ for (let i = 0; i < count; i++) {+ nowMs += 1000 / 60;+ const pending = frameCallbacks;+ frameCallbacks = [];+ for (const cb of pending) cb(nowMs);+ }+}++const originalAudioContext = (globalThis as Record<string, unknown>).AudioContext;+const originalRaf = window.requestAnimationFrame;+const originalCaf = window.cancelAnimationFrame;++beforeEach(() => {+ sources.clear();+ nowMs = 0;+ frameCallbacks = [];+ (globalThis as Record<string, unknown>).AudioContext = GraphAudioContext;+ (globalThis as typeof globalThis & { CSS?: { escape?: (v: string) => string } }).CSS ??= {};+ globalThis.CSS.escape ??= (v: string) => v;+ window.requestAnimationFrame = ((cb: FrameRequestCallback) => {+ frameCallbacks.push(cb);+ return frameCallbacks.length;+ }) as typeof window.requestAnimationFrame;+ window.cancelAnimationFrame = (() => {}) as typeof window.cancelAnimationFrame;+ vi.spyOn(performance, "now").mockImplementation(() => nowMs);+});++afterEach(() => {+ window.__hfRuntimeTeardown?.();+ document.body.innerHTML = "";+ document.documentElement.removeAttribute("style");+ document.body.removeAttribute("style");+ window.__timelines = {} as Record<string, RuntimeTimelineLike>;+ delete window.__player;+ delete window.__hf;+ vi.restoreAllMocks();+ (globalThis as Record<string, unknown>).AudioContext = originalAudioContext;+ window.requestAnimationFrame = originalRaf;+ window.cancelAnimationFrame = originalCaf;+});++describe("the audio the playhead follows", () => {+ it("keeps following a playing voice when a clip earlier in the page starts late", async () => {+ mount(+ `<audio id="sfx" data-start="1" data-duration="2" src="/assets/sfx.mp3"></audio>` ++ `<audio id="vo" data-start="0" data-duration="10" src="/assets/vo.mp3"></audio>`,+ );+ const sfx = document.getElementById("sfx") as HTMLAudioElement;+ const vo = document.getElementById("vo") as HTMLAudioElement;+ initSandboxRuntimeModular();+ await flush();+ window.__player?.play();+ await flush();+ const playedAt = nowMs;+ Object.assign(vo, { paused: false });+ let lastTime = 0;+ for (let frame = 0; frame < 90; frame++) {+ stepFrames(1);+ const t = (nowMs - playedAt) / 1000;+ vo.currentTime = t; // the voice plays in step with the wall clock+ if (t >= 1) Object.assign(sfx, { paused: false, currentTime: 0 }); // started, not moving yet+ const time = window.__player!.getTime();+ expect(time).toBeGreaterThanOrEqual(lastTime);+ lastTime = time;+ }+ expect(lastTime).toBeGreaterThan(1.4);+ });++ it("follows the next clip when one earlier in the page failed to load", async () => {+ mount(+ `<audio id="broken" data-start="0" data-duration="10" src="/assets/missing.mp3"></audio>` ++ `<audio id="vo" data-start="0" data-duration="10" src="/assets/vo.mp3"></audio>`,+ );+ const broken = document.getElementById("broken") as HTMLAudioElement;+ const vo = document.getElementById("vo") as HTMLAudioElement;+ initSandboxRuntimeModular();+ await flush();+ window.__player?.play();+ await flush();+ const playedAt = nowMs;+ Object.defineProperty(broken, "error", { value: { code: 4 } });+ Object.assign(broken, { paused: false, currentTime: 0 });+ Object.assign(vo, { paused: false });+ for (let frame = 0; frame < 90; frame++) {+ stepFrames(1);+ vo.currentTime = (nowMs - playedAt) / 1000 + 0.2; // ahead of the wall clock, so only the voice explains it+ }+ expect(window.__player!.getTime()).toBeGreaterThan((nowMs - playedAt) / 1000 + 0.1);+ });++ it("is not frozen by a clip that has no playable source", async () => {+ mount(+ `<audio id="empty" data-start="0" data-duration="10"></audio>` ++ `<audio id="vo" data-start="0" data-duration="10" src="/assets/vo.mp3"></audio>`,+ );+ const empty = document.getElementById("empty") as HTMLAudioElement;+ const vo = document.getElementById("vo") as HTMLAudioElement;+ initSandboxRuntimeModular();+ await flush();+ window.__player?.play();+ await flush();+ const playedAt = nowMs;+ // What Chrome reports for an <audio> with no source once play() is called on it.+ Object.defineProperty(empty, "networkState", { value: 0 });+ Object.defineProperty(empty, "readyState", { value: 0 });+ Object.assign(empty, { paused: false });+ Object.assign(vo, { paused: false });+ for (let frame = 0; frame < 90; frame++) {+ stepFrames(1);+ vo.currentTime = (nowMs - playedAt) / 1000;+ }+ expect(window.__player!.getTime()).toBeGreaterThan(1.4);+ });+});diff --git a/packages/core/src/runtime/clock.test.ts b/packages/core/src/runtime/clock.test.tsindex 6e4e1672f8..ddea31687b 100644--- a/packages/core/src/runtime/clock.test.ts+++ b/packages/core/src/runtime/clock.test.ts@@ -316,6 +316,99 @@ describe("TransportClock", () => {expect(clock.getSource()).toBe("monotonic");});+ it("holds, never rewinds, the playhead for a clip that started late, then follows it", () => {+ const { clock, advance } = createClock({ duration: 30 });+ clock.play();+ advancePolled(clock, advance, 6300);+ const sfx = createMockAudioEl(0, false); // at its start, 50 ms behind the playhead+ clock.attachAudioSource({ el: sfx, compositionStart: 6.25, mediaStart: 0 });+ expect(clock.now()).toBeCloseTo(6.3, 5);+ advance(100);+ sfx.currentTime = 0.02;+ expect(clock.now()).toBeCloseTo(6.3, 5);+ advance(100);+ sfx.currentTime = 0.12;+ expect(clock.now()).toBeCloseTo(6.37, 5);+ });++ it("holds the playhead while its element seeks, even to a time ahead", () => {+ const { clock, advance } = createClock({ duration: 30 });+ clock.play();+ advancePolled(clock, advance, 2000);+ const voice = Object.assign(createMockAudioEl(3, false), { seeking: true });+ clock.attachAudioSource({ el: voice, compositionStart: 2, mediaStart: 0 });+ advance(150);+ expect(clock.now()).toBeCloseTo(2, 5);+ });++ it("holds the playhead on an element that is buffering behind it", () => {+ const { clock, advance } = createClock({ duration: 30 });+ clock.play();+ advancePolled(clock, advance, 6300);+ const voice = Object.assign(createMockAudioEl(0, false), { readyState: 1 });+ clock.attachAudioSource({ el: voice, compositionStart: 5, mediaStart: 0 });+ advance(500);+ expect(clock.now()).toBeCloseTo(6.3, 5);+ });++ it("does not follow a jump back past a hard sync, such as a native loop wrap", () => {+ const { clock, advance } = createClock({ duration: 30 });+ clock.play();+ const bed = createMockAudioEl(5, false);+ clock.attachAudioSource({ el: bed, compositionStart: 0, mediaStart: 0 });+ expect(clock.now()).toBe(5);+ advance(16);+ bed.currentTime = 1; // wrapped to its loop start+ expect(clock.now()).toBeCloseTo(5.016, 5);+ });++ it("does not freeze on a short native loop that wraps", () => {+ const { clock, advance } = createClock({ duration: 30 });+ clock.play();+ const bed = Object.assign(createMockAudioEl(0.3, false), { loop: true, duration: 0.32 });+ clock.attachAudioSource({ el: bed, compositionStart: 0, mediaStart: 0 });+ expect(clock.now()).toBeCloseTo(0.3, 5);+ advance(16);+ bed.currentTime = 0.02; // wrapped to its loop start+ expect(clock.now()).toBeCloseTo(0.316, 5);+ });++ it("holds the playhead for a looping bed that starts late", () => {+ const { clock, advance } = createClock({ duration: 30 });+ clock.play();+ advancePolled(clock, advance, 2000);+ const bed = Object.assign(createMockAudioEl(0, false), { loop: true, duration: 8 });+ clock.attachAudioSource({ el: bed, compositionStart: 1.9, mediaStart: 0 });+ advance(16);+ expect(clock.now()).toBeCloseTo(2, 5);+ });++ it("continues from the last audio time when the element drops out", () => {+ const { clock, advance } = createClock({ duration: 30 });+ clock.play();+ advancePolled(clock, advance, 4000);+ const voice = createMockAudioEl(5, false);+ clock.attachAudioSource({ el: voice, compositionStart: 0, mediaStart: 0 });+ expect(clock.now()).toBe(5);+ Object.assign(voice, { paused: true });+ advance(16);+ expect(clock.now()).toBeCloseTo(5.016, 5);+ });++ it("lets a seek move the playhead back past a held time", () => {+ const { clock, advance } = createClock({ duration: 30 });+ clock.play();+ const voice = createMockAudioEl(5, false);+ clock.attachAudioSource({ el: voice, compositionStart: 0, mediaStart: 0 });+ expect(clock.now()).toBe(5);+ clock.detachAudioSource();+ clock.seek(2);+ voice.currentTime = 2;+ clock.attachAudioSource({ el: voice, compositionStart: 0, mediaStart: 0 });+ advance(16);+ expect(clock.now()).toBe(2);+ });+it("clamps audio-derived time to duration", () => {const { clock } = createClock({ duration: 10 });const audioEl = createMockAudioEl(15.0, false);@@ -364,15 +457,24 @@ describe("TransportClock", () => {expect(clock.now()).toBe(4);});- it("composition time is correct when el.playbackRate differs from clock rate", () => {- // General formula: wall_elapsed = (el.currentTime - mediaStart) / el.playbackRate- // composition_time = compositionStart + wall_elapsed * clockRate+ it("reads the element's place in its file, whatever playbackRate it has been given", () => {const { clock } = createClock({ rate: 2, duration: 20 });- // Audio at 1x, clock at 2x: after 1s wall, el.currentTime=1, comp should be 2.const audioEl = { currentTime: 1, paused: false, playbackRate: 1 } as HTMLMediaElement;clock.play();clock.attachAudioSource({ el: audioEl, compositionStart: 0, mediaStart: 0 });- expect(clock.now()).toBe(2);+ expect(clock.now()).toBe(1);+ });++ it("keeps its place when the speed changes under a clip with its own rate", () => {+ const { clock, advance } = createClock({ duration: 30 });+ clock.play();+ const voice = { currentTime: 12.5, paused: false, playbackRate: 1.25 } as HTMLMediaElement;+ clock.attachAudioSource({ el: voice, compositionStart: 0, mediaStart: 0, rate: 1.25 });+ expect(clock.now()).toBe(10);+ clock.setRate(1.5);+ voice.playbackRate = 1.5; // the transport rate lands on the element before media sync restores 1.875+ advance(16);+ expect(clock.now()).toBe(10);});});});diff --git a/packages/core/src/runtime/clock.ts b/packages/core/src/runtime/clock.tsindex 13d5a17878..1bae66df7c 100644--- a/packages/core/src/runtime/clock.ts+++ b/packages/core/src/runtime/clock.ts@@ -1,4 +1,5 @@import { timeAtSourceTime, type RateSpec } from "../speedRamp.js";+import { MEDIA_HARD_SYNC_SECONDS } from "./media.js";export type TransportClockSnapshot = {time: number;@@ -23,6 +24,7 @@ export type AudioClockSource =/** GSAP's own `lagSmoothing` default — see the PR body for why this clock needs its own copy. */const STALL_THRESHOLD_MS = 500;const STALL_ADJUSTED_LAG_MS = 33;+const HAVE_FUTURE_DATA = 3;export class TransportClock {private _baseTime = 0;@@ -33,6 +35,7 @@ export class TransportClock {private _audioSource: AudioClockSource | null = null;/** Wall-clock time of the last `now()` read while playing; null while paused. */private _lastReadMs: number | null = null;+ private _lastNow = 0;constructor(opts?: {initialTime?: number;@@ -58,22 +61,15 @@ export class TransportClock {if ("currentTimeSeconds" in this._audioSource) {audioTime = this._audioSource.currentTimeSeconds;} else {- const { el, compositionStart, mediaStart, rate } = this._audioSource;- if (!el.paused && Number.isFinite(el.currentTime)) {- audioTime =- typeof rate === "object"- ? timeAtSourceTime(rate, el.currentTime - mediaStart) + compositionStart- : ((el.currentTime - mediaStart) / (el.playbackRate > 0 ? el.playbackRate : 1)) *- this._rate +- compositionStart;- }+ audioTime = this._elementTimeNeverBehind(this._audioSource);}if (audioTime !== null) {+ const t = Number.isFinite(this._duration) ? Math.min(audioTime, this._duration) : audioTime;+ this._baseTime = Math.max(0, t);+ this._playStartMs = this._nowMs();this._lastReadMs = null;- if (Number.isFinite(this._duration) && audioTime >= this._duration) {- return this._duration;- }- return Math.max(0, audioTime);+ this._lastNow = this._baseTime;+ return this._baseTime;}}@@ -81,10 +77,25 @@ export class TransportClock {this._applyStallCorrection();const elapsed = (this._nowMs() - this._playStartMs) / 1000;const t = this._baseTime + elapsed * this._rate;- if (Number.isFinite(this._duration) && t >= this._duration) {- return this._duration;- }- return Math.max(0, t);+ this._lastNow = Math.max(0, Number.isFinite(this._duration) ? Math.min(t, this._duration) : t);+ return this._lastNow;+ }++ private _elementTimeNeverBehind(+ source: Extract<AudioClockSource, { el: HTMLMediaElement }>,+ ): number | null {+ const { el, compositionStart, mediaStart, rate } = source;+ if (el.paused || !Number.isFinite(el.currentTime)) return null;+ if (el.seeking) return this._lastNow;+ const spec = rate ?? 1;+ const time = timeAtSourceTime(spec, el.currentTime - mediaStart) + compositionStart;+ return time >= this._lastNow ? time : this._heldTimeWhenBehind(el, this._lastNow - time, spec);+ }++ private _heldTimeWhenBehind(el: HTMLMediaElement, behind: number, spec: RateSpec): number | null {+ if (el.loop && behind > timeAtSourceTime(spec, el.duration) / 2) return null;+ const buffering = el.readyState < HAVE_FUTURE_DATA;+ return buffering || behind <= MEDIA_HARD_SYNC_SECONDS ? this._lastNow : null;}/** Folds a >500ms gap since the last read into `_playStartMs` so it's never reported as played time. */@@ -104,6 +115,7 @@ export class TransportClock {if (this._playStartMs !== null) return false;if (Number.isFinite(this._duration) && this._baseTime >= this._duration) return false;this._playStartMs = this._nowMs();+ this._lastNow = this._baseTime;// Not a stall: the gap since the clock was last read (possibly a long// paused idle) says nothing about lost playback time, since none was// playing. `_applyStallCorrection` treats null as "nothing to compare@@ -124,6 +136,7 @@ export class TransportClock {? Math.max(0, Math.min(timeSeconds, this._duration)): Math.max(0, timeSeconds);this._baseTime = clamped;+ this._lastNow = clamped;if (this._playStartMs !== null) {this._playStartMs = this._nowMs();// Same reasoning as `play()`: the seek itself, not any elapsed gap@@ -176,6 +189,11 @@ export class TransportClock {this._audioSource = null;}+ /** The element the playhead currently follows, if the source is one. */+ audioElement(): HTMLMediaElement | null {+ return this._audioSource && "el" in this._audioSource ? this._audioSource.el : null;+ }+hasAudioSource(): boolean {return this._audioSource !== null;}diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.tsindex b345b74559..eae5c00c76 100644--- a/packages/core/src/runtime/init.ts+++ b/packages/core/src/runtime/init.ts@@ -30,14 +30,15 @@ import {import { sourceTimeAt } from "../speedRamp";import { createWaapiAdapter } from "./adapters/waapi";import {+ isUnplayable,readElementPlaybackRate,- readElementRateSpec,readElementPlaybackStart,+ readElementRateSpec,refreshRuntimeMediaCache,- resolveRuntimeMediaClipDuration,resolveNaturalMediaTimelineDuration,- type RuntimeMediaClip,+ resolveRuntimeMediaClipDuration,syncRuntimeMedia,+ type RuntimeMediaClip,} from "./media";import { handleErrorForProxy, handleMetadataForProxy, maybeProxyProactively } from "./mediaProxy";import { probeAndCacheElementVolume, type VolumeKeyframe } from "./mediaVolumeEnvelope.js";@@ -4511,10 +4512,12 @@ export function initSandboxRuntimeModular(): void {}} else {const audioEls = document.querySelectorAll("audio[data-start]");+ const followed = clock.audioElement();let foundActive = false;- for (const rawEl of audioEls) {+ for (const rawEl of followed ? [followed, ...audioEls] : audioEls) {if (!isMediaElement(rawEl) || !rawEl.isConnected) continue;- if (isSilencedByHidden(rawEl)) continue;+ if (isSilencedByHidden(rawEl) || isUnplayable(rawEl)) continue;+ if (!rawEl.hasAttribute("src") && !rawEl.querySelector("source[src]")) continue;const start = resolveAbsoluteMediaStartSeconds(rawEl);const durAttr = parseStrictFiniteTimingNumber(rawEl.dataset.duration);const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;@@ -4528,7 +4531,7 @@ export function initSandboxRuntimeModular(): void {rate: readElementRateSpec(rawEl),});foundActive = true;- } else if (!rawEl.error && rawEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {+ } else if (rawEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {// Audio is buffering — freeze visuals at last known position// instead of falling through to monotonic (which runs ahead).clock.attachAudioSource({ currentTimeSeconds: state.currentTime });diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.tsindex 1428f8d88e..f790727b05 100644--- a/packages/core/src/runtime/media.ts+++ b/packages/core/src/runtime/media.ts@@ -208,7 +208,7 @@ function markPlayRequested(el: HTMLMediaElement): void {const MEDIA_NETWORK_NO_SOURCE = 3;// An element that errored or has no source can't play; re-issuing play() every// tick just floods rejections. Skip it until its state changes (src reload).-function isUnplayable(el: HTMLMediaElement): boolean {+export function isUnplayable(el: HTMLMediaElement): boolean {return el.error != null || el.networkState === MEDIA_NETWORK_NO_SOURCE;}@@ -250,6 +250,8 @@ export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {);}+export const MEDIA_HARD_SYNC_SECONDS = 0.5;+/** Drift a playing audio element may carry before sync pulls it back onto the playhead. */const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;@@ -495,7 +497,7 @@ export function syncRuntimeMedia(params: {(isHeldVideoTail && drift > 0.001) ||(el.ended && canSeekEndedMediaBackward && drift > 0.001) ||staleAudioOnFirstTick ||- (drift > 0.5 && (firstTickOfClip || offsetJumped || catastrophicDrift));+ (drift > MEDIA_HARD_SYNC_SECONDS && (firstTickOfClip || offsetJumped || catastrophicDrift));// Playing videos use the browser's decoder for timing. Seeking one resets the decoder: a// ~150ms freeze while it re-buffers, as the monotonic clock advances, which loops into a// seek→freeze→drift→seek stutter. So a playing video skips strict and force sync; only hard