Skip to content

Write HTML. Render video. Built for agents.

fix(core): the playhead no longer jumps back when an audio clip starts (#4679)

Miguel Ángelfdbc5dcaf717
diff --git a/packages/core/src/runtime/audioClockSource.test.ts b/packages/core/src/runtime/audioClockSource.test.ts
new file mode 100644
index 0000000000..3ff15d5092
--- /dev/null
+++ b/packages/core/src/runtime/audioClockSource.test.ts
@@ -0,0 +1,215 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { initSandboxRuntimeModular } from "./init";
+import type { RuntimeTimelineLike } from "./types";
+
+type GraphNode = {
+ kind: string;
+ outputs: Set<GraphNode>;
+ gain?: { value: number };
+ connect(dest: GraphNode): GraphNode;
+ disconnect(dest?: GraphNode): void;
+};
+
+function makeParam(value: number) {
+ return {
+ value,
+ cancelScheduledValues() {},
+ cancelAndHoldAtTime() {},
+ setValueAtTime() {},
+ linearRampToValueAtTime() {},
+ setTargetAtTime() {},
+ };
+}
+
+function makeNode(kind: string, extra: Partial<GraphNode> = {}): GraphNode {
+ const node: GraphNode = {
+ kind,
+ outputs: new Set(),
+ connect(dest) {
+ node.outputs.add(dest);
+ return dest;
+ },
+ disconnect(dest) {
+ if (dest) node.outputs.delete(dest);
+ else node.outputs.clear();
+ },
+ ...extra,
+ };
+ return node;
+}
+
+const sources = new Map<HTMLMediaElement, GraphNode>();
+
+class GraphAudioContext {
+ state = "running";
+ currentTime = 0;
+ destination = makeNode("destination");
+ resume() {
+ return Promise.resolve();
+ }
+ close() {
+ return Promise.resolve();
+ }
+ createGain() {
+ return makeNode("gain", { gain: makeParam(1) });
+ }
+ createMediaElementSource(el: HTMLMediaElement) {
+ const node = makeNode("media-element");
+ sources.set(el, node);
+ return node;
+ }
+}
+
+function createTimeline(duration: number): RuntimeTimelineLike {
+ const s = { time: 0, paused: true };
+ return {
+ play: () => void (s.paused = false),
+ pause: () => void (s.paused = true),
+ seek: (t?: number) => (t === undefined ? s.time : (s.time = t)),
+ totalTime: (t?: number) => (t === undefined ? s.time : (s.time = t)),
+ time: () => s.time,
+ duration: () => duration,
+ add: () => {},
+ paused: (v?: boolean) => (typeof v === "boolean" ? (s.paused = v) : s.paused),
+ timeScale: () => {},
+ set: () => {},
+ getChildren: () => [],
+ };
+}
+
+function mount(bodyHtml: string): void {
+ document.body.innerHTML =
+ `<div data-composition-id="main" data-root="true" data-start="0" data-duration="10"` +
+ ` data-width="1920" data-height="1080">${bodyHtml}</div>`;
+ for (const el of document.querySelectorAll("audio")) {
+ el.load = () => {};
+ el.play = vi.fn(() => Promise.resolve());
+ el.pause = vi.fn();
+ for (const [key, value] of Object.entries({ paused: true, readyState: 4, currentTime: 0 })) {
+ Object.defineProperty(el, key, { value, writable: true, configurable: true });
+ }
+ }
+ window.__timelines = { main: createTimeline(10) };
+}
+
+async function flush() {
+ for (let i = 0; i < 4; i++) await Promise.resolve();
+}
+
+let frameCallbacks: FrameRequestCallback[] = [];
+let nowMs = 0;
+function stepFrames(count: number) {
+ for (let i = 0; i < count; i++) {
+ nowMs += 1000 / 60;
+ const pending = frameCallbacks;
+ frameCallbacks = [];
+ for (const cb of pending) cb(nowMs);
+ }
+}
+
+const originalAudioContext = (globalThis as Record<string, unknown>).AudioContext;
+const originalRaf = window.requestAnimationFrame;
+const originalCaf = window.cancelAnimationFrame;
+
+beforeEach(() => {
+ sources.clear();
+ nowMs = 0;
+ frameCallbacks = [];
+ (globalThis as Record<string, unknown>).AudioContext = GraphAudioContext;
+ (globalThis as typeof globalThis & { CSS?: { escape?: (v: string) => string } }).CSS ??= {};
+ globalThis.CSS.escape ??= (v: string) => v;
+ window.requestAnimationFrame = ((cb: FrameRequestCallback) => {
+ frameCallbacks.push(cb);
+ return frameCallbacks.length;
+ }) as typeof window.requestAnimationFrame;
+ window.cancelAnimationFrame = (() => {}) as typeof window.cancelAnimationFrame;
+ vi.spyOn(performance, "now").mockImplementation(() => nowMs);
+});
+
+afterEach(() => {
+ window.__hfRuntimeTeardown?.();
+ document.body.innerHTML = "";
+ document.documentElement.removeAttribute("style");
+ document.body.removeAttribute("style");
+ window.__timelines = {} as Record<string, RuntimeTimelineLike>;
+ delete window.__player;
+ delete window.__hf;
+ vi.restoreAllMocks();
+ (globalThis as Record<string, unknown>).AudioContext = originalAudioContext;
+ window.requestAnimationFrame = originalRaf;
+ window.cancelAnimationFrame = originalCaf;
+});
+
+describe("the audio the playhead follows", () => {
+ it("keeps following a playing voice when a clip earlier in the page starts late", async () => {
+ mount(
+ `<audio id="sfx" data-start="1" data-duration="2" src="/assets/sfx.mp3"></audio>` +
+ `<audio id="vo" data-start="0" data-duration="10" src="/assets/vo.mp3"></audio>`,
+ );
+ const sfx = document.getElementById("sfx") as HTMLAudioElement;
+ const vo = document.getElementById("vo") as HTMLAudioElement;
+ initSandboxRuntimeModular();
+ await flush();
+ window.__player?.play();
+ await flush();
+ const playedAt = nowMs;
+ Object.assign(vo, { paused: false });
+ let lastTime = 0;
+ for (let frame = 0; frame < 90; frame++) {
+ stepFrames(1);
+ const t = (nowMs - playedAt) / 1000;
+ vo.currentTime = t; // the voice plays in step with the wall clock
+ if (t >= 1) Object.assign(sfx, { paused: false, currentTime: 0 }); // started, not moving yet
+ const time = window.__player!.getTime();
+ expect(time).toBeGreaterThanOrEqual(lastTime);
+ lastTime = time;
+ }
+ expect(lastTime).toBeGreaterThan(1.4);
+ });
+
+ it("follows the next clip when one earlier in the page failed to load", async () => {
+ mount(
+ `<audio id="broken" data-start="0" data-duration="10" src="/assets/missing.mp3"></audio>` +
+ `<audio id="vo" data-start="0" data-duration="10" src="/assets/vo.mp3"></audio>`,
+ );
+ const broken = document.getElementById("broken") as HTMLAudioElement;
+ const vo = document.getElementById("vo") as HTMLAudioElement;
+ initSandboxRuntimeModular();
+ await flush();
+ window.__player?.play();
+ await flush();
+ const playedAt = nowMs;
+ Object.defineProperty(broken, "error", { value: { code: 4 } });
+ Object.assign(broken, { paused: false, currentTime: 0 });
+ Object.assign(vo, { paused: false });
+ for (let frame = 0; frame < 90; frame++) {
+ stepFrames(1);
+ vo.currentTime = (nowMs - playedAt) / 1000 + 0.2; // ahead of the wall clock, so only the voice explains it
+ }
+ expect(window.__player!.getTime()).toBeGreaterThan((nowMs - playedAt) / 1000 + 0.1);
+ });
+
+ it("is not frozen by a clip that has no playable source", async () => {
+ mount(
+ `<audio id="empty" data-start="0" data-duration="10"></audio>` +
+ `<audio id="vo" data-start="0" data-duration="10" src="/assets/vo.mp3"></audio>`,
+ );
+ const empty = document.getElementById("empty") as HTMLAudioElement;
+ const vo = document.getElementById("vo") as HTMLAudioElement;
+ initSandboxRuntimeModular();
+ await flush();
+ window.__player?.play();
+ await flush();
+ const playedAt = nowMs;
+ // What Chrome reports for an <audio> with no source once play() is called on it.
+ Object.defineProperty(empty, "networkState", { value: 0 });
+ Object.defineProperty(empty, "readyState", { value: 0 });
+ Object.assign(empty, { paused: false });
+ Object.assign(vo, { paused: false });
+ for (let frame = 0; frame < 90; frame++) {
+ stepFrames(1);
+ vo.currentTime = (nowMs - playedAt) / 1000;
+ }
+ expect(window.__player!.getTime()).toBeGreaterThan(1.4);
+ });
+});
diff --git a/packages/core/src/runtime/clock.test.ts b/packages/core/src/runtime/clock.test.ts
index 6e4e1672f8..ddea31687b 100644
--- a/packages/core/src/runtime/clock.test.ts
+++ b/packages/core/src/runtime/clock.test.ts
@@ -316,6 +316,99 @@ describe("TransportClock", () => {
expect(clock.getSource()).toBe("monotonic");
});
+ it("holds, never rewinds, the playhead for a clip that started late, then follows it", () => {
+ const { clock, advance } = createClock({ duration: 30 });
+ clock.play();
+ advancePolled(clock, advance, 6300);
+ const sfx = createMockAudioEl(0, false); // at its start, 50 ms behind the playhead
+ clock.attachAudioSource({ el: sfx, compositionStart: 6.25, mediaStart: 0 });
+ expect(clock.now()).toBeCloseTo(6.3, 5);
+ advance(100);
+ sfx.currentTime = 0.02;
+ expect(clock.now()).toBeCloseTo(6.3, 5);
+ advance(100);
+ sfx.currentTime = 0.12;
+ expect(clock.now()).toBeCloseTo(6.37, 5);
+ });
+
+ it("holds the playhead while its element seeks, even to a time ahead", () => {
+ const { clock, advance } = createClock({ duration: 30 });
+ clock.play();
+ advancePolled(clock, advance, 2000);
+ const voice = Object.assign(createMockAudioEl(3, false), { seeking: true });
+ clock.attachAudioSource({ el: voice, compositionStart: 2, mediaStart: 0 });
+ advance(150);
+ expect(clock.now()).toBeCloseTo(2, 5);
+ });
+
+ it("holds the playhead on an element that is buffering behind it", () => {
+ const { clock, advance } = createClock({ duration: 30 });
+ clock.play();
+ advancePolled(clock, advance, 6300);
+ const voice = Object.assign(createMockAudioEl(0, false), { readyState: 1 });
+ clock.attachAudioSource({ el: voice, compositionStart: 5, mediaStart: 0 });
+ advance(500);
+ expect(clock.now()).toBeCloseTo(6.3, 5);
+ });
+
+ it("does not follow a jump back past a hard sync, such as a native loop wrap", () => {
+ const { clock, advance } = createClock({ duration: 30 });
+ clock.play();
+ const bed = createMockAudioEl(5, false);
+ clock.attachAudioSource({ el: bed, compositionStart: 0, mediaStart: 0 });
+ expect(clock.now()).toBe(5);
+ advance(16);
+ bed.currentTime = 1; // wrapped to its loop start
+ expect(clock.now()).toBeCloseTo(5.016, 5);
+ });
+
+ it("does not freeze on a short native loop that wraps", () => {
+ const { clock, advance } = createClock({ duration: 30 });
+ clock.play();
+ const bed = Object.assign(createMockAudioEl(0.3, false), { loop: true, duration: 0.32 });
+ clock.attachAudioSource({ el: bed, compositionStart: 0, mediaStart: 0 });
+ expect(clock.now()).toBeCloseTo(0.3, 5);
+ advance(16);
+ bed.currentTime = 0.02; // wrapped to its loop start
+ expect(clock.now()).toBeCloseTo(0.316, 5);
+ });
+
+ it("holds the playhead for a looping bed that starts late", () => {
+ const { clock, advance } = createClock({ duration: 30 });
+ clock.play();
+ advancePolled(clock, advance, 2000);
+ const bed = Object.assign(createMockAudioEl(0, false), { loop: true, duration: 8 });
+ clock.attachAudioSource({ el: bed, compositionStart: 1.9, mediaStart: 0 });
+ advance(16);
+ expect(clock.now()).toBeCloseTo(2, 5);
+ });
+
+ it("continues from the last audio time when the element drops out", () => {
+ const { clock, advance } = createClock({ duration: 30 });
+ clock.play();
+ advancePolled(clock, advance, 4000);
+ const voice = createMockAudioEl(5, false);
+ clock.attachAudioSource({ el: voice, compositionStart: 0, mediaStart: 0 });
+ expect(clock.now()).toBe(5);
+ Object.assign(voice, { paused: true });
+ advance(16);
+ expect(clock.now()).toBeCloseTo(5.016, 5);
+ });
+
+ it("lets a seek move the playhead back past a held time", () => {
+ const { clock, advance } = createClock({ duration: 30 });
+ clock.play();
+ const voice = createMockAudioEl(5, false);
+ clock.attachAudioSource({ el: voice, compositionStart: 0, mediaStart: 0 });
+ expect(clock.now()).toBe(5);
+ clock.detachAudioSource();
+ clock.seek(2);
+ voice.currentTime = 2;
+ clock.attachAudioSource({ el: voice, compositionStart: 0, mediaStart: 0 });
+ advance(16);
+ expect(clock.now()).toBe(2);
+ });
+
it("clamps audio-derived time to duration", () => {
const { clock } = createClock({ duration: 10 });
const audioEl = createMockAudioEl(15.0, false);
@@ -364,15 +457,24 @@ describe("TransportClock", () => {
expect(clock.now()).toBe(4);
});
- it("composition time is correct when el.playbackRate differs from clock rate", () => {
- // General formula: wall_elapsed = (el.currentTime - mediaStart) / el.playbackRate
- // composition_time = compositionStart + wall_elapsed * clockRate
+ it("reads the element's place in its file, whatever playbackRate it has been given", () => {
const { clock } = createClock({ rate: 2, duration: 20 });
- // Audio at 1x, clock at 2x: after 1s wall, el.currentTime=1, comp should be 2.
const audioEl = { currentTime: 1, paused: false, playbackRate: 1 } as HTMLMediaElement;
clock.play();
clock.attachAudioSource({ el: audioEl, compositionStart: 0, mediaStart: 0 });
- expect(clock.now()).toBe(2);
+ expect(clock.now()).toBe(1);
+ });
+
+ it("keeps its place when the speed changes under a clip with its own rate", () => {
+ const { clock, advance } = createClock({ duration: 30 });
+ clock.play();
+ const voice = { currentTime: 12.5, paused: false, playbackRate: 1.25 } as HTMLMediaElement;
+ clock.attachAudioSource({ el: voice, compositionStart: 0, mediaStart: 0, rate: 1.25 });
+ expect(clock.now()).toBe(10);
+ clock.setRate(1.5);
+ voice.playbackRate = 1.5; // the transport rate lands on the element before media sync restores 1.875
+ advance(16);
+ expect(clock.now()).toBe(10);
});
});
});
diff --git a/packages/core/src/runtime/clock.ts b/packages/core/src/runtime/clock.ts
index 13d5a17878..1bae66df7c 100644
--- a/packages/core/src/runtime/clock.ts
+++ b/packages/core/src/runtime/clock.ts
@@ -1,4 +1,5 @@
import { timeAtSourceTime, type RateSpec } from "../speedRamp.js";
+import { MEDIA_HARD_SYNC_SECONDS } from "./media.js";
export type TransportClockSnapshot = {
time: number;
@@ -23,6 +24,7 @@ export type AudioClockSource =
/** GSAP's own `lagSmoothing` default — see the PR body for why this clock needs its own copy. */
const STALL_THRESHOLD_MS = 500;
const STALL_ADJUSTED_LAG_MS = 33;
+const HAVE_FUTURE_DATA = 3;
export class TransportClock {
private _baseTime = 0;
@@ -33,6 +35,7 @@ export class TransportClock {
private _audioSource: AudioClockSource | null = null;
/** Wall-clock time of the last `now()` read while playing; null while paused. */
private _lastReadMs: number | null = null;
+ private _lastNow = 0;
constructor(opts?: {
initialTime?: number;
@@ -58,22 +61,15 @@ export class TransportClock {
if ("currentTimeSeconds" in this._audioSource) {
audioTime = this._audioSource.currentTimeSeconds;
} else {
- const { el, compositionStart, mediaStart, rate } = this._audioSource;
- if (!el.paused && Number.isFinite(el.currentTime)) {
- audioTime =
- typeof rate === "object"
- ? timeAtSourceTime(rate, el.currentTime - mediaStart) + compositionStart
- : ((el.currentTime - mediaStart) / (el.playbackRate > 0 ? el.playbackRate : 1)) *
- this._rate +
- compositionStart;
- }
+ audioTime = this._elementTimeNeverBehind(this._audioSource);
}
if (audioTime !== null) {
+ const t = Number.isFinite(this._duration) ? Math.min(audioTime, this._duration) : audioTime;
+ this._baseTime = Math.max(0, t);
+ this._playStartMs = this._nowMs();
this._lastReadMs = null;
- if (Number.isFinite(this._duration) && audioTime >= this._duration) {
- return this._duration;
- }
- return Math.max(0, audioTime);
+ this._lastNow = this._baseTime;
+ return this._baseTime;
}
}
@@ -81,10 +77,25 @@ export class TransportClock {
this._applyStallCorrection();
const elapsed = (this._nowMs() - this._playStartMs) / 1000;
const t = this._baseTime + elapsed * this._rate;
- if (Number.isFinite(this._duration) && t >= this._duration) {
- return this._duration;
- }
- return Math.max(0, t);
+ this._lastNow = Math.max(0, Number.isFinite(this._duration) ? Math.min(t, this._duration) : t);
+ return this._lastNow;
+ }
+
+ private _elementTimeNeverBehind(
+ source: Extract<AudioClockSource, { el: HTMLMediaElement }>,
+ ): number | null {
+ const { el, compositionStart, mediaStart, rate } = source;
+ if (el.paused || !Number.isFinite(el.currentTime)) return null;
+ if (el.seeking) return this._lastNow;
+ const spec = rate ?? 1;
+ const time = timeAtSourceTime(spec, el.currentTime - mediaStart) + compositionStart;
+ return time >= this._lastNow ? time : this._heldTimeWhenBehind(el, this._lastNow - time, spec);
+ }
+
+ private _heldTimeWhenBehind(el: HTMLMediaElement, behind: number, spec: RateSpec): number | null {
+ if (el.loop && behind > timeAtSourceTime(spec, el.duration) / 2) return null;
+ const buffering = el.readyState < HAVE_FUTURE_DATA;
+ return buffering || behind <= MEDIA_HARD_SYNC_SECONDS ? this._lastNow : null;
}
/** Folds a >500ms gap since the last read into `_playStartMs` so it's never reported as played time. */
@@ -104,6 +115,7 @@ export class TransportClock {
if (this._playStartMs !== null) return false;
if (Number.isFinite(this._duration) && this._baseTime >= this._duration) return false;
this._playStartMs = this._nowMs();
+ this._lastNow = this._baseTime;
// Not a stall: the gap since the clock was last read (possibly a long
// paused idle) says nothing about lost playback time, since none was
// playing. `_applyStallCorrection` treats null as "nothing to compare
@@ -124,6 +136,7 @@ export class TransportClock {
? Math.max(0, Math.min(timeSeconds, this._duration))
: Math.max(0, timeSeconds);
this._baseTime = clamped;
+ this._lastNow = clamped;
if (this._playStartMs !== null) {
this._playStartMs = this._nowMs();
// Same reasoning as `play()`: the seek itself, not any elapsed gap
@@ -176,6 +189,11 @@ export class TransportClock {
this._audioSource = null;
}
+ /** The element the playhead currently follows, if the source is one. */
+ audioElement(): HTMLMediaElement | null {
+ return this._audioSource && "el" in this._audioSource ? this._audioSource.el : null;
+ }
+
hasAudioSource(): boolean {
return this._audioSource !== null;
}
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index b345b74559..eae5c00c76 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -30,14 +30,15 @@ import {
import { sourceTimeAt } from "../speedRamp";
import { createWaapiAdapter } from "./adapters/waapi";
import {
+ isUnplayable,
readElementPlaybackRate,
- readElementRateSpec,
readElementPlaybackStart,
+ readElementRateSpec,
refreshRuntimeMediaCache,
- resolveRuntimeMediaClipDuration,
resolveNaturalMediaTimelineDuration,
- type RuntimeMediaClip,
+ resolveRuntimeMediaClipDuration,
syncRuntimeMedia,
+ type RuntimeMediaClip,
} from "./media";
import { handleErrorForProxy, handleMetadataForProxy, maybeProxyProactively } from "./mediaProxy";
import { probeAndCacheElementVolume, type VolumeKeyframe } from "./mediaVolumeEnvelope.js";
@@ -4511,10 +4512,12 @@ export function initSandboxRuntimeModular(): void {
}
} else {
const audioEls = document.querySelectorAll("audio[data-start]");
+ const followed = clock.audioElement();
let foundActive = false;
- for (const rawEl of audioEls) {
+ for (const rawEl of followed ? [followed, ...audioEls] : audioEls) {
if (!isMediaElement(rawEl) || !rawEl.isConnected) continue;
- if (isSilencedByHidden(rawEl)) continue;
+ if (isSilencedByHidden(rawEl) || isUnplayable(rawEl)) continue;
+ if (!rawEl.hasAttribute("src") && !rawEl.querySelector("source[src]")) continue;
const start = resolveAbsoluteMediaStartSeconds(rawEl);
const durAttr = parseStrictFiniteTimingNumber(rawEl.dataset.duration);
const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;
@@ -4528,7 +4531,7 @@ export function initSandboxRuntimeModular(): void {
rate: readElementRateSpec(rawEl),
});
foundActive = true;
- } else if (!rawEl.error && rawEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
+ } else if (rawEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
// Audio is buffering — freeze visuals at last known position
// instead of falling through to monotonic (which runs ahead).
clock.attachAudioSource({ currentTimeSeconds: state.currentTime });
diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts
index 1428f8d88e..f790727b05 100644
--- a/packages/core/src/runtime/media.ts
+++ b/packages/core/src/runtime/media.ts
@@ -208,7 +208,7 @@ function markPlayRequested(el: HTMLMediaElement): void {
const MEDIA_NETWORK_NO_SOURCE = 3;
// An element that errored or has no source can't play; re-issuing play() every
// tick just floods rejections. Skip it until its state changes (src reload).
-function isUnplayable(el: HTMLMediaElement): boolean {
+export function isUnplayable(el: HTMLMediaElement): boolean {
return el.error != null || el.networkState === MEDIA_NETWORK_NO_SOURCE;
}
@@ -250,6 +250,8 @@ export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {
);
}
+export const MEDIA_HARD_SYNC_SECONDS = 0.5;
+
/** Drift a playing audio element may carry before sync pulls it back onto the playhead. */
const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;
@@ -495,7 +497,7 @@ export function syncRuntimeMedia(params: {
(isHeldVideoTail && drift > 0.001) ||
(el.ended && canSeekEndedMediaBackward && drift > 0.001) ||
staleAudioOnFirstTick ||
- (drift > 0.5 && (firstTickOfClip || offsetJumped || catastrophicDrift));
+ (drift > MEDIA_HARD_SYNC_SECONDS && (firstTickOfClip || offsetJumped || catastrophicDrift));
// Playing videos use the browser's decoder for timing. Seeking one resets the decoder: a
// ~150ms freeze while it re-buffers, as the monotonic clock advances, which loops into a
// seek→freeze→drift→seek stutter. So a playing video skips strict and force sync; only hard