Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(editor): hold the audio to the picture by rate, and resync it after a seek

EtienneLescotc870e3bee60f
diff --git a/src/components/ai-edition/VirtualPreview.playback.test.tsx b/src/components/ai-edition/VirtualPreview.playback.test.tsx
index 806505030..66a08bbd1 100644
--- a/src/components/ai-edition/VirtualPreview.playback.test.tsx
+++ b/src/components/ai-edition/VirtualPreview.playback.test.tsx
@@ -3,6 +3,7 @@ import "@testing-library/jest-dom";
import { act, cleanup, fireEvent, render } from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { AxcutClip, AxcutTrimRange } from "@/lib/ai-edition/schema";
+import type { SpeedRegion } from "@/lib/ai-edition/timeline/speed";
import { type VideoSource, VirtualPreview } from "./VirtualPreview";
// The rAF tick is the whole subject here, so it is driven by hand rather than by the
@@ -384,3 +385,303 @@ describe("VirtualPreview imported audio track boost", () => {
expect(audioEl.volume).toBe(1); // volume left at unity so it doesn't double-attenuate
});
});
+
+// The recording's sound is a second element over the same file: a muted <video> is the clock
+// and a separate <audio> is what is heard. Two clocks, so the rAF loop has to keep them
+// together, and it has to do it without seeking the audio: every `currentTime` write flushes
+// its pipeline, which is the stutter #898 measured (157 `seeking` events in 20 s).
+//
+// What follows runs that loop against a small model of the elements. Real time passes between
+// frames (`frame(gapSec)`), each element advances at its own `playbackRate`, and an audio seek
+// stalls the element for `seekLatencySec`, so it lands that far behind the picture, the way the
+// shipped editor was measured (~0.1 s). Ticks land 100-150 ms apart, which is what a main
+// thread ~44 % blocked does to requestAnimationFrame.
+function driveAudioClock(
+ el: HTMLAudioElement,
+ { seekLatencySec, skew }: { seekLatencySec: number; skew: number },
+) {
+ let currentTime = 0;
+ let paused = true;
+ let stallLeftSec = 0;
+ const writes: number[] = [];
+ Object.defineProperty(el, "currentTime", {
+ configurable: true,
+ get: () => currentTime,
+ set: (next: number) => {
+ currentTime = next;
+ stallLeftSec = seekLatencySec;
+ writes.push(next);
+ },
+ });
+ Object.defineProperty(el, "seeking", { configurable: true, get: () => stallLeftSec > 0 });
+ Object.defineProperty(el, "paused", { configurable: true, get: () => paused });
+ Object.defineProperty(el, "duration", { configurable: true, get: () => 60 });
+ el.play = vi.fn(() => {
+ paused = false;
+ return Promise.resolve();
+ });
+ el.pause = vi.fn(() => {
+ paused = true;
+ });
+ return {
+ writes,
+ get currentTime() {
+ return currentTime;
+ },
+ /** Put the element somewhere without a seek: how a stretch already in sync starts. */
+ place: (next: number) => {
+ currentTime = next;
+ },
+ /** Real time passing. `skew` is how much faster this element's own clock runs. */
+ advance: (dtSec: number) => {
+ const stalledSec = Math.min(dtSec, stallLeftSec);
+ stallLeftSec -= stalledSec;
+ if (!paused && stallLeftSec === 0) {
+ currentTime += (dtSec - stalledSec) * el.playbackRate * (1 + skew);
+ }
+ },
+ };
+}
+
+const LOADED_GAPS_SEC = [0.1, 0.15, 0.12, 0.1, 0.14];
+
+function mountSynced({
+ trims = [],
+ speedRegions = [],
+ seekLatencySec = 0.1,
+ skew = 0,
+}: {
+ trims?: AxcutTrimRange[];
+ speedRegions?: SpeedRegion[];
+ seekLatencySec?: number;
+ skew?: number;
+} = {}) {
+ const clips = [clip("c1", "a1", 0, 60, 0)];
+ const sources: VideoSource[] = [{ id: "a1", src: "file:///tmp/a1.mp4", label: "a1" }];
+ const onTimeChange = vi.fn();
+ let requestId = 0;
+ const tree = (seekTarget: { timeSec: number; requestId: number } | null) => (
+ <VirtualPreview
+ videoSources={sources}
+ clips={clips}
+ trimRanges={trims}
+ speedRegions={speedRegions}
+ seekTarget={seekTarget}
+ onTimeChange={onTimeChange}
+ />
+ );
+ const view = render(tree(null));
+ const videoEl = view.container.querySelector("video") as HTMLVideoElement;
+ const audioEl = view.container.querySelector(
+ '[data-testid="preview-audio-primary"]',
+ ) as HTMLAudioElement;
+ const video = driveVideo(videoEl);
+ const audio = driveAudioClock(audioEl, { seekLatencySec, skew });
+ act(() => {
+ fireEvent.loadedMetadata(videoEl);
+ });
+ /** Real time passes, then one rAF frame runs. */
+ const frame = (gapSec: number) => {
+ if (!videoEl.paused) video.seekTo(video.currentTime + gapSec * videoEl.playbackRate);
+ audio.advance(gapSec);
+ tick();
+ };
+ return {
+ video,
+ videoEl,
+ audio,
+ audioEl,
+ frame,
+ /** Frames until `seconds` of real time have passed, sampling after each one. */
+ run: (seconds: number, gaps = LOADED_GAPS_SEC, onFrame?: () => void) => {
+ let elapsedSec = 0;
+ for (let i = 0; elapsedSec < seconds; i++) {
+ const gap = gaps[i % gaps.length];
+ elapsedSec += gap;
+ frame(gap);
+ onFrame?.();
+ }
+ },
+ /** The audio's offset from the picture, in media seconds; negative is behind it. */
+ offset: () => audio.currentTime - video.currentTime,
+ /** One explicit seek published by the shell, the way a click on the ruler is. */
+ seekTo: (timeSec: number) =>
+ act(() => {
+ requestId += 1;
+ view.rerender(tree({ timeSec, requestId }));
+ }),
+ /** Playing, with the audio already in step, as it is a few seconds into a take. */
+ playInSync: (atSec: number) => {
+ video.seekTo(atSec);
+ audio.place(atSec);
+ video.play();
+ tick(); // the loop starts the audio; nothing to correct
+ audio.writes.length = 0;
+ },
+ };
+}
+
+describe("VirtualPreview primary audio keeps to the picture", () => {
+ // Every offset here is media time, which is what a lip-sync mismatch is measured in and is
+ // never smaller than the wall-clock one, so a bound on it holds for both.
+ const TOLERANCE_SEC = 0.04;
+
+ it("does not re-seek the audio under load and brings its offset inside 40 ms", () => {
+ const world = mountSynced();
+ world.video.seekTo(1);
+ world.video.play();
+
+ const offsets: number[] = [];
+ world.run(30, LOADED_GAPS_SEC, () => offsets.push(world.offset()));
+
+ // The audio is placed once, when playback starts, and never touched again: a hard seek
+ // on a playing element is what stuttered. It lands behind the picture by its own seek
+ // latency, and the loop closes that by nudging its rate.
+ expect(world.audio.writes).toHaveLength(1);
+ expect(offsets[1]).toBeLessThan(-0.05);
+ // Some 2 s to close 100 ms at a few percent of rate; from there it holds.
+ const settled = offsets.slice(30);
+ expect(Math.max(...settled.map(Math.abs))).toBeLessThan(TOLERANCE_SEC);
+ });
+
+ it("absorbs a free-running clock skew in either direction without a seek", () => {
+ for (const skew of [0.005, -0.005]) {
+ const world = mountSynced({ skew });
+ world.playInSync(1);
+
+ // Nearly a minute at 0.5 % is 275 ms of drift if nothing intervenes, past every
+ // threshold in this file, so holding it inside 40 ms is the loop's doing.
+ const offsets: number[] = [];
+ world.run(55, LOADED_GAPS_SEC, () => offsets.push(world.offset()));
+
+ expect(world.audio.writes, `skew ${skew}`).toHaveLength(0);
+ expect(Math.max(...offsets.map(Math.abs)), `skew ${skew}`).toBeLessThan(TOLERANCE_SEC);
+ cleanup();
+ }
+ });
+
+ it("pulls an audio that is AHEAD of the picture back, not only one behind it", () => {
+ const world = mountSynced();
+ world.playInSync(1);
+ // Ahead is what is noticed first, from about 45 ms.
+ world.audio.place(world.video.currentTime + 0.1);
+
+ const offsets: number[] = [];
+ world.run(10, LOADED_GAPS_SEC, () => offsets.push(world.offset()));
+
+ expect(world.audio.writes).toHaveLength(0);
+ expect(Math.max(...offsets.slice(30).map(Math.abs))).toBeLessThan(TOLERANCE_SEC);
+ });
+
+ it("resyncs on a small explicit seek during playback, whatever the drift", () => {
+ const world = mountSynced();
+ world.playInSync(5);
+ world.run(1, [0.016]);
+ expect(world.audio.writes).toHaveLength(0);
+
+ // 100 ms back: a click on the ruler next to the playhead. The audio's 100 ms lead is
+ // inside any leash the storm fix can afford, so only a resync on the seek itself moves it.
+ world.seekTo(world.video.currentTime - 0.1);
+ world.frame(0.016);
+
+ expect(world.audio.writes).toHaveLength(1);
+ expect(world.audio.writes[0]).toBeCloseTo(world.video.currentTime, 1);
+ world.run(3, [0.016]);
+ expect(Math.abs(world.offset())).toBeLessThan(TOLERANCE_SEC);
+ expect(world.audio.writes).toHaveLength(1);
+ });
+
+ it("resyncs across a skipped interval shorter than the leash", () => {
+ // A cut of 5.00 -> 5.11. The preview only calls a frame cut once it is 50 ms inside
+ // (`locateSourcePosition` gives each edge that much slack), so the picture jumps from
+ // ~5.05 to 5.11 and the audio, still playing the cut material, would sit ~60 ms off
+ // from then on.
+ const trim: AxcutTrimRange = {
+ id: "cut",
+ assetId: "a1",
+ clipId: "c1",
+ startSec: 5,
+ endSec: 5.11,
+ origin: "user",
+ reason: "",
+ };
+ const world = mountSynced({ trims: [trim] });
+ world.playInSync(4.9);
+ // Small steps, because the slack leaves only ~10 ms of the cut where a frame is
+ // recognised as inside it.
+ world.run(0.5, [0.004]);
+
+ expect(world.audio.writes).toHaveLength(1);
+ expect(world.audio.writes[0]).toBeGreaterThanOrEqual(5.11);
+ world.run(3, [0.016]);
+ expect(Math.abs(world.offset())).toBeLessThan(TOLERANCE_SEC);
+ expect(world.audio.writes).toHaveLength(1);
+ });
+
+ it("does not seek an audio element that is still seeking, and follows once it is done", () => {
+ const world = mountSynced({ seekLatencySec: 0.2 });
+ world.playInSync(5);
+
+ world.seekTo(4.9);
+ world.frame(0.016); // the first resync: the audio starts a 0.2 s seek
+ world.seekTo(6);
+ world.frame(0.016); // still seeking: this one waits instead of restarting it
+ expect(world.audio.writes).toHaveLength(1);
+
+ world.run(1, [0.1]); // the seek ends, and the loop goes where the picture went
+ expect(world.audio.writes).toHaveLength(2);
+ expect(world.audio.writes[1]).toBeGreaterThan(6);
+ });
+
+ it("keeps in step through a 2x region, entering and leaving it under load", () => {
+ const region: SpeedRegion = { id: "s", startMs: 10_000, endMs: 14_000, speed: 2 };
+ const world = mountSynced({ speedRegions: [region] });
+ world.video.seekTo(1);
+ world.video.play();
+ world.run(8); // settled before the region
+
+ // The video's rate changes at the END of a tick, so the audio's has to change with it
+ // and not at the top of the next one: over a 150 ms gap that lag alone leaves the audio
+ // 150 ms behind on the way in and 150 ms ahead on the way out.
+ const rates: number[] = [];
+ const offsets: number[] = [];
+ world.run(30, LOADED_GAPS_SEC, () => {
+ rates.push(world.audioEl.playbackRate);
+ offsets.push(world.offset());
+ });
+
+ expect(rates).toContain(2);
+ expect(world.audioEl.playbackRate).toBe(1);
+ expect(world.audio.writes).toHaveLength(1);
+ expect(Math.max(...offsets.map(Math.abs))).toBeLessThan(TOLERANCE_SEC);
+ });
+
+ it("never asks the audio for a rate the browser refuses", () => {
+ // Chromium throws on a playbackRate above 16, from inside the rAF tick. A nudge on top
+ // of a video already at that cap must stop at it.
+ const region: SpeedRegion = { id: "s", startMs: 0, endMs: 60_000, speed: 16 };
+ const world = mountSynced({ speedRegions: [region] });
+ world.playInSync(1);
+ world.frame(0.016); // the region's rate reaches the video
+ world.audio.place(world.video.currentTime - 0.1); // behind: the nudge goes UP
+
+ const rates: number[] = [];
+ world.run(1, [0.016], () => rates.push(world.audioEl.playbackRate));
+
+ expect(world.videoEl.playbackRate).toBe(16);
+ expect(Math.max(...rates)).toBeLessThanOrEqual(16);
+ });
+
+ it("goes back to the video's own rate once playback pauses", () => {
+ const world = mountSynced();
+ world.playInSync(1);
+ world.audio.place(world.video.currentTime + 0.1); // a nudge is under way
+ world.frame(0.016);
+ expect(world.audioEl.playbackRate).toBeLessThan(1);
+
+ world.videoEl.pause();
+ world.frame(0.016);
+
+ expect(world.audioEl.playbackRate).toBe(1);
+ });
+});
diff --git a/src/components/ai-edition/VirtualPreview.seekStorm.test.tsx b/src/components/ai-edition/VirtualPreview.seekStorm.test.tsx
index feebbe513..e1c88d5f5 100644
--- a/src/components/ai-edition/VirtualPreview.seekStorm.test.tsx
+++ b/src/components/ai-edition/VirtualPreview.seekStorm.test.tsx
@@ -4,9 +4,12 @@ import { act, cleanup, fireEvent, render } from "@testing-library/react";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { AxcutClip } from "@/lib/ai-edition/schema";
import {
- PRIMARY_AUDIO_PLAYING_LEASH_SEC,
- SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC,
+ AUDIO_NUDGE_START_SEC,
+ AUDIO_NUDGE_STOP_SEC,
+ AUDIO_SEEK_LEASH_SEC,
+ IMPORTED_AUDIO_PLAYING_LEASH_SEC,
shouldResyncAudio,
+ steerAudio,
type VideoSource,
VirtualPreview,
} from "./VirtualPreview";
@@ -155,49 +158,72 @@ describe("VirtualPreview keeps one demuxer seek in flight (issue #395 root cause
});
});
-describe("the primary audio element free-runs instead of being re-seeked every frame", () => {
- // Measured in the shipped editor on a Snapdragon X Elite, during ordinary playback:
- // 157 `seeking` events in 20 s with a single `seeked`, and 87 `currentTime` writes in
- // 15 s — about six a second. Each write stepped forward by ~0.1 s, which is exactly the
- // time that had passed: the audio was being sent to where it already was heading.
- //
- // The 25 ms leash assumed the rAF tick runs tight against the <video>'s own clock. It
- // does not when the renderer main thread is loaded (measured ~44 % blocked during
- // playback): ticks land 100+ ms apart, the audio element has free-run past 25 ms by
- // then, and gets yanked back. The yank stalls it, so it falls behind again — the storm
- // sustains itself, which is why the drift sat at a steady ~100 ms instead of decaying.
- it("leaves a playing element alone at the drift the storm was measured at", () => {
- expect(shouldResyncAudio(0.1, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(false);
- expect(shouldResyncAudio(-0.1, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(false);
+// What the rAF loop does to a playing audio element between two ticks. The measured story
+// behind these is on `AUDIO_NUDGE_START_SEC`; the loop itself is driven end to end in
+// VirtualPreview.playback.test.tsx ("primary audio keeps to the picture"), because a
+// decision that is right in isolation says nothing about what the call site feeds it.
+describe("steerAudio: a playing element is nudged, not seeked", () => {
+ const playing = (driftSec: number, nudged = false) =>
+ steerAudio(driftSec, true, false, false, nudged);
+
+ it("leaves an element on the picture alone", () => {
+ expect(playing(0.01)).toEqual({ seek: false, rateFactor: 1 });
+ expect(playing(-0.01)).toEqual({ seek: false, rateFactor: 1 });
});
- it("still corrects a real desync", () => {
- expect(shouldResyncAudio(0.4, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(true);
- expect(shouldResyncAudio(-0.4, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(true);
+ it("nudges the rate toward the picture instead of seeking, whichever side it is on", () => {
+ // The drift the storm was measured at: what used to be re-seeked six times a second.
+ const behind = playing(-0.1);
+ const ahead = playing(0.1);
+ expect(behind.seek).toBe(false);
+ expect(ahead.seek).toBe(false);
+ expect(behind.rateFactor).toBeGreaterThan(1);
+ expect(ahead.rateFactor).toBeLessThan(1);
+ // Symmetric: a lead is noticed sooner than a lag, so it must not be the slower to close.
+ expect(behind.rateFactor - 1).toBeCloseTo(1 - ahead.rateFactor, 10);
});
- // A parked element is not free-running, so nothing sustains its position: it has to be
- // placed exactly, and placing it costs nothing because it is not playing.
- it("keeps the tight leash when the element is not free-running", () => {
- expect(shouldResyncAudio(0.05, false, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(true);
- expect(shouldResyncAudio(0.01, false, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(false);
+ it("stops nudging closer to the picture than it starts, so an edge does not flap", () => {
+ const between = (AUDIO_NUDGE_START_SEC + AUDIO_NUDGE_STOP_SEC) / 2;
+ expect(playing(between, false).rateFactor).toBe(1);
+ expect(playing(between, true).rateFactor).not.toBe(1);
+ expect(playing(AUDIO_NUDGE_STOP_SEC / 2, true).rateFactor).toBe(1);
});
- // The same discipline the video path learned in issue #395: a write onto an element
- // that is already seeking restarts the seek instead of finishing it, so the element
- // never arrives. Measured after the leash change alone, residual writes still landed
- // on an element mid-seek.
+ it("seeks only past what a nudge can close", () => {
+ expect(playing(AUDIO_SEEK_LEASH_SEC * 0.9).seek).toBe(false);
+ expect(playing(AUDIO_SEEK_LEASH_SEC * 1.1)).toEqual({ seek: true, rateFactor: 1 });
+ expect(playing(-AUDIO_SEEK_LEASH_SEC * 1.1).seek).toBe(true);
+ });
+
+ it("places a parked element exactly", () => {
+ expect(steerAudio(0.05, false, false, false, false).seek).toBe(true);
+ expect(steerAudio(0.01, false, false, false, false).seek).toBe(false);
+ });
+
+ it("follows an explicit jump of the picture whatever the drift", () => {
+ expect(steerAudio(0.001, true, false, true, false).seek).toBe(true);
+ expect(steerAudio(0.08, false, false, true, false).seek).toBe(true);
+ });
+
+ // The discipline the video path learned in issue #395: a write onto an element that is
+ // already seeking restarts the seek instead of finishing it, so the element never arrives.
it("never stacks a write onto an element that is already seeking", () => {
- expect(shouldResyncAudio(5, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC, true)).toBe(false);
- expect(shouldResyncAudio(5, false, PRIMARY_AUDIO_PLAYING_LEASH_SEC, true)).toBe(false);
- expect(shouldResyncAudio(5, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC, false)).toBe(true);
+ expect(steerAudio(5, true, true, false, false).seek).toBe(false);
+ expect(steerAudio(5, false, true, false, false).seek).toBe(false);
+ expect(steerAudio(0.001, true, true, true, false).seek).toBe(false); // the jump waits
+ });
+});
+
+describe("shouldResyncAudio: an imported track free-runs inside a wide leash", () => {
+ it("leaves a playing track alone inside the leash and corrects a real desync", () => {
+ expect(shouldResyncAudio(0.2, true, IMPORTED_AUDIO_PLAYING_LEASH_SEC)).toBe(false);
+ expect(shouldResyncAudio(-0.2, true, IMPORTED_AUDIO_PLAYING_LEASH_SEC)).toBe(false);
+ expect(shouldResyncAudio(0.4, true, IMPORTED_AUDIO_PLAYING_LEASH_SEC)).toBe(true);
});
- // Lip sync is the constraint the primary track has and an imported one does not, so it
- // gets the shorter leash of the two. Audio behind picture is tolerated to about 125 ms
- // before it reads as out of sync, which is the ceiling this has to stay under.
- it("gives the primary track a shorter leash than an imported one", () => {
- expect(PRIMARY_AUDIO_PLAYING_LEASH_SEC).toBeLessThan(SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC);
- expect(PRIMARY_AUDIO_PLAYING_LEASH_SEC).toBeLessThan(0.125);
+ it("places a parked track exactly, and never onto one that is seeking", () => {
+ expect(shouldResyncAudio(0.05, false, IMPORTED_AUDIO_PLAYING_LEASH_SEC)).toBe(true);
+ expect(shouldResyncAudio(5, true, IMPORTED_AUDIO_PLAYING_LEASH_SEC, true)).toBe(false);
});
});
diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsx
index 97b84be9f..5980c3654 100644
--- a/src/components/ai-edition/VirtualPreview.tsx
+++ b/src/components/ai-edition/VirtualPreview.tsx
@@ -66,10 +66,11 @@ export interface VideoSource {
* nothing because it is not playing. */
const AUDIO_PARKED_LEASH_SEC = 0.025;
-/** Drift tolerated on the primary audio track while it plays.
+/** How a PLAYING audio element that rides the video's clock (the recording's own sound) is held
+ * to the picture: by its `playbackRate`, not by seeks.
*
- * This used to be `AUDIO_PARKED_LEASH_SEC` on the grounds that the primary track syncs
- * to the `<video>`'s own authoritative clock and so could be held that tight. Measured
+ * It used to be held by seeks, at `AUDIO_PARKED_LEASH_SEC`, on the grounds that the element
+ * syncs to the `<video>`'s own authoritative clock and so could be kept that tight. Measured
* in the shipped editor on a Snapdragon X Elite, during ordinary playback, it could not:
* 157 `seeking` events in 20 s against a single `seeked`, and 87 `currentTime` writes in
* 15 s — six a second, each stepping forward by ~0.1 s, which is exactly the time that had
@@ -80,20 +81,82 @@ const AUDIO_PARKED_LEASH_SEC = 0.025;
* the renderer main thread is loaded (measured ~44 % blocked during playback): ticks land
* 100+ ms apart, the element has free-run past 25 ms by then, and gets yanked back. The
* yank stalls it, so it falls behind again — which is why the drift sat at a steady ~100 ms
- * instead of decaying. A playing element already runs at the right rate from the right
- * offset, so left alone it stays in sync on its own.
+ * instead of decaying.
*
- * The ceiling is lip sync, not the storm: audio behind picture reads as out of sync at
- * roughly 125 ms, and audio ahead of it is noticed sooner still. This sits under both while
- * clearing the ~100 ms the storm was sustaining. */
-export const PRIMARY_AUDIO_PLAYING_LEASH_SEC = 0.12;
+ * Widening the leash ends the storm and keeps the offset instead: a seek lands the audio
+ * BEHIND the picture by its own latency (that ~100 ms), and both elements then run at the
+ * same rate, so nothing ever closes it. Audio is noticed from about 45 ms when it leads and
+ * about 125 ms when it lags (ITU-R BT.1359), so an offset that stays is a defect, not a
+ * tolerance.
+ *
+ * So the element is nudged. Once it drifts more than `AUDIO_NUDGE_START_SEC` its rate is
+ * trimmed by `AUDIO_NUDGE_FRACTION` toward the picture, and it goes back to the video's own
+ * rate within `AUDIO_NUDGE_STOP_SEC`; the gap between the two keeps an element at the edge
+ * from flipping its rate every frame. A rate change flushes nothing, and with
+ * `preservesPitch` (the default) it does not move the pitch. It closes ~50 ms a second, so a
+ * seek's 100 ms is gone in ~2 s and a free-running clock skew never builds up. */
+export const AUDIO_NUDGE_START_SEC = 0.02;
+export const AUDIO_NUDGE_STOP_SEC = 0.005;
+export const AUDIO_NUDGE_FRACTION = 0.05;
+
+/** Past this a nudge would take seconds and something else is wrong (a stalled or starved
+ * element): only then is a seek worth its flush. Kept above the ~100 ms a seek itself
+ * leaves, or the seek would be the thing that triggers the next one. */
+export const AUDIO_SEEK_LEASH_SEC = 0.3;
+
+/** Drift tolerated on an imported track (issue #350) while it plays. It syncs to
+ * `virtualTimeSec`, which is DERIVED from the video clock each frame and so is noisier than
+ * the recording's own audio, and it carries BGM or voiceover rather than lip sync, so it is
+ * left to free-run inside a wide leash. */
+export const IMPORTED_AUDIO_PLAYING_LEASH_SEC = 0.3;
-/** Same, for an imported track (issue #350). It syncs to `virtualTimeSec`, which is DERIVED
- * from the video clock each frame and so is noisier still — and it carries BGM or voiceover
- * rather than lip sync, so it can afford a wider leash than the primary track. */
-export const SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC = 0.3;
+/** What the rAF tick should do to one of the recording's audio elements this frame. */
+export interface AudioSteer {
+ /** Write `currentTime`. */
+ seek: boolean;
+ /** `playbackRate` as a multiple of the video's own: 1 on target, nudged while catching up. */
+ rateFactor: number;
+}
-/** Whether an audio element's drift warrants a `currentTime` write.
+/** Decide how to hold an audio element that rides the video's clock to the picture.
+ *
+ * `driftSec` is the element's position minus where the picture is: positive means the audio
+ * is AHEAD. `freeRunning` means it is playing and is meant to be. `seeking` means a write is
+ * already in flight: another one restarts that seek instead of finishing it, so the element
+ * never arrives and the drift that triggered it never closes (issue #395, which the video
+ * path took and the audio path had not). `jumped` means the picture was MOVED, on purpose,
+ * since this element was last placed: a scrub, a skipped cut, a clip junction. That is
+ * indistinguishable from clock drift by size, since a cut can be 50 ms, so the caller says
+ * so, and the audio follows whatever the drift once it is not seeking. `nudged` is whether
+ * its rate is off the video's now, which is what makes the stop threshold apply. */
+export function steerAudio(
+ driftSec: number,
+ freeRunning: boolean,
+ seeking: boolean,
+ jumped: boolean,
+ nudged: boolean,
+): AudioSteer {
+ if (seeking) {
+ return { seek: false, rateFactor: 1 };
+ }
+ if (jumped) {
+ return { seek: true, rateFactor: 1 };
+ }
+ if (!freeRunning) {
+ // Nothing sustains a parked element's position, and placing it costs nothing.
+ return { seek: Math.abs(driftSec) > AUDIO_PARKED_LEASH_SEC, rateFactor: 1 };
+ }
+ const off = Math.abs(driftSec);
+ if (off > AUDIO_SEEK_LEASH_SEC) {
+ return { seek: true, rateFactor: 1 };
+ }
+ if (off > (nudged ? AUDIO_NUDGE_STOP_SEC : AUDIO_NUDGE_START_SEC)) {
+ return { seek: false, rateFactor: 1 - Math.sign(driftSec) * AUDIO_NUDGE_FRACTION };
+ }
+ return { seek: false, rateFactor: 1 };
+}
+
+/** Whether an imported track's drift warrants a `currentTime` write.
*
* `freeRunning` means the element is playing and is supposed to be: it holds its own
* position, so only a real discontinuity — a scrub, a trim jump, a first play — is worth
@@ -102,8 +165,7 @@ export const SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC = 0.3;
*
* `seeking` is the other half of that lesson: a write onto an element that is still
* seeking restarts the seek rather than finishing it, so the element never arrives and
- * the drift that triggered the write never closes. The video path has had this guard
- * since #395; the audio path had not. */
+ * the drift that triggered the write never closes. */
export function shouldResyncAudio(
driftSec: number,
freeRunning: boolean,
@@ -388,6 +450,9 @@ export function VirtualPreview({
const videoRef = useRef<HTMLVideoElement | null>(null);
const primaryAudioRef = useRef<HTMLAudioElement | null>(null);
const supplementalAudioRef = useRef<HTMLAudioElement | null>(null);
+ // The audio elements that have not yet followed the last explicit move of the picture (see
+ // `applySourceTime` and `steerAudio`).
+ const audioJumpedRef = useRef(new WeakSet<HTMLAudioElement>());
const [primaryAudioEl, setPrimaryAudioEl] = useState<HTMLAudioElement | null>(null);
const [supplementalAudioEl, setSupplementalAudioEl] = useState<HTMLAudioElement | null>(null);
const [supplementalAudioSrc, setSupplementalAudioSrc] = useState<string | null>(null);
@@ -817,17 +882,23 @@ export function VirtualPreview({
for (const audio of [primaryAudioRef.current, supplementalAudioRef.current]) {
if (!audio) continue;
const target = resolveAudioTrackPlayback(v.currentTime, audio.duration);
- if (audio.playbackRate !== v.playbackRate) audio.playbackRate = v.playbackRate;
- if (
- shouldResyncAudio(
- audio.currentTime - target.targetTimeSec,
- !audio.paused && target.shouldPlay,
- PRIMARY_AUDIO_PLAYING_LEASH_SEC,
- audio.seeking,
- )
- ) {
+ // A paused video is not a clock to free-run against: the audio is about to be
+ // paused with it, and is placed exactly where the picture stopped.
+ const steer = steerAudio(
+ audio.currentTime - target.targetTimeSec,
+ !v.paused && !audio.paused && target.shouldPlay,
+ audio.seeking,
+ audioJumpedRef.current.has(audio),
+ audio.playbackRate !== v.playbackRate,
+ );
+ // Capped where the browser stops accepting it (it throws above 16, and that
+ // would end this frame): a video already at the cap has nothing left to nudge.
+ const rate = Math.min(v.playbackRate * steer.rateFactor, MAX_NATIVE_PLAYBACK_RATE);
+ if (audio.playbackRate !== rate) audio.playbackRate = rate;
+ if (steer.seek) {
try {
audio.currentTime = target.targetTimeSec;
+ audioJumpedRef.current.delete(audio);
} catch {
// media metadata not ready yet
}
@@ -971,7 +1042,7 @@ export function VirtualPreview({
shouldResyncAudio(
el.currentTime - trackTarget.targetTimeSec,
!el.paused && trackTarget.shouldPlay,
- SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC,
+ IMPORTED_AUDIO_PLAYING_LEASH_SEC,
el.seeking,
)
) {
@@ -1196,7 +1267,16 @@ export function VirtualPreview({
// Clamp to the browser's 16× playbackRate ceiling; >16× is rendered at
// its true speed only on the offline export path, not the live preview.
const rate = Math.min(activeRegion?.speed ?? 1, MAX_NATIVE_PLAYBACK_RATE);
- if (v.playbackRate !== rate) v.playbackRate = rate;
+ if (v.playbackRate !== rate) {
+ v.playbackRate = rate;
+ // The recording's audio rides this clock, so it takes the new rate in the same
+ // breath. Left to the next tick it plays one whole frame gap at the old one,
+ // 150 ms under load: the audio that far behind entering a 2x region, and
+ // that far ahead leaving it.
+ for (const audio of [primaryAudioRef.current, supplementalAudioRef.current]) {
+ if (audio) audio.playbackRate = rate;
+ }
+ }
}
},
[onTimeChange],
@@ -1253,6 +1333,13 @@ export function VirtualPreview({
// have its flag consumed by a frame on which no seek happened.
isProgrammaticSeekRef.current = true;
video.currentTime = sourceTimeSec;
+ // The audio elements keep their own position, so a scrub, a skipped cut or a clip
+ // junction moves the picture out from under them. The rAF tick cannot tell that from
+ // clock drift by its size (a cut can be 50 ms), so it is told: each element follows
+ // once it is not seeking, whatever the drift.
+ for (const audio of [primaryAudioRef.current, supplementalAudioRef.current]) {
+ if (audio) audioJumpedRef.current.add(audio);
+ }
}, []);
const seekToVirtualTime = useCallback(