Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(editor): hold the audio to the picture by rate, and resync it after a seek
EtienneLescotc870e3bee60f
diff --git a/src/components/ai-edition/VirtualPreview.playback.test.tsx b/src/components/ai-edition/VirtualPreview.playback.test.tsxindex 806505030..66a08bbd1 100644--- a/src/components/ai-edition/VirtualPreview.playback.test.tsx+++ b/src/components/ai-edition/VirtualPreview.playback.test.tsx@@ -3,6 +3,7 @@ import "@testing-library/jest-dom";import { act, cleanup, fireEvent, render } from "@testing-library/react";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import type { AxcutClip, AxcutTrimRange } from "@/lib/ai-edition/schema";+import type { SpeedRegion } from "@/lib/ai-edition/timeline/speed";import { type VideoSource, VirtualPreview } from "./VirtualPreview";// The rAF tick is the whole subject here, so it is driven by hand rather than by the@@ -384,3 +385,303 @@ describe("VirtualPreview imported audio track boost", () => {expect(audioEl.volume).toBe(1); // volume left at unity so it doesn't double-attenuate});});++// The recording's sound is a second element over the same file: a muted <video> is the clock+// and a separate <audio> is what is heard. Two clocks, so the rAF loop has to keep them+// together, and it has to do it without seeking the audio: every `currentTime` write flushes+// its pipeline, which is the stutter #898 measured (157 `seeking` events in 20 s).+//+// What follows runs that loop against a small model of the elements. Real time passes between+// frames (`frame(gapSec)`), each element advances at its own `playbackRate`, and an audio seek+// stalls the element for `seekLatencySec`, so it lands that far behind the picture, the way the+// shipped editor was measured (~0.1 s). Ticks land 100-150 ms apart, which is what a main+// thread ~44 % blocked does to requestAnimationFrame.+function driveAudioClock(+ el: HTMLAudioElement,+ { seekLatencySec, skew }: { seekLatencySec: number; skew: number },+) {+ let currentTime = 0;+ let paused = true;+ let stallLeftSec = 0;+ const writes: number[] = [];+ Object.defineProperty(el, "currentTime", {+ configurable: true,+ get: () => currentTime,+ set: (next: number) => {+ currentTime = next;+ stallLeftSec = seekLatencySec;+ writes.push(next);+ },+ });+ Object.defineProperty(el, "seeking", { configurable: true, get: () => stallLeftSec > 0 });+ Object.defineProperty(el, "paused", { configurable: true, get: () => paused });+ Object.defineProperty(el, "duration", { configurable: true, get: () => 60 });+ el.play = vi.fn(() => {+ paused = false;+ return Promise.resolve();+ });+ el.pause = vi.fn(() => {+ paused = true;+ });+ return {+ writes,+ get currentTime() {+ return currentTime;+ },+ /** Put the element somewhere without a seek: how a stretch already in sync starts. */+ place: (next: number) => {+ currentTime = next;+ },+ /** Real time passing. `skew` is how much faster this element's own clock runs. */+ advance: (dtSec: number) => {+ const stalledSec = Math.min(dtSec, stallLeftSec);+ stallLeftSec -= stalledSec;+ if (!paused && stallLeftSec === 0) {+ currentTime += (dtSec - stalledSec) * el.playbackRate * (1 + skew);+ }+ },+ };+}++const LOADED_GAPS_SEC = [0.1, 0.15, 0.12, 0.1, 0.14];++function mountSynced({+ trims = [],+ speedRegions = [],+ seekLatencySec = 0.1,+ skew = 0,+}: {+ trims?: AxcutTrimRange[];+ speedRegions?: SpeedRegion[];+ seekLatencySec?: number;+ skew?: number;+} = {}) {+ const clips = [clip("c1", "a1", 0, 60, 0)];+ const sources: VideoSource[] = [{ id: "a1", src: "file:///tmp/a1.mp4", label: "a1" }];+ const onTimeChange = vi.fn();+ let requestId = 0;+ const tree = (seekTarget: { timeSec: number; requestId: number } | null) => (+ <VirtualPreview+ videoSources={sources}+ clips={clips}+ trimRanges={trims}+ speedRegions={speedRegions}+ seekTarget={seekTarget}+ onTimeChange={onTimeChange}+ />+ );+ const view = render(tree(null));+ const videoEl = view.container.querySelector("video") as HTMLVideoElement;+ const audioEl = view.container.querySelector(+ '[data-testid="preview-audio-primary"]',+ ) as HTMLAudioElement;+ const video = driveVideo(videoEl);+ const audio = driveAudioClock(audioEl, { seekLatencySec, skew });+ act(() => {+ fireEvent.loadedMetadata(videoEl);+ });+ /** Real time passes, then one rAF frame runs. */+ const frame = (gapSec: number) => {+ if (!videoEl.paused) video.seekTo(video.currentTime + gapSec * videoEl.playbackRate);+ audio.advance(gapSec);+ tick();+ };+ return {+ video,+ videoEl,+ audio,+ audioEl,+ frame,+ /** Frames until `seconds` of real time have passed, sampling after each one. */+ run: (seconds: number, gaps = LOADED_GAPS_SEC, onFrame?: () => void) => {+ let elapsedSec = 0;+ for (let i = 0; elapsedSec < seconds; i++) {+ const gap = gaps[i % gaps.length];+ elapsedSec += gap;+ frame(gap);+ onFrame?.();+ }+ },+ /** The audio's offset from the picture, in media seconds; negative is behind it. */+ offset: () => audio.currentTime - video.currentTime,+ /** One explicit seek published by the shell, the way a click on the ruler is. */+ seekTo: (timeSec: number) =>+ act(() => {+ requestId += 1;+ view.rerender(tree({ timeSec, requestId }));+ }),+ /** Playing, with the audio already in step, as it is a few seconds into a take. */+ playInSync: (atSec: number) => {+ video.seekTo(atSec);+ audio.place(atSec);+ video.play();+ tick(); // the loop starts the audio; nothing to correct+ audio.writes.length = 0;+ },+ };+}++describe("VirtualPreview primary audio keeps to the picture", () => {+ // Every offset here is media time, which is what a lip-sync mismatch is measured in and is+ // never smaller than the wall-clock one, so a bound on it holds for both.+ const TOLERANCE_SEC = 0.04;++ it("does not re-seek the audio under load and brings its offset inside 40 ms", () => {+ const world = mountSynced();+ world.video.seekTo(1);+ world.video.play();++ const offsets: number[] = [];+ world.run(30, LOADED_GAPS_SEC, () => offsets.push(world.offset()));++ // The audio is placed once, when playback starts, and never touched again: a hard seek+ // on a playing element is what stuttered. It lands behind the picture by its own seek+ // latency, and the loop closes that by nudging its rate.+ expect(world.audio.writes).toHaveLength(1);+ expect(offsets[1]).toBeLessThan(-0.05);+ // Some 2 s to close 100 ms at a few percent of rate; from there it holds.+ const settled = offsets.slice(30);+ expect(Math.max(...settled.map(Math.abs))).toBeLessThan(TOLERANCE_SEC);+ });++ it("absorbs a free-running clock skew in either direction without a seek", () => {+ for (const skew of [0.005, -0.005]) {+ const world = mountSynced({ skew });+ world.playInSync(1);++ // Nearly a minute at 0.5 % is 275 ms of drift if nothing intervenes, past every+ // threshold in this file, so holding it inside 40 ms is the loop's doing.+ const offsets: number[] = [];+ world.run(55, LOADED_GAPS_SEC, () => offsets.push(world.offset()));++ expect(world.audio.writes, `skew ${skew}`).toHaveLength(0);+ expect(Math.max(...offsets.map(Math.abs)), `skew ${skew}`).toBeLessThan(TOLERANCE_SEC);+ cleanup();+ }+ });++ it("pulls an audio that is AHEAD of the picture back, not only one behind it", () => {+ const world = mountSynced();+ world.playInSync(1);+ // Ahead is what is noticed first, from about 45 ms.+ world.audio.place(world.video.currentTime + 0.1);++ const offsets: number[] = [];+ world.run(10, LOADED_GAPS_SEC, () => offsets.push(world.offset()));++ expect(world.audio.writes).toHaveLength(0);+ expect(Math.max(...offsets.slice(30).map(Math.abs))).toBeLessThan(TOLERANCE_SEC);+ });++ it("resyncs on a small explicit seek during playback, whatever the drift", () => {+ const world = mountSynced();+ world.playInSync(5);+ world.run(1, [0.016]);+ expect(world.audio.writes).toHaveLength(0);++ // 100 ms back: a click on the ruler next to the playhead. The audio's 100 ms lead is+ // inside any leash the storm fix can afford, so only a resync on the seek itself moves it.+ world.seekTo(world.video.currentTime - 0.1);+ world.frame(0.016);++ expect(world.audio.writes).toHaveLength(1);+ expect(world.audio.writes[0]).toBeCloseTo(world.video.currentTime, 1);+ world.run(3, [0.016]);+ expect(Math.abs(world.offset())).toBeLessThan(TOLERANCE_SEC);+ expect(world.audio.writes).toHaveLength(1);+ });++ it("resyncs across a skipped interval shorter than the leash", () => {+ // A cut of 5.00 -> 5.11. The preview only calls a frame cut once it is 50 ms inside+ // (`locateSourcePosition` gives each edge that much slack), so the picture jumps from+ // ~5.05 to 5.11 and the audio, still playing the cut material, would sit ~60 ms off+ // from then on.+ const trim: AxcutTrimRange = {+ id: "cut",+ assetId: "a1",+ clipId: "c1",+ startSec: 5,+ endSec: 5.11,+ origin: "user",+ reason: "",+ };+ const world = mountSynced({ trims: [trim] });+ world.playInSync(4.9);+ // Small steps, because the slack leaves only ~10 ms of the cut where a frame is+ // recognised as inside it.+ world.run(0.5, [0.004]);++ expect(world.audio.writes).toHaveLength(1);+ expect(world.audio.writes[0]).toBeGreaterThanOrEqual(5.11);+ world.run(3, [0.016]);+ expect(Math.abs(world.offset())).toBeLessThan(TOLERANCE_SEC);+ expect(world.audio.writes).toHaveLength(1);+ });++ it("does not seek an audio element that is still seeking, and follows once it is done", () => {+ const world = mountSynced({ seekLatencySec: 0.2 });+ world.playInSync(5);++ world.seekTo(4.9);+ world.frame(0.016); // the first resync: the audio starts a 0.2 s seek+ world.seekTo(6);+ world.frame(0.016); // still seeking: this one waits instead of restarting it+ expect(world.audio.writes).toHaveLength(1);++ world.run(1, [0.1]); // the seek ends, and the loop goes where the picture went+ expect(world.audio.writes).toHaveLength(2);+ expect(world.audio.writes[1]).toBeGreaterThan(6);+ });++ it("keeps in step through a 2x region, entering and leaving it under load", () => {+ const region: SpeedRegion = { id: "s", startMs: 10_000, endMs: 14_000, speed: 2 };+ const world = mountSynced({ speedRegions: [region] });+ world.video.seekTo(1);+ world.video.play();+ world.run(8); // settled before the region++ // The video's rate changes at the END of a tick, so the audio's has to change with it+ // and not at the top of the next one: over a 150 ms gap that lag alone leaves the audio+ // 150 ms behind on the way in and 150 ms ahead on the way out.+ const rates: number[] = [];+ const offsets: number[] = [];+ world.run(30, LOADED_GAPS_SEC, () => {+ rates.push(world.audioEl.playbackRate);+ offsets.push(world.offset());+ });++ expect(rates).toContain(2);+ expect(world.audioEl.playbackRate).toBe(1);+ expect(world.audio.writes).toHaveLength(1);+ expect(Math.max(...offsets.map(Math.abs))).toBeLessThan(TOLERANCE_SEC);+ });++ it("never asks the audio for a rate the browser refuses", () => {+ // Chromium throws on a playbackRate above 16, from inside the rAF tick. A nudge on top+ // of a video already at that cap must stop at it.+ const region: SpeedRegion = { id: "s", startMs: 0, endMs: 60_000, speed: 16 };+ const world = mountSynced({ speedRegions: [region] });+ world.playInSync(1);+ world.frame(0.016); // the region's rate reaches the video+ world.audio.place(world.video.currentTime - 0.1); // behind: the nudge goes UP++ const rates: number[] = [];+ world.run(1, [0.016], () => rates.push(world.audioEl.playbackRate));++ expect(world.videoEl.playbackRate).toBe(16);+ expect(Math.max(...rates)).toBeLessThanOrEqual(16);+ });++ it("goes back to the video's own rate once playback pauses", () => {+ const world = mountSynced();+ world.playInSync(1);+ world.audio.place(world.video.currentTime + 0.1); // a nudge is under way+ world.frame(0.016);+ expect(world.audioEl.playbackRate).toBeLessThan(1);++ world.videoEl.pause();+ world.frame(0.016);++ expect(world.audioEl.playbackRate).toBe(1);+ });+});diff --git a/src/components/ai-edition/VirtualPreview.seekStorm.test.tsx b/src/components/ai-edition/VirtualPreview.seekStorm.test.tsxindex feebbe513..e1c88d5f5 100644--- a/src/components/ai-edition/VirtualPreview.seekStorm.test.tsx+++ b/src/components/ai-edition/VirtualPreview.seekStorm.test.tsx@@ -4,9 +4,12 @@ import { act, cleanup, fireEvent, render } from "@testing-library/react";import { afterEach, describe, expect, it, vi } from "vitest";import type { AxcutClip } from "@/lib/ai-edition/schema";import {- PRIMARY_AUDIO_PLAYING_LEASH_SEC,- SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC,+ AUDIO_NUDGE_START_SEC,+ AUDIO_NUDGE_STOP_SEC,+ AUDIO_SEEK_LEASH_SEC,+ IMPORTED_AUDIO_PLAYING_LEASH_SEC,shouldResyncAudio,+ steerAudio,type VideoSource,VirtualPreview,} from "./VirtualPreview";@@ -155,49 +158,72 @@ describe("VirtualPreview keeps one demuxer seek in flight (issue #395 root cause});});-describe("the primary audio element free-runs instead of being re-seeked every frame", () => {- // Measured in the shipped editor on a Snapdragon X Elite, during ordinary playback:- // 157 `seeking` events in 20 s with a single `seeked`, and 87 `currentTime` writes in- // 15 s — about six a second. Each write stepped forward by ~0.1 s, which is exactly the- // time that had passed: the audio was being sent to where it already was heading.- //- // The 25 ms leash assumed the rAF tick runs tight against the <video>'s own clock. It- // does not when the renderer main thread is loaded (measured ~44 % blocked during- // playback): ticks land 100+ ms apart, the audio element has free-run past 25 ms by- // then, and gets yanked back. The yank stalls it, so it falls behind again — the storm- // sustains itself, which is why the drift sat at a steady ~100 ms instead of decaying.- it("leaves a playing element alone at the drift the storm was measured at", () => {- expect(shouldResyncAudio(0.1, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(false);- expect(shouldResyncAudio(-0.1, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(false);+// What the rAF loop does to a playing audio element between two ticks. The measured story+// behind these is on `AUDIO_NUDGE_START_SEC`; the loop itself is driven end to end in+// VirtualPreview.playback.test.tsx ("primary audio keeps to the picture"), because a+// decision that is right in isolation says nothing about what the call site feeds it.+describe("steerAudio: a playing element is nudged, not seeked", () => {+ const playing = (driftSec: number, nudged = false) =>+ steerAudio(driftSec, true, false, false, nudged);++ it("leaves an element on the picture alone", () => {+ expect(playing(0.01)).toEqual({ seek: false, rateFactor: 1 });+ expect(playing(-0.01)).toEqual({ seek: false, rateFactor: 1 });});- it("still corrects a real desync", () => {- expect(shouldResyncAudio(0.4, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(true);- expect(shouldResyncAudio(-0.4, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(true);+ it("nudges the rate toward the picture instead of seeking, whichever side it is on", () => {+ // The drift the storm was measured at: what used to be re-seeked six times a second.+ const behind = playing(-0.1);+ const ahead = playing(0.1);+ expect(behind.seek).toBe(false);+ expect(ahead.seek).toBe(false);+ expect(behind.rateFactor).toBeGreaterThan(1);+ expect(ahead.rateFactor).toBeLessThan(1);+ // Symmetric: a lead is noticed sooner than a lag, so it must not be the slower to close.+ expect(behind.rateFactor - 1).toBeCloseTo(1 - ahead.rateFactor, 10);});- // A parked element is not free-running, so nothing sustains its position: it has to be- // placed exactly, and placing it costs nothing because it is not playing.- it("keeps the tight leash when the element is not free-running", () => {- expect(shouldResyncAudio(0.05, false, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(true);- expect(shouldResyncAudio(0.01, false, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(false);+ it("stops nudging closer to the picture than it starts, so an edge does not flap", () => {+ const between = (AUDIO_NUDGE_START_SEC + AUDIO_NUDGE_STOP_SEC) / 2;+ expect(playing(between, false).rateFactor).toBe(1);+ expect(playing(between, true).rateFactor).not.toBe(1);+ expect(playing(AUDIO_NUDGE_STOP_SEC / 2, true).rateFactor).toBe(1);});- // The same discipline the video path learned in issue #395: a write onto an element- // that is already seeking restarts the seek instead of finishing it, so the element- // never arrives. Measured after the leash change alone, residual writes still landed- // on an element mid-seek.+ it("seeks only past what a nudge can close", () => {+ expect(playing(AUDIO_SEEK_LEASH_SEC * 0.9).seek).toBe(false);+ expect(playing(AUDIO_SEEK_LEASH_SEC * 1.1)).toEqual({ seek: true, rateFactor: 1 });+ expect(playing(-AUDIO_SEEK_LEASH_SEC * 1.1).seek).toBe(true);+ });++ it("places a parked element exactly", () => {+ expect(steerAudio(0.05, false, false, false, false).seek).toBe(true);+ expect(steerAudio(0.01, false, false, false, false).seek).toBe(false);+ });++ it("follows an explicit jump of the picture whatever the drift", () => {+ expect(steerAudio(0.001, true, false, true, false).seek).toBe(true);+ expect(steerAudio(0.08, false, false, true, false).seek).toBe(true);+ });++ // The discipline the video path learned in issue #395: a write onto an element that is+ // already seeking restarts the seek instead of finishing it, so the element never arrives.it("never stacks a write onto an element that is already seeking", () => {- expect(shouldResyncAudio(5, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC, true)).toBe(false);- expect(shouldResyncAudio(5, false, PRIMARY_AUDIO_PLAYING_LEASH_SEC, true)).toBe(false);- expect(shouldResyncAudio(5, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC, false)).toBe(true);+ expect(steerAudio(5, true, true, false, false).seek).toBe(false);+ expect(steerAudio(5, false, true, false, false).seek).toBe(false);+ expect(steerAudio(0.001, true, true, true, false).seek).toBe(false); // the jump waits+ });+});++describe("shouldResyncAudio: an imported track free-runs inside a wide leash", () => {+ it("leaves a playing track alone inside the leash and corrects a real desync", () => {+ expect(shouldResyncAudio(0.2, true, IMPORTED_AUDIO_PLAYING_LEASH_SEC)).toBe(false);+ expect(shouldResyncAudio(-0.2, true, IMPORTED_AUDIO_PLAYING_LEASH_SEC)).toBe(false);+ expect(shouldResyncAudio(0.4, true, IMPORTED_AUDIO_PLAYING_LEASH_SEC)).toBe(true);});- // Lip sync is the constraint the primary track has and an imported one does not, so it- // gets the shorter leash of the two. Audio behind picture is tolerated to about 125 ms- // before it reads as out of sync, which is the ceiling this has to stay under.- it("gives the primary track a shorter leash than an imported one", () => {- expect(PRIMARY_AUDIO_PLAYING_LEASH_SEC).toBeLessThan(SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC);- expect(PRIMARY_AUDIO_PLAYING_LEASH_SEC).toBeLessThan(0.125);+ it("places a parked track exactly, and never onto one that is seeking", () => {+ expect(shouldResyncAudio(0.05, false, IMPORTED_AUDIO_PLAYING_LEASH_SEC)).toBe(true);+ expect(shouldResyncAudio(5, true, IMPORTED_AUDIO_PLAYING_LEASH_SEC, true)).toBe(false);});});diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsxindex 97b84be9f..5980c3654 100644--- a/src/components/ai-edition/VirtualPreview.tsx+++ b/src/components/ai-edition/VirtualPreview.tsx@@ -66,10 +66,11 @@ export interface VideoSource {* nothing because it is not playing. */const AUDIO_PARKED_LEASH_SEC = 0.025;-/** Drift tolerated on the primary audio track while it plays.+/** How a PLAYING audio element that rides the video's clock (the recording's own sound) is held+ * to the picture: by its `playbackRate`, not by seeks.*- * This used to be `AUDIO_PARKED_LEASH_SEC` on the grounds that the primary track syncs- * to the `<video>`'s own authoritative clock and so could be held that tight. Measured+ * It used to be held by seeks, at `AUDIO_PARKED_LEASH_SEC`, on the grounds that the element+ * syncs to the `<video>`'s own authoritative clock and so could be kept that tight. Measured* in the shipped editor on a Snapdragon X Elite, during ordinary playback, it could not:* 157 `seeking` events in 20 s against a single `seeked`, and 87 `currentTime` writes in* 15 s — six a second, each stepping forward by ~0.1 s, which is exactly the time that had@@ -80,20 +81,82 @@ const AUDIO_PARKED_LEASH_SEC = 0.025;* the renderer main thread is loaded (measured ~44 % blocked during playback): ticks land* 100+ ms apart, the element has free-run past 25 ms by then, and gets yanked back. The* yank stalls it, so it falls behind again — which is why the drift sat at a steady ~100 ms- * instead of decaying. A playing element already runs at the right rate from the right- * offset, so left alone it stays in sync on its own.+ * instead of decaying.*- * The ceiling is lip sync, not the storm: audio behind picture reads as out of sync at- * roughly 125 ms, and audio ahead of it is noticed sooner still. This sits under both while- * clearing the ~100 ms the storm was sustaining. */-export const PRIMARY_AUDIO_PLAYING_LEASH_SEC = 0.12;+ * Widening the leash ends the storm and keeps the offset instead: a seek lands the audio+ * BEHIND the picture by its own latency (that ~100 ms), and both elements then run at the+ * same rate, so nothing ever closes it. Audio is noticed from about 45 ms when it leads and+ * about 125 ms when it lags (ITU-R BT.1359), so an offset that stays is a defect, not a+ * tolerance.+ *+ * So the element is nudged. Once it drifts more than `AUDIO_NUDGE_START_SEC` its rate is+ * trimmed by `AUDIO_NUDGE_FRACTION` toward the picture, and it goes back to the video's own+ * rate within `AUDIO_NUDGE_STOP_SEC`; the gap between the two keeps an element at the edge+ * from flipping its rate every frame. A rate change flushes nothing, and with+ * `preservesPitch` (the default) it does not move the pitch. It closes ~50 ms a second, so a+ * seek's 100 ms is gone in ~2 s and a free-running clock skew never builds up. */+export const AUDIO_NUDGE_START_SEC = 0.02;+export const AUDIO_NUDGE_STOP_SEC = 0.005;+export const AUDIO_NUDGE_FRACTION = 0.05;++/** Past this a nudge would take seconds and something else is wrong (a stalled or starved+ * element): only then is a seek worth its flush. Kept above the ~100 ms a seek itself+ * leaves, or the seek would be the thing that triggers the next one. */+export const AUDIO_SEEK_LEASH_SEC = 0.3;++/** Drift tolerated on an imported track (issue #350) while it plays. It syncs to+ * `virtualTimeSec`, which is DERIVED from the video clock each frame and so is noisier than+ * the recording's own audio, and it carries BGM or voiceover rather than lip sync, so it is+ * left to free-run inside a wide leash. */+export const IMPORTED_AUDIO_PLAYING_LEASH_SEC = 0.3;-/** Same, for an imported track (issue #350). It syncs to `virtualTimeSec`, which is DERIVED- * from the video clock each frame and so is noisier still — and it carries BGM or voiceover- * rather than lip sync, so it can afford a wider leash than the primary track. */-export const SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC = 0.3;+/** What the rAF tick should do to one of the recording's audio elements this frame. */+export interface AudioSteer {+ /** Write `currentTime`. */+ seek: boolean;+ /** `playbackRate` as a multiple of the video's own: 1 on target, nudged while catching up. */+ rateFactor: number;+}-/** Whether an audio element's drift warrants a `currentTime` write.+/** Decide how to hold an audio element that rides the video's clock to the picture.+ *+ * `driftSec` is the element's position minus where the picture is: positive means the audio+ * is AHEAD. `freeRunning` means it is playing and is meant to be. `seeking` means a write is+ * already in flight: another one restarts that seek instead of finishing it, so the element+ * never arrives and the drift that triggered it never closes (issue #395, which the video+ * path took and the audio path had not). `jumped` means the picture was MOVED, on purpose,+ * since this element was last placed: a scrub, a skipped cut, a clip junction. That is+ * indistinguishable from clock drift by size, since a cut can be 50 ms, so the caller says+ * so, and the audio follows whatever the drift once it is not seeking. `nudged` is whether+ * its rate is off the video's now, which is what makes the stop threshold apply. */+export function steerAudio(+ driftSec: number,+ freeRunning: boolean,+ seeking: boolean,+ jumped: boolean,+ nudged: boolean,+): AudioSteer {+ if (seeking) {+ return { seek: false, rateFactor: 1 };+ }+ if (jumped) {+ return { seek: true, rateFactor: 1 };+ }+ if (!freeRunning) {+ // Nothing sustains a parked element's position, and placing it costs nothing.+ return { seek: Math.abs(driftSec) > AUDIO_PARKED_LEASH_SEC, rateFactor: 1 };+ }+ const off = Math.abs(driftSec);+ if (off > AUDIO_SEEK_LEASH_SEC) {+ return { seek: true, rateFactor: 1 };+ }+ if (off > (nudged ? AUDIO_NUDGE_STOP_SEC : AUDIO_NUDGE_START_SEC)) {+ return { seek: false, rateFactor: 1 - Math.sign(driftSec) * AUDIO_NUDGE_FRACTION };+ }+ return { seek: false, rateFactor: 1 };+}++/** Whether an imported track's drift warrants a `currentTime` write.** `freeRunning` means the element is playing and is supposed to be: it holds its own* position, so only a real discontinuity — a scrub, a trim jump, a first play — is worth@@ -102,8 +165,7 @@ export const SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC = 0.3;** `seeking` is the other half of that lesson: a write onto an element that is still* seeking restarts the seek rather than finishing it, so the element never arrives and- * the drift that triggered the write never closes. The video path has had this guard- * since #395; the audio path had not. */+ * the drift that triggered the write never closes. */export function shouldResyncAudio(driftSec: number,freeRunning: boolean,@@ -388,6 +450,9 @@ export function VirtualPreview({const videoRef = useRef<HTMLVideoElement | null>(null);const primaryAudioRef = useRef<HTMLAudioElement | null>(null);const supplementalAudioRef = useRef<HTMLAudioElement | null>(null);+ // The audio elements that have not yet followed the last explicit move of the picture (see+ // `applySourceTime` and `steerAudio`).+ const audioJumpedRef = useRef(new WeakSet<HTMLAudioElement>());const [primaryAudioEl, setPrimaryAudioEl] = useState<HTMLAudioElement | null>(null);const [supplementalAudioEl, setSupplementalAudioEl] = useState<HTMLAudioElement | null>(null);const [supplementalAudioSrc, setSupplementalAudioSrc] = useState<string | null>(null);@@ -817,17 +882,23 @@ export function VirtualPreview({for (const audio of [primaryAudioRef.current, supplementalAudioRef.current]) {if (!audio) continue;const target = resolveAudioTrackPlayback(v.currentTime, audio.duration);- if (audio.playbackRate !== v.playbackRate) audio.playbackRate = v.playbackRate;- if (- shouldResyncAudio(- audio.currentTime - target.targetTimeSec,- !audio.paused && target.shouldPlay,- PRIMARY_AUDIO_PLAYING_LEASH_SEC,- audio.seeking,- )- ) {+ // A paused video is not a clock to free-run against: the audio is about to be+ // paused with it, and is placed exactly where the picture stopped.+ const steer = steerAudio(+ audio.currentTime - target.targetTimeSec,+ !v.paused && !audio.paused && target.shouldPlay,+ audio.seeking,+ audioJumpedRef.current.has(audio),+ audio.playbackRate !== v.playbackRate,+ );+ // Capped where the browser stops accepting it (it throws above 16, and that+ // would end this frame): a video already at the cap has nothing left to nudge.+ const rate = Math.min(v.playbackRate * steer.rateFactor, MAX_NATIVE_PLAYBACK_RATE);+ if (audio.playbackRate !== rate) audio.playbackRate = rate;+ if (steer.seek) {try {audio.currentTime = target.targetTimeSec;+ audioJumpedRef.current.delete(audio);} catch {// media metadata not ready yet}@@ -971,7 +1042,7 @@ export function VirtualPreview({shouldResyncAudio(el.currentTime - trackTarget.targetTimeSec,!el.paused && trackTarget.shouldPlay,- SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC,+ IMPORTED_AUDIO_PLAYING_LEASH_SEC,el.seeking,)) {@@ -1196,7 +1267,16 @@ export function VirtualPreview({// Clamp to the browser's 16× playbackRate ceiling; >16× is rendered at// its true speed only on the offline export path, not the live preview.const rate = Math.min(activeRegion?.speed ?? 1, MAX_NATIVE_PLAYBACK_RATE);- if (v.playbackRate !== rate) v.playbackRate = rate;+ if (v.playbackRate !== rate) {+ v.playbackRate = rate;+ // The recording's audio rides this clock, so it takes the new rate in the same+ // breath. Left to the next tick it plays one whole frame gap at the old one,+ // 150 ms under load: the audio that far behind entering a 2x region, and+ // that far ahead leaving it.+ for (const audio of [primaryAudioRef.current, supplementalAudioRef.current]) {+ if (audio) audio.playbackRate = rate;+ }+ }}},[onTimeChange],@@ -1253,6 +1333,13 @@ export function VirtualPreview({// have its flag consumed by a frame on which no seek happened.isProgrammaticSeekRef.current = true;video.currentTime = sourceTimeSec;+ // The audio elements keep their own position, so a scrub, a skipped cut or a clip+ // junction moves the picture out from under them. The rAF tick cannot tell that from+ // clock drift by its size (a cut can be 50 ms), so it is told: each element follows+ // once it is not seeking, whatever the drift.+ for (const audio of [primaryAudioRef.current, supplementalAudioRef.current]) {+ if (audio) audioJumpedRef.current.add(audio);+ }}, []);const seekToVirtualTime = useCallback(