Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(editor): let the audio element free-run instead of re-seeking it every frame
Christian Schmittelf1550dfba277
diff --git a/src/components/ai-edition/VirtualPreview.seekStorm.test.tsx b/src/components/ai-edition/VirtualPreview.seekStorm.test.tsxindex bb0d97cbd..feebbe513 100644--- a/src/components/ai-edition/VirtualPreview.seekStorm.test.tsx+++ b/src/components/ai-edition/VirtualPreview.seekStorm.test.tsx@@ -3,7 +3,13 @@ import "@testing-library/jest-dom";import { act, cleanup, fireEvent, render } from "@testing-library/react";import { afterEach, describe, expect, it, vi } from "vitest";import type { AxcutClip } from "@/lib/ai-edition/schema";-import { type VideoSource, VirtualPreview } from "./VirtualPreview";+import {+ PRIMARY_AUDIO_PLAYING_LEASH_SEC,+ SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC,+ shouldResyncAudio,+ type VideoSource,+ VirtualPreview,+} from "./VirtualPreview";// The root cause of issue #395, in a test.//@@ -148,3 +154,50 @@ describe("VirtualPreview keeps one demuxer seek in flight (issue #395 root causeexpect(video.writes).toEqual([3, 7]);});});++describe("the primary audio element free-runs instead of being re-seeked every frame", () => {+ // Measured in the shipped editor on a Snapdragon X Elite, during ordinary playback:+ // 157 `seeking` events in 20 s with a single `seeked`, and 87 `currentTime` writes in+ // 15 s — about six a second. Each write stepped forward by ~0.1 s, which is exactly the+ // time that had passed: the audio was being sent to where it already was heading.+ //+ // The 25 ms leash assumed the rAF tick runs tight against the <video>'s own clock. It+ // does not when the renderer main thread is loaded (measured ~44 % blocked during+ // playback): ticks land 100+ ms apart, the audio element has free-run past 25 ms by+ // then, and gets yanked back. The yank stalls it, so it falls behind again — the storm+ // sustains itself, which is why the drift sat at a steady ~100 ms instead of decaying.+ it("leaves a playing element alone at the drift the storm was measured at", () => {+ expect(shouldResyncAudio(0.1, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(false);+ expect(shouldResyncAudio(-0.1, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(false);+ });++ it("still corrects a real desync", () => {+ expect(shouldResyncAudio(0.4, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(true);+ expect(shouldResyncAudio(-0.4, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(true);+ });++ // A parked element is not free-running, so nothing sustains its position: it has to be+ // placed exactly, and placing it costs nothing because it is not playing.+ it("keeps the tight leash when the element is not free-running", () => {+ expect(shouldResyncAudio(0.05, false, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(true);+ expect(shouldResyncAudio(0.01, false, PRIMARY_AUDIO_PLAYING_LEASH_SEC)).toBe(false);+ });++ // The same discipline the video path learned in issue #395: a write onto an element+ // that is already seeking restarts the seek instead of finishing it, so the element+ // never arrives. Measured after the leash change alone, residual writes still landed+ // on an element mid-seek.+ it("never stacks a write onto an element that is already seeking", () => {+ expect(shouldResyncAudio(5, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC, true)).toBe(false);+ expect(shouldResyncAudio(5, false, PRIMARY_AUDIO_PLAYING_LEASH_SEC, true)).toBe(false);+ expect(shouldResyncAudio(5, true, PRIMARY_AUDIO_PLAYING_LEASH_SEC, false)).toBe(true);+ });++ // Lip sync is the constraint the primary track has and an imported one does not, so it+ // gets the shorter leash of the two. Audio behind picture is tolerated to about 125 ms+ // before it reads as out of sync, which is the ceiling this has to stay under.+ it("gives the primary track a shorter leash than an imported one", () => {+ expect(PRIMARY_AUDIO_PLAYING_LEASH_SEC).toBeLessThan(SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC);+ expect(PRIMARY_AUDIO_PLAYING_LEASH_SEC).toBeLessThan(0.125);+ });+});diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsxindex d5d2b4858..9ec3fbd1e 100644--- a/src/components/ai-edition/VirtualPreview.tsx+++ b/src/components/ai-edition/VirtualPreview.tsx@@ -74,6 +74,61 @@ export interface VideoSource {* already over: an empty extraction has to read as ended, or the rAF loop below spends the* whole timeline seeking and calling `play()` on an element that has nothing to play.*/+/** Drift tolerated before an audio element that is NOT free-running gets placed+ * exactly. Nothing sustains a parked element's position, and placing it costs+ * nothing because it is not playing. */+const AUDIO_PARKED_LEASH_SEC = 0.025;++/** Drift tolerated on the primary audio track while it plays.+ *+ * This used to be `AUDIO_PARKED_LEASH_SEC` on the grounds that the primary track syncs+ * to the `<video>`'s own authoritative clock and so could be held that tight. Measured+ * in the shipped editor on a Snapdragon X Elite, during ordinary playback, it could not:+ * 157 `seeking` events in 20 s against a single `seeked`, and 87 `currentTime` writes in+ * 15 s — six a second, each stepping forward by ~0.1 s, which is exactly the time that had+ * passed. The element was being sent to where it was already heading, and every write+ * flushed its pipeline: the stutter.+ *+ * The tight leash assumed the rAF tick runs close behind the video clock. It does not when+ * the renderer main thread is loaded (measured ~44 % blocked during playback): ticks land+ * 100+ ms apart, the element has free-run past 25 ms by then, and gets yanked back. The+ * yank stalls it, so it falls behind again — which is why the drift sat at a steady ~100 ms+ * instead of decaying. A playing element already runs at the right rate from the right+ * offset, so left alone it stays in sync on its own.+ *+ * The ceiling is lip sync, not the storm: audio behind picture reads as out of sync at+ * roughly 125 ms, and audio ahead of it is noticed sooner still. This sits under both while+ * clearing the ~100 ms the storm was sustaining. */+export const PRIMARY_AUDIO_PLAYING_LEASH_SEC = 0.12;++/** Same, for an imported track (issue #350). It syncs to `virtualTimeSec`, which is DERIVED+ * from the video clock each frame and so is noisier still — and it carries BGM or voiceover+ * rather than lip sync, so it can afford a wider leash than the primary track. */+export const SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC = 0.3;++/** Whether an audio element's drift warrants a `currentTime` write.+ *+ * `freeRunning` means the element is playing and is supposed to be: it holds its own+ * position, so only a real discontinuity — a scrub, a trim jump, a first play — is worth+ * the cost of a seek. See issue #395 and `VirtualPreview.seekStorm.test.tsx` for what+ * happens when writes are issued per frame instead.+ *+ * `seeking` is the other half of that lesson: a write onto an element that is still+ * seeking restarts the seek rather than finishing it, so the element never arrives and+ * the drift that triggered the write never closes. The video path has had this guard+ * since #395; the audio path had not. */+export function shouldResyncAudio(+ driftSec: number,+ freeRunning: boolean,+ playingLeashSec: number,+ seeking = false,+): boolean {+ if (seeking) {+ return false;+ }+ return Math.abs(driftSec) > (freeRunning ? playingLeashSec : AUDIO_PARKED_LEASH_SEC);+}+export function resolveAudioTrackPlayback(videoTimeSec: number,durationSec = Number.POSITIVE_INFINITY,@@ -763,7 +818,14 @@ export function VirtualPreview({if (!audio) continue;const target = resolveAudioTrackPlayback(v.currentTime, audio.duration);if (audio.playbackRate !== v.playbackRate) audio.playbackRate = v.playbackRate;- if (Math.abs(audio.currentTime - target.targetTimeSec) > 0.025) {+ if (+ shouldResyncAudio(+ audio.currentTime - target.targetTimeSec,+ !audio.paused && target.shouldPlay,+ PRIMARY_AUDIO_PLAYING_LEASH_SEC,+ audio.seeking,+ )+ ) {try {audio.currentTime = target.targetTimeSec;} catch {@@ -899,17 +961,20 @@ export function VirtualPreview({} else {el.volume = Math.min(1, audioGainScalar(trackGainDb) * globalGain * fade);}- // Only re-seek on a real discontinuity (a scrub, a trim jump, a first- // play), NOT on the sub-frame drift of normal playback. The primary audio- // can afford a 25 ms leash because it syncs to the <video>'s own- // authoritative clock; an imported track syncs to `virtualTimeSec`, which is- // DERIVED from that clock each frame and so is slightly noisy — at a 25 ms- // leash it re-seeks most frames, and each seek briefly stalls the element:- // the jitter. A started element already plays at the right rate from the- // right offset, so it free-runs in sync; this wide leash just catches the- // jumps. BGM/voiceover tolerates it; frame-tight sync is the video's job.- const leashSec = !el.paused && trackTarget.shouldPlay ? 0.3 : 0.025;- if (Math.abs(el.currentTime - trackTarget.targetTimeSec) > leashSec) {+ // Only re-seek on a real discontinuity (a scrub, a trim jump, a first play),+ // NOT on the sub-frame drift of normal playback: at a tight leash this+ // re-seeks most frames, and each seek briefly stalls the element — the+ // jitter. A started element already plays at the right rate from the right+ // offset, so it free-runs in sync; the wide leash just catches the jumps.+ // BGM/voiceover tolerates it; frame-tight sync is the video's job.+ if (+ shouldResyncAudio(+ el.currentTime - trackTarget.targetTimeSec,+ !el.paused && trackTarget.shouldPlay,+ SUPPLEMENTAL_AUDIO_PLAYING_LEASH_SEC,+ el.seeking,+ )+ ) {try {el.currentTime = trackTarget.targetTimeSec;} catch {