Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(audio): decode every audio stream on the media clock for peaks and STT
EtienneLescot22ec4baf9e3a
diff --git a/electron/media/audioPeaks.test.ts b/electron/media/audioPeaks.test.tsindex 485fbb034..644f9bebb 100644--- a/electron/media/audioPeaks.test.ts+++ b/electron/media/audioPeaks.test.ts@@ -1,9 +1,16 @@// @vitest-environment node+import { spawnSync } from "node:child_process";import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import path from "node:path";-import { afterEach, beforeEach, describe, expect, it } from "vitest";-import { ffmpegCandidates, peakBlockCount, resolveFfmpeg } from "./audioPeaks";+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";+import {+ countAudioStreams,+ decodePeaks,+ ffmpegCandidates,+ peakBlockCount,+ resolveFfmpeg,+} from "./audioPeaks";const ROOT = path.resolve(__dirname, "..", "..");@@ -311,3 +318,104 @@ describe.runIf(staged)("decoding a real file", () => {expect(mn).toBeCloseTo(-0.205, 1);}, 120_000);});++describe("countAudioStreams", () => {+ it("counts the audio streams in ffmpeg's listing, and nothing else", () => {+ // The layout of an older macOS recording, as ffmpeg 8.1 lists it.+ const listing = [+ "Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'recording.mp4':",+ " Duration: 00:00:12.00, start: 0.000000, bitrate: 5012 kb/s",+ " Stream #0:0[0x1](und): Video: h264 (High) (avc1 / 0x31637661), yuv420p, 60 fps (default)",+ " Stream #0:1[0x2](und): Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, stereo (default)",+ " Metadata:",+ " handler_name : Audio: system",+ " Stream #0:2[0x3](und): Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, mono (default)",+ "At least one output file must be specified",+ ].join("\n");+ expect(countAudioStreams(listing)).toBe(2);+ expect(countAudioStreams(" Stream #0:0: Video: ffv1, 16x16")).toBe(0);+ });+});++const ffmpegOnPath = spawnSync("ffmpeg", ["-version"], { timeout: 5_000 }).status === 0;++describe.skipIf(!ffmpegOnPath)("peaks on the media clock, with real FFmpeg", () => {+ let fixtureDir: string;+ let delayed: string;+ let twoTracks: string;++ /** A four-second video from zero, muxed with the given audio inputs, one stream each. */+ function makeFixture(filename: string, audioInputs: string[][]): string {+ const filePath = path.join(fixtureDir, filename);+ const result = spawnSync(+ "ffmpeg",+ [+ "-hide_banner",+ "-loglevel",+ "error",+ "-y",+ "-f",+ "lavfi",+ "-i",+ "color=c=black:s=16x16:r=1:d=4",+ ...audioInputs.flat(),+ "-map",+ "0:v",+ ...audioInputs.flatMap((_, i) => ["-map", `${i + 1}:a`]),+ "-c:v",+ "ffv1",+ "-c:a",+ "pcm_s16le",+ filePath,+ ],+ { encoding: "utf8", timeout: 10_000 },+ );+ if (result.status !== 0) throw new Error(result.error?.message ?? result.stderr);+ return filePath;+ }++ /** Seconds into the clip of the first block with signal in it, or -1. */+ function firstAudibleSec(peaks: Float32Array): number {+ for (let i = 0; i < peaks.length; i += 2) {+ if (peaks[i + 1] > 0.01) return i / 2 / 200;+ }+ return -1;+ }++ beforeAll(() => {+ fixtureDir = mkdtempSync(path.join(tmpdir(), "openscreen-peaks-clock-"));+ const delayedTone = (sec: number, duration: number) => [+ "-itsoffset",+ String(sec),+ "-f",+ "lavfi",+ "-i",+ `sine=frequency=440:sample_rate=16000:duration=${duration}`,+ ];+ // Video from zero, audio from two seconds, no encoded silence in between.+ delayed = makeFixture("delayed.mkv", [delayedTone(2, 2)]);+ // Older macOS layout: a silent stereo system track, which is the stream ffmpeg+ // picks on its own, and a mono microphone speaking from the first second.+ twoTracks = makeFixture("two-tracks.mkv", [+ ["-f", "lavfi", "-i", "anullsrc=r=16000:cl=stereo:d=4"],+ delayedTone(1, 3),+ ]);+ });++ afterAll(() => {+ if (fixtureDir) rmSync(fixtureDir, { recursive: true, force: true });+ });++ it("draws audio that starts after the video where it plays, not at zero", async () => {+ // Pulled to zero, the waveform runs two seconds ahead of the transcript and of+ // the export, both of which keep the offset.+ const peaks = await decodePeaks("ffmpeg", delayed, 4);+ expect(peaks.length).toBe(peakBlockCount(4) * 2);+ expect(firstAudibleSec(peaks)).toBe(2);+ });++ it("draws every audio stream, each on the media clock, as the export mixes them", async () => {+ const peaks = await decodePeaks("ffmpeg", twoTracks, 4);+ expect(firstAudibleSec(peaks)).toBe(1);+ });+});diff --git a/electron/media/audioPeaks.ts b/electron/media/audioPeaks.tsindex 604afd051..13f66a93f 100644--- a/electron/media/audioPeaks.ts+++ b/electron/media/audioPeaks.ts@@ -147,6 +147,60 @@ export function resolveFfmpeg(here?: string): string | null {return found;}+/** Listing streams is only opening the file: past this, the open is wedged. */+const PROBE_TIMEOUT_MS = 10_000;++/** Pads audio that starts late, so sample zero stays on the media clock. */+const ON_MEDIA_CLOCK = "aresample=first_pts=0";++/** Audio streams in ffmpeg's listing of its input (the stderr of `ffmpeg -i <file>`). */+export function countAudioStreams(listing: string): number {+ return listing.match(/^\s*Stream #0:\d+\S*: Audio:/gm)?.length ?? 0;+}++/** Any failure counts as zero: the decode that follows then runs as it always did,+ * and reports the problem itself. */+function probeAudioStreams(ffmpeg: string, filePath: string): Promise<number> {+ return new Promise((resolve) => {+ let listing = "";+ const child = spawn(ffmpeg, ["-hide_banner", "-i", filePath], {+ stdio: ["ignore", "ignore", "pipe"],+ timeout: PROBE_TIMEOUT_MS,+ killSignal: "SIGKILL",+ });+ child.stderr.on("data", (c: Buffer) => {+ listing += c.toString();+ });+ child.once("error", () => resolve(0));+ child.once("close", () => resolve(countAudioStreams(listing)));+ });+}++/**+ * The ffmpeg arguments, after `-i`, that decode a recording's audio the way the+ * compositor exports it, so the waveform and the transcript line up with the export:+ *+ * - On the media clock. Audio that starts after the video is padded, not pulled to+ * zero, or everything drawn or transcribed from it lands that much early.+ * - Every audio stream, summed. Older macOS recordings carry system audio and the+ * microphone as two streams; ffmpeg alone decodes the one it picks, usually the+ * system track, silent whenever nothing plays, while the export mixes both. `amix`+ * mixes by sample count, not timestamp, so each stream is aligned first, and+ * `normalize=0` sums like the compositor instead of averaging.+ */+export async function mediaClockAudioArgs(ffmpeg: string, filePath: string): Promise<string[]> {+ const streams = await probeAudioStreams(ffmpeg, filePath);+ if (streams < 2) return ["-vn", "-af", ON_MEDIA_CLOCK];+ const aligned = Array.from({ length: streams }, (_, i) => `[0:a:${i}]${ON_MEDIA_CLOCK}[a${i}]`);+ const labels = aligned.map((_, i) => `[a${i}]`).join("");+ return [+ "-filter_complex",+ `${aligned.join(";")};${labels}amix=inputs=${streams}:normalize=0[a]`,+ "-map",+ "[a]",+ ];+}+/** Number of min/max blocks for a clip of `durationSec`. */export function peakBlockCount(durationSec: number): number {return Math.min(MAX_PEAK_BLOCKS, Math.max(1, Math.ceil(durationSec * PEAK_BLOCKS_PER_SEC)));@@ -205,13 +259,14 @@ class PeakFolder {}/** Runs ffmpeg and folds its PCM straight into peaks. Never buffers the audio. */-async function decodePeaks(+export async function decodePeaks(ffmpeg: string,filePath: string,durationSec: number,): Promise<Float32Array> {const blocks = peakBlockCount(durationSec);const folder = new PeakFolder(blocks, durationSec * PCM_RATE);+ const audioArgs = await mediaClockAudioArgs(ffmpeg, filePath);const child = spawn(ffmpeg,[@@ -220,7 +275,7 @@ async function decodePeaks("error","-i",filePath,- "-vn",+ ...audioArgs,"-ac","1","-ar",@@ -265,11 +320,14 @@ async function decodePeaks(* Cache key: path plus size plus mtime. A recording is immutable in practice,* but keying on identity alone would serve stale peaks for a re-encoded or* replaced file, and that failure is silent and confusing.+ *+ * The leading version changes whenever the decode does: v2 put the peaks on the+ * media clock and mixed every audio stream, which the v1 entries do not.*/async function cacheKey(filePath: string): Promise<string> {const info = await stat(filePath);return createHash("sha1")- .update(`${filePath}:${info.size}:${info.mtimeMs}`)+ .update(`v2:${filePath}:${info.size}:${info.mtimeMs}`).digest("hex").slice(0, 32);}diff --git a/electron/stt/extractAudio.integration.test.ts b/electron/stt/extractAudio.integration.test.tsindex 2301f6fd3..f37059243 100644--- a/electron/stt/extractAudio.integration.test.ts+++ b/electron/stt/extractAudio.integration.test.ts@@ -5,7 +5,10 @@ import path from "node:path";import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";// Only the binary lookup is replaced: extraction still spawns and reads real FFmpeg.-vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => "ffmpeg" }));+vi.mock("../media/audioPeaks", async (importOriginal) => ({+ ...(await importOriginal<typeof import("../media/audioPeaks")>()),+ resolveFfmpeg: () => "ffmpeg",+}));import { extractMono16kPcm } from "./extractAudio";@@ -19,6 +22,7 @@ describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => {let fixtureDir: string;let delayedVideo: string;let standaloneAudio: string;+ let twoTracks: string;function makeFixture(filename: string, args: string[]): string {const filePath = path.join(fixtureDir, filename);@@ -59,6 +63,35 @@ describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => {"pcm_s16le",]);standaloneAudio = makeFixture("standalone.mka", [...tone, "-c:a", "pcm_s16le"]);+ // An older macOS recording: system audio and microphone as two streams. The+ // system track is stereo, so it is the one ffmpeg picks on its own, and it is+ // silent; the microphone, mono, speaks from the first second.+ twoTracks = makeFixture("two-tracks.mkv", [+ "-f",+ "lavfi",+ "-i",+ "color=c=black:s=16x16:r=1:d=4",+ "-f",+ "lavfi",+ "-i",+ "anullsrc=r=16000:cl=stereo:d=4",+ "-itsoffset",+ "1",+ "-f",+ "lavfi",+ "-i",+ "sine=frequency=440:sample_rate=16000:duration=3",+ "-map",+ "0:v",+ "-map",+ "1:a",+ "-map",+ "2:a",+ "-c:v",+ "ffv1",+ "-c:a",+ "pcm_s16le",+ ]);});afterAll(() => {@@ -85,4 +118,14 @@ describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => {expect(firstAudibleSample).toBeGreaterThanOrEqual(0);expect(firstAudibleSample).toBeLessThan(16);});++ it("transcribes every audio stream, each on the media clock, as the export mixes them", async () => {+ // Decoding only ffmpeg's pick yields the silent system track: no transcript,+ // while the exported video has the voice in it.+ const samples = await extractMono16kPcm(twoTracks);+ expect(samples).toHaveLength(4 * 16_000);+ const firstAudibleSample = samples.findIndex((sample) => Math.abs(sample) > 0.01);+ expect(firstAudibleSample).toBeGreaterThanOrEqual(16_000);+ expect(firstAudibleSample).toBeLessThan(16_000 + 16);+ });});diff --git a/electron/stt/extractAudio.test.ts b/electron/stt/extractAudio.test.tsindex c437b1ae9..a55b1a555 100644--- a/electron/stt/extractAudio.test.ts+++ b/electron/stt/extractAudio.test.ts@@ -6,7 +6,10 @@ const spawnMock = vi.fn();const resolveFfmpegMock = vi.fn<() => string | null>();vi.mock("node:child_process", () => ({ spawn: (...args: unknown[]) => spawnMock(...args) }));-vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => resolveFfmpegMock() }));+vi.mock("../media/audioPeaks", () => ({+ resolveFfmpeg: () => resolveFfmpegMock(),+ mediaClockAudioArgs: async () => ["-vn", "-af", "aresample=first_pts=0"],+}));const { extractMono16kPcm, FfmpegUnavailableError, MediaUnreadableError, NoAudioTrackError } =await import("./extractAudio");@@ -27,6 +30,11 @@ function fakeChild() {return child;}+/** Extraction counts the audio streams before it spawns the decode: let it get there. */+async function spawned(): Promise<void> {+ for (let i = 0; i < 100 && spawnMock.mock.calls.length === 0; i++) await Promise.resolve();+}+/** The little-endian float32 bytes ffmpeg would emit for `values`. */function f32le(values: number[]): Buffer {const buf = Buffer.alloc(values.length * 4);@@ -44,6 +52,7 @@ describe("extractMono16kPcm", () => {const child = fakeChild();spawnMock.mockReturnValue(child);const promise = extractMono16kPcm("/tmp/a.mp3");+ await spawned();child.stdout.end(f32le([0.5]));child.emit("close", 0);await promise;@@ -61,6 +70,7 @@ describe("extractMono16kPcm", () => {const child = fakeChild();spawnMock.mockReturnValue(child);const promise = extractMono16kPcm("/tmp/a.mp3");+ await spawned();child.stdout.end(f32le([0, 0.5, -0.25]));child.emit("close", 0);@@ -75,6 +85,7 @@ describe("extractMono16kPcm", () => {const child = fakeChild();spawnMock.mockReturnValue(child);const promise = extractMono16kPcm("/tmp/a.mp3");+ await spawned();const bytes = f32le([0.25, -0.75, 1]);child.stdout.write(bytes.subarray(0, 6)); // one whole float + half of the nextchild.stdout.write(bytes.subarray(6));@@ -89,6 +100,7 @@ describe("extractMono16kPcm", () => {const child = fakeChild();spawnMock.mockReturnValue(child);const promise = extractMono16kPcm("/tmp/silent.mp4");+ await spawned();child.stderr.end("Stream map '0:a' matches no streams");child.stdout.end();child.emit("close", 1);@@ -102,6 +114,7 @@ describe("extractMono16kPcm", () => {const child = fakeChild();spawnMock.mockReturnValue(child);const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4");+ await spawned();child.stderr.end("/Users/me/Downloads/a.mp4: Operation not permitted");child.stdout.end();child.emit("close", 1);@@ -116,6 +129,7 @@ describe("extractMono16kPcm", () => {const child = fakeChild();spawnMock.mockReturnValue(child);const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4");+ await spawned();const settled = expect(promise).rejects.toThrow(STT_MEDIA_UNREADABLE);await vi.advanceTimersByTimeAsync(60_000);await settled;@@ -132,6 +146,7 @@ describe("extractMono16kPcm", () => {const child = fakeChild();spawnMock.mockReturnValue(child);const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4");+ await spawned();const settled = promise.then(() => {throw new Error("expected a rejection");@@ -156,6 +171,7 @@ describe("extractMono16kPcm", () => {const child = fakeChild();spawnMock.mockReturnValue(child);const promise = extractMono16kPcm("/tmp/truncated.mp3");+ await spawned();child.stdout.end(f32le([0.1, 0.2]));child.emit("close", 1);@@ -177,6 +193,7 @@ describe("extractMono16kPcm", () => {spawnMock.mockReturnValue(child);const controller = new AbortController();const promise = extractMono16kPcm("/tmp/a.mp3", { signal: controller.signal });+ await spawned();controller.abort();await expect(promise).rejects.toMatchObject({ name: "AbortError" });diff --git a/electron/stt/extractAudio.ts b/electron/stt/extractAudio.tsindex 901078669..f9697460b 100644--- a/electron/stt/extractAudio.ts+++ b/electron/stt/extractAudio.ts@@ -20,7 +20,7 @@// that is already never repeated.import { spawn } from "node:child_process";-import { resolveFfmpeg } from "../media/audioPeaks";+import { mediaClockAudioArgs, resolveFfmpeg } from "../media/audioPeaks";import { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract";/** What whisper.cpp wants, and what `decodePeaks` already asks ffmpeg for. */@@ -82,6 +82,9 @@ export async function extractMono16kPcm(const ffmpeg = resolveFfmpeg();if (!ffmpeg) throw new FfmpegUnavailableError();if (options.signal?.aborted) throw new DOMException("Aborted", "AbortError");+ const audioArgs = await mediaClockAudioArgs(ffmpeg, filePath);+ // Again: an abort during the probe fired before the listener below existed.+ if (options.signal?.aborted) throw new DOMException("Aborted", "AbortError");const child = spawn(ffmpeg,@@ -91,14 +94,11 @@ export async function extractMono16kPcm("error","-i",filePath,- "-vn",+ ...audioArgs,"-ac","1","-ar",String(SAMPLE_RATE),- // Pad delayed audio to keep sample zero on the media clock.- "-af",- "aresample=first_pts=0","-f","f32le","-",