Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(audio): decode every audio stream on the media clock for peaks and STT

EtienneLescot22ec4baf9e3a
diff --git a/electron/media/audioPeaks.test.ts b/electron/media/audioPeaks.test.ts
index 485fbb034..644f9bebb 100644
--- a/electron/media/audioPeaks.test.ts
+++ b/electron/media/audioPeaks.test.ts
@@ -1,9 +1,16 @@
// @vitest-environment node
+import { spawnSync } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { ffmpegCandidates, peakBlockCount, resolveFfmpeg } from "./audioPeaks";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
+import {
+ countAudioStreams,
+ decodePeaks,
+ ffmpegCandidates,
+ peakBlockCount,
+ resolveFfmpeg,
+} from "./audioPeaks";
const ROOT = path.resolve(__dirname, "..", "..");
@@ -311,3 +318,104 @@ describe.runIf(staged)("decoding a real file", () => {
expect(mn).toBeCloseTo(-0.205, 1);
}, 120_000);
});
+
+describe("countAudioStreams", () => {
+ it("counts the audio streams in ffmpeg's listing, and nothing else", () => {
+ // The layout of an older macOS recording, as ffmpeg 8.1 lists it.
+ const listing = [
+ "Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'recording.mp4':",
+ " Duration: 00:00:12.00, start: 0.000000, bitrate: 5012 kb/s",
+ " Stream #0:0[0x1](und): Video: h264 (High) (avc1 / 0x31637661), yuv420p, 60 fps (default)",
+ " Stream #0:1[0x2](und): Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, stereo (default)",
+ " Metadata:",
+ " handler_name : Audio: system",
+ " Stream #0:2[0x3](und): Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, mono (default)",
+ "At least one output file must be specified",
+ ].join("\n");
+ expect(countAudioStreams(listing)).toBe(2);
+ expect(countAudioStreams(" Stream #0:0: Video: ffv1, 16x16")).toBe(0);
+ });
+});
+
+const ffmpegOnPath = spawnSync("ffmpeg", ["-version"], { timeout: 5_000 }).status === 0;
+
+describe.skipIf(!ffmpegOnPath)("peaks on the media clock, with real FFmpeg", () => {
+ let fixtureDir: string;
+ let delayed: string;
+ let twoTracks: string;
+
+ /** A four-second video from zero, muxed with the given audio inputs, one stream each. */
+ function makeFixture(filename: string, audioInputs: string[][]): string {
+ const filePath = path.join(fixtureDir, filename);
+ const result = spawnSync(
+ "ffmpeg",
+ [
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-y",
+ "-f",
+ "lavfi",
+ "-i",
+ "color=c=black:s=16x16:r=1:d=4",
+ ...audioInputs.flat(),
+ "-map",
+ "0:v",
+ ...audioInputs.flatMap((_, i) => ["-map", `${i + 1}:a`]),
+ "-c:v",
+ "ffv1",
+ "-c:a",
+ "pcm_s16le",
+ filePath,
+ ],
+ { encoding: "utf8", timeout: 10_000 },
+ );
+ if (result.status !== 0) throw new Error(result.error?.message ?? result.stderr);
+ return filePath;
+ }
+
+ /** Seconds into the clip of the first block with signal in it, or -1. */
+ function firstAudibleSec(peaks: Float32Array): number {
+ for (let i = 0; i < peaks.length; i += 2) {
+ if (peaks[i + 1] > 0.01) return i / 2 / 200;
+ }
+ return -1;
+ }
+
+ beforeAll(() => {
+ fixtureDir = mkdtempSync(path.join(tmpdir(), "openscreen-peaks-clock-"));
+ const delayedTone = (sec: number, duration: number) => [
+ "-itsoffset",
+ String(sec),
+ "-f",
+ "lavfi",
+ "-i",
+ `sine=frequency=440:sample_rate=16000:duration=${duration}`,
+ ];
+ // Video from zero, audio from two seconds, no encoded silence in between.
+ delayed = makeFixture("delayed.mkv", [delayedTone(2, 2)]);
+ // Older macOS layout: a silent stereo system track, which is the stream ffmpeg
+ // picks on its own, and a mono microphone speaking from the first second.
+ twoTracks = makeFixture("two-tracks.mkv", [
+ ["-f", "lavfi", "-i", "anullsrc=r=16000:cl=stereo:d=4"],
+ delayedTone(1, 3),
+ ]);
+ });
+
+ afterAll(() => {
+ if (fixtureDir) rmSync(fixtureDir, { recursive: true, force: true });
+ });
+
+ it("draws audio that starts after the video where it plays, not at zero", async () => {
+ // Pulled to zero, the waveform runs two seconds ahead of the transcript and of
+ // the export, both of which keep the offset.
+ const peaks = await decodePeaks("ffmpeg", delayed, 4);
+ expect(peaks.length).toBe(peakBlockCount(4) * 2);
+ expect(firstAudibleSec(peaks)).toBe(2);
+ });
+
+ it("draws every audio stream, each on the media clock, as the export mixes them", async () => {
+ const peaks = await decodePeaks("ffmpeg", twoTracks, 4);
+ expect(firstAudibleSec(peaks)).toBe(1);
+ });
+});
diff --git a/electron/media/audioPeaks.ts b/electron/media/audioPeaks.ts
index 604afd051..13f66a93f 100644
--- a/electron/media/audioPeaks.ts
+++ b/electron/media/audioPeaks.ts
@@ -147,6 +147,60 @@ export function resolveFfmpeg(here?: string): string | null {
return found;
}
+/** Listing streams is only opening the file: past this, the open is wedged. */
+const PROBE_TIMEOUT_MS = 10_000;
+
+/** Pads audio that starts late, so sample zero stays on the media clock. */
+const ON_MEDIA_CLOCK = "aresample=first_pts=0";
+
+/** Audio streams in ffmpeg's listing of its input (the stderr of `ffmpeg -i <file>`). */
+export function countAudioStreams(listing: string): number {
+ return listing.match(/^\s*Stream #0:\d+\S*: Audio:/gm)?.length ?? 0;
+}
+
+/** Any failure counts as zero: the decode that follows then runs as it always did,
+ * and reports the problem itself. */
+function probeAudioStreams(ffmpeg: string, filePath: string): Promise<number> {
+ return new Promise((resolve) => {
+ let listing = "";
+ const child = spawn(ffmpeg, ["-hide_banner", "-i", filePath], {
+ stdio: ["ignore", "ignore", "pipe"],
+ timeout: PROBE_TIMEOUT_MS,
+ killSignal: "SIGKILL",
+ });
+ child.stderr.on("data", (c: Buffer) => {
+ listing += c.toString();
+ });
+ child.once("error", () => resolve(0));
+ child.once("close", () => resolve(countAudioStreams(listing)));
+ });
+}
+
+/**
+ * The ffmpeg arguments, after `-i`, that decode a recording's audio the way the
+ * compositor exports it, so the waveform and the transcript line up with the export:
+ *
+ * - On the media clock. Audio that starts after the video is padded, not pulled to
+ * zero, or everything drawn or transcribed from it lands that much early.
+ * - Every audio stream, summed. Older macOS recordings carry system audio and the
+ * microphone as two streams; ffmpeg alone decodes the one it picks, usually the
+ * system track, silent whenever nothing plays, while the export mixes both. `amix`
+ * mixes by sample count, not timestamp, so each stream is aligned first, and
+ * `normalize=0` sums like the compositor instead of averaging.
+ */
+export async function mediaClockAudioArgs(ffmpeg: string, filePath: string): Promise<string[]> {
+ const streams = await probeAudioStreams(ffmpeg, filePath);
+ if (streams < 2) return ["-vn", "-af", ON_MEDIA_CLOCK];
+ const aligned = Array.from({ length: streams }, (_, i) => `[0:a:${i}]${ON_MEDIA_CLOCK}[a${i}]`);
+ const labels = aligned.map((_, i) => `[a${i}]`).join("");
+ return [
+ "-filter_complex",
+ `${aligned.join(";")};${labels}amix=inputs=${streams}:normalize=0[a]`,
+ "-map",
+ "[a]",
+ ];
+}
+
/** Number of min/max blocks for a clip of `durationSec`. */
export function peakBlockCount(durationSec: number): number {
return Math.min(MAX_PEAK_BLOCKS, Math.max(1, Math.ceil(durationSec * PEAK_BLOCKS_PER_SEC)));
@@ -205,13 +259,14 @@ class PeakFolder {
}
/** Runs ffmpeg and folds its PCM straight into peaks. Never buffers the audio. */
-async function decodePeaks(
+export async function decodePeaks(
ffmpeg: string,
filePath: string,
durationSec: number,
): Promise<Float32Array> {
const blocks = peakBlockCount(durationSec);
const folder = new PeakFolder(blocks, durationSec * PCM_RATE);
+ const audioArgs = await mediaClockAudioArgs(ffmpeg, filePath);
const child = spawn(
ffmpeg,
[
@@ -220,7 +275,7 @@ async function decodePeaks(
"error",
"-i",
filePath,
- "-vn",
+ ...audioArgs,
"-ac",
"1",
"-ar",
@@ -265,11 +320,14 @@ async function decodePeaks(
* Cache key: path plus size plus mtime. A recording is immutable in practice,
* but keying on identity alone would serve stale peaks for a re-encoded or
* replaced file, and that failure is silent and confusing.
+ *
+ * The leading version changes whenever the decode does: v2 put the peaks on the
+ * media clock and mixed every audio stream, which the v1 entries do not.
*/
async function cacheKey(filePath: string): Promise<string> {
const info = await stat(filePath);
return createHash("sha1")
- .update(`${filePath}:${info.size}:${info.mtimeMs}`)
+ .update(`v2:${filePath}:${info.size}:${info.mtimeMs}`)
.digest("hex")
.slice(0, 32);
}
diff --git a/electron/stt/extractAudio.integration.test.ts b/electron/stt/extractAudio.integration.test.ts
index 2301f6fd3..f37059243 100644
--- a/electron/stt/extractAudio.integration.test.ts
+++ b/electron/stt/extractAudio.integration.test.ts
@@ -5,7 +5,10 @@ import path from "node:path";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
// Only the binary lookup is replaced: extraction still spawns and reads real FFmpeg.
-vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => "ffmpeg" }));
+vi.mock("../media/audioPeaks", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("../media/audioPeaks")>()),
+ resolveFfmpeg: () => "ffmpeg",
+}));
import { extractMono16kPcm } from "./extractAudio";
@@ -19,6 +22,7 @@ describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => {
let fixtureDir: string;
let delayedVideo: string;
let standaloneAudio: string;
+ let twoTracks: string;
function makeFixture(filename: string, args: string[]): string {
const filePath = path.join(fixtureDir, filename);
@@ -59,6 +63,35 @@ describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => {
"pcm_s16le",
]);
standaloneAudio = makeFixture("standalone.mka", [...tone, "-c:a", "pcm_s16le"]);
+ // An older macOS recording: system audio and microphone as two streams. The
+ // system track is stereo, so it is the one ffmpeg picks on its own, and it is
+ // silent; the microphone, mono, speaks from the first second.
+ twoTracks = makeFixture("two-tracks.mkv", [
+ "-f",
+ "lavfi",
+ "-i",
+ "color=c=black:s=16x16:r=1:d=4",
+ "-f",
+ "lavfi",
+ "-i",
+ "anullsrc=r=16000:cl=stereo:d=4",
+ "-itsoffset",
+ "1",
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=440:sample_rate=16000:duration=3",
+ "-map",
+ "0:v",
+ "-map",
+ "1:a",
+ "-map",
+ "2:a",
+ "-c:v",
+ "ffv1",
+ "-c:a",
+ "pcm_s16le",
+ ]);
});
afterAll(() => {
@@ -85,4 +118,14 @@ describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => {
expect(firstAudibleSample).toBeGreaterThanOrEqual(0);
expect(firstAudibleSample).toBeLessThan(16);
});
+
+ it("transcribes every audio stream, each on the media clock, as the export mixes them", async () => {
+ // Decoding only ffmpeg's pick yields the silent system track: no transcript,
+ // while the exported video has the voice in it.
+ const samples = await extractMono16kPcm(twoTracks);
+ expect(samples).toHaveLength(4 * 16_000);
+ const firstAudibleSample = samples.findIndex((sample) => Math.abs(sample) > 0.01);
+ expect(firstAudibleSample).toBeGreaterThanOrEqual(16_000);
+ expect(firstAudibleSample).toBeLessThan(16_000 + 16);
+ });
});
diff --git a/electron/stt/extractAudio.test.ts b/electron/stt/extractAudio.test.ts
index c437b1ae9..a55b1a555 100644
--- a/electron/stt/extractAudio.test.ts
+++ b/electron/stt/extractAudio.test.ts
@@ -6,7 +6,10 @@ const spawnMock = vi.fn();
const resolveFfmpegMock = vi.fn<() => string | null>();
vi.mock("node:child_process", () => ({ spawn: (...args: unknown[]) => spawnMock(...args) }));
-vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => resolveFfmpegMock() }));
+vi.mock("../media/audioPeaks", () => ({
+ resolveFfmpeg: () => resolveFfmpegMock(),
+ mediaClockAudioArgs: async () => ["-vn", "-af", "aresample=first_pts=0"],
+}));
const { extractMono16kPcm, FfmpegUnavailableError, MediaUnreadableError, NoAudioTrackError } =
await import("./extractAudio");
@@ -27,6 +30,11 @@ function fakeChild() {
return child;
}
+/** Extraction counts the audio streams before it spawns the decode: let it get there. */
+async function spawned(): Promise<void> {
+ for (let i = 0; i < 100 && spawnMock.mock.calls.length === 0; i++) await Promise.resolve();
+}
+
/** The little-endian float32 bytes ffmpeg would emit for `values`. */
function f32le(values: number[]): Buffer {
const buf = Buffer.alloc(values.length * 4);
@@ -44,6 +52,7 @@ describe("extractMono16kPcm", () => {
const child = fakeChild();
spawnMock.mockReturnValue(child);
const promise = extractMono16kPcm("/tmp/a.mp3");
+ await spawned();
child.stdout.end(f32le([0.5]));
child.emit("close", 0);
await promise;
@@ -61,6 +70,7 @@ describe("extractMono16kPcm", () => {
const child = fakeChild();
spawnMock.mockReturnValue(child);
const promise = extractMono16kPcm("/tmp/a.mp3");
+ await spawned();
child.stdout.end(f32le([0, 0.5, -0.25]));
child.emit("close", 0);
@@ -75,6 +85,7 @@ describe("extractMono16kPcm", () => {
const child = fakeChild();
spawnMock.mockReturnValue(child);
const promise = extractMono16kPcm("/tmp/a.mp3");
+ await spawned();
const bytes = f32le([0.25, -0.75, 1]);
child.stdout.write(bytes.subarray(0, 6)); // one whole float + half of the next
child.stdout.write(bytes.subarray(6));
@@ -89,6 +100,7 @@ describe("extractMono16kPcm", () => {
const child = fakeChild();
spawnMock.mockReturnValue(child);
const promise = extractMono16kPcm("/tmp/silent.mp4");
+ await spawned();
child.stderr.end("Stream map '0:a' matches no streams");
child.stdout.end();
child.emit("close", 1);
@@ -102,6 +114,7 @@ describe("extractMono16kPcm", () => {
const child = fakeChild();
spawnMock.mockReturnValue(child);
const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4");
+ await spawned();
child.stderr.end("/Users/me/Downloads/a.mp4: Operation not permitted");
child.stdout.end();
child.emit("close", 1);
@@ -116,6 +129,7 @@ describe("extractMono16kPcm", () => {
const child = fakeChild();
spawnMock.mockReturnValue(child);
const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4");
+ await spawned();
const settled = expect(promise).rejects.toThrow(STT_MEDIA_UNREADABLE);
await vi.advanceTimersByTimeAsync(60_000);
await settled;
@@ -132,6 +146,7 @@ describe("extractMono16kPcm", () => {
const child = fakeChild();
spawnMock.mockReturnValue(child);
const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4");
+ await spawned();
const settled = promise.then(
() => {
throw new Error("expected a rejection");
@@ -156,6 +171,7 @@ describe("extractMono16kPcm", () => {
const child = fakeChild();
spawnMock.mockReturnValue(child);
const promise = extractMono16kPcm("/tmp/truncated.mp3");
+ await spawned();
child.stdout.end(f32le([0.1, 0.2]));
child.emit("close", 1);
@@ -177,6 +193,7 @@ describe("extractMono16kPcm", () => {
spawnMock.mockReturnValue(child);
const controller = new AbortController();
const promise = extractMono16kPcm("/tmp/a.mp3", { signal: controller.signal });
+ await spawned();
controller.abort();
await expect(promise).rejects.toMatchObject({ name: "AbortError" });
diff --git a/electron/stt/extractAudio.ts b/electron/stt/extractAudio.ts
index 901078669..f9697460b 100644
--- a/electron/stt/extractAudio.ts
+++ b/electron/stt/extractAudio.ts
@@ -20,7 +20,7 @@
// that is already never repeated.
import { spawn } from "node:child_process";
-import { resolveFfmpeg } from "../media/audioPeaks";
+import { mediaClockAudioArgs, resolveFfmpeg } from "../media/audioPeaks";
import { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract";
/** What whisper.cpp wants, and what `decodePeaks` already asks ffmpeg for. */
@@ -82,6 +82,9 @@ export async function extractMono16kPcm(
const ffmpeg = resolveFfmpeg();
if (!ffmpeg) throw new FfmpegUnavailableError();
if (options.signal?.aborted) throw new DOMException("Aborted", "AbortError");
+ const audioArgs = await mediaClockAudioArgs(ffmpeg, filePath);
+ // Again: an abort during the probe fired before the listener below existed.
+ if (options.signal?.aborted) throw new DOMException("Aborted", "AbortError");
const child = spawn(
ffmpeg,
@@ -91,14 +94,11 @@ export async function extractMono16kPcm(
"error",
"-i",
filePath,
- "-vn",
+ ...audioArgs,
"-ac",
"1",
"-ar",
String(SAMPLE_RATE),
- // Pad delayed audio to keep sample zero on the media clock.
- "-af",
- "aresample=first_pts=0",
"-f",
"f32le",
"-",