Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(stt): preserve media-clock audio offsets
Satwik Sharma42bcce276916
diff --git a/electron/stt/extractAudio.integration.test.ts b/electron/stt/extractAudio.integration.test.tsnew file mode 100644index 000000000..2301f6fd3--- /dev/null+++ b/electron/stt/extractAudio.integration.test.ts@@ -0,0 +1,88 @@+import { spawnSync } from "node:child_process";+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import path from "node:path";+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";++// Only the binary lookup is replaced: extraction still spawns and reads real FFmpeg.+vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => "ffmpeg" }));++import { extractMono16kPcm } from "./extractAudio";++const ffmpegAvailable = spawnSync("ffmpeg", ["-version"], { timeout: 5_000 }).status === 0;++if (!ffmpegAvailable) {+ console.warn("Skipping audio-origin integration tests: FFmpeg is unavailable on PATH.");+}++describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => {+ let fixtureDir: string;+ let delayedVideo: string;+ let standaloneAudio: string;++ function makeFixture(filename: string, args: string[]): string {+ const filePath = path.join(fixtureDir, filename);+ const result = spawnSync(+ "ffmpeg",+ ["-hide_banner", "-loglevel", "error", "-y", ...args, filePath],+ { encoding: "utf8", timeout: 10_000 },+ );+ if (result.status !== 0) {+ throw new Error(result.error?.message ?? result.stderr);+ }+ return filePath;+ }++ beforeAll(() => {+ fixtureDir = mkdtempSync(path.join(tmpdir(), "openscreen-stt-audio-origin-"));+ const tone = [+ "-itsoffset",+ "2",+ "-f",+ "lavfi",+ "-i",+ "sine=frequency=440:sample_rate=16000:duration=2",+ ];+ delayedVideo = makeFixture("delayed.mkv", [+ "-f",+ "lavfi",+ "-i",+ "color=c=black:s=16x16:r=1:d=4",+ ...tone,+ "-map",+ "0:v",+ "-map",+ "1:a",+ "-c:v",+ "ffv1",+ "-c:a",+ "pcm_s16le",+ ]);+ standaloneAudio = makeFixture("standalone.mka", [...tone, "-c:a", "pcm_s16le"]);+ });++ afterAll(() => {+ if (fixtureDir) rmSync(fixtureDir, { recursive: true, force: true });+ });++ it("keeps delayed audio on the video's clock", async () => {+ // The audio stream starts two seconds after video, without encoded silence.+ // Dropping that timestamp gap makes every word and text cut two seconds early.+ const samples = await extractMono16kPcm(delayedVideo);+ expect(samples).toHaveLength(4 * 16_000);+ expect(samples.subarray(0, 2 * 16_000).every((sample) => sample === 0)).toBe(true);+ const firstAudibleSample = samples.findIndex((sample) => Math.abs(sample) > 0.01);+ expect(firstAudibleSample).toBeGreaterThanOrEqual(2 * 16_000);+ expect(firstAudibleSample).toBeLessThan(2 * 16_000 + 16);+ });++ it("keeps standalone audio relative to its own media origin", async () => {+ // This container starts at two seconds too, but has no earlier video stream.+ // Its own origin is still zero: padding it would invent two seconds of silence.+ const samples = await extractMono16kPcm(standaloneAudio);+ expect(samples).toHaveLength(2 * 16_000);+ const firstAudibleSample = samples.findIndex((sample) => Math.abs(sample) > 0.01);+ expect(firstAudibleSample).toBeGreaterThanOrEqual(0);+ expect(firstAudibleSample).toBeLessThan(16);+ });+});diff --git a/electron/stt/extractAudio.ts b/electron/stt/extractAudio.tsindex 28dd9f177..901078669 100644--- a/electron/stt/extractAudio.ts+++ b/electron/stt/extractAudio.ts@@ -96,6 +96,9 @@ export async function extractMono16kPcm("1","-ar",String(SAMPLE_RATE),+ // Pad delayed audio to keep sample zero on the media clock.+ "-af",+ "aresample=first_pts=0","-f","f32le","-",