Create polished demo videos without editing skills. Mac/Windows/Linux
fix(captions): preserve fallback audio and speech between microphone words
webadderalle49a1aba7475
diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.tsindex 2613d3f9..f394769a 100644--- a/electron/ipc/captions/generate.ts+++ b/electron/ipc/captions/generate.ts@@ -340,7 +340,9 @@ export async function generateAutoCaptionsFromVideo(options: {const candidates = await resolveCaptionAudioCandidates(options.videoPath);const microphone = candidates.filter((source) => source.label === "microphone audio sidecar");const system = candidates.filter((source) => source.label === "system audio sidecar");- const recording = candidates.filter((source) => source.label === "recording");+ const secondary = candidates.filter(+ (source) => !microphone.includes(source) && !system.includes(source),+ );if (microphone.length === 0) {return generateCaptionsForSource({ ...options, candidates: [...system, ...candidates] });}@@ -355,7 +357,7 @@ export async function generateAutoCaptionsFromVideo(options: {}};const micCues = await transcribeTrack(microphone);- const systemCues = await transcribeTrack([...system, ...recording]);+ const systemCues = await transcribeTrack([...system, ...secondary]);if (micCues === null && systemCues === null)throw new NoCaptionAudioError("No audio could be extracted from the recording.");return {diff --git a/electron/ipc/captions/generation.test.ts b/electron/ipc/captions/generation.test.tsindex 71807d80..70ba2658 100644--- a/electron/ipc/captions/generation.test.ts+++ b/electron/ipc/captions/generation.test.ts@@ -6,6 +6,7 @@ const mocks = vi.hoisted(() => ({rm: vi.fn(),companions: vi.fn(),delay: vi.fn(),+ session: vi.fn(),}));vi.mock("node:child_process", () => ({execFile: (...args: unknown[]) => {@@ -29,7 +30,7 @@ vi.mock("node:fs/promises", () => ({vi.mock("electron", () => ({ app: { getPath: () => "/tmp" } }));vi.mock("../ffmpeg/binary", () => ({ getFfmpegBinaryPath: () => "/ffmpeg" }));vi.mock("../paths/binaries", () => ({ getBundledWhisperExecutableCandidates: () => ["/whisper"] }));-vi.mock("../project/session", () => ({ resolveRecordingSession: vi.fn().mockResolvedValue(null) }));+vi.mock("../project/session", () => ({ resolveRecordingSession: mocks.session }));vi.mock("../recording/diagnostics", () => ({getUsableCompanionAudioCandidates: mocks.companions,getCompanionAudioStartDelayMs: mocks.delay,@@ -55,6 +56,7 @@ const json = JSON.stringify({beforeEach(() => {vi.clearAllMocks();+ mocks.session.mockReset().mockResolvedValue(null);mocks.companions.mockReset().mockResolvedValue([]);mocks.delay.mockReset().mockResolvedValue(null);mocks.exec.mockReset().mockResolvedValue({ stderr: "" });@@ -135,3 +137,30 @@ describe("caption generation pipeline", () => {expect((await generateAutoCaptionsFromVideo(options)).cues[0].text).toBe("Hello world.");});});++it("falls back to linked webcam audio when mic exists and other secondary sources fail", async () => {+ mocks.session.mockResolvedValue({ webcamPath: "/webcam.mp4" });+ mocks.companions.mockResolvedValue([+ {+ platform: "mac",+ micPath: "/video.mic.wav",+ systemPath: "/video.system.wav",+ usablePaths: ["/video.mic.wav", "/video.system.wav"],+ },+ ]);+ mocks.exec.mockImplementation(async (file: string, args: string[]) => {+ if (file === "/ffmpeg" && args.includes("pcm_s16le")) {+ const source = args[args.indexOf("-i") + 1];+ if (source === "/video.mp4" || source === "/video.system.wav")+ throw new Error("No audio");+ }+ return { stderr: "" };+ });+ const result = await generateAutoCaptionsFromVideo(options);+ expect(result.cues.length).toBeGreaterThan(0);+ expect(+ mocks.exec.mock.calls.some(+ ([file, args]) => file === "/ffmpeg" && args.includes("/webcam.mp4"),+ ),+ ).toBe(true);+});diff --git a/electron/ipc/captions/mergeSources.test.ts b/electron/ipc/captions/mergeSources.test.tsindex 41c1b914..4e7bb524 100644--- a/electron/ipc/captions/mergeSources.test.ts+++ b/electron/ipc/captions/mergeSources.test.ts@@ -73,3 +73,22 @@ it("preserves untimed speech next to timed words", async () => {);expect(result.map((c) => c.text).join(" ")).toContain("There is a timeline editor.");});++it("preserves system speech between timed microphone words", () => {+ const result = mergeCaptionSources(+ [+ {+ id: "mic",+ startMs: 0,+ endMs: 3000,+ text: "Hello again",+ words: [+ { text: "Hello", startMs: 0, endMs: 600 },+ { text: "again", startMs: 2400, endMs: 3000 },+ ],+ },+ ],+ [{ id: "system", startMs: 1000, endMs: 1800, text: "In the gap" }],+ );+ expect(result.map((cue) => cue.text)).toContain("In the gap");+});diff --git a/electron/ipc/captions/mergeSources.ts b/electron/ipc/captions/mergeSources.tsindex 35f427d0..6d44622c 100644--- a/electron/ipc/captions/mergeSources.ts+++ b/electron/ipc/captions/mergeSources.ts@@ -8,8 +8,9 @@ export function mergeCaptionSources(): CaptionCuePayload[] {// Sound-event labels are not competing microphone speech.const micSpeech = microphone.filter((cue) => !/^(?:\s*[[(][^\])]+[\])]\s*)+$/.test(cue.text));+ const micSpans = micSpeech.flatMap((cue) => (cue.words?.length ? cue.words : [cue]));const overlapsMic = (startMs: number, endMs: number) =>- micSpeech.some((cue) => startMs < cue.endMs && endMs > cue.startMs);+ micSpans.some((span) => startMs < span.endMs && endMs > span.startMs);const systemCues: CaptionCuePayload[] = [];for (const cue of system) {if (!overlapsMic(cue.startMs, cue.endMs)) {diff --git a/tests/ui/caption-speed.spec.ts b/tests/ui/caption-speed.spec.tsindex 812e833d..87fc629a 100644--- a/tests/ui/caption-speed.spec.ts+++ b/tests/ui/caption-speed.spec.ts@@ -4,8 +4,11 @@ import { installDesktopBridge } from "./bridge";async function seek(page: Page, clip: Locator, timeMs: number) {const end = Number(await clip.getAttribute("data-end-ms"));const box = (await clip.locator(".timeline-block").boundingBox())!;- const row = (await page.locator('[data-timeline-row="row-clip"]').boundingBox())!;- await page.mouse.click(box.x + (box.width * timeMs) / end, row.y - 8);+ const cap = (await page.getByTestId("playhead-cap").boundingBox())!;+ await page.mouse.move(cap.x + cap.width / 2, cap.y + cap.height / 2);+ await page.mouse.down();+ await page.mouse.move(box.x + (box.width * timeMs) / end, cap.y + cap.height / 2);+ await page.mouse.up();}test("captions render in preview and remain synchronized at 1x, 2x and 4x", async ({ page }) => {@@ -100,6 +103,13 @@ test("captions render in preview and remain synchronized at 1x, 2x and 4x", asynawait seek(page, clip, 0);await page.getByRole("button", { name: "Play", exact: true }).click();await expect(visibleCaption).toBeVisible();+ await expect+ .poll(() =>+ page+ .locator('video[aria-hidden="true"]')+ .evaluate((video: HTMLVideoElement) => video.currentTime * 1000),+ )+ .toBeGreaterThan(sourceEnd);await expect(visibleCaption).toHaveCount(0);expect(errors).toEqual([]);});