Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

fix(captions): preserve fallback audio and speech between microphone words

webadderalle49a1aba7475
diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.ts
index 2613d3f9..f394769a 100644
--- a/electron/ipc/captions/generate.ts
+++ b/electron/ipc/captions/generate.ts
@@ -340,7 +340,9 @@ export async function generateAutoCaptionsFromVideo(options: {
const candidates = await resolveCaptionAudioCandidates(options.videoPath);
const microphone = candidates.filter((source) => source.label === "microphone audio sidecar");
const system = candidates.filter((source) => source.label === "system audio sidecar");
- const recording = candidates.filter((source) => source.label === "recording");
+ const secondary = candidates.filter(
+ (source) => !microphone.includes(source) && !system.includes(source),
+ );
if (microphone.length === 0) {
return generateCaptionsForSource({ ...options, candidates: [...system, ...candidates] });
}
@@ -355,7 +357,7 @@ export async function generateAutoCaptionsFromVideo(options: {
}
};
const micCues = await transcribeTrack(microphone);
- const systemCues = await transcribeTrack([...system, ...recording]);
+ const systemCues = await transcribeTrack([...system, ...secondary]);
if (micCues === null && systemCues === null)
throw new NoCaptionAudioError("No audio could be extracted from the recording.");
return {
diff --git a/electron/ipc/captions/generation.test.ts b/electron/ipc/captions/generation.test.ts
index 71807d80..70ba2658 100644
--- a/electron/ipc/captions/generation.test.ts
+++ b/electron/ipc/captions/generation.test.ts
@@ -6,6 +6,7 @@ const mocks = vi.hoisted(() => ({
rm: vi.fn(),
companions: vi.fn(),
delay: vi.fn(),
+ session: vi.fn(),
}));
vi.mock("node:child_process", () => ({
execFile: (...args: unknown[]) => {
@@ -29,7 +30,7 @@ vi.mock("node:fs/promises", () => ({
vi.mock("electron", () => ({ app: { getPath: () => "/tmp" } }));
vi.mock("../ffmpeg/binary", () => ({ getFfmpegBinaryPath: () => "/ffmpeg" }));
vi.mock("../paths/binaries", () => ({ getBundledWhisperExecutableCandidates: () => ["/whisper"] }));
-vi.mock("../project/session", () => ({ resolveRecordingSession: vi.fn().mockResolvedValue(null) }));
+vi.mock("../project/session", () => ({ resolveRecordingSession: mocks.session }));
vi.mock("../recording/diagnostics", () => ({
getUsableCompanionAudioCandidates: mocks.companions,
getCompanionAudioStartDelayMs: mocks.delay,
@@ -55,6 +56,7 @@ const json = JSON.stringify({
beforeEach(() => {
vi.clearAllMocks();
+ mocks.session.mockReset().mockResolvedValue(null);
mocks.companions.mockReset().mockResolvedValue([]);
mocks.delay.mockReset().mockResolvedValue(null);
mocks.exec.mockReset().mockResolvedValue({ stderr: "" });
@@ -135,3 +137,30 @@ describe("caption generation pipeline", () => {
expect((await generateAutoCaptionsFromVideo(options)).cues[0].text).toBe("Hello world.");
});
});
+
+it("falls back to linked webcam audio when mic exists and other secondary sources fail", async () => {
+ mocks.session.mockResolvedValue({ webcamPath: "/webcam.mp4" });
+ mocks.companions.mockResolvedValue([
+ {
+ platform: "mac",
+ micPath: "/video.mic.wav",
+ systemPath: "/video.system.wav",
+ usablePaths: ["/video.mic.wav", "/video.system.wav"],
+ },
+ ]);
+ mocks.exec.mockImplementation(async (file: string, args: string[]) => {
+ if (file === "/ffmpeg" && args.includes("pcm_s16le")) {
+ const source = args[args.indexOf("-i") + 1];
+ if (source === "/video.mp4" || source === "/video.system.wav")
+ throw new Error("No audio");
+ }
+ return { stderr: "" };
+ });
+ const result = await generateAutoCaptionsFromVideo(options);
+ expect(result.cues.length).toBeGreaterThan(0);
+ expect(
+ mocks.exec.mock.calls.some(
+ ([file, args]) => file === "/ffmpeg" && args.includes("/webcam.mp4"),
+ ),
+ ).toBe(true);
+});
diff --git a/electron/ipc/captions/mergeSources.test.ts b/electron/ipc/captions/mergeSources.test.ts
index 41c1b914..4e7bb524 100644
--- a/electron/ipc/captions/mergeSources.test.ts
+++ b/electron/ipc/captions/mergeSources.test.ts
@@ -73,3 +73,22 @@ it("preserves untimed speech next to timed words", async () => {
);
expect(result.map((c) => c.text).join(" ")).toContain("There is a timeline editor.");
});
+
+it("preserves system speech between timed microphone words", () => {
+ const result = mergeCaptionSources(
+ [
+ {
+ id: "mic",
+ startMs: 0,
+ endMs: 3000,
+ text: "Hello again",
+ words: [
+ { text: "Hello", startMs: 0, endMs: 600 },
+ { text: "again", startMs: 2400, endMs: 3000 },
+ ],
+ },
+ ],
+ [{ id: "system", startMs: 1000, endMs: 1800, text: "In the gap" }],
+ );
+ expect(result.map((cue) => cue.text)).toContain("In the gap");
+});
diff --git a/electron/ipc/captions/mergeSources.ts b/electron/ipc/captions/mergeSources.ts
index 35f427d0..6d44622c 100644
--- a/electron/ipc/captions/mergeSources.ts
+++ b/electron/ipc/captions/mergeSources.ts
@@ -8,8 +8,9 @@ export function mergeCaptionSources(
): CaptionCuePayload[] {
// Sound-event labels are not competing microphone speech.
const micSpeech = microphone.filter((cue) => !/^(?:\s*[[(][^\])]+[\])]\s*)+$/.test(cue.text));
+ const micSpans = micSpeech.flatMap((cue) => (cue.words?.length ? cue.words : [cue]));
const overlapsMic = (startMs: number, endMs: number) =>
- micSpeech.some((cue) => startMs < cue.endMs && endMs > cue.startMs);
+ micSpans.some((span) => startMs < span.endMs && endMs > span.startMs);
const systemCues: CaptionCuePayload[] = [];
for (const cue of system) {
if (!overlapsMic(cue.startMs, cue.endMs)) {
diff --git a/tests/ui/caption-speed.spec.ts b/tests/ui/caption-speed.spec.ts
index 812e833d..87fc629a 100644
--- a/tests/ui/caption-speed.spec.ts
+++ b/tests/ui/caption-speed.spec.ts
@@ -4,8 +4,11 @@ import { installDesktopBridge } from "./bridge";
async function seek(page: Page, clip: Locator, timeMs: number) {
const end = Number(await clip.getAttribute("data-end-ms"));
const box = (await clip.locator(".timeline-block").boundingBox())!;
- const row = (await page.locator('[data-timeline-row="row-clip"]').boundingBox())!;
- await page.mouse.click(box.x + (box.width * timeMs) / end, row.y - 8);
+ const cap = (await page.getByTestId("playhead-cap").boundingBox())!;
+ await page.mouse.move(cap.x + cap.width / 2, cap.y + cap.height / 2);
+ await page.mouse.down();
+ await page.mouse.move(box.x + (box.width * timeMs) / end, cap.y + cap.height / 2);
+ await page.mouse.up();
}
test("captions render in preview and remain synchronized at 1x, 2x and 4x", async ({ page }) => {
@@ -100,6 +103,13 @@ test("captions render in preview and remain synchronized at 1x, 2x and 4x", asyn
await seek(page, clip, 0);
await page.getByRole("button", { name: "Play", exact: true }).click();
await expect(visibleCaption).toBeVisible();
+ await expect
+ .poll(() =>
+ page
+ .locator('video[aria-hidden="true"]')
+ .evaluate((video: HTMLVideoElement) => video.currentTime * 1000),
+ )
+ .toBeGreaterThan(sourceEnd);
await expect(visibleCaption).toHaveCount(0);
expect(errors).toEqual([]);
});