Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

Fix cross-platform caption generation

webadderall17150671f608
diff --git a/electron/ipc/captions/audioCandidates.test.ts b/electron/ipc/captions/audioCandidates.test.ts
new file mode 100644
index 00000000..c08b49ff
--- /dev/null
+++ b/electron/ipc/captions/audioCandidates.test.ts
@@ -0,0 +1,33 @@
+import { describe, expect, it } from "vitest";
+import { getCaptionCompanionAudioCandidates } from "./audioCandidates";
+
+describe("getCaptionCompanionAudioCandidates", () => {
+ it("prefers microphone audio and includes system audio when both sidecars exist", () => {
+ expect(
+ getCaptionCompanionAudioCandidates([
+ {
+ platform: "win",
+ micPath: "recording.mic.wav",
+ systemPath: "recording.system.wav",
+ usablePaths: ["recording.system.wav", "recording.mic.wav"],
+ },
+ ]),
+ ).toEqual([
+ { path: "recording.mic.wav", label: "microphone audio sidecar" },
+ { path: "recording.system.wav", label: "system audio sidecar" },
+ ]);
+ });
+
+ it("omits companion paths that were not found on disk", () => {
+ expect(
+ getCaptionCompanionAudioCandidates([
+ {
+ platform: "win",
+ micPath: "recording.mic.wav",
+ systemPath: "recording.system.wav",
+ usablePaths: ["recording.mic.wav"],
+ },
+ ]),
+ ).toEqual([{ path: "recording.mic.wav", label: "microphone audio sidecar" }]);
+ });
+});
diff --git a/electron/ipc/captions/audioCandidates.ts b/electron/ipc/captions/audioCandidates.ts
new file mode 100644
index 00000000..33e62d2e
--- /dev/null
+++ b/electron/ipc/captions/audioCandidates.ts
@@ -0,0 +1,21 @@
+import type { CompanionAudioCandidate } from "../types";
+
+export type CaptionAudioCandidate = {
+ path: string;
+ label: string;
+};
+
+export function getCaptionCompanionAudioCandidates(
+ companions: CompanionAudioCandidate[],
+): CaptionAudioCandidate[] {
+ return companions.flatMap((companion) => {
+ const candidates: CaptionAudioCandidate[] = [];
+ if (companion.usablePaths.includes(companion.micPath)) {
+ candidates.push({ path: companion.micPath, label: "microphone audio sidecar" });
+ }
+ if (companion.usablePaths.includes(companion.systemPath)) {
+ candidates.push({ path: companion.systemPath, label: "system audio sidecar" });
+ }
+ return candidates;
+ });
+}
diff --git a/electron/ipc/captions/generate.test.ts b/electron/ipc/captions/generate.test.ts
new file mode 100644
index 00000000..9844fecc
--- /dev/null
+++ b/electron/ipc/captions/generate.test.ts
@@ -0,0 +1,16 @@
+import { describe, expect, it } from "vitest";
+import { isMissingWindowsWhisperRuntimeDependency } from "./runtimeErrors";
+
+describe("isMissingWindowsWhisperRuntimeDependency", () => {
+ it("recognizes the signed and unsigned STATUS_DLL_NOT_FOUND exit codes on Windows", () => {
+ if (process.platform !== "win32") return;
+
+ expect(isMissingWindowsWhisperRuntimeDependency({ code: -1073741515 })).toBe(true);
+ expect(isMissingWindowsWhisperRuntimeDependency({ code: 3221225781 })).toBe(true);
+ });
+
+ it("does not classify ordinary Whisper failures as missing runtimes", () => {
+ expect(isMissingWindowsWhisperRuntimeDependency({ code: 1 })).toBe(false);
+ expect(isMissingWindowsWhisperRuntimeDependency(new Error("bad model"))).toBe(false);
+ });
+});
diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.ts
index 2a3f49cd..246c4f1a 100644
--- a/electron/ipc/captions/generate.ts
+++ b/electron/ipc/captions/generate.ts
@@ -7,8 +7,11 @@ import { app } from "electron";
import { getFfmpegBinaryPath } from "../ffmpeg/binary";
import { getBundledWhisperExecutableCandidates } from "../paths/binaries";
import { resolveRecordingSession } from "../project/session";
+import { getUsableCompanionAudioCandidates } from "../recording/diagnostics";
import { normalizeVideoSourcePath } from "../utils";
+import { getCaptionCompanionAudioCandidates } from "./audioCandidates";
import { parseSrtCues, parseWhisperJsonCues, shouldRetryWhisperWithoutJson } from "./parser";
+import { isMissingWindowsWhisperRuntimeDependency } from "./runtimeErrors";
import { segmentCuesIntoPhrases } from "./segment";
import {
parseSilenceIntervals,
@@ -19,6 +22,22 @@ import {
const execFileAsync = promisify(execFile);
+async function executeWhisper(whisperExecutablePath: string, args: string[]) {
+ try {
+ await execFileAsync(whisperExecutablePath, args, {
+ timeout: 30 * 60 * 1000,
+ maxBuffer: 20 * 1024 * 1024,
+ });
+ } catch (error) {
+ if (isMissingWindowsWhisperRuntimeDependency(error)) {
+ throw new Error(
+ "Whisper could not start because the Microsoft Visual C++ x64 Redistributable is missing. Install it from https://aka.ms/vc14/vc_redist.x64.exe, then restart Recordly.",
+ );
+ }
+ throw error;
+ }
+}
+
export async function ensureReadableFile(filePath: string, options?: { executable?: boolean }) {
await fs.access(filePath, fsConstants.R_OK);
if (options?.executable) {
@@ -78,7 +97,8 @@ export async function resolveWhisperExecutablePath(preferredPath?: string | null
}
throw new Error(
- "No Whisper runtime was found. Recordly looked for a bundled binary first, then checked common system install locations.",
+ `No Whisper runtime was found for ${process.platform}/${process.arch}. ` +
+ "This Recordly build is missing its bundled caption runtime. Reinstall or update Recordly, or select a whisper-cli executable in Caption settings.",
);
}
@@ -97,6 +117,10 @@ export async function resolveCaptionAudioCandidates(videoPath: string) {
};
pushCandidate(videoPath, "recording");
+ const companionAudio = await getUsableCompanionAudioCandidates(videoPath);
+ for (const candidate of getCaptionCompanionAudioCandidates(companionAudio)) {
+ pushCandidate(candidate.path, candidate.label);
+ }
const requestedRecordingSession = await resolveRecordingSession(videoPath);
pushCandidate(requestedRecordingSession?.webcamPath, "linked webcam recording");
@@ -236,10 +260,7 @@ export async function generateAutoCaptionsFromVideo(options: {
let jsonEnabled = true;
try {
- await execFileAsync(whisperExecutablePath, [...whisperBaseArgs, "-ojf"], {
- timeout: 30 * 60 * 1000,
- maxBuffer: 20 * 1024 * 1024,
- });
+ await executeWhisper(whisperExecutablePath, [...whisperBaseArgs, "-ojf"]);
} catch (error) {
if (!shouldRetryWhisperWithoutJson(error)) {
throw error;
@@ -250,10 +271,7 @@ export async function generateAutoCaptionsFromVideo(options: {
"[auto-captions] Whisper runtime does not support JSON full output, retrying with SRT only:",
error,
);
- await execFileAsync(whisperExecutablePath, whisperBaseArgs, {
- timeout: 30 * 60 * 1000,
- maxBuffer: 20 * 1024 * 1024,
- });
+ await executeWhisper(whisperExecutablePath, whisperBaseArgs);
}
const timedCues = jsonEnabled
diff --git a/electron/ipc/captions/runtimeErrors.ts b/electron/ipc/captions/runtimeErrors.ts
new file mode 100644
index 00000000..707848dd
--- /dev/null
+++ b/electron/ipc/captions/runtimeErrors.ts
@@ -0,0 +1,10 @@
+const WINDOWS_MISSING_RUNTIME_EXIT_CODES = new Set([-1073741515, 3221225781]);
+
+export function isMissingWindowsWhisperRuntimeDependency(error: unknown) {
+ if (process.platform !== "win32" || !error || typeof error !== "object") {
+ return false;
+ }
+
+ const code = (error as { code?: unknown }).code;
+ return typeof code === "number" && WINDOWS_MISSING_RUNTIME_EXIT_CODES.has(code);
+}
diff --git a/electron/ipc/register/captions.ts b/electron/ipc/register/captions.ts
index fe93afd7..7dfe5d67 100644
--- a/electron/ipc/register/captions.ts
+++ b/electron/ipc/register/captions.ts
@@ -247,7 +247,8 @@ export function registerCaptionHandlers() {
return {
success: false,
error: String(error),
- message: "Failed to generate auto captions",
+ message:
+ error instanceof Error ? error.message : "Failed to generate auto captions",
};
}
},
diff --git a/package.json b/package.json
index b23ab48f..f1ef508f 100644
--- a/package.json
+++ b/package.json
@@ -26,7 +26,7 @@
"rebuild:native": "node ./node_modules/@electron/rebuild/lib/cli.js --force --only uiohook-napi",
"build:native-helpers": "node scripts/build-native-helpers.mjs",
"build:whisper-runtime": "node scripts/build-whisper-runtime.mjs",
- "build:platform-native-helpers": "npm run build:native-helpers && npm run build:windows-capture && npm run build:windows-gpu-export && npm run build:nvidia-cuda-compositor && npm run build:cursor-monitor && npm run build:whisper-runtime",
+ "build:platform-native-helpers": "npm run build:whisper-runtime && npm run build:native-helpers && npm run build:windows-capture && npm run build:windows-gpu-export && npm run build:nvidia-cuda-compositor && npm run build:cursor-monitor",
"build:windows-gpu-export": "node scripts/build-windows-gpu-export.mjs",
"build:nvidia-cuda-compositor": "node scripts/build-nvidia-cuda-compositor.mjs",
"build:windows-capture": "node scripts/build-windows-capture.mjs",
diff --git a/scripts/build-whisper-runtime.mjs b/scripts/build-whisper-runtime.mjs
index e5e46c99..f1f76f7a 100644
--- a/scripts/build-whisper-runtime.mjs
+++ b/scripts/build-whisper-runtime.mjs
@@ -1,4 +1,5 @@
import { execFileSync, execSync } from "node:child_process";
+import { createHash } from "node:crypto";
import { createWriteStream, existsSync, rmSync } from "node:fs";
import { chmod, cp, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
import { get as httpsGet } from "node:https";
@@ -15,6 +16,8 @@ const nativeRoot = path.join(projectRoot, "electron", "native");
const cacheRoot = path.join(projectRoot, ".tmp", "whisper-runtime");
const archivePath = path.join(cacheRoot, `${whisperVersion}.tar.gz`);
const extractRoot = path.join(cacheRoot, `src-${whisperVersion}`);
+const windowsX64ArchivePath = path.join(cacheRoot, `${whisperVersion}-windows-x64.zip`);
+const windowsX64ArchiveSha256 = "74f973345cb52ef5ba3ec9e7e7af8e48cc8c71722d1528603b80588a11f82e3e";
function getHostArch() {
return process.arch === "arm64" ? "arm64" : "x64";
@@ -225,6 +228,75 @@ async function downloadFile(url, destinationPath) {
});
}
+async function getFileSha256(filePath) {
+ return createHash("sha256")
+ .update(await readFile(filePath))
+ .digest("hex");
+}
+
+async function findDirectoryContaining(rootPath, fileName) {
+ const pending = [rootPath];
+ while (pending.length > 0) {
+ const currentPath = pending.shift();
+ const entries = await readdir(currentPath, { withFileTypes: true });
+ if (entries.some((entry) => entry.isFile() && entry.name === fileName)) {
+ return currentPath;
+ }
+ for (const entry of entries) {
+ if (entry.isDirectory()) {
+ pending.push(path.join(currentPath, entry.name));
+ }
+ }
+ }
+ return null;
+}
+
+async function stageWindowsX64PrebuiltRuntime(target) {
+ if (target.platform !== "win32" || target.arch !== "x64") {
+ return false;
+ }
+
+ await mkdir(cacheRoot, { recursive: true });
+ const archiveUrl = `https://github.com/ggml-org/whisper.cpp/releases/download/${whisperVersion}/whisper-bin-x64.zip`;
+ let archiveIsValid =
+ existsSync(windowsX64ArchivePath) &&
+ (await getFileSha256(windowsX64ArchivePath)) === windowsX64ArchiveSha256;
+ if (!archiveIsValid) {
+ await rm(windowsX64ArchivePath, { force: true });
+ console.log(
+ `[build-whisper-runtime] Downloading official whisper.cpp ${whisperVersion} Windows x64 runtime...`,
+ );
+ await downloadFile(archiveUrl, windowsX64ArchivePath);
+ archiveIsValid = (await getFileSha256(windowsX64ArchivePath)) === windowsX64ArchiveSha256;
+ }
+
+ if (!archiveIsValid) {
+ throw new Error(
+ "[build-whisper-runtime] Windows x64 runtime archive failed its SHA-256 integrity check.",
+ );
+ }
+
+ const prebuiltExtractRoot = path.join(cacheRoot, `prebuilt-${target.archTag}`);
+ await rm(prebuiltExtractRoot, { recursive: true, force: true });
+ await mkdir(prebuiltExtractRoot, { recursive: true });
+ execFileSync("tar", ["-xf", windowsX64ArchivePath, "-C", prebuiltExtractRoot], {
+ stdio: "inherit",
+ });
+
+ const runtimeDir = await findDirectoryContaining(prebuiltExtractRoot, "whisper-cli.exe");
+ if (!runtimeDir) {
+ throw new Error(
+ "[build-whisper-runtime] Official Windows archive did not contain whisper-cli.exe.",
+ );
+ }
+
+ const runtimeEntries = (await readdir(runtimeDir)).filter(
+ (entry) => /^(whisper|ggml)/i.test(entry) || entry.toLowerCase().endsWith(".dll"),
+ );
+ await stageRuntimeArtifacts(target, runtimeDir, runtimeEntries);
+ return true;
+}
+
async function ensureSourceTree() {
const extractedSourceDir = path.join(
extractRoot,
@@ -366,6 +438,31 @@ async function stageRuntimeArtifacts(target, candidateDir, runtimeEntries) {
async function main() {
const targets = getTargetConfigs();
+
+ // Official whisper.cpp releases include a signed, portable Windows x64
+ // runtime. Prefer it so developers and packaged builds do not require a full
+ // Visual Studio C++ installation just to enable captions.
+ for (const target of targets) {
+ if (!(await shouldSkipBuild(target))) {
+ try {
+ await stageWindowsX64PrebuiltRuntime(target);
+ } catch (error) {
+ console.warn(
+ "[build-whisper-runtime] Failed to stage the official Windows runtime; falling back to a source build:",
+ error,
+ );
+ }
+ }
+ }
+
+ const stagedChecks = await Promise.all(targets.map((target) => shouldSkipBuild(target)));
+ if (stagedChecks.every(Boolean)) {
+ console.log(
+ `[build-whisper-runtime] Whisper runtime ${whisperVersion} is staged for ${targets.map((target) => target.archTag).join(", ")}.`,
+ );
+ return;
+ }
+
const cmake = findCmake();
if (!cmake) {
diff --git a/src/components/video-editor/captions/useAutoCaptionController.ts b/src/components/video-editor/captions/useAutoCaptionController.ts
index 3ec81ee7..a008f135 100644
--- a/src/components/video-editor/captions/useAutoCaptionController.ts
+++ b/src/components/video-editor/captions/useAutoCaptionController.ts
@@ -195,8 +195,8 @@ export function useAutoCaptionController({
});
if (!result.success || !result.cues) {
toast.error(
- result.message ||
- getErrorMessage(result.error) ||
+ getErrorMessage(result.error) ||
+ result.message ||
"Failed to generate captions",
);
return;