Create polished demo videos without editing skills. Mac/Windows/Linux
Fix cross-platform caption generation
webadderall17150671f608
diff --git a/electron/ipc/captions/audioCandidates.test.ts b/electron/ipc/captions/audioCandidates.test.tsnew file mode 100644index 00000000..c08b49ff--- /dev/null+++ b/electron/ipc/captions/audioCandidates.test.ts@@ -0,0 +1,33 @@+import { describe, expect, it } from "vitest";+import { getCaptionCompanionAudioCandidates } from "./audioCandidates";++describe("getCaptionCompanionAudioCandidates", () => {+ it("prefers microphone audio and includes system audio when both sidecars exist", () => {+ expect(+ getCaptionCompanionAudioCandidates([+ {+ platform: "win",+ micPath: "recording.mic.wav",+ systemPath: "recording.system.wav",+ usablePaths: ["recording.system.wav", "recording.mic.wav"],+ },+ ]),+ ).toEqual([+ { path: "recording.mic.wav", label: "microphone audio sidecar" },+ { path: "recording.system.wav", label: "system audio sidecar" },+ ]);+ });++ it("omits companion paths that were not found on disk", () => {+ expect(+ getCaptionCompanionAudioCandidates([+ {+ platform: "win",+ micPath: "recording.mic.wav",+ systemPath: "recording.system.wav",+ usablePaths: ["recording.mic.wav"],+ },+ ]),+ ).toEqual([{ path: "recording.mic.wav", label: "microphone audio sidecar" }]);+ });+});diff --git a/electron/ipc/captions/audioCandidates.ts b/electron/ipc/captions/audioCandidates.tsnew file mode 100644index 00000000..33e62d2e--- /dev/null+++ b/electron/ipc/captions/audioCandidates.ts@@ -0,0 +1,21 @@+import type { CompanionAudioCandidate } from "../types";++export type CaptionAudioCandidate = {+ path: string;+ label: string;+};++export function getCaptionCompanionAudioCandidates(+ companions: CompanionAudioCandidate[],+): CaptionAudioCandidate[] {+ return companions.flatMap((companion) => {+ const candidates: CaptionAudioCandidate[] = [];+ if (companion.usablePaths.includes(companion.micPath)) {+ candidates.push({ path: companion.micPath, label: "microphone audio sidecar" });+ }+ if (companion.usablePaths.includes(companion.systemPath)) {+ candidates.push({ path: companion.systemPath, label: "system audio sidecar" });+ }+ return candidates;+ });+}diff --git a/electron/ipc/captions/generate.test.ts b/electron/ipc/captions/generate.test.tsnew file mode 100644index 00000000..9844fecc--- /dev/null+++ b/electron/ipc/captions/generate.test.ts@@ -0,0 +1,16 @@+import { describe, expect, it } from "vitest";+import { isMissingWindowsWhisperRuntimeDependency } from "./runtimeErrors";++describe("isMissingWindowsWhisperRuntimeDependency", () => {+ it("recognizes the signed and unsigned STATUS_DLL_NOT_FOUND exit codes on Windows", () => {+ if (process.platform !== "win32") return;++ expect(isMissingWindowsWhisperRuntimeDependency({ code: -1073741515 })).toBe(true);+ expect(isMissingWindowsWhisperRuntimeDependency({ code: 3221225781 })).toBe(true);+ });++ it("does not classify ordinary Whisper failures as missing runtimes", () => {+ expect(isMissingWindowsWhisperRuntimeDependency({ code: 1 })).toBe(false);+ expect(isMissingWindowsWhisperRuntimeDependency(new Error("bad model"))).toBe(false);+ });+});diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.tsindex 2a3f49cd..246c4f1a 100644--- a/electron/ipc/captions/generate.ts+++ b/electron/ipc/captions/generate.ts@@ -7,8 +7,11 @@ import { app } from "electron";import { getFfmpegBinaryPath } from "../ffmpeg/binary";import { getBundledWhisperExecutableCandidates } from "../paths/binaries";import { resolveRecordingSession } from "../project/session";+import { getUsableCompanionAudioCandidates } from "../recording/diagnostics";import { normalizeVideoSourcePath } from "../utils";+import { getCaptionCompanionAudioCandidates } from "./audioCandidates";import { parseSrtCues, parseWhisperJsonCues, shouldRetryWhisperWithoutJson } from "./parser";+import { isMissingWindowsWhisperRuntimeDependency } from "./runtimeErrors";import { segmentCuesIntoPhrases } from "./segment";import {parseSilenceIntervals,@@ -19,6 +22,22 @@ import {const execFileAsync = promisify(execFile);+async function executeWhisper(whisperExecutablePath: string, args: string[]) {+ try {+ await execFileAsync(whisperExecutablePath, args, {+ timeout: 30 * 60 * 1000,+ maxBuffer: 20 * 1024 * 1024,+ });+ } catch (error) {+ if (isMissingWindowsWhisperRuntimeDependency(error)) {+ throw new Error(+ "Whisper could not start because the Microsoft Visual C++ x64 Redistributable is missing. Install it from https://aka.ms/vc14/vc_redist.x64.exe, then restart Recordly.",+ );+ }+ throw error;+ }+}+export async function ensureReadableFile(filePath: string, options?: { executable?: boolean }) {await fs.access(filePath, fsConstants.R_OK);if (options?.executable) {@@ -78,7 +97,8 @@ export async function resolveWhisperExecutablePath(preferredPath?: string | null}throw new Error(- "No Whisper runtime was found. Recordly looked for a bundled binary first, then checked common system install locations.",+ `No Whisper runtime was found for ${process.platform}/${process.arch}. ` ++ "This Recordly build is missing its bundled caption runtime. Reinstall or update Recordly, or select a whisper-cli executable in Caption settings.",);}@@ -97,6 +117,10 @@ export async function resolveCaptionAudioCandidates(videoPath: string) {};pushCandidate(videoPath, "recording");+ const companionAudio = await getUsableCompanionAudioCandidates(videoPath);+ for (const candidate of getCaptionCompanionAudioCandidates(companionAudio)) {+ pushCandidate(candidate.path, candidate.label);+ }const requestedRecordingSession = await resolveRecordingSession(videoPath);pushCandidate(requestedRecordingSession?.webcamPath, "linked webcam recording");@@ -236,10 +260,7 @@ export async function generateAutoCaptionsFromVideo(options: {let jsonEnabled = true;try {- await execFileAsync(whisperExecutablePath, [...whisperBaseArgs, "-ojf"], {- timeout: 30 * 60 * 1000,- maxBuffer: 20 * 1024 * 1024,- });+ await executeWhisper(whisperExecutablePath, [...whisperBaseArgs, "-ojf"]);} catch (error) {if (!shouldRetryWhisperWithoutJson(error)) {throw error;@@ -250,10 +271,7 @@ export async function generateAutoCaptionsFromVideo(options: {"[auto-captions] Whisper runtime does not support JSON full output, retrying with SRT only:",error,);- await execFileAsync(whisperExecutablePath, whisperBaseArgs, {- timeout: 30 * 60 * 1000,- maxBuffer: 20 * 1024 * 1024,- });+ await executeWhisper(whisperExecutablePath, whisperBaseArgs);}const timedCues = jsonEnableddiff --git a/electron/ipc/captions/runtimeErrors.ts b/electron/ipc/captions/runtimeErrors.tsnew file mode 100644index 00000000..707848dd--- /dev/null+++ b/electron/ipc/captions/runtimeErrors.ts@@ -0,0 +1,10 @@+const WINDOWS_MISSING_RUNTIME_EXIT_CODES = new Set([-1073741515, 3221225781]);++export function isMissingWindowsWhisperRuntimeDependency(error: unknown) {+ if (process.platform !== "win32" || !error || typeof error !== "object") {+ return false;+ }++ const code = (error as { code?: unknown }).code;+ return typeof code === "number" && WINDOWS_MISSING_RUNTIME_EXIT_CODES.has(code);+}diff --git a/electron/ipc/register/captions.ts b/electron/ipc/register/captions.tsindex fe93afd7..7dfe5d67 100644--- a/electron/ipc/register/captions.ts+++ b/electron/ipc/register/captions.ts@@ -247,7 +247,8 @@ export function registerCaptionHandlers() {return {success: false,error: String(error),- message: "Failed to generate auto captions",+ message:+ error instanceof Error ? error.message : "Failed to generate auto captions",};}},diff --git a/package.json b/package.jsonindex b23ab48f..f1ef508f 100644--- a/package.json+++ b/package.json@@ -26,7 +26,7 @@"rebuild:native": "node ./node_modules/@electron/rebuild/lib/cli.js --force --only uiohook-napi","build:native-helpers": "node scripts/build-native-helpers.mjs","build:whisper-runtime": "node scripts/build-whisper-runtime.mjs",- "build:platform-native-helpers": "npm run build:native-helpers && npm run build:windows-capture && npm run build:windows-gpu-export && npm run build:nvidia-cuda-compositor && npm run build:cursor-monitor && npm run build:whisper-runtime",+ "build:platform-native-helpers": "npm run build:whisper-runtime && npm run build:native-helpers && npm run build:windows-capture && npm run build:windows-gpu-export && npm run build:nvidia-cuda-compositor && npm run build:cursor-monitor","build:windows-gpu-export": "node scripts/build-windows-gpu-export.mjs","build:nvidia-cuda-compositor": "node scripts/build-nvidia-cuda-compositor.mjs","build:windows-capture": "node scripts/build-windows-capture.mjs",diff --git a/scripts/build-whisper-runtime.mjs b/scripts/build-whisper-runtime.mjsindex e5e46c99..f1f76f7a 100644--- a/scripts/build-whisper-runtime.mjs+++ b/scripts/build-whisper-runtime.mjs@@ -1,4 +1,5 @@import { execFileSync, execSync } from "node:child_process";+import { createHash } from "node:crypto";import { createWriteStream, existsSync, rmSync } from "node:fs";import { chmod, cp, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";import { get as httpsGet } from "node:https";@@ -15,6 +16,8 @@ const nativeRoot = path.join(projectRoot, "electron", "native");const cacheRoot = path.join(projectRoot, ".tmp", "whisper-runtime");const archivePath = path.join(cacheRoot, `${whisperVersion}.tar.gz`);const extractRoot = path.join(cacheRoot, `src-${whisperVersion}`);+const windowsX64ArchivePath = path.join(cacheRoot, `${whisperVersion}-windows-x64.zip`);+const windowsX64ArchiveSha256 = "74f973345cb52ef5ba3ec9e7e7af8e48cc8c71722d1528603b80588a11f82e3e";function getHostArch() {return process.arch === "arm64" ? "arm64" : "x64";@@ -225,6 +228,75 @@ async function downloadFile(url, destinationPath) {});}+async function getFileSha256(filePath) {+ return createHash("sha256")+ .update(await readFile(filePath))+ .digest("hex");+}++async function findDirectoryContaining(rootPath, fileName) {+ const pending = [rootPath];+ while (pending.length > 0) {+ const currentPath = pending.shift();+ const entries = await readdir(currentPath, { withFileTypes: true });+ if (entries.some((entry) => entry.isFile() && entry.name === fileName)) {+ return currentPath;+ }+ for (const entry of entries) {+ if (entry.isDirectory()) {+ pending.push(path.join(currentPath, entry.name));+ }+ }+ }+ return null;+}++async function stageWindowsX64PrebuiltRuntime(target) {+ if (target.platform !== "win32" || target.arch !== "x64") {+ return false;+ }++ await mkdir(cacheRoot, { recursive: true });+ const archiveUrl = `https://github.com/ggml-org/whisper.cpp/releases/download/${whisperVersion}/whisper-bin-x64.zip`;+ let archiveIsValid =+ existsSync(windowsX64ArchivePath) &&+ (await getFileSha256(windowsX64ArchivePath)) === windowsX64ArchiveSha256;+ if (!archiveIsValid) {+ await rm(windowsX64ArchivePath, { force: true });+ console.log(+ `[build-whisper-runtime] Downloading official whisper.cpp ${whisperVersion} Windows x64 runtime...`,+ );+ await downloadFile(archiveUrl, windowsX64ArchivePath);+ archiveIsValid = (await getFileSha256(windowsX64ArchivePath)) === windowsX64ArchiveSha256;+ }++ if (!archiveIsValid) {+ throw new Error(+ "[build-whisper-runtime] Windows x64 runtime archive failed its SHA-256 integrity check.",+ );+ }++ const prebuiltExtractRoot = path.join(cacheRoot, `prebuilt-${target.archTag}`);+ await rm(prebuiltExtractRoot, { recursive: true, force: true });+ await mkdir(prebuiltExtractRoot, { recursive: true });+ execFileSync("tar", ["-xf", windowsX64ArchivePath, "-C", prebuiltExtractRoot], {+ stdio: "inherit",+ });++ const runtimeDir = await findDirectoryContaining(prebuiltExtractRoot, "whisper-cli.exe");+ if (!runtimeDir) {+ throw new Error(+ "[build-whisper-runtime] Official Windows archive did not contain whisper-cli.exe.",+ );+ }++ const runtimeEntries = (await readdir(runtimeDir)).filter(+ (entry) => /^(whisper|ggml)/i.test(entry) || entry.toLowerCase().endsWith(".dll"),+ );+ await stageRuntimeArtifacts(target, runtimeDir, runtimeEntries);+ return true;+}+async function ensureSourceTree() {const extractedSourceDir = path.join(extractRoot,@@ -366,6 +438,31 @@ async function stageRuntimeArtifacts(target, candidateDir, runtimeEntries) {async function main() {const targets = getTargetConfigs();++ // Official whisper.cpp releases include a signed, portable Windows x64+ // runtime. Prefer it so developers and packaged builds do not require a full+ // Visual Studio C++ installation just to enable captions.+ for (const target of targets) {+ if (!(await shouldSkipBuild(target))) {+ try {+ await stageWindowsX64PrebuiltRuntime(target);+ } catch (error) {+ console.warn(+ "[build-whisper-runtime] Failed to stage the official Windows runtime; falling back to a source build:",+ error,+ );+ }+ }+ }++ const stagedChecks = await Promise.all(targets.map((target) => shouldSkipBuild(target)));+ if (stagedChecks.every(Boolean)) {+ console.log(+ `[build-whisper-runtime] Whisper runtime ${whisperVersion} is staged for ${targets.map((target) => target.archTag).join(", ")}.`,+ );+ return;+ }+const cmake = findCmake();if (!cmake) {diff --git a/src/components/video-editor/captions/useAutoCaptionController.ts b/src/components/video-editor/captions/useAutoCaptionController.tsindex 3ec81ee7..a008f135 100644--- a/src/components/video-editor/captions/useAutoCaptionController.ts+++ b/src/components/video-editor/captions/useAutoCaptionController.ts@@ -195,8 +195,8 @@ export function useAutoCaptionController({});if (!result.success || !result.cues) {toast.error(- result.message ||- getErrorMessage(result.error) ||+ getErrorMessage(result.error) ||+ result.message ||"Failed to generate captions",);return;