Skip to content

Write HTML. Render video. Built for agents.

fix(cli): non-English transcription reaches whisper, and whisper failures say why (#5892)

Miguel Ángel9ad12da18a15
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx
index b99c64f791..2c4b74a486 100644
--- a/docs/packages/cli.mdx
+++ b/docs/packages/cli.mdx
@@ -303,9 +303,9 @@ npx hyperframes transcribe video.mp4 --to vtt
| Flag | Description |
| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `--dir, -d` | Project directory (default: current directory) |
-| `--engine, -e` | ASR engine: `auto` (default — Parakeet if installed and the language is one of its 25, else whisper), `parakeet` (fails if Parakeet is missing, broken, or lacks the `--language`), or `whisper` |
+| `--engine, -e` | ASR engine: `auto` (default — Parakeet if installed and the language is one of its 25, else whisper), `parakeet` (fails if Parakeet is missing, broken, or lacks the `--language`), or `whisper`. Under `auto` or `parakeet`, a `--language` other than English runs whisper when it is installed, with a note, because Parakeet detects the language itself |
| `--model, -m` | Whisper model (default `small.en`): `tiny.en`, `base.en`, `small.en`, `medium.en`, `large-v3` |
-| `--language, -l` | Language code (`en`, `es`, `ja`, …). Whisper transcribes as this language; Parakeet, used when it covers it, detects the language itself. |
+| `--language, -l` | Language code (`en`, `es`, `ja`, …). Whisper transcribes as this language; Parakeet detects the language itself, so it only takes a non-English `--language` when whisper is not installed. |
| `--to` | Write an `srt` or `vtt` sidecar: from a transcript file it exports instead of importing; from audio or video it is written beside `transcript.json`, with cues for languages without spaces grouped, which exporting that `transcript.json` later may split differently |
| `--output, -o` | Where the exported SRT/VTT goes |
| `--preserve-cues` | Keep each transcript entry as its own caption cue, skipping word-level grouping |
diff --git a/packages/cli/src/api.test.ts b/packages/cli/src/api.test.ts
index 26bb8dcb06..a10b982a37 100644
--- a/packages/cli/src/api.test.ts
+++ b/packages/cli/src/api.test.ts
@@ -370,6 +370,7 @@ describe("transcribe", () => {
"utils/stoppableExecFile.ts",
"utils/stoppableExecFile.test.ts",
"whisper/manager.ts",
+ "whisper/modelFile.ts",
"whisper/modelForLanguage.ts",
"whisper/parakeet.ts",
"whisper/sherpa.ts",
diff --git a/packages/cli/src/api/transcribe.ts b/packages/cli/src/api/transcribe.ts
index b10e4c6ee9..7054668cd7 100644
--- a/packages/cli/src/api/transcribe.ts
+++ b/packages/cli/src/api/transcribe.ts
@@ -5,7 +5,7 @@ import {
createRenderCancellationScope,
stoppedByCancelSignal,
} from "../utils/renderCancellation.js";
-import { DEFAULT_MODEL, isWhisperUnavailable } from "../whisper/manager.js";
+import { DEFAULT_MODEL, findWhisper, isWhisperUnavailable } from "../whisper/manager.js";
import { initialModelForLanguage } from "../whisper/modelForLanguage.js";
import {
formatSrt,
@@ -60,6 +60,8 @@ export interface TranscribeEngineEvent {
model: string;
/** Set when auto mode gave up on Parakeet and this runner takes over. */
fallbackReason?: string;
+ /** One line on why this runner was chosen over the one asked for. */
+ notice?: string;
}
export interface TranscribeOptions {
@@ -325,11 +327,19 @@ function pickRunner(
engine: string,
parakeet: () => ParakeetRunner | null,
language?: string,
+ whisperForLanguage = false,
): TranscribeRunner {
- if (engine === "whisper" || !parakeetSpeaks(language)) return "whisper";
+ if (engine === "whisper" || whisperForLanguage || !parakeetSpeaks(language)) return "whisper";
return parakeet() ?? "whisper";
}
+/** Parakeet picks its own language, so a non-English --language goes to an installed whisper. */
+function needsWhisperForLanguage(engine: string, language?: string): boolean {
+ const base = language?.trim().toLowerCase().split(/[-_]/, 1)[0];
+ if (engine === "whisper" || !base || base === "en" || !parakeetSpeaks(language)) return false;
+ return findWhisper() !== undefined;
+}
+
/** When Parakeet fails, only auto falls back; an explicit parakeet engine fails with the error. */
const parakeetFallsBack = (engine: string) => engine === "auto";
@@ -372,8 +382,14 @@ async function transcribeAudio(
);
}
const unsupported = sherpaUnsupportedReason();
- let runner = pickRunner(engine, () => parakeetRunner({ unsupported }), options.language);
- if (engine === "parakeet" && runner === "whisper") {
+ const whisperForLanguage = needsWhisperForLanguage(engine, options.language);
+ let runner = pickRunner(
+ engine,
+ () => parakeetRunner({ unsupported }),
+ options.language,
+ whisperForLanguage,
+ );
+ if (engine === "parakeet" && runner === "whisper" && !whisperForLanguage) {
throw new TranscribeError(
"parakeet_unavailable",
!parakeetSpeaks(options.language)
@@ -385,7 +401,14 @@ async function transcribeAudio(
const model = options.model ?? DEFAULT_MODEL;
const whisperModel = initialModelForLanguage(model, options.language);
- options.onEngine?.({ runner, engine: engineOf(runner), model: whisperModel });
+ options.onEngine?.({
+ runner,
+ engine: engineOf(runner),
+ model: whisperModel,
+ ...(whisperForLanguage && {
+ notice: `Parakeet chooses its own language, so whisper transcribes --language ${options.language}.`,
+ }),
+ });
const { onStatus, onProgress } = options;
const startScope = (): CancellationScope =>
options.stopOnProcessSignals
diff --git a/packages/cli/src/commands/transcribe.test.ts b/packages/cli/src/commands/transcribe.test.ts
index 4a8b98e135..0d52d1460b 100644
--- a/packages/cli/src/commands/transcribe.test.ts
+++ b/packages/cli/src/commands/transcribe.test.ts
@@ -18,6 +18,7 @@ import { basename, dirname, join } from "node:path";
import { syncBuiltinESMExports } from "node:module";
import { tmpdir } from "node:os";
import { pathToFileURL } from "node:url";
+import * as manager from "../whisper/manager.js";
import { WhisperUnavailableError } from "../whisper/manager.js";
import { formatVtt, loadTranscript } from "../whisper/normalize.js";
import * as parakeet from "../whisper/parakeet.js";
@@ -32,7 +33,7 @@ const prepareWavMock = vi.fn((input: string) => input);
let audioSeconds = 1;
// Engine selection: which runners look installed, and the sherpa decode child it spawns.
-const runners = { sherpa: false, mlx: false };
+const runners: { sherpa: boolean; mlx: boolean; whisper?: boolean } = { sherpa: false, mlx: false };
let runtimeDir: string;
const realTranscribeWithSherpa = sherpa.transcribeWithSherpa;
const realTranscribeWithParakeet = parakeet.transcribeWithParakeet;
@@ -62,6 +63,9 @@ function replaceRunners(): void {
runners.mlx ? "/bin/parakeet-mlx" : undefined,
);
vi.spyOn(parakeet, "transcribeWithParakeet").mockImplementation((...a) => mlxMock(...a));
+ vi.spyOn(manager, "findWhisper").mockImplementation(() =>
+ runners.whisper ? { executablePath: "/bin/whisper-cli", source: "system" } : undefined,
+ );
vi.spyOn(childProcess, "execFile").mockImplementation(((...a: unknown[]) =>
execFileMock(...a)) as unknown as typeof childProcess.execFile);
// Node's named exports of a builtin follow its default export only once synced.
@@ -103,7 +107,7 @@ describe("transcribe command", () => {
prepareWavMock.mockReset().mockImplementation((input: string) => input);
trackTranscribeUnavailable.mockReset();
mlxMock.mockReset();
- Object.assign(runners, { sherpa: false, mlx: false });
+ Object.assign(runners, { sherpa: false, mlx: false, whisper: false });
transcribeMock.mockRejectedValue(
new WhisperUnavailableError("whisper-cpp not found. Install: brew install whisper-cpp"),
);
@@ -325,6 +329,24 @@ echo '{"sentences":[{"tokens":[{"text":" hola","start":0,"end":1}]}]}' > "$2/$(b
expect(transcribeMock).toHaveBeenCalledTimes(1);
});
+ it("sends a non-English language to an installed whisper, since Parakeet picks its own", async () => {
+ const all = { sherpa: true, mlx: true, whisper: true };
+ expect(await transcribeWith("auto", all, "fr")).toMatchObject({ engine: "whisper" });
+ expect(await transcribeWith("parakeet", all, "es")).toMatchObject({ engine: "whisper" });
+ expect(await transcribeWith("auto", all, "en-GB")).toMatchObject({ engine: "parakeet" });
+ expect(await transcribeWith("auto", all)).toMatchObject({ engine: "parakeet" });
+ const noWhisper = { sherpa: true, mlx: true, whisper: false };
+ expect(await transcribeWith("auto", noWhisper, "fr")).toMatchObject({ engine: "parakeet" });
+
+ Object.assign(runners, all);
+ const { dir, input } = dummyAudio();
+ dirs.push(dir);
+ await transcribeCmd.run!({ args: { input, engine: "auto", language: "fr" } } as never);
+ expect(vi.mocked(console.error).mock.calls.flat().join("\n")).toContain(
+ "Parakeet chooses its own language, so whisper transcribes --language fr.",
+ );
+ });
+
it("--engine parakeet with a language it does not transcribe fails without falling back", async () => {
Object.assign(runners, { sherpa: true, mlx: true });
const { exitCode, out } = await transcribeFails("parakeet", { language: "ja" });
diff --git a/packages/cli/src/commands/transcribe.ts b/packages/cli/src/commands/transcribe.ts
index 6c12eb08f5..bc9bec032c 100644
--- a/packages/cli/src/commands/transcribe.ts
+++ b/packages/cli/src/commands/transcribe.ts
@@ -127,6 +127,7 @@ export default defineCommand({
onStatus: json ? undefined : (message) => ui.spin?.message(message),
onProgress: json ? createProgressWriter(process.stderr) : undefined,
onEngine: (event) => {
+ if (event.notice && !json) console.error(c.dim(` Note: ${event.notice}`));
ui.spin = showEngine(event, ui.spin, { json, model: args.model });
},
});
diff --git a/packages/cli/src/whisper/manager.model.test.ts b/packages/cli/src/whisper/manager.model.test.ts
new file mode 100644
index 0000000000..7e144dec39
--- /dev/null
+++ b/packages/cli/src/whisper/manager.model.test.ts
@@ -0,0 +1,73 @@
+import {
+ existsSync,
+ mkdirSync,
+ readFileSync,
+ readdirSync,
+ rmSync,
+ truncateSync,
+ writeFileSync,
+} from "node:fs";
+import { join } from "node:path";
+import { afterEach, beforeEach, expect, it, vi } from "vitest";
+import { downloadFile } from "../utils/download.js";
+import { verifies } from "./modelFile.js";
+import { ensureModel } from "./manager.js";
+
+const home = vi.hoisted(() => `${process.env.TMPDIR ?? "/tmp"}/hf-whisper-model-${process.pid}`);
+vi.mock("node:os", async (original) => ({
+ ...(await original<typeof import("node:os")>()),
+ homedir: () => home,
+}));
+vi.mock("../utils/download.js", () => ({
+ downloadFile: vi.fn(async (_url: string, dest: string) => writeFileSync(dest, "fresh")),
+}));
+vi.mock("./modelFile.js", async (original) => ({
+ ...(await original<typeof import("./modelFile.js")>()),
+ verifies: vi.fn(async () => true),
+}));
+const modelsDir = join(home, ".cache", "hyperframes", "whisper", "models");
+const tiny = join(modelsDir, "ggml-tiny.bin");
+
+beforeEach(() => mkdirSync(modelsDir, { recursive: true }));
+afterEach(() => {
+ rmSync(home, { recursive: true, force: true });
+ vi.clearAllMocks();
+});
+
+it("downloads a pinned model again when the cached file has the wrong size", async () => {
+ writeFileSync(tiny, "cut");
+
+ await expect(ensureModel("tiny")).resolves.toBe(tiny);
+
+ expect(readFileSync(tiny, "utf8")).toBe("fresh");
+ expect(vi.mocked(downloadFile).mock.calls[0]![0]).toMatch(
+ /huggingface\.co\/ggerganov\/whisper\.cpp\/resolve\/[0-9a-f]{40}\/ggml-tiny\.bin$/,
+ );
+});
+
+it("discards a download that does not match its pinned size and sha256", async () => {
+ vi.mocked(verifies).mockResolvedValueOnce(false);
+
+ await expect(ensureModel("tiny")).rejects.toThrow(/ggml-tiny\.bin did not match its pinned/);
+
+ expect(readdirSync(modelsDir)).toEqual([]);
+});
+
+it("keeps a pinned model of the right size without downloading it", async () => {
+ writeFileSync(tiny, "");
+ truncateSync(tiny, 77_691_713);
+
+ await expect(ensureModel("tiny")).resolves.toBe(tiny);
+
+ expect(downloadFile).not.toHaveBeenCalled();
+});
+
+it("uses a model that is not pinned as it was downloaded", async () => {
+ const custom = join(modelsDir, "ggml-large-v3-turbo-q5_0.bin");
+ writeFileSync(custom, "any size");
+
+ await expect(ensureModel("large-v3-turbo-q5_0")).resolves.toBe(custom);
+
+ expect(downloadFile).not.toHaveBeenCalled();
+ expect(existsSync(custom)).toBe(true);
+});
diff --git a/packages/cli/src/whisper/manager.ts b/packages/cli/src/whisper/manager.ts
index bf507e683d..b1dcdf1a20 100644
--- a/packages/cli/src/whisper/manager.ts
+++ b/packages/cli/src/whisper/manager.ts
@@ -1,11 +1,12 @@
// fallow-ignore-file code-duplication complexity
import { execFileSync } from "node:child_process";
-import { existsSync, mkdirSync, readdirSync, rmSync } from "node:fs";
+import { existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync } from "node:fs";
import { homedir, platform } from "node:os";
import { join } from "node:path";
import { findFFmpeg } from "../browser/ffmpeg.js";
import { downloadFile } from "../utils/download.js";
import { stoppableExecFile } from "../utils/stoppableExecFile.js";
+import { verifies, type ModelFile } from "./modelFile.js";
const MODELS_DIR = join(homedir(), ".cache", "hyperframes", "whisper", "models");
const DEFAULT_MODEL = "small.en";
@@ -35,8 +36,67 @@ export function isWhisperUnavailable(err: unknown): err is WhisperUnavailableErr
return err instanceof Error && "code" in err && err.code === "WHISPER_UNAVAILABLE";
}
+const MODEL_REVISION = "5359861c739e955e79d9a303bcbc70fb988958b1";
+
function getModelUrl(model: string): string {
- return `https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-${model}.bin`;
+ return `https://huggingface.co/ggerganov/whisper.cpp/resolve/${MODEL_REVISION}/ggml-${model}.bin`;
+}
+
+const pinned = (model: string, bytes: number, sha256: string): [string, ModelFile] => [
+ model,
+ { name: `ggml-${model}.bin`, bytes, sha256 },
+];
+
+/** Sizes and sha256 at MODEL_REVISION; a model not listed here is used as downloaded. */
+const PINNED_MODELS: ReadonlyMap<string, ModelFile> = new Map([
+ pinned("tiny", 77_691_713, "be07e048e1e599ad46341c8d2a135645097a538221678b7acdd1b1919c6e1b21"),
+ pinned("tiny.en", 77_704_715, "921e4cf8686fdd993dcd081a5da5b6c365bfde1162e72b08d75ac75289920b1f"),
+ pinned("base", 147_951_465, "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe"),
+ pinned(
+ "base.en",
+ 147_964_211,
+ "a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002",
+ ),
+ pinned("small", 487_601_967, "1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b"),
+ pinned(
+ "small.en",
+ 487_614_201,
+ "c6138d6d58ecc8322097e0f987c32f1be8bb0a18532a3f88f734d1bbf9c41e5d",
+ ),
+ pinned(
+ "medium",
+ 1_533_763_059,
+ "6c14d5adee5f86394037b4e4e8b59f1673b6cee10e3cf0b11bbdbee79c156208",
+ ),
+ pinned(
+ "medium.en",
+ 1_533_774_781,
+ "cc37e93478338ec7700281a7ac30a10128929eb8f427dda2e865faa8f6da4356",
+ ),
+ pinned(
+ "large-v1",
+ 3_094_623_691,
+ "7d99f41a10525d0206bddadd86760181fa920438b6b33237e3118ff6c83bb53d",
+ ),
+ pinned(
+ "large-v2",
+ 3_094_623_691,
+ "9a423fe4d40c82774b6af34115b8b935f34152246eb19e80e376071d3f999487",
+ ),
+ pinned(
+ "large-v3",
+ 3_095_033_483,
+ "64d182b440b98d5203c4f9bd541544d84c605196c4f7b845dfa11fb23594d1e2",
+ ),
+ pinned(
+ "large-v3-turbo",
+ 1_624_555_275,
+ "1fc70f774d38eb169993ac391eea357ef47c88757ef72ee5943879b7e8e2bc69",
+ ),
+]);
+
+export function pinnedWhisperModel(model: string): ModelFile | undefined {
+ return PINNED_MODELS.get(model);
}
// --- Find helpers -----------------------------------------------------------
@@ -249,20 +309,32 @@ export async function ensureModel(
},
): Promise<string> {
const modelPath = join(MODELS_DIR, `ggml-${model}.bin`);
- if (existsSync(modelPath)) return modelPath;
+ const pin = PINNED_MODELS.get(model);
+ const cachedBytes = statSync(modelPath, { throwIfNoEntry: false })?.size;
+ if (cachedBytes !== undefined && (!pin || cachedBytes === pin.bytes)) return modelPath;
mkdirSync(MODELS_DIR, { recursive: true });
-
- options?.onProgress?.(`Downloading model ${model}...`);
- await downloadFile(getModelUrl(model), modelPath, {
- signal: options?.signal,
- onProgress: options?.onDownloadProgress,
- });
-
- if (!existsSync(modelPath)) {
- throw new Error(`Model download failed: ${model}`);
+ rmSync(modelPath, { force: true });
+ options?.onProgress?.(
+ cachedBytes === undefined
+ ? `Downloading model ${model}...`
+ : `Model ${model} is incomplete; downloading it again...`,
+ );
+ const downloaded = `${modelPath}.${process.pid}.download`;
+ try {
+ await downloadFile(getModelUrl(model), downloaded, {
+ signal: options?.signal,
+ onProgress: options?.onDownloadProgress,
+ });
+ if (pin && !(await verifies(downloaded, pin))) {
+ throw new Error(
+ `${pin.name} did not match its pinned size and sha256, so it was discarded. Re-run to retry.`,
+ );
+ }
+ renameSync(downloaded, modelPath);
+ } finally {
+ rmSync(downloaded, { force: true });
}
-
return modelPath;
}
diff --git a/packages/cli/src/whisper/modelFile.ts b/packages/cli/src/whisper/modelFile.ts
new file mode 100644
index 0000000000..c17d381cd6
--- /dev/null
+++ b/packages/cli/src/whisper/modelFile.ts
@@ -0,0 +1,20 @@
+import { createHash } from "node:crypto";
+import { createReadStream, statSync } from "node:fs";
+
+export interface ModelFile {
+ name: string;
+ bytes: number;
+ sha256: string;
+}
+
+export async function sha256File(path: string): Promise<string> {
+ const hash = createHash("sha256");
+ for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer);
+ return hash.digest("hex");
+}
+
+/** The file at `path` has the pinned size and sha256. */
+export async function verifies(path: string, file: ModelFile): Promise<boolean> {
+ if (statSync(path, { throwIfNoEntry: false })?.size !== file.bytes) return false;
+ return (await sha256File(path)) === file.sha256;
+}
diff --git a/packages/cli/src/whisper/sherpa.ts b/packages/cli/src/whisper/sherpa.ts
index 61fe2a76d5..f18a6c0937 100644
--- a/packages/cli/src/whisper/sherpa.ts
+++ b/packages/cli/src/whisper/sherpa.ts
@@ -1,11 +1,12 @@
import { execFile, spawn, type ExecFileException } from "node:child_process";
-import { createHash, randomUUID } from "node:crypto";
-import { createReadStream, mkdirSync, renameSync, rmSync, statSync } from "node:fs";
+import { randomUUID } from "node:crypto";
+import { mkdirSync, renameSync, rmSync, statSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import { createInterface } from "node:readline";
import { fileURLToPath } from "node:url";
import { downloadToFile } from "../cloud/download.js";
+import { verifies, type ModelFile } from "./modelFile.js";
import { stoppedByCancelSignal } from "../utils/renderCancellation.js";
import {
CACHE_DIR,
@@ -52,11 +53,7 @@ export const PARAKEET_MODEL_DIR = join(
"parakeet-tdt-0.6b-v3-int8",
);
-export interface ModelFile {
- name: string;
- bytes: number;
- sha256: string;
-}
+export type { ModelFile };
/** Order matters: encoder, decoder, joiner, tokens (see recognizerConfig). */
const PARAKEET_MODEL_FILES: readonly ModelFile[] = [
@@ -280,17 +277,6 @@ export function sherpaParakeetInstalled(cliUrl = import.meta.url): boolean {
);
}
-async function sha256File(path: string): Promise<string> {
- const hash = createHash("sha256");
- for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer);
- return hash.digest("hex");
-}
-
-async function verifies(path: string, file: ModelFile): Promise<boolean> {
- if (statSync(path, { throwIfNoEntry: false })?.size !== file.bytes) return false;
- return (await sha256File(path)) === file.sha256;
-}
-
interface EnsureModelOptions {
dir?: string;
files?: readonly ModelFile[];
diff --git a/packages/cli/src/whisper/transcribe.metadata.test.ts b/packages/cli/src/whisper/transcribe.metadata.test.ts
index 1317129b95..348e1281a0 100644
--- a/packages/cli/src/whisper/transcribe.metadata.test.ts
+++ b/packages/cli/src/whisper/transcribe.metadata.test.ts
@@ -1,5 +1,5 @@
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
+import { homedir, tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { transcribe, type TranscribeProgress } from "./transcribe.js";
@@ -45,9 +45,15 @@ vi.mock("./manager.js", () => ({
options: { onDownloadProgress?: (received: number, total: number | null) => void },
) => {
options.onDownloadProgress?.(5, null);
- return `ggml-${model}.bin`;
+ return join(dir, `ggml-${model}.bin`);
},
hasFFmpeg: () => true,
+ // sha256 of "abc".
+ pinnedWhisperModel: (model: string) => ({
+ name: `ggml-${model}.bin`,
+ bytes: 3,
+ sha256: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+ }),
}));
vi.mock("../browser/ffmpeg.js", () => ({
findFFprobe: () => "ffprobe",
@@ -69,6 +75,7 @@ beforeEach(() => {
],
});
if (command !== "whisper-cli") throw new Error(`Unexpected executable: ${command}`);
+ if (args.includes("--version")) return "whisper.cpp version: 1.9.5\n";
if (args.includes("--detect-language")) return "";
const selected = args[args.indexOf("--language") + 1];
let language = args.includes("--language") ? selected : "en";
@@ -240,19 +247,41 @@ it("installs the stop handling only once setup is done and whisper starts", asyn
expect(order).toEqual(["runtime", "download", "stop handling"]);
});
-it("keeps whisper's log out of the error message, leaving its last lines to the command", async () => {
+it("says how whisper-cli failed, which build and model ran, and its last lines without paths", async () => {
const previous = native.exec.getMockImplementation()!;
native.exec.mockImplementation((command: string, args: string[]) => {
- if (command === "whisper-cli")
- throw new Error("Command failed: whisper-cli\nload log\nbad model");
+ if (command === "whisper-cli" && !args.includes("--version"))
+ throw Object.assign(new Error(`Command failed: ${homedir()}/bin/whisper-cli --model x`), {
+ code: null,
+ signal: "SIGSEGV",
+ });
return previous(command, args);
});
- native.printed.stderr = "load log\nbad model\n";
+ writeFileSync(join(dir, "ggml-small.bin"), "abd");
+ native.printed.stderr = `whisper_init_from_file_with_params_no_state: loading model from '${homedir()}/.cache/m/ggml-small.bin'\nggml_metal_init: found device\n`;
+
const failure = await transcribe(join(dir, "audio.wav"), dir, { model: "small" }).catch((e) => e);
- expect(failure).toMatchObject({
- message: "Command failed: whisper-cli",
- stderr: "load log\nbad model\n",
- });
+
+ expect(failure.message).toBe(
+ [
+ "whisper-cli crashed (SIGSEGV).",
+ "whisper-cli 1.9.5 (env: whisper-cli)",
+ "model small: 3 bytes, sha256 a52d159f262b (damaged: expected 3 bytes, sha256 ba7816bf8f01; delete it and re-run)",
+ "whisper-cli said:",
+ "whisper_init_from_file_with_params_no_state: loading model from 'ggml-small.bin'",
+ "ggml_metal_init: found device",
+ ].join("\n"),
+ );
+ expect(failure.message).not.toContain(homedir());
+ expect(failure).not.toHaveProperty("stderr");
+});
+
+it("asks whisper-cli for no DTW alignment, whose timings no transcript reads", async () => {
+ await transcribe(join(dir, "audio.wav"), dir, { model: "large-v3-turbo", language: "fr" });
+ const run = native.exec.mock.calls.find(
+ ([command, args]) => command === "whisper-cli" && args.includes("--output-file"),
+ );
+ expect(run?.[1]).not.toContain("--dtw");
});
it("names the timeout knob when whisper outlives its own timeout", async () => {
diff --git a/packages/cli/src/whisper/transcribe.test.ts b/packages/cli/src/whisper/transcribe.test.ts
index 90032e8f2b..4d2f29cc4a 100644
--- a/packages/cli/src/whisper/transcribe.test.ts
+++ b/packages/cli/src/whisper/transcribe.test.ts
@@ -1,9 +1,8 @@
import { chmodSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
+import { homedir, tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it, test } from "vitest";
import {
- dtwPresetForModel,
getPreparedWavDurationSeconds,
initialModelForLanguage,
isPcm16kMono,
@@ -12,29 +11,20 @@ import {
resolveAudioPreparationTimeoutMs,
resolveWhisperTimeoutMs,
whisperModelSlowdownFactor,
+ withoutPaths,
wrapWhisperTimeoutError,
} from "./transcribe.js";
-describe("dtwPresetForModel", () => {
- // The large family is the regression: model files are hyphenated but
- // whisper.cpp's --dtw preset is dotted, so `--dtw large-v3` used to abort
- // with "unknown DTW preset 'large-v3'".
- test.each([
- ["large-v3", "large.v3"],
- ["large-v2", "large.v2"],
- ["large-v1", "large.v1"],
- ["large-v3-turbo", "large.v3.turbo"],
- ])("maps hyphenated large model %s to dotted preset %s", (model, preset) => {
- expect(dtwPresetForModel(model)).toBe(preset);
- });
-
- // tiny/base/small/medium (+.en) already match their preset — must be unchanged.
- test.each(["tiny", "base.en", "small.en", "medium.en", "small"])(
- "leaves preset-identical model %s unchanged",
- (model) => {
- expect(dtwPresetForModel(model)).toBe(model);
- },
- );
+describe("withoutPaths", () => {
+ it("leaves only file names of absolute paths, and leaves other slashes alone", () => {
+ expect(
+ withoutPaths(
+ `loading '${join(homedir(), ".cache", "m", "ggml-tiny.bin")}' from /opt/x/lib/libwhisper.1.dylib (C:\\Users\\me\\w.exe), file:/srv/a/talk.wav [/srv/b/c.wav] 1/2 done`,
+ ),
+ ).toBe(
+ "loading 'ggml-tiny.bin' from libwhisper.1.dylib (w.exe), file:talk.wav [c.wav] 1/2 done",
+ );
+ });
});
describe("resolveWhisperTimeoutMs", () => {
diff --git a/packages/cli/src/whisper/transcribe.ts b/packages/cli/src/whisper/transcribe.ts
index 1e7c834da9..c4e75bb608 100644
--- a/packages/cli/src/whisper/transcribe.ts
+++ b/packages/cli/src/whisper/transcribe.ts
@@ -5,12 +5,20 @@ import { createInterface } from "node:readline";
import type { Readable } from "node:stream";
import { existsSync, readFileSync, mkdirSync, rmSync, statSync, unlinkSync } from "node:fs";
import { basename, join, extname } from "node:path";
-import { tmpdir } from "node:os";
+import { homedir, tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";
import { stoppedByCancelSignal } from "../utils/renderCancellation.js";
import { stoppableExecFile } from "../utils/stoppableExecFile.js";
-import { ensureWhisper, ensureModel, hasFFmpeg, DEFAULT_MODEL } from "./manager.js";
+import {
+ ensureWhisper,
+ ensureModel,
+ hasFFmpeg,
+ pinnedWhisperModel,
+ DEFAULT_MODEL,
+ type WhisperResult,
+} from "./manager.js";
+import { sha256File } from "./modelFile.js";
import { initialModelForLanguage } from "./modelForLanguage.js";
import { TRANSCRIPT_FILE } from "./transcriptFile.js";
import { findWavChunk } from "./wav.js";
@@ -411,21 +419,6 @@ async function prepareAudio(audioPath: string, signal?: AbortSignal): Promise<st
return wavPath;
}
-/**
- * Map a ggml model file-stem to whisper.cpp's `--dtw` alignment-heads preset.
- *
- * The two mostly coincide, so the stem was long passed straight to `--dtw` — but
- * they diverge for the large family: the model files are hyphenated
- * (`ggml-large-v3.bin`) while the DTW presets are dotted (`large.v3`,
- * `large.v3.turbo`). `--dtw large-v3` makes whisper-cli abort with
- * "unknown DTW preset 'large-v3'", surfacing as "Transcription failed". The
- * tiny/base/small/medium (+`.en`) families have no hyphen, so `-`→`.` is a no-op
- * for them and correct for the large family.
- */
-export function dtwPresetForModel(model: string): string {
- return model.replace(/-/g, ".");
-}
-
/** Aborting `signal` stops ffmpeg, removes its partial WAV and rejects with the reason. */
export async function prepareWav(
inputPath: string,
@@ -512,8 +505,6 @@ export async function transcribe(
"--output-json-full",
"--output-file",
outputBase,
- "--dtw",
- dtwPresetForModel(model),
"--suppress-nst",
];
whisperArgs.push("--language", language);
@@ -552,13 +543,15 @@ export async function transcribe(
} catch (err) {
// Surface the timeout knob when the child was killed by our own timeout —
// otherwise the reporter sees a bare ETIMEDOUT / SIGTERM with no hint that
- // `--timeout` even exists. Non-timeout errors flow through unchanged so the
- // existing stderr-tail handling in `transcribeAudio` still applies.
- throw wrapWhisperTimeoutError(err, {
- effectiveTimeoutMs: whisperTimeoutMs,
- model,
- wasOverride: options?.timeoutMs != null,
- });
+ // `--timeout` even exists. Any other failure names the build, model and stderr.
+ if (isWhisperTimeoutError(err) || stoppedByCancelSignal(err as { signal?: string })) {
+ throw wrapWhisperTimeoutError(err, {
+ effectiveTimeoutMs: whisperTimeoutMs,
+ model,
+ wasOverride: options?.timeoutMs != null,
+ });
+ }
+ throw await diagnoseWhisperFailure(err, whisper, modelPath, model);
}
// 6. Read and validate output
@@ -668,6 +661,67 @@ async function runWhisper(
await Promise.all([exited, read(stdout, options.onStdout), read(stderr, options.onStderr)]);
}
+/** Every absolute path becomes its file name, so a pasted error names no user or folder. */
+export function withoutPaths(text: string): string {
+ return text
+ .split(homedir())
+ .join("~")
+ .replace(/(?<=^|[\s'"(=:[])(?:~|[A-Za-z]:)?[\\/](?:[^\s'"`\\/]+[\\/])*([^\s'"`\\/]+)/g, "$1");
+}
+
+function whisperVersion(executable: string): string {
+ try {
+ const said = execFileSync(executable, ["--version"], {
+ encoding: "utf-8",
+ stdio: ["ignore", "pipe", "pipe"],
+ timeout: 5000,
+ });
+ return /version:?\s*(\S+)/i.exec(said)?.[1] ?? "unknown version";
+ } catch {
+ return "unknown version";
+ }
+}
+
+async function describeModel(modelPath: string, model: string): Promise<string> {
+ const bytes = statSync(modelPath, { throwIfNoEntry: false })?.size;
+ if (bytes === undefined) return `model ${model}: file missing`;
+ const sha256 = await sha256File(modelPath).catch(() => "unreadable");
+ const pin = pinnedWhisperModel(model);
+ const verdict = !pin
+ ? "not a pinned model"
+ : pin.bytes === bytes && pin.sha256 === sha256
+ ? "matches the pinned file"
+ : `damaged: expected ${pin.bytes} bytes, sha256 ${pin.sha256.slice(0, 12)}; delete it and re-run`;
+ return `model ${model}: ${bytes} bytes, sha256 ${sha256.slice(0, 12)} (${verdict})`;
+}
+
+/** A whisper-cli failure that says which build, which model file and what whisper printed, for a bug report. */
+async function diagnoseWhisperFailure(
+ err: unknown,
+ whisper: WhisperResult,
+ modelPath: string,
+ model: string,
+): Promise<Error> {
+ const failed = err as { code?: unknown; signal?: string | null; stderr?: unknown };
+ const how = failed.signal
+ ? `crashed (${failed.signal})`
+ : typeof failed.code === "number"
+ ? `exited with code ${failed.code}`
+ : `failed: ${withoutPaths(err instanceof Error ? err.message : String(err))}`;
+ const said = withoutPaths(String(failed.stderr ?? ""))
+ .trim()
+ .split("\n")
+ .slice(-12)
+ .join("\n");
+ const lines = [
+ `whisper-cli ${how}.`,
+ `whisper-cli ${whisperVersion(whisper.executablePath)} (${whisper.source}: ${basename(whisper.executablePath)})`,
+ await describeModel(modelPath, model),
+ said ? `whisper-cli said:\n${said}` : "whisper-cli printed nothing.",
+ ];
+ return new Error(lines.join("\n"), { cause: err });
+}
+
// ---------------------------------------------------------------------------
// Timeout error discoverability
// ---------------------------------------------------------------------------
diff --git a/skills-manifest.json b/skills-manifest.json
index c2d4df14e0..e5dcbbe1f0 100644
--- a/skills-manifest.json
+++ b/skills-manifest.json
@@ -54,7 +54,7 @@
"files": 1
},
"media-use": {
- "hash": "1a2c3f1c16ab3ef7",
+ "hash": "5a8f4a204aa1e159",
"files": 113
},
"motion-graphics": {
diff --git a/skills/media-use/references/operations.md b/skills/media-use/references/operations.md
index 463238b5f7..c9c7c5900f 100644
--- a/skills/media-use/references/operations.md
+++ b/skills/media-use/references/operations.md
@@ -121,7 +121,9 @@ VERIFIED on 24GB: accurate, ~3s (cached) for 8s audio. Parakeet covers 25
European languages, English included. For other languages, or when parakeet-mlx is not
installed, transcribe.mjs falls back to `hyperframes transcribe`, which uses the
CLI's Parakeet when `models install parakeet` has run and whisper.cpp (99
-languages) otherwise. `--engine parakeet|whisper` forces one. (Cohere
+languages) otherwise. `--engine parakeet|whisper` forces one, except that a
+`--language` other than English runs whisper whenever it is installed, since
+Parakeet detects the language itself. (Cohere
Transcribe tops the leaderboard on paper but its mlx-audio quants produced
garbage and ran 40-70x slower on a Mac in testing, so it is not wired in.)