Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(cli): non-English transcription reaches whisper, and whisper failures say why (#5892)
Miguel Ángel9ad12da18a15
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdxindex b99c64f791..2c4b74a486 100644--- a/docs/packages/cli.mdx+++ b/docs/packages/cli.mdx@@ -303,9 +303,9 @@ npx hyperframes transcribe video.mp4 --to vtt| Flag | Description || ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- || `--dir, -d` | Project directory (default: current directory) |-| `--engine, -e` | ASR engine: `auto` (default — Parakeet if installed and the language is one of its 25, else whisper), `parakeet` (fails if Parakeet is missing, broken, or lacks the `--language`), or `whisper` |+| `--engine, -e` | ASR engine: `auto` (default — Parakeet if installed and the language is one of its 25, else whisper), `parakeet` (fails if Parakeet is missing, broken, or lacks the `--language`), or `whisper`. Under `auto` or `parakeet`, a `--language` other than English runs whisper when it is installed, with a note, because Parakeet detects the language itself || `--model, -m` | Whisper model (default `small.en`): `tiny.en`, `base.en`, `small.en`, `medium.en`, `large-v3` |-| `--language, -l` | Language code (`en`, `es`, `ja`, …). Whisper transcribes as this language; Parakeet, used when it covers it, detects the language itself. |+| `--language, -l` | Language code (`en`, `es`, `ja`, …). Whisper transcribes as this language; Parakeet detects the language itself, so it only takes a non-English `--language` when whisper is not installed. || `--to` | Write an `srt` or `vtt` sidecar: from a transcript file it exports instead of importing; from audio or video it is written beside `transcript.json`, with cues for languages without spaces grouped, which exporting that `transcript.json` later may split differently || `--output, -o` | Where the exported SRT/VTT goes || `--preserve-cues` | Keep each transcript entry as its own caption cue, skipping word-level grouping |diff --git a/packages/cli/src/api.test.ts b/packages/cli/src/api.test.tsindex 26bb8dcb06..a10b982a37 100644--- a/packages/cli/src/api.test.ts+++ b/packages/cli/src/api.test.ts@@ -370,6 +370,7 @@ describe("transcribe", () => {"utils/stoppableExecFile.ts","utils/stoppableExecFile.test.ts","whisper/manager.ts",+ "whisper/modelFile.ts","whisper/modelForLanguage.ts","whisper/parakeet.ts","whisper/sherpa.ts",diff --git a/packages/cli/src/api/transcribe.ts b/packages/cli/src/api/transcribe.tsindex b10e4c6ee9..7054668cd7 100644--- a/packages/cli/src/api/transcribe.ts+++ b/packages/cli/src/api/transcribe.ts@@ -5,7 +5,7 @@ import {createRenderCancellationScope,stoppedByCancelSignal,} from "../utils/renderCancellation.js";-import { DEFAULT_MODEL, isWhisperUnavailable } from "../whisper/manager.js";+import { DEFAULT_MODEL, findWhisper, isWhisperUnavailable } from "../whisper/manager.js";import { initialModelForLanguage } from "../whisper/modelForLanguage.js";import {formatSrt,@@ -60,6 +60,8 @@ export interface TranscribeEngineEvent {model: string;/** Set when auto mode gave up on Parakeet and this runner takes over. */fallbackReason?: string;+ /** One line on why this runner was chosen over the one asked for. */+ notice?: string;}export interface TranscribeOptions {@@ -325,11 +327,19 @@ function pickRunner(engine: string,parakeet: () => ParakeetRunner | null,language?: string,+ whisperForLanguage = false,): TranscribeRunner {- if (engine === "whisper" || !parakeetSpeaks(language)) return "whisper";+ if (engine === "whisper" || whisperForLanguage || !parakeetSpeaks(language)) return "whisper";return parakeet() ?? "whisper";}+/** Parakeet picks its own language, so a non-English --language goes to an installed whisper. */+function needsWhisperForLanguage(engine: string, language?: string): boolean {+ const base = language?.trim().toLowerCase().split(/[-_]/, 1)[0];+ if (engine === "whisper" || !base || base === "en" || !parakeetSpeaks(language)) return false;+ return findWhisper() !== undefined;+}+/** When Parakeet fails, only auto falls back; an explicit parakeet engine fails with the error. */const parakeetFallsBack = (engine: string) => engine === "auto";@@ -372,8 +382,14 @@ async function transcribeAudio();}const unsupported = sherpaUnsupportedReason();- let runner = pickRunner(engine, () => parakeetRunner({ unsupported }), options.language);- if (engine === "parakeet" && runner === "whisper") {+ const whisperForLanguage = needsWhisperForLanguage(engine, options.language);+ let runner = pickRunner(+ engine,+ () => parakeetRunner({ unsupported }),+ options.language,+ whisperForLanguage,+ );+ if (engine === "parakeet" && runner === "whisper" && !whisperForLanguage) {throw new TranscribeError("parakeet_unavailable",!parakeetSpeaks(options.language)@@ -385,7 +401,14 @@ async function transcribeAudio(const model = options.model ?? DEFAULT_MODEL;const whisperModel = initialModelForLanguage(model, options.language);- options.onEngine?.({ runner, engine: engineOf(runner), model: whisperModel });+ options.onEngine?.({+ runner,+ engine: engineOf(runner),+ model: whisperModel,+ ...(whisperForLanguage && {+ notice: `Parakeet chooses its own language, so whisper transcribes --language ${options.language}.`,+ }),+ });const { onStatus, onProgress } = options;const startScope = (): CancellationScope =>options.stopOnProcessSignalsdiff --git a/packages/cli/src/commands/transcribe.test.ts b/packages/cli/src/commands/transcribe.test.tsindex 4a8b98e135..0d52d1460b 100644--- a/packages/cli/src/commands/transcribe.test.ts+++ b/packages/cli/src/commands/transcribe.test.ts@@ -18,6 +18,7 @@ import { basename, dirname, join } from "node:path";import { syncBuiltinESMExports } from "node:module";import { tmpdir } from "node:os";import { pathToFileURL } from "node:url";+import * as manager from "../whisper/manager.js";import { WhisperUnavailableError } from "../whisper/manager.js";import { formatVtt, loadTranscript } from "../whisper/normalize.js";import * as parakeet from "../whisper/parakeet.js";@@ -32,7 +33,7 @@ const prepareWavMock = vi.fn((input: string) => input);let audioSeconds = 1;// Engine selection: which runners look installed, and the sherpa decode child it spawns.-const runners = { sherpa: false, mlx: false };+const runners: { sherpa: boolean; mlx: boolean; whisper?: boolean } = { sherpa: false, mlx: false };let runtimeDir: string;const realTranscribeWithSherpa = sherpa.transcribeWithSherpa;const realTranscribeWithParakeet = parakeet.transcribeWithParakeet;@@ -62,6 +63,9 @@ function replaceRunners(): void {runners.mlx ? "/bin/parakeet-mlx" : undefined,);vi.spyOn(parakeet, "transcribeWithParakeet").mockImplementation((...a) => mlxMock(...a));+ vi.spyOn(manager, "findWhisper").mockImplementation(() =>+ runners.whisper ? { executablePath: "/bin/whisper-cli", source: "system" } : undefined,+ );vi.spyOn(childProcess, "execFile").mockImplementation(((...a: unknown[]) =>execFileMock(...a)) as unknown as typeof childProcess.execFile);// Node's named exports of a builtin follow its default export only once synced.@@ -103,7 +107,7 @@ describe("transcribe command", () => {prepareWavMock.mockReset().mockImplementation((input: string) => input);trackTranscribeUnavailable.mockReset();mlxMock.mockReset();- Object.assign(runners, { sherpa: false, mlx: false });+ Object.assign(runners, { sherpa: false, mlx: false, whisper: false });transcribeMock.mockRejectedValue(new WhisperUnavailableError("whisper-cpp not found. Install: brew install whisper-cpp"),);@@ -325,6 +329,24 @@ echo '{"sentences":[{"tokens":[{"text":" hola","start":0,"end":1}]}]}' > "$2/$(bexpect(transcribeMock).toHaveBeenCalledTimes(1);});+ it("sends a non-English language to an installed whisper, since Parakeet picks its own", async () => {+ const all = { sherpa: true, mlx: true, whisper: true };+ expect(await transcribeWith("auto", all, "fr")).toMatchObject({ engine: "whisper" });+ expect(await transcribeWith("parakeet", all, "es")).toMatchObject({ engine: "whisper" });+ expect(await transcribeWith("auto", all, "en-GB")).toMatchObject({ engine: "parakeet" });+ expect(await transcribeWith("auto", all)).toMatchObject({ engine: "parakeet" });+ const noWhisper = { sherpa: true, mlx: true, whisper: false };+ expect(await transcribeWith("auto", noWhisper, "fr")).toMatchObject({ engine: "parakeet" });++ Object.assign(runners, all);+ const { dir, input } = dummyAudio();+ dirs.push(dir);+ await transcribeCmd.run!({ args: { input, engine: "auto", language: "fr" } } as never);+ expect(vi.mocked(console.error).mock.calls.flat().join("\n")).toContain(+ "Parakeet chooses its own language, so whisper transcribes --language fr.",+ );+ });+it("--engine parakeet with a language it does not transcribe fails without falling back", async () => {Object.assign(runners, { sherpa: true, mlx: true });const { exitCode, out } = await transcribeFails("parakeet", { language: "ja" });diff --git a/packages/cli/src/commands/transcribe.ts b/packages/cli/src/commands/transcribe.tsindex 6c12eb08f5..bc9bec032c 100644--- a/packages/cli/src/commands/transcribe.ts+++ b/packages/cli/src/commands/transcribe.ts@@ -127,6 +127,7 @@ export default defineCommand({onStatus: json ? undefined : (message) => ui.spin?.message(message),onProgress: json ? createProgressWriter(process.stderr) : undefined,onEngine: (event) => {+ if (event.notice && !json) console.error(c.dim(` Note: ${event.notice}`));ui.spin = showEngine(event, ui.spin, { json, model: args.model });},});diff --git a/packages/cli/src/whisper/manager.model.test.ts b/packages/cli/src/whisper/manager.model.test.tsnew file mode 100644index 0000000000..7e144dec39--- /dev/null+++ b/packages/cli/src/whisper/manager.model.test.ts@@ -0,0 +1,73 @@+import {+ existsSync,+ mkdirSync,+ readFileSync,+ readdirSync,+ rmSync,+ truncateSync,+ writeFileSync,+} from "node:fs";+import { join } from "node:path";+import { afterEach, beforeEach, expect, it, vi } from "vitest";+import { downloadFile } from "../utils/download.js";+import { verifies } from "./modelFile.js";+import { ensureModel } from "./manager.js";++const home = vi.hoisted(() => `${process.env.TMPDIR ?? "/tmp"}/hf-whisper-model-${process.pid}`);+vi.mock("node:os", async (original) => ({+ ...(await original<typeof import("node:os")>()),+ homedir: () => home,+}));+vi.mock("../utils/download.js", () => ({+ downloadFile: vi.fn(async (_url: string, dest: string) => writeFileSync(dest, "fresh")),+}));+vi.mock("./modelFile.js", async (original) => ({+ ...(await original<typeof import("./modelFile.js")>()),+ verifies: vi.fn(async () => true),+}));+const modelsDir = join(home, ".cache", "hyperframes", "whisper", "models");+const tiny = join(modelsDir, "ggml-tiny.bin");++beforeEach(() => mkdirSync(modelsDir, { recursive: true }));+afterEach(() => {+ rmSync(home, { recursive: true, force: true });+ vi.clearAllMocks();+});++it("downloads a pinned model again when the cached file has the wrong size", async () => {+ writeFileSync(tiny, "cut");++ await expect(ensureModel("tiny")).resolves.toBe(tiny);++ expect(readFileSync(tiny, "utf8")).toBe("fresh");+ expect(vi.mocked(downloadFile).mock.calls[0]![0]).toMatch(+ /huggingface\.co\/ggerganov\/whisper\.cpp\/resolve\/[0-9a-f]{40}\/ggml-tiny\.bin$/,+ );+});++it("discards a download that does not match its pinned size and sha256", async () => {+ vi.mocked(verifies).mockResolvedValueOnce(false);++ await expect(ensureModel("tiny")).rejects.toThrow(/ggml-tiny\.bin did not match its pinned/);++ expect(readdirSync(modelsDir)).toEqual([]);+});++it("keeps a pinned model of the right size without downloading it", async () => {+ writeFileSync(tiny, "");+ truncateSync(tiny, 77_691_713);++ await expect(ensureModel("tiny")).resolves.toBe(tiny);++ expect(downloadFile).not.toHaveBeenCalled();+});++it("uses a model that is not pinned as it was downloaded", async () => {+ const custom = join(modelsDir, "ggml-large-v3-turbo-q5_0.bin");+ writeFileSync(custom, "any size");++ await expect(ensureModel("large-v3-turbo-q5_0")).resolves.toBe(custom);++ expect(downloadFile).not.toHaveBeenCalled();+ expect(existsSync(custom)).toBe(true);+});diff --git a/packages/cli/src/whisper/manager.ts b/packages/cli/src/whisper/manager.tsindex bf507e683d..b1dcdf1a20 100644--- a/packages/cli/src/whisper/manager.ts+++ b/packages/cli/src/whisper/manager.ts@@ -1,11 +1,12 @@// fallow-ignore-file code-duplication complexityimport { execFileSync } from "node:child_process";-import { existsSync, mkdirSync, readdirSync, rmSync } from "node:fs";+import { existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync } from "node:fs";import { homedir, platform } from "node:os";import { join } from "node:path";import { findFFmpeg } from "../browser/ffmpeg.js";import { downloadFile } from "../utils/download.js";import { stoppableExecFile } from "../utils/stoppableExecFile.js";+import { verifies, type ModelFile } from "./modelFile.js";const MODELS_DIR = join(homedir(), ".cache", "hyperframes", "whisper", "models");const DEFAULT_MODEL = "small.en";@@ -35,8 +36,67 @@ export function isWhisperUnavailable(err: unknown): err is WhisperUnavailableErrreturn err instanceof Error && "code" in err && err.code === "WHISPER_UNAVAILABLE";}+const MODEL_REVISION = "5359861c739e955e79d9a303bcbc70fb988958b1";+function getModelUrl(model: string): string {- return `https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-${model}.bin`;+ return `https://huggingface.co/ggerganov/whisper.cpp/resolve/${MODEL_REVISION}/ggml-${model}.bin`;+}++const pinned = (model: string, bytes: number, sha256: string): [string, ModelFile] => [+ model,+ { name: `ggml-${model}.bin`, bytes, sha256 },+];++/** Sizes and sha256 at MODEL_REVISION; a model not listed here is used as downloaded. */+const PINNED_MODELS: ReadonlyMap<string, ModelFile> = new Map([+ pinned("tiny", 77_691_713, "be07e048e1e599ad46341c8d2a135645097a538221678b7acdd1b1919c6e1b21"),+ pinned("tiny.en", 77_704_715, "921e4cf8686fdd993dcd081a5da5b6c365bfde1162e72b08d75ac75289920b1f"),+ pinned("base", 147_951_465, "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe"),+ pinned(+ "base.en",+ 147_964_211,+ "a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002",+ ),+ pinned("small", 487_601_967, "1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b"),+ pinned(+ "small.en",+ 487_614_201,+ "c6138d6d58ecc8322097e0f987c32f1be8bb0a18532a3f88f734d1bbf9c41e5d",+ ),+ pinned(+ "medium",+ 1_533_763_059,+ "6c14d5adee5f86394037b4e4e8b59f1673b6cee10e3cf0b11bbdbee79c156208",+ ),+ pinned(+ "medium.en",+ 1_533_774_781,+ "cc37e93478338ec7700281a7ac30a10128929eb8f427dda2e865faa8f6da4356",+ ),+ pinned(+ "large-v1",+ 3_094_623_691,+ "7d99f41a10525d0206bddadd86760181fa920438b6b33237e3118ff6c83bb53d",+ ),+ pinned(+ "large-v2",+ 3_094_623_691,+ "9a423fe4d40c82774b6af34115b8b935f34152246eb19e80e376071d3f999487",+ ),+ pinned(+ "large-v3",+ 3_095_033_483,+ "64d182b440b98d5203c4f9bd541544d84c605196c4f7b845dfa11fb23594d1e2",+ ),+ pinned(+ "large-v3-turbo",+ 1_624_555_275,+ "1fc70f774d38eb169993ac391eea357ef47c88757ef72ee5943879b7e8e2bc69",+ ),+]);++export function pinnedWhisperModel(model: string): ModelFile | undefined {+ return PINNED_MODELS.get(model);}// --- Find helpers -----------------------------------------------------------@@ -249,20 +309,32 @@ export async function ensureModel(},): Promise<string> {const modelPath = join(MODELS_DIR, `ggml-${model}.bin`);- if (existsSync(modelPath)) return modelPath;+ const pin = PINNED_MODELS.get(model);+ const cachedBytes = statSync(modelPath, { throwIfNoEntry: false })?.size;+ if (cachedBytes !== undefined && (!pin || cachedBytes === pin.bytes)) return modelPath;mkdirSync(MODELS_DIR, { recursive: true });-- options?.onProgress?.(`Downloading model ${model}...`);- await downloadFile(getModelUrl(model), modelPath, {- signal: options?.signal,- onProgress: options?.onDownloadProgress,- });-- if (!existsSync(modelPath)) {- throw new Error(`Model download failed: ${model}`);+ rmSync(modelPath, { force: true });+ options?.onProgress?.(+ cachedBytes === undefined+ ? `Downloading model ${model}...`+ : `Model ${model} is incomplete; downloading it again...`,+ );+ const downloaded = `${modelPath}.${process.pid}.download`;+ try {+ await downloadFile(getModelUrl(model), downloaded, {+ signal: options?.signal,+ onProgress: options?.onDownloadProgress,+ });+ if (pin && !(await verifies(downloaded, pin))) {+ throw new Error(+ `${pin.name} did not match its pinned size and sha256, so it was discarded. Re-run to retry.`,+ );+ }+ renameSync(downloaded, modelPath);+ } finally {+ rmSync(downloaded, { force: true });}-return modelPath;}diff --git a/packages/cli/src/whisper/modelFile.ts b/packages/cli/src/whisper/modelFile.tsnew file mode 100644index 0000000000..c17d381cd6--- /dev/null+++ b/packages/cli/src/whisper/modelFile.ts@@ -0,0 +1,20 @@+import { createHash } from "node:crypto";+import { createReadStream, statSync } from "node:fs";++export interface ModelFile {+ name: string;+ bytes: number;+ sha256: string;+}++export async function sha256File(path: string): Promise<string> {+ const hash = createHash("sha256");+ for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer);+ return hash.digest("hex");+}++/** The file at `path` has the pinned size and sha256. */+export async function verifies(path: string, file: ModelFile): Promise<boolean> {+ if (statSync(path, { throwIfNoEntry: false })?.size !== file.bytes) return false;+ return (await sha256File(path)) === file.sha256;+}diff --git a/packages/cli/src/whisper/sherpa.ts b/packages/cli/src/whisper/sherpa.tsindex 61fe2a76d5..f18a6c0937 100644--- a/packages/cli/src/whisper/sherpa.ts+++ b/packages/cli/src/whisper/sherpa.ts@@ -1,11 +1,12 @@import { execFile, spawn, type ExecFileException } from "node:child_process";-import { createHash, randomUUID } from "node:crypto";-import { createReadStream, mkdirSync, renameSync, rmSync, statSync } from "node:fs";+import { randomUUID } from "node:crypto";+import { mkdirSync, renameSync, rmSync, statSync } from "node:fs";import { homedir } from "node:os";import { join } from "node:path";import { createInterface } from "node:readline";import { fileURLToPath } from "node:url";import { downloadToFile } from "../cloud/download.js";+import { verifies, type ModelFile } from "./modelFile.js";import { stoppedByCancelSignal } from "../utils/renderCancellation.js";import {CACHE_DIR,@@ -52,11 +53,7 @@ export const PARAKEET_MODEL_DIR = join("parakeet-tdt-0.6b-v3-int8",);-export interface ModelFile {- name: string;- bytes: number;- sha256: string;-}+export type { ModelFile };/** Order matters: encoder, decoder, joiner, tokens (see recognizerConfig). */const PARAKEET_MODEL_FILES: readonly ModelFile[] = [@@ -280,17 +277,6 @@ export function sherpaParakeetInstalled(cliUrl = import.meta.url): boolean {);}-async function sha256File(path: string): Promise<string> {- const hash = createHash("sha256");- for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer);- return hash.digest("hex");-}--async function verifies(path: string, file: ModelFile): Promise<boolean> {- if (statSync(path, { throwIfNoEntry: false })?.size !== file.bytes) return false;- return (await sha256File(path)) === file.sha256;-}-interface EnsureModelOptions {dir?: string;files?: readonly ModelFile[];diff --git a/packages/cli/src/whisper/transcribe.metadata.test.ts b/packages/cli/src/whisper/transcribe.metadata.test.tsindex 1317129b95..348e1281a0 100644--- a/packages/cli/src/whisper/transcribe.metadata.test.ts+++ b/packages/cli/src/whisper/transcribe.metadata.test.ts@@ -1,5 +1,5 @@import { mkdtempSync, rmSync, writeFileSync } from "node:fs";-import { tmpdir } from "node:os";+import { homedir, tmpdir } from "node:os";import { join } from "node:path";import { afterEach, beforeEach, expect, it, vi } from "vitest";import { transcribe, type TranscribeProgress } from "./transcribe.js";@@ -45,9 +45,15 @@ vi.mock("./manager.js", () => ({options: { onDownloadProgress?: (received: number, total: number | null) => void },) => {options.onDownloadProgress?.(5, null);- return `ggml-${model}.bin`;+ return join(dir, `ggml-${model}.bin`);},hasFFmpeg: () => true,+ // sha256 of "abc".+ pinnedWhisperModel: (model: string) => ({+ name: `ggml-${model}.bin`,+ bytes: 3,+ sha256: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",+ }),}));vi.mock("../browser/ffmpeg.js", () => ({findFFprobe: () => "ffprobe",@@ -69,6 +75,7 @@ beforeEach(() => {],});if (command !== "whisper-cli") throw new Error(`Unexpected executable: ${command}`);+ if (args.includes("--version")) return "whisper.cpp version: 1.9.5\n";if (args.includes("--detect-language")) return "";const selected = args[args.indexOf("--language") + 1];let language = args.includes("--language") ? selected : "en";@@ -240,19 +247,41 @@ it("installs the stop handling only once setup is done and whisper starts", asynexpect(order).toEqual(["runtime", "download", "stop handling"]);});-it("keeps whisper's log out of the error message, leaving its last lines to the command", async () => {+it("says how whisper-cli failed, which build and model ran, and its last lines without paths", async () => {const previous = native.exec.getMockImplementation()!;native.exec.mockImplementation((command: string, args: string[]) => {- if (command === "whisper-cli")- throw new Error("Command failed: whisper-cli\nload log\nbad model");+ if (command === "whisper-cli" && !args.includes("--version"))+ throw Object.assign(new Error(`Command failed: ${homedir()}/bin/whisper-cli --model x`), {+ code: null,+ signal: "SIGSEGV",+ });return previous(command, args);});- native.printed.stderr = "load log\nbad model\n";+ writeFileSync(join(dir, "ggml-small.bin"), "abd");+ native.printed.stderr = `whisper_init_from_file_with_params_no_state: loading model from '${homedir()}/.cache/m/ggml-small.bin'\nggml_metal_init: found device\n`;+const failure = await transcribe(join(dir, "audio.wav"), dir, { model: "small" }).catch((e) => e);- expect(failure).toMatchObject({- message: "Command failed: whisper-cli",- stderr: "load log\nbad model\n",- });++ expect(failure.message).toBe(+ [+ "whisper-cli crashed (SIGSEGV).",+ "whisper-cli 1.9.5 (env: whisper-cli)",+ "model small: 3 bytes, sha256 a52d159f262b (damaged: expected 3 bytes, sha256 ba7816bf8f01; delete it and re-run)",+ "whisper-cli said:",+ "whisper_init_from_file_with_params_no_state: loading model from 'ggml-small.bin'",+ "ggml_metal_init: found device",+ ].join("\n"),+ );+ expect(failure.message).not.toContain(homedir());+ expect(failure).not.toHaveProperty("stderr");+});++it("asks whisper-cli for no DTW alignment, whose timings no transcript reads", async () => {+ await transcribe(join(dir, "audio.wav"), dir, { model: "large-v3-turbo", language: "fr" });+ const run = native.exec.mock.calls.find(+ ([command, args]) => command === "whisper-cli" && args.includes("--output-file"),+ );+ expect(run?.[1]).not.toContain("--dtw");});it("names the timeout knob when whisper outlives its own timeout", async () => {diff --git a/packages/cli/src/whisper/transcribe.test.ts b/packages/cli/src/whisper/transcribe.test.tsindex 90032e8f2b..4d2f29cc4a 100644--- a/packages/cli/src/whisper/transcribe.test.ts+++ b/packages/cli/src/whisper/transcribe.test.ts@@ -1,9 +1,8 @@import { chmodSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";-import { tmpdir } from "node:os";+import { homedir, tmpdir } from "node:os";import { join } from "node:path";import { describe, expect, it, test } from "vitest";import {- dtwPresetForModel,getPreparedWavDurationSeconds,initialModelForLanguage,isPcm16kMono,@@ -12,29 +11,20 @@ import {resolveAudioPreparationTimeoutMs,resolveWhisperTimeoutMs,whisperModelSlowdownFactor,+ withoutPaths,wrapWhisperTimeoutError,} from "./transcribe.js";-describe("dtwPresetForModel", () => {- // The large family is the regression: model files are hyphenated but- // whisper.cpp's --dtw preset is dotted, so `--dtw large-v3` used to abort- // with "unknown DTW preset 'large-v3'".- test.each([- ["large-v3", "large.v3"],- ["large-v2", "large.v2"],- ["large-v1", "large.v1"],- ["large-v3-turbo", "large.v3.turbo"],- ])("maps hyphenated large model %s to dotted preset %s", (model, preset) => {- expect(dtwPresetForModel(model)).toBe(preset);- });-- // tiny/base/small/medium (+.en) already match their preset — must be unchanged.- test.each(["tiny", "base.en", "small.en", "medium.en", "small"])(- "leaves preset-identical model %s unchanged",- (model) => {- expect(dtwPresetForModel(model)).toBe(model);- },- );+describe("withoutPaths", () => {+ it("leaves only file names of absolute paths, and leaves other slashes alone", () => {+ expect(+ withoutPaths(+ `loading '${join(homedir(), ".cache", "m", "ggml-tiny.bin")}' from /opt/x/lib/libwhisper.1.dylib (C:\\Users\\me\\w.exe), file:/srv/a/talk.wav [/srv/b/c.wav] 1/2 done`,+ ),+ ).toBe(+ "loading 'ggml-tiny.bin' from libwhisper.1.dylib (w.exe), file:talk.wav [c.wav] 1/2 done",+ );+ });});describe("resolveWhisperTimeoutMs", () => {diff --git a/packages/cli/src/whisper/transcribe.ts b/packages/cli/src/whisper/transcribe.tsindex 1e7c834da9..c4e75bb608 100644--- a/packages/cli/src/whisper/transcribe.ts+++ b/packages/cli/src/whisper/transcribe.ts@@ -5,12 +5,20 @@ import { createInterface } from "node:readline";import type { Readable } from "node:stream";import { existsSync, readFileSync, mkdirSync, rmSync, statSync, unlinkSync } from "node:fs";import { basename, join, extname } from "node:path";-import { tmpdir } from "node:os";+import { homedir, tmpdir } from "node:os";import { randomUUID } from "node:crypto";import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";import { stoppedByCancelSignal } from "../utils/renderCancellation.js";import { stoppableExecFile } from "../utils/stoppableExecFile.js";-import { ensureWhisper, ensureModel, hasFFmpeg, DEFAULT_MODEL } from "./manager.js";+import {+ ensureWhisper,+ ensureModel,+ hasFFmpeg,+ pinnedWhisperModel,+ DEFAULT_MODEL,+ type WhisperResult,+} from "./manager.js";+import { sha256File } from "./modelFile.js";import { initialModelForLanguage } from "./modelForLanguage.js";import { TRANSCRIPT_FILE } from "./transcriptFile.js";import { findWavChunk } from "./wav.js";@@ -411,21 +419,6 @@ async function prepareAudio(audioPath: string, signal?: AbortSignal): Promise<streturn wavPath;}-/**- * Map a ggml model file-stem to whisper.cpp's `--dtw` alignment-heads preset.- *- * The two mostly coincide, so the stem was long passed straight to `--dtw` — but- * they diverge for the large family: the model files are hyphenated- * (`ggml-large-v3.bin`) while the DTW presets are dotted (`large.v3`,- * `large.v3.turbo`). `--dtw large-v3` makes whisper-cli abort with- * "unknown DTW preset 'large-v3'", surfacing as "Transcription failed". The- * tiny/base/small/medium (+`.en`) families have no hyphen, so `-`→`.` is a no-op- * for them and correct for the large family.- */-export function dtwPresetForModel(model: string): string {- return model.replace(/-/g, ".");-}-/** Aborting `signal` stops ffmpeg, removes its partial WAV and rejects with the reason. */export async function prepareWav(inputPath: string,@@ -512,8 +505,6 @@ export async function transcribe("--output-json-full","--output-file",outputBase,- "--dtw",- dtwPresetForModel(model),"--suppress-nst",];whisperArgs.push("--language", language);@@ -552,13 +543,15 @@ export async function transcribe(} catch (err) {// Surface the timeout knob when the child was killed by our own timeout —// otherwise the reporter sees a bare ETIMEDOUT / SIGTERM with no hint that- // `--timeout` even exists. Non-timeout errors flow through unchanged so the- // existing stderr-tail handling in `transcribeAudio` still applies.- throw wrapWhisperTimeoutError(err, {- effectiveTimeoutMs: whisperTimeoutMs,- model,- wasOverride: options?.timeoutMs != null,- });+ // `--timeout` even exists. Any other failure names the build, model and stderr.+ if (isWhisperTimeoutError(err) || stoppedByCancelSignal(err as { signal?: string })) {+ throw wrapWhisperTimeoutError(err, {+ effectiveTimeoutMs: whisperTimeoutMs,+ model,+ wasOverride: options?.timeoutMs != null,+ });+ }+ throw await diagnoseWhisperFailure(err, whisper, modelPath, model);}// 6. Read and validate output@@ -668,6 +661,67 @@ async function runWhisper(await Promise.all([exited, read(stdout, options.onStdout), read(stderr, options.onStderr)]);}+/** Every absolute path becomes its file name, so a pasted error names no user or folder. */+export function withoutPaths(text: string): string {+ return text+ .split(homedir())+ .join("~")+ .replace(/(?<=^|[\s'"(=:[])(?:~|[A-Za-z]:)?[\\/](?:[^\s'"`\\/]+[\\/])*([^\s'"`\\/]+)/g, "$1");+}++function whisperVersion(executable: string): string {+ try {+ const said = execFileSync(executable, ["--version"], {+ encoding: "utf-8",+ stdio: ["ignore", "pipe", "pipe"],+ timeout: 5000,+ });+ return /version:?\s*(\S+)/i.exec(said)?.[1] ?? "unknown version";+ } catch {+ return "unknown version";+ }+}++async function describeModel(modelPath: string, model: string): Promise<string> {+ const bytes = statSync(modelPath, { throwIfNoEntry: false })?.size;+ if (bytes === undefined) return `model ${model}: file missing`;+ const sha256 = await sha256File(modelPath).catch(() => "unreadable");+ const pin = pinnedWhisperModel(model);+ const verdict = !pin+ ? "not a pinned model"+ : pin.bytes === bytes && pin.sha256 === sha256+ ? "matches the pinned file"+ : `damaged: expected ${pin.bytes} bytes, sha256 ${pin.sha256.slice(0, 12)}; delete it and re-run`;+ return `model ${model}: ${bytes} bytes, sha256 ${sha256.slice(0, 12)} (${verdict})`;+}++/** A whisper-cli failure that says which build, which model file and what whisper printed, for a bug report. */+async function diagnoseWhisperFailure(+ err: unknown,+ whisper: WhisperResult,+ modelPath: string,+ model: string,+): Promise<Error> {+ const failed = err as { code?: unknown; signal?: string | null; stderr?: unknown };+ const how = failed.signal+ ? `crashed (${failed.signal})`+ : typeof failed.code === "number"+ ? `exited with code ${failed.code}`+ : `failed: ${withoutPaths(err instanceof Error ? err.message : String(err))}`;+ const said = withoutPaths(String(failed.stderr ?? ""))+ .trim()+ .split("\n")+ .slice(-12)+ .join("\n");+ const lines = [+ `whisper-cli ${how}.`,+ `whisper-cli ${whisperVersion(whisper.executablePath)} (${whisper.source}: ${basename(whisper.executablePath)})`,+ await describeModel(modelPath, model),+ said ? `whisper-cli said:\n${said}` : "whisper-cli printed nothing.",+ ];+ return new Error(lines.join("\n"), { cause: err });+}+// ---------------------------------------------------------------------------// Timeout error discoverability// ---------------------------------------------------------------------------diff --git a/skills-manifest.json b/skills-manifest.jsonindex c2d4df14e0..e5dcbbe1f0 100644--- a/skills-manifest.json+++ b/skills-manifest.json@@ -54,7 +54,7 @@"files": 1},"media-use": {- "hash": "1a2c3f1c16ab3ef7",+ "hash": "5a8f4a204aa1e159","files": 113},"motion-graphics": {diff --git a/skills/media-use/references/operations.md b/skills/media-use/references/operations.mdindex 463238b5f7..c9c7c5900f 100644--- a/skills/media-use/references/operations.md+++ b/skills/media-use/references/operations.md@@ -121,7 +121,9 @@ VERIFIED on 24GB: accurate, ~3s (cached) for 8s audio. Parakeet covers 25European languages, English included. For other languages, or when parakeet-mlx is notinstalled, transcribe.mjs falls back to `hyperframes transcribe`, which uses theCLI's Parakeet when `models install parakeet` has run and whisper.cpp (99-languages) otherwise. `--engine parakeet|whisper` forces one. (Cohere+languages) otherwise. `--engine parakeet|whisper` forces one, except that a+`--language` other than English runs whisper whenever it is installed, since+Parakeet detects the language itself. (CohereTranscribe tops the leaderboard on paper but its mlx-audio quants producedgarbage and ran 40-70x slower on a Mac in testing, so it is not wired in.)