Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
feat(cli): models list shows which speech models transcribe can use (#5323)
Miguel Ángel0576b24c75a6
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdxindex ac8bf8c599..b9a3f30569 100644--- a/docs/packages/cli.mdx+++ b/docs/packages/cli.mdx@@ -309,14 +309,21 @@ cues with no internal spaces — single-word or CJK captions.### `models`-Download an on-device model. Only asked-for installs happen; nothing downloads+List or download on-device models. Only asked-for installs happen; nothing downloadson first use.```bash+npx hyperframes models list # Parakeet and any downloaded whisper models+npx hyperframes models list --json # { ok, models: [{ engine, model, installed, runner, path }] }npx hyperframes models install parakeet # progress on stderrnpx hyperframes models install parakeet --json # one JSON result```+`list` reports Parakeet as installed only when `transcribe` would use it, through+the CLI's own runtime or `parakeet-mlx` (`runner`), and gives the reason when this+machine cannot run it. Whisper models appear once+`transcribe` has downloaded them.+`parakeet` installs the sherpa-onnx runtime from npm and NVIDIA's Parakeet TDT0.6B v3 speech model (int8, about 640 MB, CC-BY-4.0) into`~/.cache/hyperframes`. Every file is checked against a pinned size anddiff --git a/packages/cli/src/commands/models.test.ts b/packages/cli/src/commands/models.test.tsindex f7c6ec7755..47af5a73c9 100644--- a/packages/cli/src/commands/models.test.ts+++ b/packages/cli/src/commands/models.test.ts@@ -1,3 +1,4 @@+import { runCommand } from "citty";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { CliRuntimeError, consumeCommandResult } from "../utils/commandResult.js";@@ -9,9 +10,17 @@ const sherpa = {sherpaUnsupportedReason: vi.fn(),installSherpaRuntime: vi.fn(),ensureParakeetModel: vi.fn(),+ sherpaParakeetInstalled: vi.fn(),DecodeCancelled: class extends Error {},};vi.mock("../whisper/sherpa.js", () => sherpa);+const manager = { listWhisperModels: vi.fn() };+vi.mock("../whisper/manager.js", () => manager);+const findParakeet = vi.hoisted(() => vi.fn());+vi.mock("../whisper/parakeet.js", async (importOriginal) => ({+ ...(await importOriginal<typeof import("../whisper/parakeet.js")>()),+ findParakeet,+}));// The command's Ctrl-C scope, driven by the test instead of a real signal.let cancel = new AbortController();@@ -129,3 +138,64 @@ describe("models install parakeet --json", () => {expect(sherpa.installSherpaRuntime).toHaveBeenCalledWith({ signal: cancel.signal });});});++describe("models list", () => {+ const smallEn = { model: "small.en", path: "/cache/whisper/models/ggml-small.en.bin" };++ beforeEach(() => {+ vi.clearAllMocks();+ consumeCommandResult();+ vi.spyOn(console, "log").mockImplementation(() => {});+ sherpa.sherpaUnsupportedReason.mockReturnValue(null);+ sherpa.sherpaParakeetInstalled.mockReturnValue(true);+ findParakeet.mockReturnValue(undefined);+ manager.listWhisperModels.mockReturnValue([smallEn]);+ });+ afterEach(() => vi.restoreAllMocks());++ const listJson = async () => {+ await runCommand(modelsCmd, { rawArgs: ["list", "--json"] });+ expect(consumeCommandResult().exitCode).toBe(0);+ return JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0]));+ };++ it("parses `models list --json` without a model name and prints the inventory", async () => {+ expect(await listJson()).toEqual({+ ok: true,+ models: [+ {+ engine: "parakeet",+ model: "parakeet-tdt-0.6b-v3",+ installed: true,+ runner: "sherpa",+ path: sherpa.PARAKEET_MODEL_DIR,+ },+ { engine: "whisper", model: "small.en", installed: true, path: smallEn.path },+ ],+ });+ });++ it("reports Parakeet as not installed, with the reason, where this system cannot run it", async () => {+ sherpa.sherpaUnsupportedReason.mockReturnValue("Parakeet needs glibc 2.32 or newer.");+ manager.listWhisperModels.mockReturnValue([]);+ expect((await listJson()).models).toEqual([+ expect.objectContaining({+ engine: "parakeet",+ installed: false,+ unsupported: "Parakeet needs glibc 2.32 or newer.",+ }),+ ]);+ expect(sherpa.sherpaParakeetInstalled).not.toHaveBeenCalled();+ });++ it("counts Parakeet as installed through parakeet-mlx, as transcribe would run it", async () => {+ sherpa.sherpaUnsupportedReason.mockReturnValue("Parakeet needs glibc 2.32 or newer.");+ findParakeet.mockReturnValue("/venv/bin/parakeet-mlx");+ expect((await listJson()).models[0]).toEqual({+ engine: "parakeet",+ model: "parakeet-tdt-0.6b-v3",+ installed: true,+ runner: "parakeet-mlx",+ });+ });+});diff --git a/packages/cli/src/commands/models.ts b/packages/cli/src/commands/models.tsindex 7fcee8c9f2..8a7a88fb5e 100644--- a/packages/cli/src/commands/models.ts+++ b/packages/cli/src/commands/models.ts@@ -5,13 +5,12 @@ import { c } from "../ui/colors.js";import { formatBytes } from "../ui/format.js";import { failCommand, setCommandExitCode } from "../utils/commandResult.js";import { createRenderCancellationScope } from "../utils/renderCancellation.js";-import { PARAKEET_MODEL_LABEL } from "../whisper/parakeet.js";+import { PARAKEET_INSTALL_COMMAND, PARAKEET_MODEL_LABEL } from "../whisper/parakeet.js";+import type { ParakeetRunner } from "../whisper/parakeetRunner.js";export const examples: Example[] = [- [- "Download the Parakeet speech model that transcribe uses",- "hyperframes models install parakeet",- ],+ ["List the speech models transcribe can use", "hyperframes models list --json"],+ ["Download the Parakeet speech model that transcribe uses", PARAKEET_INSTALL_COMMAND],];function fail(message: string, json: boolean): never {@@ -95,17 +94,73 @@ async function installParakeet(json: boolean): Promise<void> {}}+type ModelRow = {+ engine: "parakeet" | "whisper";+ model: string;+ installed: boolean;+ runner?: ParakeetRunner;+ path?: string;+ unsupported?: string;+};++function parakeetRow(+ runner: ParakeetRunner | null,+ unsupported: string | null,+ modelDir: string,+): ModelRow {+ const row: ModelRow = { engine: "parakeet", model: PARAKEET_MODEL_LABEL, installed: !!runner };+ if (runner) row.runner = runner;+ // parakeet-mlx keeps its model in its own cache; this is where models install puts ours.+ if (runner !== "parakeet-mlx") row.path = modelDir;+ if (!runner && unsupported) row.unsupported = unsupported;+ return row;+}++function describeState(m: ModelRow): string {+ if (!m.installed) return c.dim(m.unsupported ?? `not installed: ${PARAKEET_INSTALL_COMMAND}`);+ return c.success(m.runner === "parakeet-mlx" ? "installed (parakeet-mlx)" : "installed");+}++async function listModels(json: boolean): Promise<void> {+ const [sherpa, { parakeetRunner }, { listWhisperModels }] = await Promise.all([+ import("../whisper/sherpa.js"),+ import("../whisper/parakeetRunner.js"),+ import("../whisper/manager.js"),+ ]);+ const unsupported = sherpa.sherpaUnsupportedReason();+ const models: ModelRow[] = [+ parakeetRow(parakeetRunner({ unsupported }), unsupported, sherpa.PARAKEET_MODEL_DIR),+ ...listWhisperModels().map(({ model, path }) => ({+ engine: "whisper" as const,+ model,+ installed: true,+ path,+ })),+ ];+ if (json) {+ console.log(JSON.stringify({ ok: true, models }));+ return;+ }+ for (const m of models) {+ console.log(`${m.engine.padEnd(9)}${m.model.padEnd(22)}${describeState(m)}`);+ }+}+export default defineCommand({- meta: { name: "models", description: "Download on-device models (models install parakeet)" },+ meta: {+ name: "models",+ description: "List or download on-device models (models list, models install parakeet)",+ },args: {- action: { type: "positional", description: "install", required: true },- name: { type: "positional", description: "Model to install: parakeet", required: true },+ action: { type: "positional", description: "list or install", required: true },+ name: { type: "positional", description: "Model to install: parakeet", required: false },json: { type: "boolean", description: "Print one JSON result, no progress", default: false },},async run({ args }) {+ if (args.action === "list") return listModels(args.json);if (args.action !== "install" || args.name !== "parakeet") {fail(- `Unknown: models ${args.action} ${args.name}. Try: hyperframes models install parakeet`,+ `Unknown: models ${[args.action, args.name].filter(Boolean).join(" ")}. Try: hyperframes models list, or ${PARAKEET_INSTALL_COMMAND}`,args.json,);}diff --git a/packages/cli/src/commands/transcribe.ts b/packages/cli/src/commands/transcribe.tsindex 56f4856ab4..0f2bde7892 100644--- a/packages/cli/src/commands/transcribe.ts+++ b/packages/cli/src/commands/transcribe.ts@@ -6,7 +6,7 @@ import { defineCommand } from "citty";import type { Example } from "./_examples.js";import { existsSync, rmSync, writeFileSync } from "node:fs";import {- findParakeet,+ PARAKEET_INSTALL_COMMAND,PARAKEET_LANGUAGES,parakeetSpeaks,transcribeWithParakeet,@@ -31,6 +31,7 @@ import { resolve, join, extname, dirname } from "node:path";import * as clack from "@clack/prompts";import { c } from "../ui/colors.js";import { DEFAULT_MODEL, isWhisperUnavailable } from "../whisper/manager.js";+import type { ParakeetRunner } from "../whisper/parakeetRunner.js";// Minimum accepted value for `--timeout` / `HYPERFRAMES_TRANSCRIBE_TIMEOUT_MS`.// Kept out of `whisper/transcribe.ts` (avoids a top-level import into this@@ -280,15 +281,16 @@ async function exportTranscript(// Transcribe audio/video with whisper// ----------------------------------------------------------------------------type Runner = "sherpa" | "parakeet-mlx" | "whisper";--const PARAKEET_INSTALL_COMMAND = "hyperframes models install parakeet";+type Runner = ParakeetRunner | "whisper";/** auto and parakeet prefer sherpa-onnx, then parakeet-mlx, then whisper, in Parakeet's languages. */-function pickRunner(engine: string, sherpaUsable: () => boolean, language?: string): Runner {+function pickRunner(+ engine: string,+ parakeet: () => ParakeetRunner | null,+ language?: string,+): Runner {if (engine === "whisper" || !parakeetSpeaks(language)) return "whisper";- if (sherpaUsable()) return "sherpa";- return findParakeet() ? "parakeet-mlx" : "whisper";+ return parakeet() ?? "whisper";}/** When Parakeet fails, only auto falls back; an explicit --engine parakeet fails with the error. */@@ -311,13 +313,9 @@ async function transcribeAudio(const { transcribe } = await import("../whisper/transcribe.js");const { loadTranscript, patchCaptionHtml } = await import("../whisper/normalize.js");- const {- DecodeCancelled,- prepareSherpaWav,- sherpaParakeetInstalled,- sherpaUnsupportedReason,- transcribeWithSherpa,- } = await import("../whisper/sherpa.js");+ const { DecodeCancelled, prepareSherpaWav, sherpaUnsupportedReason, transcribeWithSherpa } =+ await import("../whisper/sherpa.js");+ const { parakeetRunner } = await import("../whisper/parakeetRunner.js");const { createRenderCancellationScope, stoppedByCancelSignal } =await import("../utils/renderCancellation.js");@@ -326,8 +324,7 @@ async function transcribeAudio(failWith(`Unknown --engine: ${opts.engine}. Use auto, parakeet, or whisper.`, !!opts.json);}const unsupported = sherpaUnsupportedReason();- const sherpaUsable = () => !unsupported && sherpaParakeetInstalled();- let runner = pickRunner(engine, sherpaUsable, opts.language);+ let runner = pickRunner(engine, () => parakeetRunner({ unsupported }), opts.language);if (engine === "parakeet" && runner === "whisper") {failWith(!parakeetSpeaks(opts.language)@@ -394,7 +391,11 @@ async function transcribeAudio(const reason = normalizeErrorMessage(err).replace(/\.+$/, "");const parakeetError = `Parakeet failed: ${reason}. To repair it, run: ${PARAKEET_INSTALL_COMMAND}`;if (!parakeetFallsBack(engine)) throw new Error(parakeetError);- runner = pickRunner(engine, () => false, opts.language);+ runner = pickRunner(+ engine,+ () => parakeetRunner({ unsupported, skipSherpa: true }),+ opts.language,+ );spin?.clear();console.error(c.warn(`${parakeetError}. Using ${runner} for this run.`));spin?.start(`Transcribing with ${label(runner)}...`);diff --git a/packages/cli/src/help.ts b/packages/cli/src/help.tsindex 686a1b901a..68e2539543 100644--- a/packages/cli/src/help.ts+++ b/packages/cli/src/help.ts@@ -93,7 +93,7 @@ const GROUPS: Group[] = ["transcribe","Transcribe audio/video to word-level timestamps, or import an existing transcript",],- ["models", "Download on-device models (models install parakeet)"],+ ["models", "List or download on-device models (models list, models install parakeet)"],["tts", "Generate speech audio from text using a local AI model (Kokoro-82M)"],["remove-background", "Remove background from a video or image to produce transparent media"],],diff --git a/packages/cli/src/whisper/manager.list.test.ts b/packages/cli/src/whisper/manager.list.test.tsnew file mode 100644index 0000000000..1ddd36f83b--- /dev/null+++ b/packages/cli/src/whisper/manager.list.test.ts@@ -0,0 +1,36 @@+import { mkdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";+import { join } from "node:path";+import { afterEach, expect, it, vi } from "vitest";+import { listWhisperModels } from "./manager.js";++const home = vi.hoisted(() => `${process.env.TMPDIR ?? "/tmp"}/hf-whisper-list-${process.pid}`);+vi.mock("node:os", async (original) => ({+ ...(await original<typeof import("node:os")>()),+ homedir: () => home,+}));+const modelsDir = join(home, ".cache", "hyperframes", "whisper", "models");++afterEach(() => rmSync(home, { recursive: true, force: true }));++it("lists nothing before any whisper model was downloaded", () => {+ expect(listWhisperModels()).toEqual([]);+});++it("lists each downloaded ggml model by the name ensureModel takes, and nothing else", () => {+ mkdirSync(modelsDir, { recursive: true });+ for (const file of ["ggml-small.en.bin", "ggml-base.bin", "notes.txt", "ggml-tiny.bin.part"])+ writeFileSync(join(modelsDir, file), "");+ expect(listWhisperModels()).toEqual([+ { model: "base", path: join(modelsDir, "ggml-base.bin") },+ { model: "small.en", path: join(modelsDir, "ggml-small.en.bin") },+ ]);+});++it.skipIf(process.platform === "win32")(+ "skips a dangling link that ensureModel would download again",+ () => {+ mkdirSync(modelsDir, { recursive: true });+ symlinkSync(join(home, "gone.bin"), join(modelsDir, "ggml-tiny.bin"));+ expect(listWhisperModels()).toEqual([]);+ },+);diff --git a/packages/cli/src/whisper/manager.ts b/packages/cli/src/whisper/manager.tsindex e9cf61304f..63731ed0a0 100644--- a/packages/cli/src/whisper/manager.ts+++ b/packages/cli/src/whisper/manager.ts@@ -1,6 +1,6 @@// fallow-ignore-file code-duplication complexityimport { execFileSync } from "node:child_process";-import { existsSync, mkdirSync, rmSync } from "node:fs";+import { existsSync, mkdirSync, readdirSync, rmSync } from "node:fs";import { homedir, platform } from "node:os";import { join } from "node:path";import { findFFmpeg } from "../browser/ffmpeg.js";@@ -150,6 +150,17 @@ export function findWhisper(): WhisperResult | undefined {return findFromEnv() ?? findFromSystem() ?? findBuiltBinary();}+export function listWhisperModels(): { model: string; path: string }[] {+ if (!existsSync(MODELS_DIR)) return [];+ return readdirSync(MODELS_DIR)+ .flatMap((file) => {+ const model = /^ggml-(.+)\.bin$/.exec(file)?.[1];+ const path = join(MODELS_DIR, file);+ return model && existsSync(path) ? [{ model, path }] : [];+ })+ .sort((a, b) => a.model.localeCompare(b.model));+}+export function getInstallInstructions(): string {if (platform() === "darwin") {return "brew install whisper-cpp";diff --git a/packages/cli/src/whisper/parakeet.ts b/packages/cli/src/whisper/parakeet.tsindex 14809d5b40..a46eb780a8 100644--- a/packages/cli/src/whisper/parakeet.ts+++ b/packages/cli/src/whisper/parakeet.ts@@ -22,6 +22,7 @@ import type { TranscribeProgress, TranscribeResult } from "./transcribe.js";/** The model name `transcribe --json` reports for every Parakeet runner. */export const PARAKEET_MODEL_LABEL = "parakeet-tdt-0.6b-v3";+export const PARAKEET_INSTALL_COMMAND = "hyperframes models install parakeet";/** Stdout/stderr line prefixes of the sherpa-onnx decode worker. */export const SHERPA_RESULT_PREFIX = "HYPERFRAMES_PARAKEET_RESULT:";export const SHERPA_ERROR_PREFIX = "HYPERFRAMES_PARAKEET_ERROR:";diff --git a/packages/cli/src/whisper/parakeetRunner.ts b/packages/cli/src/whisper/parakeetRunner.tsnew file mode 100644index 0000000000..2c8dce8431--- /dev/null+++ b/packages/cli/src/whisper/parakeetRunner.ts@@ -0,0 +1,13 @@+import { findParakeet } from "./parakeet.js";+import { sherpaParakeetInstalled, sherpaUnsupportedReason } from "./sherpa.js";++export type ParakeetRunner = "sherpa" | "parakeet-mlx";++/** Transcribe's Parakeet runner here, sherpa-onnx first; `skipSherpa` once sherpa failed. */+export function parakeetRunner({+ unsupported = sherpaUnsupportedReason(),+ skipSherpa = false,+}: { unsupported?: string | null; skipSherpa?: boolean } = {}): ParakeetRunner | null {+ if (!skipSherpa && !unsupported && sherpaParakeetInstalled()) return "sherpa";+ return findParakeet() ? "parakeet-mlx" : null;+}