Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
/**
* Parakeet-TDT transcription engine (via parakeet-mlx on Apple Silicon).
*
* The higher-accuracy alternative to the whisper.cpp engine: NVIDIA Parakeet
* beats whisper-large-v3 on the Open ASR Leaderboard (~6.05% vs 7.44% avg WER,
* and 4.73% vs 5.96% on noisy audio where whisper-v3 hallucinates), while being
* 5-10x faster. Covers English + 25 European languages; whisper stays the
* multilingual fallback.
*
* Like the Kokoro TTS path, this is a user-installed local model: we DETECT it
* and, if absent, tell the user how to enable it (no auto-install). parakeet-mlx
* emits sub-word TOKENS; we merge them into the word timestamps the rest of the
* pipeline consumes.
*/
import { execFileSync } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { homedir, tmpdir } from "node:os";
import { basename, extname, join } from "node:path";
import { stoppableExecFile } from "../utils/stoppableExecFile.js";
import { TRANSCRIPT_FILE } from "./transcriptFile.js";
import type { Word } from "./normalize.js";
import type { TranscribeProgress, RunnerResult } from "./transcribe.js";
/** The model name `transcribe --json` reports for every Parakeet runner. */
export const PARAKEET_MODEL_LABEL = "parakeet-tdt-0.6b-v3";
export const PARAKEET_INSTALL_COMMAND = "hyperframes models install parakeet";
/** Stdout/stderr line prefixes of the sherpa-onnx decode worker. */
export const SHERPA_RESULT_PREFIX = "HYPERFRAMES_PARAKEET_RESULT:";
export const SHERPA_ERROR_PREFIX = "HYPERFRAMES_PARAKEET_ERROR:";
export const SHERPA_WINDOW_PREFIX = "HYPERFRAMES_PARAKEET_WINDOW:";
/** Parakeet TDT 0.6B v3's languages, as listed on its model card. */
export const PARAKEET_LANGUAGES =
"en es fr de bg hr cs da nl et fi el hu it lv lt mt pl pt ro sk sl sv ru uk";
/** True when no language was asked for, or Parakeet transcribes it ("pt-BR" counts as pt). */
export function parakeetSpeaks(language?: string): boolean {
const base = language?.trim().toLowerCase().split(/[-_]/, 1)[0];
return !base || PARAKEET_LANGUAGES.split(" ").includes(base);
}
const DEFAULT_MODEL = "mlx-community/parakeet-tdt-0.6b-v3";
const PARAKEET_INSTALL =
"uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx";
/** Verify a candidate binary actually runs (mirrors the --version gate on
* HYPERFRAMES_PYTHON) so a stale $HYPERFRAMES_PARAKEET path can't shadow a
* working install on PATH. */
function isRunnable(bin: string): boolean {
try {
execFileSync(bin, ["--help"], { stdio: ["ignore", "ignore", "ignore"], timeout: 10000 });
return true;
} catch {
return false;
}
}
/** Locate the `parakeet-mlx` runner: env override, the documented venv, then PATH. */
export function findParakeet(): string | undefined {
const candidates = [
process.env.HYPERFRAMES_PARAKEET,
join(homedir(), ".venvs", "parakeet", "bin", "parakeet-mlx"),
].filter((p): p is string => Boolean(p));
for (const path of candidates) {
if (existsSync(path) && isRunnable(path)) return path;
}
try {
const which = process.platform === "win32" ? "where" : "which";
const out = execFileSync(which, ["parakeet-mlx"], {
encoding: "utf-8",
stdio: ["ignore", "pipe", "ignore"],
timeout: 5000,
});
const first = out
.split(/\r?\n/)
.map((s) => s.trim())
.find(Boolean);
if (first && existsSync(first)) return first;
} catch {
// not on PATH
}
return undefined;
}
interface ParakeetToken {
text?: string;
start?: number;
end?: number;
}
interface ParakeetJson {
text?: string;
sentences?: { tokens?: ParakeetToken[] }[];
}
/**
* Merge Parakeet's sub-word tokens (" H", "ello", ...) into words on the space
* boundary: a token starting with a space (or the first token) begins a word;
* the rest append. Produces the { text, start, end } words the pipeline uses.
*/
function tokenBounds(token: ParakeetToken): { text: string; start: number; end: number } {
const text = typeof token.text === "string" ? token.text : "";
const start = typeof token.start === "number" ? token.start : 0;
const end = typeof token.end === "number" ? token.end : start;
return { text, start, end };
}
export function mergeTokensToWords(parakeet: ParakeetJson): Word[] {
const words: Word[] = [];
for (const sentence of parakeet.sentences ?? []) {
for (const token of sentence.tokens ?? []) {
const { text, start, end } = tokenBounds(token);
if (text.startsWith(" ") || words.length === 0) {
words.push({ text: text.trim(), start, end });
} else {
const w = words[words.length - 1]!;
w.text += text;
w.end = end;
}
}
}
return words.filter((w) => w.text.length > 0);
}
/** One decoded window from sherpa-onnx: token times are relative to the window's `offset` (s). */
export interface SherpaWindow {
offset: number;
tokens: string[];
timestamps: number[];
durations?: number[];
}
const roundToMs = (seconds: number) => Math.round(seconds * 1000) / 1000;
export function mergeWindowsToWords(windows: SherpaWindow[]): Word[] {
return mergeTokensToWords({
sentences: windows.map((w) => ({
tokens: w.tokens.map((text, i) => {
const start = w.offset + (w.timestamps[i] ?? 0);
return { text, start: roundToMs(start), end: roundToMs(start + (w.durations?.[i] ?? 0)) };
}),
})),
});
}
function meanEnergy(samples: Float32Array, from: number, to: number): number {
let energy = 0;
for (let i = from; i < to; i++) energy += samples[i]! * samples[i]!;
return energy / Math.max(1, to - from);
}
/**
* Stretches (s) of at least 1 s without tokens that are about as loud as the window: skipped speech.
* A window with no tokens at all is music or noise, not a skip.
*/
export function droppedSpeechGaps(
samples: Float32Array,
sampleRate: number,
decoded: Omit<SherpaWindow, "offset">,
): [number, number][] {
if (decoded.tokens.length === 0) return [];
const loud = 0.3 * meanEnergy(samples, 0, samples.length);
const { timestamps, durations } = decoded;
const gapStarts = [0, ...timestamps.map((t, i) => t + (durations?.[i] ?? 0))];
const gapEnds = [...timestamps, samples.length / sampleRate];
const at = (seconds: number) => Math.round(seconds * sampleRate);
return gapStarts
.map((from, i): [number, number] => [from, gapEnds[i]!])
.filter(([from, to]) => to - from >= 1 && meanEnergy(samples, at(from), at(to)) > loud);
}
/** Number of trailing `a` entries that equal the leading `b` entries. */
function overlap(a: string[], b: string[]): number {
for (let n = Math.min(a.length, b.length); n > 0; n--) {
if (a.slice(-n).every((text, i) => text === b[i])) return n;
}
return 0;
}
interface Token {
text: string;
start: number;
duration: number;
}
/** Tokens grouped into words: a token with a leading space starts one. */
function wordsOf(tokens: Token[]): Token[][] {
const words: Token[][] = [];
for (const t of tokens) {
if (t.text.startsWith(" ") || words.length === 0) words.push([t]);
else words.at(-1)!.push(t);
}
return words;
}
const letters = (word: Token[]) =>
word
.map((t) => t.text)
.join("")
.toLowerCase()
.replace(/[^\p{L}\p{N}]/gu, "");
/** How far from a gap edge the patch may re-hear the neighbouring word (measured: 0.26 s). */
const EDGE_SECONDS = 0.3;
/**
* Adds the `patch` words, decoded from `offset` s into the window, that start inside the gap, minus
* the gap's neighbours re-heard near its edges (compared as whole words, by letters).
*/
export function spliceGap(
decoded: Omit<SherpaWindow, "offset">,
patch: Omit<SherpaWindow, "offset">,
offset: number,
[from, to]: [number, number],
): Omit<SherpaWindow, "offset"> {
const tokens = (w: Omit<SherpaWindow, "offset">, shift: number): Token[] =>
w.tokens.map((text, i) => ({
text,
start: shift + w.timestamps[i]!,
duration: w.durations?.[i] ?? 0,
}));
const kept = wordsOf(tokens(decoded, 0));
const added = wordsOf(tokens(patch, offset)).filter(
([first]) => first!.start >= from && first!.start < to,
);
const before = kept.filter(([first]) => first!.start < from).map(letters);
const after = kept.filter(([first]) => first!.start >= to).map(letters);
const head = added.filter(([first]) => first!.start < from + EDGE_SECONDS).map(letters);
const tail = added.filter(([first]) => first!.start >= to - EDGE_SECONDS).map(letters);
const fresh = added
.slice(overlap(before, head), added.length - overlap(tail, after))
.map(([first, ...rest]) => [{ ...first!, text: ` ${first!.text.trimStart()}` }, ...rest]);
const merged = [...kept, ...fresh].sort((a, b) => a[0]!.start - b[0]!.start).flat();
return {
tokens: merged.map((t) => t.text),
timestamps: merged.map((t) => t.start),
durations: merged.map((t) => t.duration),
};
}
function quietestBlockCenter(samples: Float32Array, lo: number, hi: number, block: number): number {
let best = hi;
let bestEnergy = Infinity;
for (let s = lo; s + block <= hi; s += block) {
const energy = meanEnergy(samples, s, s + block);
if (energy < bestEnergy) {
bestEnergy = energy;
best = s + block / 2;
}
}
return best;
}
/**
* Window boundaries (sample indices, first 0, last samples.length) about every `windowSeconds`,
* each moved to the quietest 100 ms within ±5 s, so no cut lands inside a word.
*/
export function silenceCuts(
samples: Float32Array,
sampleRate: number,
windowSeconds = 60,
): number[] {
const window = windowSeconds * sampleRate;
const slack = 5 * sampleRate;
const cuts = [0];
while (cuts.at(-1)! + window < samples.length) {
const target = cuts.at(-1)! + window;
const hi = Math.min(target + slack, samples.length);
cuts.push(quietestBlockCenter(samples, target - slack, hi, sampleRate / 10));
}
if (samples.length > 0) cuts.push(samples.length);
return cuts;
}
interface ParakeetOptions {
model?: string;
onProgress?: (message: string) => void;
onEvent?: (event: TranscribeProgress) => void;
signal?: AbortSignal;
}
/** Transcribe with Parakeet and write `transcript.json` (Word[]) into `dir`. */
export async function transcribeWithParakeet(
inputPath: string,
dir: string,
options?: ParakeetOptions,
): Promise<RunnerResult> {
const runner = findParakeet();
if (!runner) {
throw new Error(
`parakeet-mlx not found. Enable the Parakeet engine with:\n ${PARAKEET_INSTALL}\n(or use --engine whisper)`,
);
}
const model = options?.model ?? DEFAULT_MODEL;
// First run pulls the model from HuggingFace (~600MB) — cue it so the wait
// doesn't read as a hang. HF caches at ~/.cache/huggingface/hub/models--<slug>.
const cached = existsSync(
join(homedir(), ".cache", "huggingface", "hub", `models--${model.replace(/\//g, "--")}`),
);
options?.onProgress?.(
cached ? "Transcribing with Parakeet..." : "Downloading Parakeet model (first run, ~600MB)...",
);
const workDir = mkdtempSync(join(tmpdir(), "hyperframes-parakeet-"));
try {
const argv = [inputPath, "--model", model, "--output-format", "json", "--output-dir", workDir];
options?.onEvent?.({
type: "progress",
phase: "transcription",
model: PARAKEET_MODEL_LABEL,
status: "started",
});
await stoppableExecFile(runner, argv, {
stdio: ["ignore", "pipe", "pipe"],
timeout: 1_800_000,
signal: options?.signal,
});
const produced = join(workDir, `${basename(inputPath, extname(inputPath))}.json`);
if (!existsSync(produced)) throw new Error("Parakeet did not produce output.");
const result = writeParakeetTranscript(
dir,
mergeTokensToWords(JSON.parse(readFileSync(produced, "utf-8")) as ParakeetJson),
);
options?.onEvent?.({
type: "progress",
phase: "transcription",
model: result.model,
status: "completed",
});
return result;
} finally {
rmSync(workDir, { recursive: true, force: true });
}
}
export function writeParakeetTranscript(dir: string, words: Word[]): RunnerResult {
mkdirSync(dir, { recursive: true });
const transcriptPath = join(dir, TRANSCRIPT_FILE);
writeFileSync(transcriptPath, JSON.stringify(words, null, 2));
const durationSeconds = words.length > 0 ? words[words.length - 1]!.end : 0;
return {
model: PARAKEET_MODEL_LABEL,
detectedLanguage: null,
transcriptPath,
wordCount: words.length,
durationSeconds,
speechOnsetSeconds: null,
};
}