Skip to content

Write HTML. Render video. Built for agents.

16.3 KB
Raw Blame
import { execFile, spawn, type ExecFileException } from "node:child_process";
import { randomUUID } from "node:crypto";
import { mkdirSync, renameSync, rmSync, statSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import { createInterface } from "node:readline";
import { fileURLToPath } from "node:url";
import { downloadToFile } from "../cloud/download.js";
import { verifies, type ModelFile } from "./modelFile.js";
import { stoppedByCancelSignal } from "../utils/renderCancellation.js";
import {
  CACHE_DIR,
  PINNED_PACKAGES,
  pinnedPackageBesideCli,
  installedPackagePath,
  install,
  isInstalled,
  runNpm,
  sweepStaleStaging,
} from "../utils/optionalPackages.js";
import {
  mergeWindowsToWords,
  SHERPA_ERROR_PREFIX,
  SHERPA_RESULT_PREFIX,
  SHERPA_WINDOW_PREFIX,
  PARAKEET_MODEL_LABEL,
  writeParakeetTranscript,
  type SherpaWindow,
} from "./parakeet.js";
import { emitWords } from "./progress.js";
import {
  getPreparedWavDurationSeconds,
  prepareWav,
  type TranscribeProgress,
  type RunnerResult,
} from "./transcribe.js";

const RUNTIME = "sherpa-onnx-node";
const RUNTIME_VERSION = PINNED_PACKAGES[RUNTIME];
/** Per platform and arch, like the native package inside, so Rosetta never shadows arm64. */
export const SHERPA_RUNTIME_DIR = join(
  CACHE_DIR,
  `${RUNTIME}@${RUNTIME_VERSION}-${process.platform}-${process.arch}`,
);

const MODEL_REPO = "csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8";
const MODEL_REVISION = "2bda32ec70b097a55adaa07d9a7173915b43cc78";
export const PARAKEET_MODEL_DIR = join(
  homedir(),
  ".cache",
  "hyperframes",
  "parakeet",
  "parakeet-tdt-0.6b-v3-int8",
);

export type { ModelFile };

/** Order matters: encoder, decoder, joiner, tokens (see recognizerConfig). */
const PARAKEET_MODEL_FILES: readonly ModelFile[] = [
  {
    name: "encoder.int8.onnx",
    bytes: 652_184_281,
    sha256: "acfc2b4456377e15d04f0243af540b7fe7c992f8d898d751cf134c3a55fd2247",
  },
  {
    name: "decoder.int8.onnx",
    bytes: 11_845_275,
    sha256: "179e50c43d1a9de79c8a24149a2f9bac6eb5981823f2a2ed88d655b24248db4e",
  },
  {
    name: "joiner.int8.onnx",
    bytes: 6_355_277,
    sha256: "3164c13fc2821009440d20fcb5fdc78bff28b4db2f8d0f0b329101719c0948b3",
  },
  {
    name: "tokens.txt",
    bytes: 93_939,
    sha256: "d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d",
  },
];

/** At least 30 minutes, twice the audio length for slow CPUs; whole ms, within Node's timer limit. */
export const decodeTimeoutMs = (audioSeconds: number) =>
  Math.min(2 ** 31 - 1, Math.max(1_800_000, Math.ceil(audioSeconds * 2000)));

/** The platforms sherpa-onnx-node publishes a native package for. */
const SUPPORTED_TARGETS = [
  "darwin-arm64",
  "darwin-x64",
  "linux-x64",
  "linux-arm64",
  "win32-x64",
  "win32-ia32",
];

interface Host {
  platform: string;
  arch: string;
  glibc?: string;
}

function currentHost(): Host {
  const report = process.report.getReport() as { header?: { glibcVersionRuntime?: string } };
  return {
    platform: process.platform,
    arch: process.arch,
    glibc: report.header?.glibcVersionRuntime,
  };
}

/** Why sherpa-onnx cannot run here, or null. Its Linux prebuilt needs glibc 2.32 (so no musl). */
export function sherpaUnsupportedReason(host: Host = currentHost()): string | null {
  const target = `${host.platform}-${host.arch}`;
  if (!SUPPORTED_TARGETS.includes(target)) {
    return `Parakeet runs on ${SUPPORTED_TARGETS.join(", ")}; this system is ${target}.`;
  }
  if (host.platform !== "linux") return null;
  if (!host.glibc) return "Parakeet needs glibc 2.32 or newer; this Linux has no glibc (musl?).";
  const [major = 0, minor = 0] = host.glibc.split(".").map(Number);
  if (major > 2 || (major === 2 && minor >= 32)) return null;
  return `Parakeet needs glibc 2.32 or newer; this system has glibc ${host.glibc}.`;
}

const nativePackageName = (platform: string, arch: string) =>
  `sherpa-onnx-${platform === "win32" ? "win" : platform}-${arch}`;

const LOAD_RUNTIME = `try {
  require(process.env.HF_SHERPA_RUNTIME_PATH);
} catch (e) {
  process.stderr.write(String(e && e.message).replace(/\\s+/g, " "));
  process.exitCode = 1;
}`;

/** Native bindings are probed only in a child, as they are loaded in the decode worker. */
function runtimeLoadError(
  runtimePath: string,
  timeoutMs: number,
  signal?: AbortSignal,
): Promise<string | null> {
  if (signal?.aborted)
    return Promise.reject(new DecodeCancelled("Parakeet runtime check cancelled"));
  return new Promise((resolve, reject) => {
    const probe = spawn(process.execPath, ["-e", LOAD_RUNTIME], {
      env: { ...process.env, HF_SHERPA_RUNTIME_PATH: runtimePath },
      stdio: ["ignore", "ignore", "pipe"],
      timeout: timeoutMs,
      signal,
    });
    let stderr = "";
    probe.stderr.setEncoding("utf8");
    probe.stderr.on("data", (chunk: string) => {
      stderr = bounded(stderr + chunk);
    });
    probe.on("error", (error) => {
      if (signal?.aborted) reject(new DecodeCancelled("Parakeet runtime check cancelled"));
      else resolve(bounded(error.message));
    });
    probe.on("close", (code, childSignal) => {
      if (
        signal?.aborted ||
        stoppedByCancelSignal({ code, signal: childSignal, killed: probe.killed })
      ) {
        reject(new DecodeCancelled("Parakeet runtime check cancelled"));
        return;
      }
      if (code === 0 && !probe.killed) {
        resolve(null);
        return;
      }
      let reason: string;
      if (probe.killed) reason = `loading it timed out after ${timeoutMs / 1000} s`;
      else if (childSignal) reason = `loading it crashed (${childSignal})`;
      else reason = `it exited ${code} with no output`;
      resolve(bounded(stderr.trim() || reason));
    });
  });
}

type RuntimeSelection =
  | { status: "ready"; path: string }
  | { status: "unavailable"; error: string };

async function cachedRuntime(
  dir: string,
  timeoutMs: number,
  signal?: AbortSignal,
): Promise<RuntimeSelection> {
  let path: string | null;
  try {
    path = installedPackagePath(dir, RUNTIME);
  } catch (err) {
    return {
      status: "unavailable",
      error: bounded(err instanceof Error ? err.message : String(err)),
    };
  }
  if (path === null)
    return { status: "unavailable", error: `${RUNTIME} is not installed in ${dir}` };
  const error = await runtimeLoadError(path, timeoutMs, signal);
  return error === null ? { status: "ready", path } : { status: "unavailable", error };
}

async function selectRuntime(
  dir: string,
  cliUrl: string,
  timeoutMs = 60_000,
  signal?: AbortSignal,
): Promise<RuntimeSelection> {
  if (signal?.aborted) throw new DecodeCancelled("Parakeet runtime check cancelled");
  const copy = pinnedPackageBesideCli(RUNTIME, cliUrl);
  if (copy !== null) {
    const path = copy.path;
    const error = await runtimeLoadError(path, timeoutMs, signal);
    if (error === null) return { status: "ready", path };
    console.error(
      `Parakeet runtime at ${path.replace(/\s+/g, " ")} did not load: ${error.replace(/\s+/g, " ")}`,
    );
  }
  return cachedRuntime(dir, timeoutMs, signal);
}

/** Returns the load failure after checking the pinned copy beside the CLI, then its cache. */
export async function sherpaRuntimeLoadError(
  dir = SHERPA_RUNTIME_DIR,
  timeoutMs = 60_000,
  cliUrl = import.meta.url,
  signal?: AbortSignal,
): Promise<string | null> {
  const selected = await selectRuntime(dir, cliUrl, timeoutMs, signal);
  return selected.status === "ready" ? null : selected.error;
}

/** The native package, pinned too: the runtime's own optionalDependencies accept any 1.13.x. */
export function sherpaPlatformPackage(platform = process.platform, arch = process.arch): string {
  return `${nativePackageName(platform, arch)}@${RUNTIME_VERSION}`;
}

/** Installs if needed and returns the healthy entry selected for native child loading. */
export async function installSherpaRuntime({
  run = runNpm,
  signal,
  dir = SHERPA_RUNTIME_DIR,
  cliUrl = import.meta.url,
}: {
  run?: typeof runNpm;
  signal?: AbortSignal;
  dir?: string;
  cliUrl?: string;
} = {}): Promise<{ installed: boolean; runtimePath: string }> {
  const present = await selectRuntime(dir, cliUrl, 60_000, signal);
  if (present.status === "ready") return { installed: false, runtimePath: present.path };
  // install() keeps any dir holding the runtime manifest, so a broken one goes first.
  rmSync(dir, { recursive: true, force: true });
  const native = sherpaPlatformPackage();
  await install(dir, RUNTIME, RUNTIME_VERSION, (args) => run([...args, native], signal));
  const selected = await cachedRuntime(dir, 60_000, signal);
  if (selected.status === "unavailable") {
    throw new Error(
      `The sherpa-onnx runtime was reinstalled but still does not load (${selected.error}). Use --engine whisper for now.`,
    );
  }
  return { installed: true, runtimePath: selected.path };
}

/**
 * Model sizes only (install verified the hashes). The runtime counts once its manifest is there, even
 * if broken: transcribe must then fail with the repair, not quietly pick whisper.
 */
export function sherpaParakeetInstalled(cliUrl = import.meta.url): boolean {
  return (
    (pinnedPackageBesideCli(RUNTIME, cliUrl) !== null ||
      isInstalled(SHERPA_RUNTIME_DIR, RUNTIME)) &&
    PARAKEET_MODEL_FILES.every(
      (f) =>
        statSync(join(PARAKEET_MODEL_DIR, f.name), { throwIfNoEntry: false })?.size === f.bytes,
    )
  );
}

interface EnsureModelOptions {
  dir?: string;
  files?: readonly ModelFile[];
  download?: typeof downloadToFile;
  onBytes?: (done: number, total: number) => void;
  signal?: AbortSignal;
}

/**
 * Fetches each file that does not verify into a pid-named staging dir, checks size and sha256, then
 * renames it into place. False when all already verified.
 */
export async function ensureParakeetModel({
  dir = PARAKEET_MODEL_DIR,
  files = PARAKEET_MODEL_FILES,
  download = downloadToFile,
  onBytes,
  signal,
}: EnsureModelOptions = {}): Promise<boolean> {
  sweepStaleStaging(dir);
  const missing: ModelFile[] = [];
  for (const file of files) if (!(await verifies(join(dir, file.name), file))) missing.push(file);
  const total = missing.reduce((sum, f) => sum + f.bytes, 0);
  const staging = `${dir}.tmp-${process.pid}-${randomUUID().slice(0, 8)}`;
  let done = 0;
  try {
    for (const file of missing) {
      signal?.throwIfAborted();
      const dest = join(dir, file.name);
      const temp = join(staging, file.name);
      rmSync(dest, { force: true });
      const url = `https://huggingface.co/${MODEL_REPO}/resolve/${MODEL_REVISION}/${file.name}`;
      await download(url, temp, {
        signal,
        onProgress: (bytes) => onBytes?.(done + bytes, total),
      }).catch((err: unknown) => {
        if (signal?.aborted) throw err;
        const why = err instanceof Error ? err.message : String(err);
        throw new Error(
          `Could not download ${file.name} from huggingface.co (${why}). Check your network and re-run.`,
          { cause: err },
        );
      });
      if (!(await verifies(temp, file))) {
        throw new Error(
          `${file.name} did not match its pinned size and sha256, so it was discarded. Re-run to retry.`,
        );
      }
      mkdirSync(dir, { recursive: true });
      renameSync(temp, dest);
      done += file.bytes;
    }
  } finally {
    rmSync(staging, { recursive: true, force: true });
  }
  return missing.length > 0;
}

function recognizerConfig(): object {
  const [encoder, decoder, joiner, tokens] = PARAKEET_MODEL_FILES.map((f) =>
    join(PARAKEET_MODEL_DIR, f.name),
  );
  return {
    featConfig: { sampleRate: 16000, featureDim: 80 },
    modelConfig: {
      transducer: { encoder, decoder, joiner },
      tokens,
      numThreads: 4,
      provider: "cpu",
      debug: 0,
      modelType: "nemo_transducer",
    },
    decodingMethod: "greedy_search",
  };
}

/** Ctrl-C or a SIGTERM stopped a Parakeet child: the user's cancel, never a reason to fall back. */
export class DecodeCancelled extends Error {}

const MAX_REASON_CHARS = 600;
const bounded = (why: string) =>
  why.length > MAX_REASON_CHARS ? `${why.slice(0, MAX_REASON_CHARS)}…` : why;

function failureReason(err: ExecFileException | null, stderr: string): string {
  const how = !err
    ? "exited without a result"
    : err.killed
      ? "was stopped (timeout or output limit)"
      : err.signal
        ? `crashed (${err.signal})`
        : `exited with code ${err.code}`;
  const lines = stderr
    .split("\n")
    .map((line) => line.trim())
    .filter(Boolean);
  const why =
    lines.find((line) => line.startsWith(SHERPA_ERROR_PREFIX))?.slice(SHERPA_ERROR_PREFIX.length) ??
    lines.at(-1);
  if (!why) return how;
  return `${how}: ${bounded(why)}`;
}

/** Decodes in a child process: onnxruntime aborts the whole process on some inputs, uncatchably. */
function decode(
  wavPath: string,
  runtimePath: string,
  signal: AbortSignal,
  onWindow?: (window: SherpaWindow, through: number) => void,
): Promise<SherpaWindow[]> {
  const sourceMode = import.meta.url.endsWith(".ts");
  const worker = new URL(sourceMode ? "./sherpaWorker.ts" : "./sherpaWorker.js", import.meta.url);
  const args = [...(sourceMode ? ["--import", "tsx"] : []), fileURLToPath(worker)];
  const input = { wavPath, runtimePath, config: recognizerConfig() };
  return new Promise((resolve, reject) => {
    const child = execFile(
      process.execPath,
      args,
      {
        env: { ...process.env, HYPERFRAMES_PARAKEET_INPUT: JSON.stringify(input) },
        maxBuffer: 256 * 1024 * 1024,
        timeout: decodeTimeoutMs(getPreparedWavDurationSeconds(wavPath) ?? 0),
        signal,
      },
      (err, stdout, stderr) => {
        if (signal.aborted || (err && stoppedByCancelSignal(err))) {
          reject(new DecodeCancelled("Transcription cancelled"));
          return;
        }
        const line = stdout.split("\n").find((l) => l.startsWith(SHERPA_RESULT_PREFIX));
        if (!err && line) resolve(JSON.parse(line.slice(SHERPA_RESULT_PREFIX.length)));
        else reject(new Error(`Parakeet decoder ${failureReason(err, stderr)}`));
      },
    );
    if (onWindow && child.stdout) {
      createInterface({ input: child.stdout }).on("line", (l) => {
        if (!l.startsWith(SHERPA_WINDOW_PREFIX)) return;
        let streamed: { window: SherpaWindow; through: number };
        try {
          streamed = JSON.parse(l.slice(SHERPA_WINDOW_PREFIX.length));
        } catch {
          return;
        }
        onWindow(streamed.window, streamed.through);
      });
    }
  });
}

/** Prepares the audio for Parakeet and its fallback alike; ffmpeg stopped by Ctrl-C is a cancel. */
export async function prepareSherpaWav(
  inputPath: string,
  onProgress?: (message: string) => void,
  signal?: AbortSignal,
): Promise<string> {
  try {
    return await prepareWav(inputPath, onProgress, signal);
  } catch (err) {
    if ((err as { cancelled?: boolean }).cancelled) {
      throw new DecodeCancelled("Transcription cancelled");
    }
    throw err;
  }
}

/** The caller owns the cancellation scope: it must already cover audio preparation. */
export async function transcribeWithSherpa(
  wavPath: string,
  dir: string,
  options: {
    signal: AbortSignal;
    onProgress?: (message: string) => void;
    runtimeDir?: string;
    cliUrl?: string;
    onEvent?: (event: TranscribeProgress) => void;
  },
): Promise<RunnerResult> {
  options.onProgress?.("Transcribing with Parakeet...");
  options.onEvent?.({
    type: "progress",
    phase: "transcription",
    model: PARAKEET_MODEL_LABEL,
    status: "started",
    durationSeconds: getPreparedWavDurationSeconds(wavPath),
  });
  if (options.signal.aborted) throw new DecodeCancelled("Transcription cancelled");
  const selected = await selectRuntime(
    options.runtimeDir ?? SHERPA_RUNTIME_DIR,
    options.cliUrl ?? import.meta.url,
    60_000,
    options.signal,
  );
  if (selected.status === "unavailable") {
    throw new Error(
      `Parakeet runtime does not load (${selected.error}). Run hyperframes models install parakeet to repair it, or use --engine whisper.`,
    );
  }
  const onEvent = options.onEvent;
  const windows = await decode(
    wavPath,
    selected.path,
    options.signal,
    onEvent &&
      ((window, through) =>
        emitWords(onEvent, PARAKEET_MODEL_LABEL, mergeWindowsToWords([window]), through)),
  );
  const result = writeParakeetTranscript(dir, mergeWindowsToWords(windows));
  options.onEvent?.({
    type: "progress",
    phase: "transcription",
    model: result.model,
    status: "completed",
  });
  return result;
}