Skip to content

Open source CapCut alternative.

TypeScript
8.2 KB
Raw Blame
import { spawn, type ChildProcess } from "node:child_process";
import { assertLocalRuntime } from "./local-only";
import { createJobRegistry } from "./jobRegistry";
import { ensureStt } from "./transcribe";
import { errorFields, UserError, type ErrorFields } from "../lib/userError";

// Live mic dictation for the AI chat composer. The browser captures the mic
// (it already holds that permission) and streams 16 kHz mono s16le PCM to the
// engine; this spawns `cut-stt --live`, forwards the PCM to its stdin, and
// exposes the evolving transcript. On-device only — nothing leaves the Mac.

export interface MicJob extends ErrorFields {
  id: string;
  status: "running" | "done" | "error" | "canceled";
  /** Latest transcript: the evolving partial while running, the final text once done. */
  text: string;
  proc: ChildProcess;
  /** Resolves once the process has emitted its final text (or died). */
  finished: Promise<void>;
  /** Reaps an orphaned session: a live dictation feeds PCM every ~250ms, so a
   * gap this long means the client is gone (a tab refresh that never cancelled). */
  idle?: ReturnType<typeof setTimeout>;
  /** When PCM last arrived — the liveness signal that separates a live
   * dictation (recent audio) from a refresh orphan (silence). */
  lastFeedAt: number;
  /** Set when stopMic ends the input: the session is flushing its final text,
   * so silence is expected — the reclaimer must not read it as staleness. */
  finalizeAt?: number;
}

// A live dictation feeds continuously; no audio for this long means the tab that
// owned it went away without cancelling. The session self-reaps so a refresh
// never leaves a zombie cut-stt blocking the next dictation.
const IDLE_MS = 15000;
// A session with PCM more recent than this is live (a live client feeds every
// ~250ms) — a new start must never kill it mid-speech. Anything staler is an
// orphan a new start may reclaim without waiting out the idle reaper.
const STALE_MS = 1500;
// How long a confirmed session may take to flush its final text. cut-stt bounds
// its own finish well inside this, so passing it means the process is gone.
const STOP_MS = 8000;
// A confirmed session flushing its final text feeds no PCM by design; stopMic
// waits out STOP_MS for the flush, so only past that is the process fair game.
const FINALIZE_MS = STOP_MS + 1000;
const { jobs, retire } = createJobRegistry<MicJob>("__cutMicJobs");

function clearIdle(job: MicJob): void {
  if (job.idle) clearTimeout(job.idle);
  job.idle = undefined;
}

/** (Re)arm the orphan watchdog: if no PCM arrives within IDLE_MS the session is
 * abandoned, so kill it. */
function armIdle(job: MicJob): void {
  clearIdle(job);
  job.idle = setTimeout(() => {
    if (job.status === "running") {
      job.status = "canceled";
      job.proc.kill("SIGKILL");
    }
  }, IDLE_MS);
  job.idle.unref();
}

/** Reap running sessions whose audio went silent (a refresh that never reached
 * cancel), and report whether a genuinely live one remains. A live dictation —
 * PCM within STALE_MS — is someone speaking right now; killing it would
 * silently truncate their speech, so the caller rejects instead. */
function reclaimStale(): { liveRemains: boolean } {
  let liveRemains = false;
  for (const job of jobs.values()) {
    if (job.status !== "running") continue;
    if (job.finalizeAt) {
      // Confirmed and flushing its final text: silence is expected, not
      // staleness, and it doesn't block a new dictation from starting. Only a
      // flush hung past stopMic's own wait gets reaped.
      if (Date.now() - job.finalizeAt > FINALIZE_MS) {
        clearIdle(job);
        job.status = "canceled";
        job.proc.kill("SIGKILL");
      }
      continue;
    }
    if (Date.now() - job.lastFeedAt > STALE_MS) {
      clearIdle(job);
      job.status = "canceled";
      job.proc.kill("SIGKILL");
    } else {
      liveRemains = true;
    }
  }
  return { liveRemains };
}

export function getMicJob(id: string): MicJob | undefined {
  return jobs.get(id);
}

export async function startMicJob(locale: string): Promise<MicJob> {
  assertLocalRuntime();
  // One dictation at a time. A new start reclaims silent orphans (a refresh
  // that never reached cancel) so nobody is rejected forever, but a session
  // that is still receiving audio is someone speaking — reject instead of
  // killing their dictation mid-sentence.
  if (reclaimStale().liveRemains) {
    throw new UserError("dictation_busy", "A dictation is already running — finish it first.");
  }
  const bin = await ensureStt();
  const id = crypto.randomUUID().slice(0, 12);
  const proc = spawn(bin, ["--live", locale], { stdio: ["pipe", "pipe", "pipe"] });

  let resolveFinished!: () => void;
  const finished = new Promise<void>((r) => (resolveFinished = r));
  const job: MicJob = { id, status: "running", text: "", proc, finished, lastFeedAt: Date.now() };
  jobs.set(id, job);
  armIdle(job);

  // Parse the process's NDJSON stdout line by line.
  let buf = "";
  proc.stdout?.on("data", (d: Buffer) => {
    buf += d.toString();
    let nl: number;
    while ((nl = buf.indexOf("\n")) >= 0) {
      const line = buf.slice(0, nl).trim();
      buf = buf.slice(nl + 1);
      if (!line) continue;
      try {
        const ev = JSON.parse(line) as { type: string; text?: string };
        if (ev.type === "partial") job.text = ev.text ?? job.text;
        else if (ev.type === "final") {
          job.text = ev.text ?? job.text;
          if (job.status === "running") job.status = "done";
        } else if (ev.type === "error") {
          Object.assign(job, errorFields(ev.text, new UserError("transcription_failed", "Transcription failed.")));
          if (job.status === "running") job.status = "error";
        }
      } catch {
        // Ignore a malformed line rather than tearing down a live dictation.
      }
    }
  });

  let stderr = "";
  proc.stderr?.on("data", (d: Buffer) => (stderr += d.toString()));
  proc.on("error", (e) => {
    clearIdle(job);
    if (job.status === "running") {
      job.status = "error";
      job.error = e.message;
    }
    resolveFinished();
    retire(job);
  });
  proc.on("close", (code) => {
    clearIdle(job);
    if (job.status === "running") {
      // Closed without a final line — surface whatever the process complained about.
      job.status = code === 0 ? "done" : "error";
      if (job.status === "error") {
        // The tool's own complaint stays in its words; a silent exit names itself.
        const tail = stderr.trim().split("\n").slice(-2).join("\n");
        Object.assign(job, errorFields(tail, new UserError("tool_exited", `cut-stt exited ${code}`, { tool: "cut-stt", code: String(code) })));
      }
    }
    resolveFinished();
    retire(job);
  });

  return job;
}

/** Forward a chunk of PCM to the running process. Returns false for an unknown/ended job. */
export function feedMic(id: string, pcm: Buffer): boolean {
  const job = jobs.get(id);
  if (!job || job.status !== "running" || !job.proc.stdin?.writable) return false;
  job.proc.stdin.write(pcm);
  job.lastFeedAt = Date.now();
  armIdle(job); // fresh audio — the client is still here
  return true;
}

/** Close the input so the model flushes its final text, then wait for it. A
 * process that misses the deadline is not coming back, so it is killed here
 * rather than left holding the next dictation's slot. */
export async function stopMic(id: string): Promise<string | null> {
  const job = jobs.get(id);
  if (!job) return null;
  clearIdle(job);
  if (job.status === "running") {
    job.finalizeAt = Date.now();
    job.proc.stdin?.end();
  }
  if (!(await withTimeout(job.finished, STOP_MS))) job.proc.kill("SIGKILL");
  return job.text;
}

/** Discard a dictation in progress. */
export function cancelMic(id: string): boolean {
  const job = jobs.get(id);
  if (!job) return false;
  clearIdle(job);
  if (job.status === "running") {
    job.status = "canceled";
    job.proc.kill("SIGKILL");
  }
  return true;
}

/** Resolves true when `p` settled in time, false when the deadline won. */
function withTimeout(p: Promise<void>, ms: number): Promise<boolean> {
  return new Promise((resolve) => {
    const t = setTimeout(() => resolve(false), ms);
    t.unref();
    void p.finally(() => {
      clearTimeout(t);
      resolve(true);
    });
  });
}