Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

13.6 KB
Raw Blame
/**
 * Live STT canary. Per keyed provider, acquires a session credential through
 * the app's REAL token registry (realtimeTokenProviders.js) and, where the
 * transport is a plain WebSocket, performs a live handshake against the same
 * endpoint the app dials — so a provider changing its auth, endpoint, or
 * handshake surfaces here on a schedule instead of in a customer report
 * (#1624 sat in shipped builds for three days as a swallowed warmup warning).
 * Providers whose request shape is theirs alone rather than the shared
 * OpenAI-compatible multipart (Gemini, batch and Live) send a real
 * transcription through the shipped module for the same reason, and OpenAI's
 * batch default is posted the way the app builds it so a retired model or
 * field surfaces here too.
 *
 * Run: node scripts/stt-canary.mjs
 * Keys come from STT_CANARY_<PROVIDER>_KEY env vars; providers without a key
 * are skipped and listed. Exit 1 when any probe on a keyed provider fails, or
 * when no key is configured at all — an all-skip run probed nothing and must
 * not report green.
 * Corti is not probed (needs tenant/environment credentials beyond a key).
 */
import { spawnSync } from "child_process";
import fs from "fs";
import os from "os";
import path from "path";
import WebSocket from "ws";
import ffmpegPath from "ffmpeg-static";
import tokenProviders from "../src/helpers/realtimeTokenProviders.js";
import audioUtils from "../src/utils/audioUtils.js";
import geminiTranscription from "../src/helpers/geminiTranscription.js";
import geminiLive from "../src/helpers/geminiLiveStreaming.js";
import AssemblyAiStreaming from "../src/helpers/assemblyAiStreaming.js";
import DeepgramStreaming from "../src/helpers/deepgramStreaming.js";
import modelRegistryData from "../src/models/modelRegistryData.json" with { type: "json" };

const { fetchRealtimeTokenForProvider } = tokenProviders;
const { pcm16ToWav } = audioUtils;
const { transcribeWithGemini } = geminiTranscription;
const { GeminiLiveStreaming } = geminiLive;

// The registry's first model is what a fresh AssemblyAI selection dictates with
// (ModelRegistry.getDefaultTranscriptionModel), so that is the id to probe.
const ASSEMBLYAI_DEFAULT_MODEL = modelRegistryData.transcriptionProviders.find(
  (provider) => provider.id === "assemblyai"
).models[0].id;

const HANDSHAKE_TIMEOUT_MS = 15000;
// Half a second of 16 kHz mono silence: enough for a provider to accept and
// decode the container, short enough to stay far inside every request cap.
const SILENT_WAV = () => pcm16ToWav(Buffer.alloc(16000));

// Mirrors the app's dial: openaiRealtimeStreaming.js connects with a bare
// Bearer header; a null token means the credential already rides in the URL
// (AssemblyAI). `authorization` overrides both, for a provider whose scheme is
// not Bearer; passing the client's own header keeps the probe from drifting.
// `awaitServerEvent` is for providers that authenticate AFTER the upgrade:
// OpenAI opens the socket for any key and only then sends an `error` event
// for a bad one, so resolving on `open` validates nothing (#1624 class).
// `verifyServerEvent` inspects that first event and returns a failure detail,
// or null to accept — for servers that acknowledge the requested configuration
// rather than reject a bad one.
function probeWebSocket(
  url,
  token,
  { awaitServerEvent = false, verifyServerEvent = null, authorization = null } = {}
) {
  return new Promise((resolve) => {
    const credential = authorization ?? (token ? `Bearer ${token}` : null);
    const ws = new WebSocket(
      url,
      credential ? { headers: { Authorization: credential } } : undefined
    );
    const timer = setTimeout(() => {
      ws.terminate();
      resolve({ ok: false, detail: `no handshake within ${HANDSHAKE_TIMEOUT_MS}ms` });
    }, HANDSHAKE_TIMEOUT_MS);
    const finish = (result) => {
      clearTimeout(timer);
      ws.terminate();
      resolve(result);
    };
    ws.on("open", () => {
      if (!awaitServerEvent) finish({ ok: true });
    });
    ws.on("message", (data) => {
      if (!awaitServerEvent) return;
      let event;
      try {
        event = JSON.parse(data);
      } catch {
        event = {};
      }
      if (event.type === "error") {
        finish({ ok: false, detail: `server error event: ${event.error?.code ?? "unknown"}` });
        return;
      }
      const detail = verifyServerEvent?.(event);
      finish(detail ? { ok: false, detail } : { ok: true });
    });
    ws.on("close", (code) => {
      if (awaitServerEvent)
        finish({ ok: false, detail: `closed before a server event (code ${code})` });
    });
    ws.on("unexpected-response", (_req, res) => {
      finish({ ok: false, detail: `handshake rejected: HTTP ${res.statusCode}` });
    });
    ws.on("error", (err) => {
      clearTimeout(timer);
      resolve({ ok: false, detail: err.message });
    });
  });
}

// Dictation records WebM/Opus, and Gemini's documented input formats do not
// list it — so wav and webm are probed separately: a wav-only probe would stay
// green while every BYOK dictation failed.
function toWebmOpus(wav) {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "stt-canary-"));
  const input = path.join(dir, "in.wav");
  const output = path.join(dir, "out.webm");
  try {
    fs.writeFileSync(input, wav);
    const result = spawnSync(ffmpegPath, ["-y", "-i", input, "-c:a", "libopus", output], {
      stdio: "ignore",
    });
    if (result.status !== 0) return null;
    return fs.readFileSync(output);
  } finally {
    fs.rmSync(dir, { recursive: true, force: true });
  }
}

// A completed interaction proves auth, endpoint, and container decode; silence
// legitimately transcribes to empty text, so text content is not asserted.
async function probeGeminiBatch(key, audio, contentType) {
  if (!audio) return { ok: false, detail: `could not build ${contentType} fixture` };
  await transcribeWithGemini({ audioBuffer: audio, contentType, apiKey: key }, fetch);
  return { ok: true };
}

// A completed Live turn proves the raw-key handshake, the setup message, the
// audio frame shape and the transcript events — the socket opening proves none
// of it, and server-side VAD suppresses silence, so the probe has to speak.
const CANARY_PHRASE = "The quick brown fox. OpenWhispr transcription test.";
const LIVE_FRAME_MS = 50;
const LIVE_FRAME_BYTES = 1600; // one 50ms dictation worklet frame at 16kHz s16le

// Gemini's own TTS keeps the fixture out of the repo and needs no second key.
async function synthesizeCanarySpeech(key) {
  const response = await fetch(
    "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-preview-tts:generateContent",
    {
      method: "POST",
      headers: { "Content-Type": "application/json", "x-goog-api-key": key },
      body: JSON.stringify({
        contents: [{ parts: [{ text: `Say clearly: ${CANARY_PHRASE}` }] }],
        generationConfig: { responseModalities: ["AUDIO"] },
      }),
    }
  );
  if (!response.ok) throw new Error(`speech fixture request failed: HTTP ${response.status}`);
  const data = await response.json();
  const audio = data?.candidates?.[0]?.content?.parts?.[0]?.inlineData?.data;
  if (!audio) throw new Error("speech fixture returned no audio");
  // TTS answers with 24kHz L16; dictation captures 16kHz.
  const resampled = spawnSync(
    ffmpegPath,
    [
      "-f",
      "s16le",
      "-ar",
      "24000",
      "-ac",
      "1",
      "-i",
      "pipe:0",
      "-ar",
      "16000",
      "-f",
      "s16le",
      "pipe:1",
    ],
    { input: Buffer.from(audio, "base64"), maxBuffer: 64 * 1024 * 1024 }
  );
  if (resampled.status !== 0) throw new Error("could not resample the speech fixture to 16kHz");
  return resampled.stdout;
}

async function probeGeminiLive(key) {
  const token = await fetchRealtimeTokenForProvider("gemini-realtime", tokenDeps(key), {
    mode: "byok",
  });
  const pcm = await synthesizeCanarySpeech(key);
  const streaming = new GeminiLiveStreaming();
  let partials = 0;
  streaming.onPartialTranscript = () => {
    partials += 1;
  };
  try {
    await streaming.connect({ token, mode: "byok", keyterms: ["OpenWhispr"] });
    // Paced like the mic worklet: the server finalizes relative to real-time
    // ingest, so a burst followed by audioStreamEnd starves the final.
    for (let offset = 0; offset < pcm.length; offset += LIVE_FRAME_BYTES) {
      streaming.sendAudio(pcm.subarray(offset, offset + LIVE_FRAME_BYTES));
      await new Promise((resolve) => setTimeout(resolve, LIVE_FRAME_MS));
    }
    const { text } = await streaming.disconnect(true);
    if (!/quick brown fox/i.test(text)) {
      return { ok: false, detail: `no usable transcript: ${JSON.stringify(text)}` };
    }
    return { ok: true, note: `${partials} partials, final "${text}"` };
  } finally {
    streaming.cleanup();
  }
}

// gpt-transcribe is the BYOK batch default and takes the custom dictionary on
// its `keywords[]` channel, so one rides along: the request fails here first if
// OpenAI retires either. Silence transcribes to empty text, so only acceptance
// is asserted.
async function probeOpenAiBatch(key, audio) {
  const body = new FormData();
  body.append("file", new Blob([audio], { type: "audio/wav" }), "audio.wav");
  body.append("model", "gpt-transcribe");
  body.append("keywords[]", "OpenWhispr");
  const response = await fetch("https://api.openai.com/v1/audio/transcriptions", {
    method: "POST",
    headers: { Authorization: `Bearer ${key}` },
    body,
  });
  return response.ok ? { ok: true } : { ok: false, detail: `HTTP ${response.status}` };
}

const tokenDeps = (key) => ({
  environmentManager: {
    getOpenAIKey: () => key,
    getTinfoilKey: () => key,
    getDeepgramKey: () => key,
    getAssemblyAIKey: () => key,
    getGeminiKey: () => key,
  },
  proxyFetch: fetch,
});

const PROBES = [
  {
    id: "openai-realtime",
    keyEnv: "STT_CANARY_OPENAI_KEY",
    run: async (key) => {
      const token = await fetchRealtimeTokenForProvider("openai-realtime", tokenDeps(key), {
        mode: "byok",
      });
      return probeWebSocket("wss://api.openai.com/v1/realtime?intent=transcription", token, {
        awaitServerEvent: true,
      });
    },
  },
  {
    id: "openai-batch",
    keyEnv: "STT_CANARY_OPENAI_KEY",
    run: (key) => probeOpenAiBatch(key, SILENT_WAV()),
  },
  {
    id: "deepgram-realtime",
    keyEnv: "STT_CANARY_DEEPGRAM_KEY",
    run: async (key) => {
      const token = await fetchRealtimeTokenForProvider("deepgram-realtime", tokenDeps(key), {
        mode: "byok",
      });
      const url = new DeepgramStreaming().buildWebSocketUrl({
        sampleRate: 16000,
        keyterms: ["OpenWhispr"],
      });
      return probeWebSocket(url, null, {
        authorization: DeepgramStreaming.authorizationHeader("byok", token),
      });
    },
  },
  {
    id: "assemblyai-realtime",
    keyEnv: "STT_CANARY_ASSEMBLYAI_KEY",
    // Dials the exact URL the client builds (token, encoding, speech_model).
    // AssemblyAI ignores query params it does not recognise instead of
    // rejecting them, so a retired or misspelt default model would silently
    // downgrade every session — only Begin's echoed configuration.model proves
    // the requested model was applied.
    run: async (key) => {
      const token = await fetchRealtimeTokenForProvider("assemblyai-realtime", tokenDeps(key), {
        mode: "byok",
      });
      const model = ASSEMBLYAI_DEFAULT_MODEL;
      const url = new AssemblyAiStreaming().buildWebSocketUrl({ token, model });
      return probeWebSocket(url, null, {
        awaitServerEvent: true,
        verifyServerEvent: (event) =>
          event.type === "Begin" && event.configuration?.model === model
            ? null
            : `expected Begin with model ${model}, got ${event.type ?? "unknown"} (model ${event.configuration?.model ?? "none"})`,
      });
    },
  },
  {
    id: "tinfoil-realtime",
    keyEnv: "STT_CANARY_TINFOIL_KEY",
    // The attested socket needs the Tinfoil SDK's enclave verification; the
    // canary stops at credential resolution, so this row only proves the
    // secret exists.
    run: async (key) => {
      const token = await fetchRealtimeTokenForProvider("tinfoil-realtime", tokenDeps(key), {
        mode: "byok",
      });
      return token
        ? { ok: true, note: "key present (transport not probed)" }
        : { ok: false, detail: "empty token" };
    },
  },
  {
    id: "gemini-live-streaming",
    keyEnv: "STT_CANARY_GEMINI_KEY",
    run: (key) => probeGeminiLive(key),
  },
  {
    id: "gemini-batch-wav",
    keyEnv: "STT_CANARY_GEMINI_KEY",
    run: (key) => probeGeminiBatch(key, SILENT_WAV(), "audio/wav"),
  },
  {
    id: "gemini-batch-webm",
    keyEnv: "STT_CANARY_GEMINI_KEY",
    run: (key) => probeGeminiBatch(key, toWebmOpus(SILENT_WAV()), "audio/webm"),
  },
];

const report = [];
const failures = [];
const skipped = [];

for (const probe of PROBES) {
  const key = process.env[probe.keyEnv];
  if (!key) {
    skipped.push(probe.id);
    continue;
  }
  const result = await probe.run(key).catch((err) => ({ ok: false, detail: err.message }));
  const cell = result.ok ? `✅${result.note ? ` ${result.note}` : ""}` : `❌ ${result.detail}`;
  report.push(`| ${probe.id} | ${cell} |`);
  if (!result.ok) failures.push(`${probe.id}: ${result.detail}`);
}

if (skipped.length === PROBES.length) {
  failures.push("no canary secrets configured — every provider was skipped, nothing was probed");
}

console.log("## STT canary\n");
console.log("| provider | result |\n|---|---|");
for (const line of report) console.log(line);
if (skipped.length) console.log(`\nSkipped (no key configured): ${skipped.join(", ")}`);
if (failures.length) {
  console.log(`\n### ${failures.length} failure(s)\n`);
  for (const f of failures) console.log(`- ${f}`);
  process.exit(1);
}
console.log("\nAll probes passed.");