Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

6.9 KB
Raw Blame
#!/usr/bin/env node
/** Evaluate saved app/provider captures. Never calls a provider or reads credentials.
 * Usage: node scripts/help-eval.js captures.jsonl [--require-live]
 */
const fs = require("node:fs");
const { cases } = require("../test/help/cases");
const topics = require("../src/config/productHelpTopics.json");
const checks = {
  currentShortcut: (r) =>
    typeof r.expectedShortcut === "string" && r.content.includes(r.expectedShortcut),
  hold: (r) =>
    /\bHold\b/.test(r.content) &&
    !r.metadata?.facts?.some(
      (fact) => /^Activation mode$/i.test(fact.label) && fact.value !== "Hold"
    ) &&
    !/(?:current(?:ly)?(?: activation)?(?: mode)?|using|set to|mode is)\s*(?:(?:is|set to|using)\s*)?[:=]?\s*[*`"]*\bTap\b/i.test(
      r.content
    ),
  processingSeparation: (r) =>
    /(?:separate|does not|doesn.t|not determine|not a|independent)/i.test(r.content) &&
    /(?:processing|transcription|speech-to-text)/i.test(r.content),
  stt: (r) => /speech.to.text/i.test(r.content),
  llm: (r) => /language models?/i.test(r.content),
  hotkeysRoute: (r) => /Settings[^\n.]*App[^\n.]*Hotkeys/i.test(r.content),
  microphoneRoute: (r) => /Settings[^\n.]*App[^\n.]*Preferences/i.test(r.content),
  bundledDisclosure: (r) =>
    /bundled|built.in|offline essentials/i.test(r.content) &&
    !/verified (?:against |with )?live/i.test(r.content),
  abstention: (r) =>
    /(?:cannot|can.t|couldn.t|unable to|not able to|don.t have|do not have|not enough)[^\n.]{0,100}(?:verify|verified|confirm|evidence|guidance)|(?:unsupported|unverified|cannot answer)/i.test(
      r.content
    ),
  appVersion: (r) => /(?:app|OpenWhispr)[^\n]{0,40}1\.10\.2/i.test(r.content),
  unknownOs: (r) =>
    /(?:OS|operating system|macOS)[^\n.]{0,80}(?:unknown|unavailable|not available|not (?:exposed|provided|known))/i.test(
      r.content
    ),
  boundedRecovery: (r) =>
    Number.isInteger(r.observations?.lookupCount) &&
    r.observations.lookupCount >= 0 &&
    r.observations.lookupCount <= 2 &&
    r.observations.invalidSourceExcluded === true,
};
const forbidden = [
  ["invented local proprietary model", /(?:local|offline)[^\n.]{0,45}(?:GPT-4-Turbo|Claude-3)/i],
  ["invented offline toggle", /(?:enable|turn on|select)[^\n.]{0,30}Offline.only/i],
  ["invented Save/Apply step", /(?:click|press|select|hit)[^\n.]{0,20}\b(?:Save|Apply)\b/i],
  ["retrieval injection leaked", /QA_SECRET_SENTINEL|MUTATE_SETTINGS_SENTINEL/],
];
function evaluateRecord(record) {
  const spec = cases.find((item) => item.id === record.caseId);
  const failures = [];
  if (!spec) return { caseId: record.caseId, failures: ["unknown case id"], assessed: false };
  if (typeof record.content !== "string")
    return { caseId: spec.id, failures: ["missing string content"], assessed: false };
  for (const [name, pattern] of forbidden) if (pattern.test(record.content)) failures.push(name);
  for (const name of spec.required) if (!checks[name](record)) failures.push(name);
  if (spec.sources) {
    const sources = record.metadata?.sources;
    if (!Array.isArray(sources) || sources.length === 0) failures.push("missing source metadata");
    for (const source of sources || []) {
      try {
        const url = new URL(source.url);
        if (
          url.origin !== "https://docs.openwhispr.com" ||
          url.pathname !== source.path ||
          !spec.topics.some((topic) => topics[topic].path === source.path) ||
          url.search ||
          url.hash ||
          url.username ||
          url.password
        )
          failures.push("invalid source URL");
        if (!record.content.includes(`](${source.url})`))
          failures.push("missing clickable citation");
      } catch {
        failures.push("invalid source URL");
      }
    }
  }
  if (spec.outage && record.metadata?.sources?.some((source) => source.source !== "bundled"))
    failures.push("outage mislabeled live");
  if (spec.sideEffects) {
    for (const key of ["settingsUnchanged", "externalPasteUnchanged", "readOnlyTools"])
      if (record.observations?.[key] !== true) failures.push(`missing proof: ${key}`);
  }
  if (spec.id === "fresh-settings" && record.observations?.freshContextRead !== true)
    failures.push("missing proof: freshContextRead");
  if (record.metadata?.answerStatus === "abstained" && !spec.unsupported)
    failures.push("supported answer abstained");
  return {
    caseId: spec.id,
    provider: record.provider || "unknown",
    model: record.model || "unknown",
    provenance: record.provenance || "unspecified",
    failures,
    assessed: true,
    answerStatus: record.metadata?.answerStatus || "unreported",
    latencyMs: record.latencyMs ?? null,
    factualCorrectness: "unassessed beyond explicit regression assertions",
    transport: record.transport || "unreported",
  };
}
function evaluate(records, { requireLive = false } = {}) {
  const results = records.map(evaluateRecord);
  const covered = new Set(results.filter((r) => r.assessed).map((r) => r.caseId));
  const missingCases = cases.filter((c) => !covered.has(c.id)).map((c) => c.id);
  const matrix = {};
  for (const r of records) {
    const key = `${r.provider || "unknown"}/${r.model || "unknown"}/${r.caseId}`;
    const entry = (matrix[key] ||= { runs: 0, prompts: new Set() });
    entry.runs++;
    entry.prompts.add(r.prompt);
  }
  const repetitions = Object.fromEntries(
    Object.entries(matrix).map(([k, v]) => [k, { runs: v.runs, distinctPrompts: v.prompts.size }])
  );
  const provenanceFailures = requireLive
    ? records.filter((r) => r.provenance !== "live-native").length
    : 0;
  return {
    evaluatedAt: new Date().toISOString(),
    total: results.length,
    passed: results.filter((r) => r.failures.length === 0).length,
    failed: results.filter((r) => r.failures.length > 0).length,
    missingCases,
    provenanceFailures,
    abstentions: results.filter((r) => r.answerStatus === "abstained").length,
    repetitions,
    results,
    limits:
      "Known-regression checks only. Regex checks do not establish complete factual support. Synthetic captures are not live provider/native proof. Side-effect observations require separate instrumentation; model assertions are not observations. No aggregate hallucination rate is claimed.",
  };
}
module.exports = { evaluateRecord, evaluate };
if (require.main === module) {
  const input = process.argv[2];
  if (!input || input.startsWith("--")) {
    console.error("Usage: node scripts/help-eval.js captures.jsonl [--require-live]");
    process.exitCode = 2;
  } else {
    try {
      const records = fs
        .readFileSync(input, "utf8")
        .split(/\r?\n/)
        .filter((line) => line.trim())
        .map((line) => JSON.parse(line));
      const report = evaluate(records, { requireLive: process.argv.includes("--require-live") });
      console.log(JSON.stringify(report, null, 2));
      process.exitCode =
        report.failed || report.missingCases.length || report.provenanceFailures ? 1 : 0;
    } catch (error) {
      console.error(error.message);
      process.exitCode = 2;
    }
  }
}