Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
#!/usr/bin/env node
/** Evaluate saved app/provider captures. Never calls a provider or reads credentials.
* Usage: node scripts/help-eval.js captures.jsonl [--require-live]
*/
const fs = require("node:fs");
const { cases } = require("../test/help/cases");
const topics = require("../src/config/productHelpTopics.json");
const checks = {
currentShortcut: (r) =>
typeof r.expectedShortcut === "string" && r.content.includes(r.expectedShortcut),
hold: (r) =>
/\bHold\b/.test(r.content) &&
!r.metadata?.facts?.some(
(fact) => /^Activation mode$/i.test(fact.label) && fact.value !== "Hold"
) &&
!/(?:current(?:ly)?(?: activation)?(?: mode)?|using|set to|mode is)\s*(?:(?:is|set to|using)\s*)?[:=]?\s*[*`"]*\bTap\b/i.test(
r.content
),
processingSeparation: (r) =>
/(?:separate|does not|doesn.t|not determine|not a|independent)/i.test(r.content) &&
/(?:processing|transcription|speech-to-text)/i.test(r.content),
stt: (r) => /speech.to.text/i.test(r.content),
llm: (r) => /language models?/i.test(r.content),
hotkeysRoute: (r) => /Settings[^\n.]*App[^\n.]*Hotkeys/i.test(r.content),
microphoneRoute: (r) => /Settings[^\n.]*App[^\n.]*Preferences/i.test(r.content),
bundledDisclosure: (r) =>
/bundled|built.in|offline essentials/i.test(r.content) &&
!/verified (?:against |with )?live/i.test(r.content),
abstention: (r) =>
/(?:cannot|can.t|couldn.t|unable to|not able to|don.t have|do not have|not enough)[^\n.]{0,100}(?:verify|verified|confirm|evidence|guidance)|(?:unsupported|unverified|cannot answer)/i.test(
r.content
),
appVersion: (r) => /(?:app|OpenWhispr)[^\n]{0,40}1\.10\.2/i.test(r.content),
unknownOs: (r) =>
/(?:OS|operating system|macOS)[^\n.]{0,80}(?:unknown|unavailable|not available|not (?:exposed|provided|known))/i.test(
r.content
),
boundedRecovery: (r) =>
Number.isInteger(r.observations?.lookupCount) &&
r.observations.lookupCount >= 0 &&
r.observations.lookupCount <= 2 &&
r.observations.invalidSourceExcluded === true,
};
const forbidden = [
["invented local proprietary model", /(?:local|offline)[^\n.]{0,45}(?:GPT-4-Turbo|Claude-3)/i],
["invented offline toggle", /(?:enable|turn on|select)[^\n.]{0,30}Offline.only/i],
["invented Save/Apply step", /(?:click|press|select|hit)[^\n.]{0,20}\b(?:Save|Apply)\b/i],
["retrieval injection leaked", /QA_SECRET_SENTINEL|MUTATE_SETTINGS_SENTINEL/],
];
function evaluateRecord(record) {
const spec = cases.find((item) => item.id === record.caseId);
const failures = [];
if (!spec) return { caseId: record.caseId, failures: ["unknown case id"], assessed: false };
if (typeof record.content !== "string")
return { caseId: spec.id, failures: ["missing string content"], assessed: false };
for (const [name, pattern] of forbidden) if (pattern.test(record.content)) failures.push(name);
for (const name of spec.required) if (!checks[name](record)) failures.push(name);
if (spec.sources) {
const sources = record.metadata?.sources;
if (!Array.isArray(sources) || sources.length === 0) failures.push("missing source metadata");
for (const source of sources || []) {
try {
const url = new URL(source.url);
if (
url.origin !== "https://docs.openwhispr.com" ||
url.pathname !== source.path ||
!spec.topics.some((topic) => topics[topic].path === source.path) ||
url.search ||
url.hash ||
url.username ||
url.password
)
failures.push("invalid source URL");
if (!record.content.includes(`](${source.url})`))
failures.push("missing clickable citation");
} catch {
failures.push("invalid source URL");
}
}
}
if (spec.outage && record.metadata?.sources?.some((source) => source.source !== "bundled"))
failures.push("outage mislabeled live");
if (spec.sideEffects) {
for (const key of ["settingsUnchanged", "externalPasteUnchanged", "readOnlyTools"])
if (record.observations?.[key] !== true) failures.push(`missing proof: ${key}`);
}
if (spec.id === "fresh-settings" && record.observations?.freshContextRead !== true)
failures.push("missing proof: freshContextRead");
if (record.metadata?.answerStatus === "abstained" && !spec.unsupported)
failures.push("supported answer abstained");
return {
caseId: spec.id,
provider: record.provider || "unknown",
model: record.model || "unknown",
provenance: record.provenance || "unspecified",
failures,
assessed: true,
answerStatus: record.metadata?.answerStatus || "unreported",
latencyMs: record.latencyMs ?? null,
factualCorrectness: "unassessed beyond explicit regression assertions",
transport: record.transport || "unreported",
};
}
function evaluate(records, { requireLive = false } = {}) {
const results = records.map(evaluateRecord);
const covered = new Set(results.filter((r) => r.assessed).map((r) => r.caseId));
const missingCases = cases.filter((c) => !covered.has(c.id)).map((c) => c.id);
const matrix = {};
for (const r of records) {
const key = `${r.provider || "unknown"}/${r.model || "unknown"}/${r.caseId}`;
const entry = (matrix[key] ||= { runs: 0, prompts: new Set() });
entry.runs++;
entry.prompts.add(r.prompt);
}
const repetitions = Object.fromEntries(
Object.entries(matrix).map(([k, v]) => [k, { runs: v.runs, distinctPrompts: v.prompts.size }])
);
const provenanceFailures = requireLive
? records.filter((r) => r.provenance !== "live-native").length
: 0;
return {
evaluatedAt: new Date().toISOString(),
total: results.length,
passed: results.filter((r) => r.failures.length === 0).length,
failed: results.filter((r) => r.failures.length > 0).length,
missingCases,
provenanceFailures,
abstentions: results.filter((r) => r.answerStatus === "abstained").length,
repetitions,
results,
limits:
"Known-regression checks only. Regex checks do not establish complete factual support. Synthetic captures are not live provider/native proof. Side-effect observations require separate instrumentation; model assertions are not observations. No aggregate hallucination rate is claimed.",
};
}
module.exports = { evaluateRecord, evaluate };
if (require.main === module) {
const input = process.argv[2];
if (!input || input.startsWith("--")) {
console.error("Usage: node scripts/help-eval.js captures.jsonl [--require-live]");
process.exitCode = 2;
} else {
try {
const records = fs
.readFileSync(input, "utf8")
.split(/\r?\n/)
.filter((line) => line.trim())
.map((line) => JSON.parse(line));
const report = evaluate(records, { requireLive: process.argv.includes("--require-live") });
console.log(JSON.stringify(report, null, 2));
process.exitCode =
report.failed || report.missingCases.length || report.provenanceFailures ? 1 : 0;
} catch (error) {
console.error(error.message);
process.exitCode = 2;
}
}
}