Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
const test = require("node:test");
const assert = require("node:assert/strict");
const { loadAudioManager: loadAudioManagerHarness } = require("./harness/audioManager");
// Standalone voice-assistant commands never run the dictation-agent model:
// processAgentCommand banks a pendingAssistantConversation directive (the
// side-channel the emission sites attach to the result) and returns the raw
// transcript. Selection edits keep the in-place path.
async function loadAudioManager(t, { cachePrefix, settingsKey }) {
const { createManager } = await loadAudioManagerHarness(t, {
cachePrefix,
settingsKey,
mockModules: {
// This suite exercises paths that call the ReasoningService default
// export directly, so it stubs the object form rather than the class.
"/services/ReasoningService": 'export default { processText: async () => "" };',
},
});
return { createManager };
}
function managerWithCapture(createManager, capture) {
const modelCalls = [];
return {
modelCalls,
// _bankAssistantDirective only ever runs mid-pipeline, so it guards on
// isProcessing — set it true here to match that real precondition.
manager: createManager({
isProcessing: true,
pendingAssistantConversation: null,
consumeSelectionCapture: async () => capture,
processWithReasoningModel: async (...args) => {
modelCalls.push(args);
return "MODEL_RESULT";
},
}),
};
}
test("a standalone command banks a panel directive and returns the transcript", async (t) => {
const { createManager } = await loadAudioManager(t, {
cachePrefix: "openwhispr-assistant-standalone-",
settingsKey: "__assistantStandaloneSettings",
});
const { manager, modelCalls } = managerWithCapture(createManager, null);
const result = await manager.processAgentCommand("draft a reply", "gpt", "Aria", {
selectionEditReachable: true,
});
assert.equal(result, "draft a reply");
assert.deepEqual(manager.pendingAssistantConversation, {
transcript: "draft a reply",
screenContext: null,
});
assert.equal(modelCalls.length, 0, "the dictation-agent model must not run");
});
test("a verified caret banks targeted response delivery when auto-paste is enabled", async (t) => {
const { createManager } = await loadAudioManagerHarness(t, {
cachePrefix: "openwhispr-assistant-caret-delivery-",
settingsKey: "__assistantCaretDeliverySettings",
settings: { autoPasteEnabled: true },
mockModules: {
"/services/ReasoningService": 'export default { processText: async () => "" };',
},
});
const { manager, modelCalls } = managerWithCapture(createManager, {
status: "editable",
sessionId: "caret-session",
});
const result = await manager.processAgentCommand("draft a reply", "gpt", "Aria", {
selectionEditReachable: true,
});
assert.equal(result, "draft a reply");
assert.deepEqual(manager.pendingAssistantConversation, {
transcript: "draft a reply",
screenContext: null,
deliverySessionId: "caret-session",
deliveryAcceptsMarkdown: false,
});
assert.equal(modelCalls.length, 0, "caret delivery must retain the chat Agent route");
});
test("a caret in a markdown-friendly app carries that verdict with the session id", async (t) => {
const { createManager } = await loadAudioManagerHarness(t, {
cachePrefix: "openwhispr-assistant-caret-markdown-",
settingsKey: "__assistantCaretMarkdownSettings",
settings: { autoPasteEnabled: true },
mockModules: {
"/services/ReasoningService": 'export default { processText: async () => "" };',
},
});
const { manager } = managerWithCapture(createManager, {
status: "editable",
sessionId: "caret-session",
acceptsMarkdown: true,
});
await manager.processAgentCommand("draft a reply", "gpt", "Aria", {
selectionEditReachable: true,
});
assert.deepEqual(manager.pendingAssistantConversation, {
transcript: "draft a reply",
screenContext: null,
deliverySessionId: "caret-session",
deliveryAcceptsMarkdown: true,
});
});
test("a verified caret stays panel-first when auto-paste is disabled", async (t) => {
const { createManager } = await loadAudioManagerHarness(t, {
cachePrefix: "openwhispr-assistant-caret-panel-",
settingsKey: "__assistantCaretPanelSettings",
settings: { autoPasteEnabled: false },
mockModules: {
"/services/ReasoningService": 'export default { processText: async () => "" };',
},
});
const { manager } = managerWithCapture(createManager, {
status: "editable",
sessionId: "caret-session",
});
await manager.processAgentCommand("draft a reply", "gpt", "Aria", {
selectionEditReachable: true,
});
assert.deepEqual(manager.pendingAssistantConversation, {
transcript: "draft a reply",
screenContext: null,
});
});
test("a wake-word command is banked without the address", async (t) => {
const { createManager } = await loadAudioManager(t, {
cachePrefix: "openwhispr-assistant-wakeword-",
settingsKey: "__assistantWakewordSettings",
});
const { manager } = managerWithCapture(createManager, null);
manager.voiceAgentRequested = false;
const result = await manager.processAgentCommand("Hey Aria, draft a reply", "gpt", "Aria", {
selectionEditReachable: true,
});
assert.equal(result, "Hey Aria, draft a reply");
assert.equal(manager.pendingAssistantConversation.transcript, "draft a reply");
});
test("a policy-restricted org never gets a panel command banked", async (t) => {
const { createManager } = await loadAudioManager(t, {
cachePrefix: "openwhispr-assistant-policy-",
settingsKey: "__assistantPolicySettings",
});
const { manager } = managerWithCapture(createManager, null);
manager.assertAgentAllowedByPolicy = () => {
const error = new Error("AI agent use is restricted by your organization.");
error.code = "POLICY_RESTRICTED";
throw error;
};
await assert.rejects(
manager.processAgentCommand("Hey Aria, summarize this", "gpt", "Aria", {
selectionEditReachable: true,
}),
/restricted/
);
assert.equal(manager.pendingAssistantConversation, null);
});
test("the directive carries the raw screenshot past the attach gate", async (t) => {
const { createManager } = await loadAudioManager(t, {
cachePrefix: "openwhispr-assistant-screenshot-",
settingsKey: "__assistantScreenshotSettings",
});
const { manager } = managerWithCapture(createManager, null);
const screenshot = { mediaType: "image/jpeg", data: "b64" };
await manager.processAgentCommand("what is on screen", "gpt", "Aria", {
selectionEditReachable: true,
rawScreenContext: screenshot,
});
assert.equal(manager.pendingAssistantConversation.screenContext, screenshot);
});
test("an attached screenshot still reaches the panel through the raw carry", async (t) => {
const { createManager } = await loadAudioManager(t, {
cachePrefix: "openwhispr-assistant-attached-",
settingsKey: "__assistantAttachedSettings",
});
const { manager } = managerWithCapture(createManager, null);
// resolveReasoningRoute mirrors the attached screenContext into
// rawScreenContext (same object), so the raw carry is the single source
// the banking path reads.
const screenshot = { mediaType: "image/jpeg", data: "attached" };
await manager.processAgentCommand("read this", "gpt", "Aria", {
selectionEditReachable: true,
screenContext: screenshot,
rawScreenContext: screenshot,
});
assert.equal(manager.pendingAssistantConversation.screenContext, screenshot);
});
test("an Agent-panel selection stays on the panel route without touching external selection editing", async (t) => {
const { createManager } = await loadAudioManager(t, {
cachePrefix: "openwhispr-assistant-internal-selection-",
settingsKey: "__assistantInternalSelectionSettings",
});
let externalCaptureCalls = 0;
const manager = createManager({
isProcessing: true,
consumeSelectionCapture: async () => {
externalCaptureCalls += 1;
throw new Error("external selection should not be read");
},
});
manager.setAssistantSelectionContext({
text: "the selected Agent response",
sourceMessageId: "assistant-1",
});
const result = await manager.processAgentCommand("make this friendlier", "gpt", "Aria", {
selectionEditReachable: true,
});
assert.equal(result, "make this friendlier");
assert.deepEqual(manager.pendingAssistantConversation, {
transcript: "make this friendlier",
screenContext: null,
selectedContext: {
text: "the selected Agent response",
sourceMessageId: "assistant-1",
},
});
assert.equal(externalCaptureCalls, 0);
assert.equal(manager.assistantSelectionContext, null, "the context must be one-shot");
assert.ok(!manager.pendingSelectionEdit, "no in-place replacement session may be armed");
});
test("a selection without a reachable dictation agent routes to the panel with the selection quoted", async (t) => {
const { createManager } = await loadAudioManager(t, {
cachePrefix: "openwhispr-assistant-sel-unreachable-",
settingsKey: "__assistantSelUnreachableSettings",
});
const capture = { status: "selected", text: "the selected paragraph", sessionId: "s1" };
const { manager, modelCalls } = managerWithCapture(createManager, capture);
const result = await manager.processAgentCommand("make this shorter", "gpt", "Aria", {
selectionEditReachable: false,
});
assert.equal(result, "make this shorter");
assert.equal(
manager.pendingAssistantConversation.transcript,
'make this shorter\n\n"the selected paragraph"'
);
assert.equal(modelCalls.length, 0);
assert.ok(!manager.pendingSelectionEdit, "no in-place replacement session may be armed");
});
test("a too-large selection with no dictation editor goes to the panel as a plain command", async (t) => {
const { createManager } = await loadAudioManager(t, {
cachePrefix: "openwhispr-assistant-toolarge-",
settingsKey: "__assistantTooLargeSettings",
});
const { manager } = managerWithCapture(createManager, {
status: "too_large",
maxCharacters: 6000,
});
const result = await manager.processAgentCommand("summarize this", "gpt", "Aria", {
selectionEditReachable: false,
});
assert.equal(result, "summarize this");
assert.equal(manager.pendingAssistantConversation.transcript, "summarize this");
});
test("an ambiguous capture with no dictation editor goes to the panel as a plain command", async (t) => {
const { createManager } = await loadAudioManager(t, {
cachePrefix: "openwhispr-assistant-ambiguous-",
settingsKey: "__assistantAmbiguousSettings",
});
const { manager } = managerWithCapture(createManager, { status: "target_changed" });
await manager.processAgentCommand("summarize this", "gpt", "Aria", {
selectionEditReachable: false,
});
assert.equal(manager.pendingAssistantConversation.transcript, "summarize this");
});
// Focus moved while the voice assistant hotkey's keys were still held (#2113),
// so the window the command is about was never checked. That is no selection to
// edit, not a failed edit: the command still runs in the panel.
test("a capture whose focus moved during the modifier wait runs as a panel command", async (t) => {
const { createManager } = await loadAudioManager(t, {
cachePrefix: "openwhispr-assistant-focus-moved-",
settingsKey: "__assistantFocusMovedSettings",
});
const { manager, modelCalls } = managerWithCapture(createManager, {
status: "target_changed",
code: "focus_moved",
});
const result = await manager.processAgentCommand("summarize this", "gpt", "Aria", {
selectionEditReachable: true,
});
assert.equal(result, "summarize this");
assert.equal(manager.pendingAssistantConversation.transcript, "summarize this");
assert.equal(modelCalls.length, 0, "the dictation-agent model must not run");
});
test("a selection with the dictation agent reachable keeps the in-place edit path", async (t) => {
const { createManager } = await loadAudioManager(t, {
cachePrefix: "openwhispr-assistant-sel-reachable-",
settingsKey: "__assistantSelReachableSettings",
});
const capture = { status: "selected", text: "the selected paragraph", sessionId: "s1" };
const { manager, modelCalls } = managerWithCapture(createManager, capture);
try {
await manager.processAgentCommand("make this shorter", "gpt", "Aria", {
selectionEditReachable: true,
rawScreenContext: { mediaType: "image/jpeg", data: "raw" },
systemPrompt: "base prompt",
});
} catch {
// The mocked model result carries no completion marker; the selection
// pipeline may reject it. The routing assertion below is what matters.
}
assert.equal(modelCalls.length, 1, "the dictation-agent model runs the selection edit");
assert.equal(manager.pendingAssistantConversation ?? null, null);
// Routing-only directives must not leak into the reasoning layer.
const selectionConfig = modelCalls[0][3];
assert.ok(!("selectionEditReachable" in selectionConfig));
assert.ok(!("rawScreenContext" in selectionConfig));
});
// "Explain this" over highlighted text asks for an answer, not an edit: the
// editor's question verdict must open the panel with the selection quoted and
// never arm the replacement that would paste the answer over the selection.
test("a question about a selection goes to the panel instead of replacing it", async (t) => {
const { createManager } = await loadAudioManagerHarness(t, {
cachePrefix: "openwhispr-assistant-sel-question-",
settingsKey: "__assistantSelQuestionSettings",
// The local edit prompt reads these.
settings: { customPrompts: {}, snippets: [] },
mockModules: {
"/services/ReasoningService": 'export default { processText: async () => "" };',
},
});
const { SELECTION_QUESTION_MARKER } = await import("../../src/helpers/selectionEditing.js");
const capture = { status: "selected", text: "the selected paragraph", sessionId: "s1" };
const screenshot = { mediaType: "image/jpeg", data: "raw" };
for (const [provider, reply] of [
["openai", (marker) => `${SELECTION_QUESTION_MARKER}${marker}`],
["local", () => JSON.stringify({ replacement: SELECTION_QUESTION_MARKER })],
]) {
const modelCalls = [];
const manager = createManager({
isProcessing: true,
pendingAssistantConversation: null,
voiceAgentRequested: false,
consumeSelectionCapture: async () => capture,
processWithReasoningModel: async (...args) => {
modelCalls.push(args);
const marker = args[3].systemPrompt?.match(/__OPENWHISPR_SELECTION_COMPLETE_[^\s]+__/);
return reply(marker?.[0] ?? "");
},
});
const result = await manager.processAgentCommand("Hey Aria, explain this", "gpt", "Aria", {
provider,
selectionEditReachable: true,
rawScreenContext: screenshot,
systemPrompt: "base prompt",
});
assert.equal(result, "Hey Aria, explain this", provider);
assert.equal(modelCalls.length, 1, provider);
assert.ok(!manager.pendingSelectionEdit, `${provider}: no replacement may be armed`);
assert.deepEqual(manager.pendingAssistantConversation, {
transcript: 'explain this\n\n"the selected paragraph"',
screenContext: screenshot,
});
}
});
test("a question verdict arriving after cancellation banks nothing", async (t) => {
const { createManager } = await loadAudioManager(t, {
cachePrefix: "openwhispr-assistant-sel-question-cancel-",
settingsKey: "__assistantSelQuestionCancelSettings",
});
const { SELECTION_QUESTION_MARKER } = await import("../../src/helpers/selectionEditing.js");
let cancelled = false;
const manager = createManager({
isProcessing: true,
pendingAssistantConversation: null,
consumeSelectionCapture: async () => ({ status: "selected", text: "x", sessionId: "s1" }),
processWithReasoningModel: async () => {
cancelled = true;
return SELECTION_QUESTION_MARKER;
},
});
await manager.processAgentCommand(
"explain this",
"gpt",
"Aria",
{ provider: "openai", selectionEditReachable: true, systemPrompt: "base" },
() => cancelled
);
assert.equal(manager.pendingAssistantConversation, null);
assert.ok(!manager.pendingSelectionEdit);
});
// Tone & Style: the app captured at recording start rides with a caret
// delivery, so the panel writes the answer in that app's style.
const TONE_SETTINGS = {
autoPasteEnabled: true,
toneStyleEnabled: true,
toneStyles: { personal: "very_casual", work: "casual", email: "formal", other: "formal" },
};
test("a caret delivery carries the kind of app the answer lands in", async (t) => {
const { createManager } = await loadAudioManagerHarness(t, {
cachePrefix: "openwhispr-assistant-caret-tone-",
settingsKey: "__assistantCaretToneSettings",
settings: TONE_SETTINGS,
mockModules: {
"/services/ReasoningService": 'export default { processText: async () => "" };',
},
});
const { manager } = managerWithCapture(createManager, {
status: "editable",
sessionId: "caret-session",
});
manager.toneStylePromise = Promise.resolve("work");
await manager.processAgentCommand("tell them the deploy is done", "gpt", "Aria", {
selectionEditReachable: true,
});
assert.equal(manager.pendingAssistantConversation.deliveryAppCategory, "work");
assert.equal(manager.toneStylePromise, null, "the capture is consumed once");
});
test("a panel-only answer carries no app to write for", async (t) => {
const { createManager } = await loadAudioManagerHarness(t, {
cachePrefix: "openwhispr-assistant-panel-tone-",
settingsKey: "__assistantPanelToneSettings",
settings: TONE_SETTINGS,
mockModules: {
"/services/ReasoningService": 'export default { processText: async () => "" };',
},
});
const { manager } = managerWithCapture(createManager, null);
manager.toneStylePromise = Promise.resolve("work");
await manager.processAgentCommand("what's on my calendar", "gpt", "Aria", {
selectionEditReachable: true,
});
assert.equal("deliveryAppCategory" in manager.pendingAssistantConversation, false);
});
test("a selection edit asks for the selection's app style only on a rewrite", async (t) => {
const { createManager } = await loadAudioManagerHarness(t, {
cachePrefix: "openwhispr-assistant-sel-tone-",
settingsKey: "__assistantSelToneSettings",
settings: TONE_SETTINGS,
mockModules: {
"/services/ReasoningService": 'export default { processText: async () => "" };',
},
});
const capture = { status: "selected", text: "the selected paragraph", sessionId: "s1" };
const { manager, modelCalls } = managerWithCapture(createManager, capture);
manager.toneStylePromise = Promise.resolve("email");
try {
await manager.processAgentCommand("rewrite this", "gpt", "Aria", {
selectionEditReachable: true,
systemPrompt: "base prompt",
textOnlySystemPrompt: "base prompt",
});
} catch {
// The mocked result has no completion marker; only the request matters here.
}
const { systemPrompt, textOnlySystemPrompt } = modelCalls[0][3];
for (const prompt of [systemPrompt, textOnlySystemPrompt]) {
assert.match(
prompt,
/^base prompt\n\nIf the instruction asks you to rewrite, reword or draft the text and names no tone, write it in the user's email style: formal and polished\./
);
// IDEs count as "other", so a rewrite there may be code the tone must not touch.
assert.match(prompt, /code and commands, with their strings and comments, are never restyled/);
assert.ok(prompt.indexOf("If the instruction asks") < prompt.indexOf("SELECTION EDITING MODE"));
}
});