Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
const test = require("node:test");
const assert = require("node:assert/strict");
const Module = require("node:module");
const fs = require("node:fs/promises");
const http = require("node:http");
const os = require("node:os");
const path = require("node:path");
const modelRegistryData = require("../../src/models/modelRegistryData.json");
const { buildGguf, LLAMA_3_2_3B_ENTRIES } = require("./harness/ggufFixtures");
// Drives modelManagerBridge.runInference against a stub llama-server, to pin
// how the context window is chosen for a request (#2142). The model file on
// disk is a REAL GGUF header padded to a plausible size, so the ceiling is
// computed through the production path rather than from a fixture object.
const originalLoad = Module._load;
const CHAIN_MODULES = [
"../../src/helpers/modelManagerBridge.js",
"../../src/helpers/modelDirUtils.js",
"../../src/helpers/llamaServer.js",
].map((relative) => require.resolve(relative));
let electronHome = os.tmpdir();
function loadModelManager() {
for (const modulePath of CHAIN_MODULES) delete require.cache[modulePath];
Module._load = function loadWithMocks(request, parent, isMain) {
if (request === "electron") {
return {
app: {
isReady: () => true,
getAppPath: () => process.cwd(),
getPath: (name) => (name === "home" ? electronHome : path.join(electronHome, name)),
},
net: {},
};
}
return originalLoad.call(this, request, parent, isMain);
};
try {
require("../../src/helpers/modelDirUtils.js");
return require("../../src/helpers/modelManagerBridge.js").default;
} finally {
Module._load = originalLoad;
}
}
const GIB = 1024 * 1024 * 1024;
const SHORT_PROMPT = "Clean up this sentence.";
// ~20k estimated tokens, the size that fails on main today.
const LONG_PROMPT = "word ".repeat(12000);
async function setup(
t,
{ tokenCount = null, totalMemoryBytes = 48 * GIB, finishReason = "stop" } = {}
) {
const home = await fs.mkdtemp(path.join(os.tmpdir(), "openwhispr-ctx-sizing-"));
electronHome = home;
t.after(() => fs.rm(home, { recursive: true, force: true }));
const calls = { tokenize: 0, props: 0, completions: 0, template: 0 };
// The completion body is where the output allowance actually lands, so tests
// about it have to read the wire rather than the config object.
let completionBody = null;
const server = http.createServer((req, res) => {
let raw = "";
req.on("data", (chunk) => (raw += chunk));
req.on("end", () => {
let status = 200;
let payload;
if (req.url === "/props") {
calls.props += 1;
payload = { default_generation_settings: { n_ctx: serverManager.contextSize } };
} else if (req.url === "/apply-template") {
calls.template += 1;
payload = { prompt: "RENDERED" };
} else if (req.url === "/tokenize") {
calls.tokenize += 1;
if (tokenCount === null) {
status = 500;
payload = {};
} else {
payload = { tokens: new Array(tokenCount).fill(0) };
}
} else {
calls.completions += 1;
completionBody = JSON.parse(raw);
payload = { choices: [{ finish_reason: finishReason, message: { content: "done" } }] };
}
res.writeHead(status, { "Content-Type": "application/json" });
res.end(JSON.stringify(payload));
});
});
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
t.after(() => new Promise((resolve) => server.close(resolve)));
const modelManager = loadModelManager();
const model = modelRegistryData.localProviders[0].models[0];
modelManager.ensureInitialized();
await fs.mkdir(modelManager.modelsDir, { recursive: true });
// A valid GGUF header, padded past the minimum-size check.
const header = buildGguf(LLAMA_3_2_3B_ENTRIES);
await fs.writeFile(
path.join(modelManager.modelsDir, model.fileName),
Buffer.concat([header, Buffer.alloc(1_000_001, 1)])
);
const serverManager = modelManager.serverManager;
serverManager.cachedServerBinaryPaths = { default: "/stub/llama-server" };
serverManager.ready = true;
serverManager.process = {};
serverManager.port = server.address().port;
serverManager.contextSize = 16384;
modelManager.currentServerModelId = model.id;
modelManager._systemMemoryBytes = () => totalMemoryBytes;
t.after(() => serverManager.clearIdleTimer());
// Stand in for the spawn. The real _doStart leaves the manager ready on a
// live port, so the stub must too, or the preflight would see a dead server.
const restarts = [];
serverManager._doStart = async (modelPath, options = {}) => {
restarts.push(options.contextSize);
serverManager.ready = true;
serverManager.process = {};
serverManager.port = server.address().port;
};
// Avoid signalling a pid that does not belong to us; stop() itself is
// covered in llamaServerContext.test.js.
serverManager.stop = async () => {
serverManager.ready = false;
serverManager.process = null;
serverManager.contextSize = null;
};
return {
modelManager,
modelId: model.id,
calls,
restarts,
serverManager,
completionBody: () => completionBody,
};
}
test("a short request costs no measurement and no restart", async (t) => {
// The dictation-cleanup invariant. Every dictation goes through this path;
// adding a round trip or a restart here would be a far worse regression than
// the bug being fixed.
const { modelManager, modelId, calls, restarts, completionBody } = await setup(t, {
tokenCount: 40,
});
const result = await modelManager.runInference(modelId, SHORT_PROMPT, {
systemPrompt: "Clean up dictation.",
maxTokens: 900,
});
assert.equal(result, "done");
assert.equal(restarts.length, 0, "a short request must not restart the server");
assert.equal(calls.tokenize, 0, "a short request must not pay for tokenization");
assert.equal(calls.completions, 1);
// A request that comfortably fits must reach the model with the allowance
// the caller asked for. Without this, trimming everything to a stub would
// still pass every other test in this file.
assert.equal(completionBody().max_tokens, 900);
});
test("a long request grows the context once and then succeeds", async (t) => {
const { modelManager, modelId, calls, restarts, serverManager } = await setup(t, {
tokenCount: 20514, // the customer's measured prompt
});
const result = await modelManager.runInference(modelId, LONG_PROMPT, {
systemPrompt: "Write meeting notes.",
});
assert.equal(result, "done");
assert.deepEqual(restarts, [32768], "exactly one restart, onto the rung that fits");
assert.equal(serverManager.contextSize, 32768);
assert.equal(calls.completions, 1);
});
test("a request too large for the machine fails before it reaches the model", async (t) => {
const { modelManager, modelId, calls } = await setup(t, { tokenCount: 200000 });
await assert.rejects(
() => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),
(error) => {
assert.equal(error.code, "CONTEXT_TOO_LARGE");
assert.ok(error.details.neededTokens >= 200000);
assert.ok(error.details.maxContextTokens > 0);
assert.ok(!error.message.includes("{"), `raw JSON leaked: ${error.message}`);
return true;
}
);
// Sending it anyway would burn minutes of prefill to earn a 400.
assert.equal(calls.completions, 0, "must not send a request that cannot fit");
});
test("a broken measurement falls back to the estimate instead of failing the request", async (t) => {
const { modelManager, modelId, calls } = await setup(t, { tokenCount: null }); // /tokenize 500s
const result = await modelManager.runInference(modelId, LONG_PROMPT, {
systemPrompt: "Write notes.",
});
assert.equal(result, "done");
assert.ok(calls.tokenize > 0, "it should have tried to measure");
assert.equal(calls.completions, 1, "and still completed on the estimate");
});
test("a short request after a long one reuses the grown context", async (t) => {
const { modelManager, modelId, restarts } = await setup(t, { tokenCount: 20514 });
await modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." });
assert.deepEqual(restarts, [32768]);
await modelManager.runInference(modelId, SHORT_PROMPT, { systemPrompt: "Clean up." });
assert.deepEqual(restarts, [32768], "the short request must not shrink or restart the server");
});
test("a small machine refuses what a large one accepts, for the same request", async (t) => {
// The #1203 guarantee, expressed end to end: the ceiling is a property of
// the machine, so the same transcript is allowed on 48 GB and refused on 8.
const small = await setup(t, { tokenCount: 60000, totalMemoryBytes: 8 * GIB });
await assert.rejects(
() => small.modelManager.runInference(small.modelId, LONG_PROMPT, { systemPrompt: "Notes." }),
(error) => error.code === "CONTEXT_TOO_LARGE"
);
assert.equal(small.calls.completions, 0);
});
test("a running server whose context is unknown is not restarted for a short request", async (t) => {
// A ready server always has at least the starting context. Treating an
// unknown value as zero restarted every warm server on its next request.
const { modelManager, modelId, restarts, serverManager } = await setup(t, { tokenCount: 40 });
serverManager.contextSize = null;
await modelManager.runInference(modelId, SHORT_PROMPT, { systemPrompt: "Clean up." });
assert.deepEqual(restarts, []);
});
test("a caller's context floor cannot breach the machine's ceiling", async (t) => {
// ReasoningConfig.contextSize raises the floor a request starts at, but the
// ceiling is the memory guarantee: honouring a floor the machine cannot
// afford is exactly the unbounded KV cache that panicked #1203.
//
// 12 GiB puts the ceiling strictly between the baseline and the floor, so
// the three possible outcomes — clamp to the ceiling, collapse to the
// baseline, obey the floor — are told apart. The assertions stay relational
// because the exact ceiling is the policy's arithmetic, not this test's, and
// because it must hold on Linux CI too (below the 65536 unverified-GPU cap,
// so memory-bound on either platform).
const { modelManager, modelId, restarts, serverManager } = await setup(t, {
tokenCount: 40,
totalMemoryBytes: 12 * GIB,
});
await modelManager.runInference(modelId, SHORT_PROMPT, {
systemPrompt: "Edit this selection.",
contextSize: 131072,
});
assert.equal(restarts.length, 1, "the raised floor should grow the window once");
const [started] = restarts;
assert.ok(started > 16384, `should grow past the baseline, started at ${started}`);
assert.ok(started < 131072, `must never reach the caller's floor, started at ${started}`);
assert.equal(serverManager.contextSize, started);
});
test("a prompt that fits gets a smaller answer rather than a refusal", async (t) => {
// llama-server only rejects a prompt that overflows the window on its own,
// so charging the full output reservation up front refused notes that work
// today. Shrink the answer to what is left instead (#2142).
const { modelManager, modelId, calls, completionBody } = await setup(t, {
tokenCount: 13000,
totalMemoryBytes: 8 * GIB, // ceiling collapses to the 16384 floor
});
const result = await modelManager.runInference(modelId, LONG_PROMPT, {
systemPrompt: "Write meeting notes.",
maxTokens: 4096,
});
assert.equal(result, "done");
assert.equal(calls.completions, 1, "the request must still be sent");
// Every token the window has left, not a token less: the prompt was
// measured exactly, and llama.cpp stops generation at the limit rather than
// rejecting it, so slack here would only shorten the note for nothing.
assert.equal(completionBody().max_tokens, 16384 - 13000);
});
test("a prompt leaving no room for a usable answer is still refused", async (t) => {
const { modelManager, modelId, calls } = await setup(t, {
tokenCount: 16000,
totalMemoryBytes: 8 * GIB,
});
await assert.rejects(
() => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),
(error) => error.code === "CONTEXT_TOO_LARGE"
);
assert.equal(calls.completions, 0, "must not send a request that cannot answer");
});
test("a selection edit is refused rather than given a clipped answer", async (t) => {
// requireCompleteOutput means a partial replacement corrupts the user's own
// text, so trading output room for a smaller answer is not an option there.
const { modelManager, modelId, calls } = await setup(t, {
tokenCount: 13000,
totalMemoryBytes: 8 * GIB,
});
await assert.rejects(
() =>
modelManager.runInference(modelId, LONG_PROMPT, {
systemPrompt: "Rewrite the selection.",
maxTokens: 8192,
requireCompleteOutput: true,
}),
(error) => error.code === "CONTEXT_TOO_LARGE"
);
assert.equal(calls.completions, 0);
});
test("a summary rewrite gets a smaller answer, but never a clipped one", async (t) => {
// A rewrite replaces the whole summary, so it refuses any cut-off reply, but
// a shorter allowance is fine: most edits (Shorten, TL;DR) need far less
// room than they ask for.
const rewrite = {
systemPrompt: "Revise the summary.",
maxTokens: 8192,
requireCompleteOutput: true,
refuseClippedByWindow: true,
};
const fits = await setup(t, { tokenCount: 13000, totalMemoryBytes: 8 * GIB });
assert.equal(await fits.modelManager.runInference(fits.modelId, LONG_PROMPT, rewrite), "done");
assert.equal(fits.completionBody().max_tokens, 16384 - 13000);
const clipped = await setup(t, {
tokenCount: 13000,
totalMemoryBytes: 8 * GIB,
finishReason: "length",
});
await assert.rejects(
() => clipped.modelManager.runInference(clipped.modelId, LONG_PROMPT, rewrite),
(error) => error.code === "CONTEXT_TOO_LARGE"
);
const cappedWithRoom = await setup(t, { tokenCount: 40, finishReason: "length" });
await assert.rejects(
() => cappedWithRoom.modelManager.runInference(cappedWithRoom.modelId, SHORT_PROMPT, rewrite),
(error) => error.code === "OUTPUT_TRUNCATED"
);
});
test("a reply the window clipped is refused when the caller asks, so it can be split", async (t) => {
// A long note trades output room for prompt room. A summary that then fills
// the smaller allowance was cut short by the window, and a note action would
// rather summarise in parts than save it (#2142).
const { modelManager, modelId, calls, completionBody } = await setup(t, {
tokenCount: 13000,
totalMemoryBytes: 8 * GIB,
finishReason: "length",
});
await assert.rejects(
() =>
modelManager.runInference(modelId, LONG_PROMPT, {
systemPrompt: "Write meeting notes.",
maxTokens: 4096,
refuseClippedByWindow: true,
}),
(error) => {
assert.equal(error.code, "CONTEXT_TOO_LARGE");
assert.equal(error.details.modelId, modelId);
return true;
}
);
assert.equal(calls.completions, 1);
assert.equal(completionBody().max_tokens, 16384 - 13000);
});
test("a trimmed reply that finishes is kept when the caller refuses window clips", async (t) => {
const { modelManager, modelId } = await setup(t, {
tokenCount: 13000,
totalMemoryBytes: 8 * GIB,
});
const result = await modelManager.runInference(modelId, LONG_PROMPT, {
systemPrompt: "Write meeting notes.",
maxTokens: 4096,
refuseClippedByWindow: true,
});
assert.equal(result, "done");
});
test("a reply cut off at the caller's own cap is kept when only window clips are refused", async (t) => {
// A summary clipped at NOTE_OUTPUT_MAX_TOKENS is still worth keeping, so the
// flag must not behave like requireCompleteOutput when the window had room.
const { modelManager, modelId, completionBody } = await setup(t, {
tokenCount: 40,
finishReason: "length",
});
const result = await modelManager.runInference(modelId, SHORT_PROMPT, {
maxTokens: 4096,
refuseClippedByWindow: true,
});
assert.equal(result, "done");
assert.equal(completionBody().max_tokens, 4096);
});
test("a grow that fails comes back at the window that was working", async (t) => {
// start() stops the running server before it spawns the bigger one, so a
// grow that dies takes a working window with it. Restoring it keeps local
// inference alive for the next request instead of until the next launch.
const { modelManager, modelId, serverManager } = await setup(t, { tokenCount: 20514 });
const port = serverManager.port;
const attempts = [];
serverManager._doStart = async (modelPath, options = {}) => {
attempts.push(options.contextSize);
if (options.contextSize > 16384) {
throw new Error(
"llama-server process died during startup (signal: SIGKILL)\n" +
"Process output: ggml_metal: failed to allocate buffer of size 4096.00 MiB"
);
}
serverManager.ready = true;
serverManager.process = {};
serverManager.port = port;
};
await assert.rejects(
() => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),
(error) => error.code === "CONTEXT_TOO_LARGE"
);
assert.deepEqual(attempts, [32768, 16384], "one failed grow, then back to what worked");
assert.equal(serverManager.ready, true, "the user must not be left without a server");
});
test("a server that cannot come back reports a typed failure, not llama.cpp stderr", async (t) => {
const { modelManager, modelId, serverManager } = await setup(t, { tokenCount: 20514 });
serverManager._doStart = async () => {
throw new Error(
"llama-server process died during startup (signal: SIGKILL)\n" +
"Process output: ggml_metal_graph_compute: command buffer 0 failed with status 5"
);
};
await assert.rejects(
() => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),
(error) => {
assert.equal(error.code, "LOCAL_SERVER_UNAVAILABLE");
assert.ok(
!error.message.includes("Process output"),
`server output leaked: ${error.message}`
);
// details crosses IPC to the renderer, so the startup dump must not ride
// along in there either — it belongs in the main-process log.
assert.ok(
!JSON.stringify(error.details).includes("Process output"),
`server output leaked through details: ${JSON.stringify(error.details)}`
);
return true;
}
);
});
test("an answer too small to be worth generating is refused rather than stubbed", async (t) => {
// 16100 of a 16384 window leaves 284 tokens, a few sentences for a whole
// meeting. Below the floor the honest answer is that it does not fit, not a
// stub saved as the note.
const { modelManager, modelId, calls } = await setup(t, {
tokenCount: 16100,
totalMemoryBytes: 8 * GIB,
});
await assert.rejects(
() =>
modelManager.runInference(modelId, LONG_PROMPT, {
systemPrompt: "Write notes.",
maxTokens: 4096,
}),
(error) => error.code === "CONTEXT_TOO_LARGE"
);
assert.equal(calls.completions, 0);
});
test("a start that steps the context down is not retried at the size that stepped down", async (t) => {
// The GPU ladder can load a smaller window than it was asked for without
// ever throwing. Treating that as success left the preflight to ask for the
// same window again, paying a second full model load to be stepped down
// again — the wasted restart the memo exists to prevent.
const { modelManager, modelId, serverManager } = await setup(t, { tokenCount: 20514 });
const port = serverManager.port;
const attempts = [];
serverManager._doStart = async (modelPath, options = {}) => {
attempts.push(options.contextSize);
serverManager.ready = true;
serverManager.process = {};
serverManager.port = port;
// What _startWithGpuFallback records when the reduced-context rung wins.
serverManager.activeContextSize = 16384;
};
await assert.rejects(
() => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),
(error) => error.code === "CONTEXT_TOO_LARGE"
);
assert.deepEqual(attempts, [32768], "the step-down must be remembered, not re-attempted");
});
test("a server that already died is not restored to a window it no longer has", async (t) => {
// llamaServer clears contextSize only in stop(); a crash or a failed health
// check leaves it set while ready goes false. Reading it blind made a failed
// start pay for a second one that was never going to help.
const { modelManager, modelId, serverManager } = await setup(t, { tokenCount: 20514 });
serverManager.ready = false;
serverManager.process = null;
const attempts = [];
serverManager._doStart = async (modelPath, options = {}) => {
attempts.push(options.contextSize);
throw new Error("llama-server process died during startup (signal: SIGKILL)");
};
await assert.rejects(
() => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),
(error) => error.code === "LOCAL_SERVER_UNAVAILABLE"
);
assert.deepEqual(attempts, [32768], "no second attempt at a window that was already gone");
});
test("only Apple Silicon is treated as memory the GPU already shares", async (t) => {
// discreteGpuUnverified reads darwin as unified memory, but an Intel Mac's
// Radeon has its own few GB. Sizing against system RAM there hands the card
// a context it cannot hold — and the darwin start path has no reduced-context
// rung to fall back on.
const real = { platform: process.platform, arch: process.arch };
const pretend = (platform, arch) => {
Object.defineProperty(process, "platform", { value: platform, configurable: true });
Object.defineProperty(process, "arch", { value: arch, configurable: true });
};
t.after(() => pretend(real.platform, real.arch));
// Both managers are built before any pretending, so only the ceiling
// decision sees the faked platform.
const roomy = { tokenCount: 40, totalMemoryBytes: 64 * GIB };
const apple = await setup(t, roomy);
const intel = await setup(t, roomy);
const model = modelRegistryData.localProviders[0].models[0];
const ceilingFor = (manager) =>
manager.contextCeiling({ model }, path.join(manager.modelsDir, model.fileName));
pretend("darwin", "arm64");
const unified = await ceilingFor(apple.modelManager);
pretend("darwin", "x64");
const discrete = await ceilingFor(intel.modelManager);
assert.notEqual(unified.reason, "gpu-cap", "unified memory needs no unverified-GPU cap");
assert.equal(discrete.reason, "gpu-cap", "an Intel Mac's GPU memory is not system memory");
assert.ok(discrete.ceiling < unified.ceiling);
});
test("a reply cut off at the token cap keeps its code through the bridge", async (t) => {
// The bridge rewraps failures; the renderer maps this code to the cleanup toast (#2091).
const { modelManager, modelId } = await setup(t, { tokenCount: 40, finishReason: "length" });
await assert.rejects(
() => modelManager.runInference(modelId, SHORT_PROMPT, { requireCompleteOutput: true }),
(error) => {
assert.equal(error.code, "OUTPUT_TRUNCATED");
assert.equal(error.details.modelId, modelId);
return true;
}
);
});