Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
const test = require("node:test");
const assert = require("node:assert/strict");
const http = require("node:http");
const LlamaServerManager = require("../../src/helpers/llamaServer.js");
// A manager whose spawn is stubbed at the _doStart boundary, recording the
// options each start was asked for.
function makeManager() {
const manager = new LlamaServerManager();
const starts = [];
manager._doStart = async (modelPath, options = {}) => {
manager.ready = true;
manager.modelPath = modelPath;
manager.draftModelPath = options.draftModelPath || null;
starts.push({ modelPath, contextSize: options.contextSize });
};
return { manager, starts };
}
// --- grow-only restart ---------------------------------------------------
//
// Before #2142 the context was a single constant, so start() could ignore
// options entirely once a server was ready. Now that the context varies per
// request, it has to restart to grow — but only to grow, or a long note
// followed by a short dictation would restart the server twice.
test("a larger context restarts the server", async () => {
const { manager, starts } = makeManager();
await manager.start("/models/main.gguf", { contextSize: 16384 });
assert.equal(starts.length, 1);
await manager.start("/models/main.gguf", { contextSize: 32768 });
assert.equal(starts.length, 2);
assert.equal(manager.contextSize, 32768);
});
test("an equal or smaller context reuses the running server", async () => {
const { manager, starts } = makeManager();
await manager.start("/models/main.gguf", { contextSize: 32768 });
assert.equal(starts.length, 1);
// Exactly the same → no restart.
await manager.start("/models/main.gguf", { contextSize: 32768 });
assert.equal(starts.length, 1);
// Smaller → no restart, and the larger window is kept. This is what stops a
// short dictation right after a long note from costing a second restart.
await manager.start("/models/main.gguf", { contextSize: 16384 });
assert.equal(starts.length, 1);
assert.equal(manager.contextSize, 32768);
});
test("growing the context still restarts when the drafter is unchanged", async () => {
const { manager, starts } = makeManager();
await manager.start("/models/main.gguf", { contextSize: 16384, draftModelPath: "/d.gguf" });
await manager.start("/models/main.gguf", { contextSize: 65536, draftModelPath: "/d.gguf" });
assert.equal(starts.length, 2);
assert.equal(manager.contextSize, 65536);
});
test("a context grown for one model does not carry over to another", async () => {
const { manager, starts } = makeManager();
await manager.start("/models/big.gguf", { contextSize: 65536 });
await manager.start("/models/other.gguf", { contextSize: 16384 });
assert.equal(starts.length, 2);
assert.equal(manager.contextSize, 16384, "the new model starts at its own size");
});
test("stopping the server forgets the grown context", async () => {
const { manager } = makeManager();
manager._killCurrentProcess = async () => {};
manager.process = {};
await manager.start("/models/main.gguf", { contextSize: 65536 });
assert.equal(manager.contextSize, 65536);
await manager.stop();
assert.equal(manager.contextSize, null);
});
// --- spawn arguments -----------------------------------------------------
test("every GPU ladder rung inherits the requested context and cache bounds", async () => {
const manager = new LlamaServerManager();
const seen = [];
manager._buildEnv = () => ({});
manager._killCurrentProcess = async () => {};
manager.findAvailablePort = async () => 8221;
manager.draftModelPath = null;
manager._startWithBinary = async (binary, args) => {
seen.push(args);
throw new Error("stub fail");
};
const baseArgs = manager._buildBaseArgs("/models/main.gguf", 8221, {
contextSize: 32768,
cacheRamMiB: 1024,
});
await assert.rejects(() =>
manager._startWithGpuFallback({ vulkan: "/bin/vulkan", cpu: "/bin/cpu" }, baseArgs, {}, [])
);
assert.ok(seen.length >= 2, "expected the ladder to try more than one rung");
for (const args of seen) {
assert.equal(args[args.indexOf("--ctx-size") + 1], "32768");
// llama-server defaults --cache-ram to 8192 MiB of host RAM on top of the
// KV cache. Left unbounded it would undo the memory budget entirely.
assert.equal(args[args.indexOf("--cache-ram") + 1], "1024");
}
});
// --- context-overflow errors --------------------------------------------
test("a context overflow is reported as a typed error, not as llama.cpp JSON", async () => {
const overflowBody = JSON.stringify({
error: {
code: 400,
message: "request (20514 tokens) exceeds the available context size (16384 tokens)",
type: "exceed_context_size_error",
n_prompt_tokens: 20514,
n_ctx: 16384,
},
});
const server = http.createServer((req, res) => {
res.writeHead(400, { "Content-Type": "application/json" });
res.end(overflowBody);
});
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
const manager = new LlamaServerManager();
manager.ready = true;
manager.process = {};
manager.port = server.address().port;
try {
await assert.rejects(
() => manager.inference([{ role: "user", content: "hi" }], {}),
(error) => {
assert.equal(error.code, "CONTEXT_TOO_LARGE");
assert.equal(error.neededTokens, 20514);
assert.equal(error.maxContextTokens, 16384);
// The whole point: a customer must never be shown a JSON blob.
assert.ok(!error.message.includes("{"), `raw JSON leaked: ${error.message}`);
return true;
}
);
} finally {
manager.clearIdleTimer();
await new Promise((resolve) => server.close(resolve));
}
});
test("other llama-server failures keep their existing shape", async () => {
const server = http.createServer((req, res) => {
res.writeHead(500, { "Content-Type": "text/plain" });
res.end("internal error");
});
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
const manager = new LlamaServerManager();
manager.ready = true;
manager.process = {};
manager.port = server.address().port;
try {
await assert.rejects(
() => manager.inference([{ role: "user", content: "hi" }], {}),
(error) => {
assert.equal(error.code, undefined);
assert.match(error.message, /llama-server returned status 500/);
return true;
}
);
} finally {
manager.clearIdleTimer();
await new Promise((resolve) => server.close(resolve));
}
});
// --- measuring the running server ----------------------------------------
//
// The estimate only picks the size the server starts at. These two calls are
// what make the decision exact: /props reports the window actually in force
// (which can differ from what we asked for), and /apply-template + /tokenize
// price the real prompt, chat template and all.
function withStubServer(routes) {
// A Map rather than indexing the object: req.url is request-controlled, so a
// plain lookup answers /toString or /constructor with an inherited
// Object.prototype method and then calls it.
const routeTable = new Map(Object.entries(routes));
return async (run) => {
const server = http.createServer((req, res) => {
const handler = routeTable.get(req.url);
if (typeof handler !== "function") {
res.writeHead(404).end();
return;
}
let body = "";
req.on("data", (chunk) => {
body += chunk;
});
req.on("end", () => {
const { status = 200, payload } = handler(body);
res.writeHead(status, { "Content-Type": "application/json" });
res.end(JSON.stringify(payload));
});
});
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
const manager = new LlamaServerManager();
manager.ready = true;
manager.process = {};
manager.port = server.address().port;
try {
await run(manager);
} finally {
manager.clearIdleTimer();
await new Promise((resolve) => server.close(resolve));
}
};
}
test("usableContextSize reports the window the server actually has", async () => {
await withStubServer({
"/props": () => ({ payload: { default_generation_settings: { n_ctx: 32768 } } }),
})(async (manager) => {
assert.equal(await manager.usableContextSize(), 32768);
});
});
test("usableContextSize returns null rather than guessing when props is unreadable", async () => {
// A broken measurement must degrade to the estimate, never fail the request.
await withStubServer({ "/props": () => ({ status: 500, payload: {} }) })(async (manager) => {
assert.equal(await manager.usableContextSize(), null);
});
await withStubServer({})(async (manager) => {
assert.equal(await manager.usableContextSize(), null);
});
});
test("countPromptTokens prices the rendered template, not just the message text", async () => {
let templatedWith = null;
let tokenized = null;
await withStubServer({
"/apply-template": (body) => {
templatedWith = JSON.parse(body);
// The real server renders role tags and, with thinking disabled, an
// empty <think></think> block. All of it costs context.
return { payload: { prompt: "<|im_start|>system\nSYS<|im_end|>\nhello" } };
},
"/tokenize": (body) => {
tokenized = JSON.parse(body);
return { payload: { tokens: new Array(20514).fill(0) } };
},
})(async (manager) => {
const messages = [
{ role: "system", content: "SYS" },
{ role: "user", content: "hello" },
];
assert.equal(await manager.countPromptTokens(messages, { disableThinking: true }), 20514);
assert.deepEqual(templatedWith.messages, messages);
assert.deepEqual(templatedWith.chat_template_kwargs, { enable_thinking: false });
assert.equal(tokenized.content, "<|im_start|>system\nSYS<|im_end|>\nhello");
});
});
test("countPromptTokens returns null when the server cannot measure", async () => {
await withStubServer({ "/apply-template": () => ({ status: 500, payload: {} }) })(
async (manager) => {
assert.equal(await manager.countPromptTokens([{ role: "user", content: "hi" }], {}), null);
}
);
});
// --- reading the reply ---------------------------------------------------
//
// With thinking on, `reasoning_content` is the reasoning, not the answer (#2187).
const MESSAGES = [{ role: "user", content: "hi" }];
// Stubs one chat completion and records the request bodies it receives.
function withReply(message, finishReason = "stop") {
const requests = [];
const run = withStubServer({
"/v1/chat/completions": (body) => {
requests.push(JSON.parse(body));
return { payload: { choices: [{ message, finish_reason: finishReason }] } };
},
});
return { run, requests };
}
const TRUNCATED = { code: "OUTPUT_TRUNCATED" };
test("reasoning never stands in for the answer when thinking was requested", async () => {
const reasoningOnly = { content: "", reasoning_content: "Thinking Process:\n\n1. Analyze" };
// Cut off mid-reasoning: there is no answer to return, so every caller gets
// the truncation error rather than an empty string it would have to explain.
const truncated = withReply(reasoningOnly, "length");
await truncated.run(async (manager) => {
await assert.rejects(manager.inference(MESSAGES, { disableThinking: false }), TRUNCATED);
const strict = { disableThinking: false, requireCompleteOutput: true };
await assert.rejects(manager.inference(MESSAGES, strict), TRUNCATED);
});
assert.equal(truncated.requests[0].chat_template_kwargs, undefined);
// Finished with no answer: reaches strict callers such as dictation cleanup too.
const finished = withReply(reasoningOnly, "stop");
await finished.run(async (manager) => {
const options = { disableThinking: false, requireCompleteOutput: true };
assert.equal(await manager.inference(MESSAGES, options), "");
});
});
test("a cut-off reply with no content is truncated even when thinking was suppressed", async () => {
// gpt-oss ignores enable_thinking and reasons into reasoning_content anyway;
// a budget spent entirely there is a cut-off reply, not a misrouted answer.
const analysisOnly = withReply({ content: "", reasoning_content: "analysis..." }, "length");
await analysisOnly.run(async (manager) => {
await assert.rejects(manager.inference(MESSAGES, { disableThinking: true }), TRUNCATED);
});
});
test("a cut-off reply that has content is still returned to lenient callers", async () => {
// Only requireCompleteOutput callers reject a partial answer (#2130).
const partial = withReply({ content: "Cleaned te", reasoning_content: "" }, "length");
await partial.run(async (manager) => {
assert.equal(await manager.inference(MESSAGES, { disableThinking: true }), "Cleaned te");
await assert.rejects(manager.inference(MESSAGES, { requireCompleteOutput: true }), TRUNCATED);
});
});
test("an answer routed into reasoning_content with thinking suppressed is still returned", async () => {
// Some llama-server builds do this despite enable_thinking: false (#809).
const misrouted = withReply({ content: "", reasoning_content: "Cleaned text." });
await misrouted.run(async (manager) => {
assert.equal(await manager.inference(MESSAGES, { disableThinking: true }), "Cleaned text.");
assert.equal(await manager.inference(MESSAGES, {}), "Cleaned text.");
});
assert.deepEqual(misrouted.requests[0].chat_template_kwargs, { enable_thinking: false });
});
test("content is returned over reasoning_content whether or not thinking was on", async () => {
const both = withReply({ content: "Cleaned text.", reasoning_content: "Thinking Process:" });
await both.run(async (manager) => {
assert.equal(await manager.inference(MESSAGES, { disableThinking: false }), "Cleaned text.");
assert.equal(await manager.inference(MESSAGES, {}), "Cleaned text.");
});
});
// --- Vulkan context step-down --------------------------------------------
//
// Sizing the context to the request is bounded by SYSTEM RAM, but a discrete
// GPU has its own memory that the app does not probe. A machine with plenty of
// RAM and a small card can therefore now ask for a context that will not fit
// in VRAM — a failure this change introduces. Dropping straight to CPU costs
// 10-30x; halving the context is far cheaper, so try that first.
function makeLadder(shouldFail, { contextSize = 32768 } = {}) {
const manager = new LlamaServerManager();
const attempts = [];
manager._buildEnv = () => ({});
manager._killCurrentProcess = async () => {};
manager.findAvailablePort = async () => 8221;
manager.draftModelPath = null;
manager._startWithBinary = async (binary, args) => {
const attempt = { binary, ctx: Number(args[args.indexOf("--ctx-size") + 1]) };
attempts.push(attempt);
if (shouldFail(attempt)) throw new Error(`stub fail: ${binary} ctx=${attempt.ctx}`);
};
const baseArgs = manager._buildBaseArgs("/models/main.gguf", 8221, { contextSize });
return { manager, attempts, baseArgs, options: { contextSize } };
}
const BINARIES = { vulkan: "/bin/vulkan", cpu: "/bin/cpu" };
test("a Vulkan start that fails at a grown context retries smaller before dropping to CPU", async () => {
// Vulkan only succeeds at the baseline context.
const { manager, attempts, baseArgs, options } = makeLadder(
(attempt) => attempt.binary === "/bin/vulkan" && attempt.ctx > 16384
);
await manager._startWithGpuFallback(BINARIES, baseArgs, options, []);
assert.deepEqual(attempts, [
{ binary: "/bin/vulkan", ctx: 32768 },
{ binary: "/bin/vulkan", ctx: 16384 },
]);
assert.equal(manager.activeBackend, "vulkan", "a live GPU beats a CPU fallback");
});
test("a baseline-context start keeps today's exact ladder, with no extra rung", async () => {
// The step-down must not add an attempt when there is nothing to step down
// to, or every existing failure path gets slower.
const { manager, attempts, baseArgs, options } = makeLadder(() => true, {
contextSize: 16384,
});
await assert.rejects(() => manager._startWithGpuFallback(BINARIES, baseArgs, options, []));
assert.deepEqual(attempts, [
{ binary: "/bin/vulkan", ctx: 16384 },
{ binary: "/bin/cpu", ctx: 16384 },
]);
});
test("a context step-down is recorded, so the server reports what it actually has", async () => {
const { manager, baseArgs, options } = makeLadder(
(attempt) => attempt.binary === "/bin/vulkan" && attempt.ctx > 16384
);
await manager._startWithGpuFallback(BINARIES, baseArgs, options, []);
// Otherwise the preflight would believe the request still fits and send a
// prompt the server cannot take.
assert.equal(manager.contextSize, 16384);
});