Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(llama): size the local context to the request instead of capping it at 16384 (#2155)

Chad Piha4335972b96a6
diff --git a/src/components/notes/BackgroundActionToastListener.tsx b/src/components/notes/BackgroundActionToastListener.tsx
index f61ac34a6..c8bb090a1 100644
--- a/src/components/notes/BackgroundActionToastListener.tsx
+++ b/src/components/notes/BackgroundActionToastListener.tsx
@@ -18,7 +18,7 @@ export default function BackgroundActionToastListener() {
for (const event of consumeErrorEvents()) {
toast({
title: t("notes.enhance.title"),
- description: event.message,
+ description: event.messageKey ? t(event.messageKey, event.messageParams) : event.message,
variant: "destructive",
});
}
diff --git a/src/helpers/ggufMetadata.js b/src/helpers/ggufMetadata.js
new file mode 100644
index 000000000..b245b54d1
--- /dev/null
+++ b/src/helpers/ggufMetadata.js
@@ -0,0 +1,282 @@
+/**
+ * Minimal GGUF metadata reader.
+ *
+ * Only the header is parsed — the magic, version, counts, and the metadata
+ * key/value section. Tensor descriptors and tensor data are never touched, so
+ * this reads a few megabytes off the front of a multi-gigabyte file.
+ *
+ * It exists to answer one question: how many bytes of KV cache does one token
+ * of context cost for this model? That number varies ~12x across the bundled
+ * registry and is anti-correlated with file size, so it cannot be guessed from
+ * anything the model registry records (#2142). The GGUF is also the file
+ * llama.cpp actually loads, which makes it the honest source for the trained
+ * context length as well.
+ *
+ * Every failure returns null. Callers must read null as "fall back to the
+ * historical context size", never as "assume this model is cheap".
+ */
+const fs = require("fs");
+
+const GGUF_MAGIC = 0x46554747; // "GGUF" little-endian
+const MIN_SUPPORTED_VERSION = 2; // v1 used 32-bit lengths; no bundled model uses it
+const MAX_SUPPORTED_VERSION = 3;
+
+const READ_CHUNK_BYTES = 1024 * 1024;
+// A real header is ~10 MiB, dominated by the tokenizer token array. These are
+// guards against a corrupt file steering us into an unbounded allocation.
+const MAX_METADATA_BYTES = 64 * 1024 * 1024;
+const MAX_KV_COUNT = 4096;
+const MAX_STRING_BYTES = 64 * 1024 * 1024;
+const MAX_CAPTURED_ARRAY_LENGTH = 1024;
+
+const TYPE = {
+ UINT8: 0,
+ INT8: 1,
+ UINT16: 2,
+ INT16: 3,
+ UINT32: 4,
+ INT32: 5,
+ FLOAT32: 6,
+ BOOL: 7,
+ STRING: 8,
+ ARRAY: 9,
+ UINT64: 10,
+ INT64: 11,
+ FLOAT64: 12,
+};
+
+const SCALAR_BYTES = {
+ [TYPE.UINT8]: 1,
+ [TYPE.INT8]: 1,
+ [TYPE.UINT16]: 2,
+ [TYPE.INT16]: 2,
+ [TYPE.UINT32]: 4,
+ [TYPE.INT32]: 4,
+ [TYPE.FLOAT32]: 4,
+ [TYPE.BOOL]: 1,
+ [TYPE.UINT64]: 8,
+ [TYPE.INT64]: 8,
+ [TYPE.FLOAT64]: 8,
+};
+
+// Suffixes whose values decide KV cost. Matched against the architecture
+// prefix the file declares, so this works for any arch without an allow-list.
+const WANTED_SUFFIXES = [
+ ".block_count",
+ ".context_length",
+ ".embedding_length",
+ ".attention.head_count",
+ ".attention.head_count_kv",
+ ".attention.key_length",
+ ".attention.value_length",
+];
+
+/** Forward-only cursor over the front of the file, buffered in chunks. */
+class HeaderCursor {
+ constructor(source) {
+ this.source = source;
+ this.chunk = Buffer.alloc(0);
+ this.chunkStart = 0;
+ this.position = 0;
+ }
+
+ async ensure(length) {
+ if (this.position + length > MAX_METADATA_BYTES) {
+ throw new Error("gguf metadata section is implausibly large");
+ }
+ const withinChunk =
+ this.position >= this.chunkStart &&
+ this.position + length <= this.chunkStart + this.chunk.length;
+ if (withinChunk) return;
+
+ const buffer = await this.source.read(this.position, Math.max(length, READ_CHUNK_BYTES));
+ if (!buffer || buffer.length < length) throw new Error("unexpected end of gguf header");
+ this.chunk = buffer;
+ this.chunkStart = this.position;
+ }
+
+ async take(length) {
+ await this.ensure(length);
+ const start = this.position - this.chunkStart;
+ this.position += length;
+ return this.chunk.subarray(start, start + length);
+ }
+
+ /** Advances without reading, so huge arrays cost nothing. */
+ skip(length) {
+ if (length < 0) throw new Error("negative skip in gguf header");
+ this.position += length;
+ }
+
+ async readUint32() {
+ return (await this.take(4)).readUInt32LE(0);
+ }
+
+ async readUint64() {
+ const value = (await this.take(8)).readBigUInt64LE(0);
+ if (value > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error("gguf length out of range");
+ return Number(value);
+ }
+
+ async readString() {
+ const length = await this.readUint64();
+ if (length > MAX_STRING_BYTES) throw new Error("gguf string is implausibly long");
+ return (await this.take(length)).toString("utf8");
+ }
+
+ async skipString() {
+ const length = await this.readUint64();
+ if (length > MAX_STRING_BYTES) throw new Error("gguf string is implausibly long");
+ this.skip(length);
+ }
+
+ async readScalar(type) {
+ const buffer = await this.take(SCALAR_BYTES[type]);
+ switch (type) {
+ case TYPE.UINT8:
+ return buffer.readUInt8(0);
+ case TYPE.INT8:
+ return buffer.readInt8(0);
+ case TYPE.UINT16:
+ return buffer.readUInt16LE(0);
+ case TYPE.INT16:
+ return buffer.readInt16LE(0);
+ case TYPE.UINT32:
+ return buffer.readUInt32LE(0);
+ case TYPE.INT32:
+ return buffer.readInt32LE(0);
+ case TYPE.FLOAT32:
+ return buffer.readFloatLE(0);
+ case TYPE.BOOL:
+ return buffer.readUInt8(0) !== 0;
+ case TYPE.UINT64:
+ return Number(buffer.readBigUInt64LE(0));
+ case TYPE.INT64:
+ return Number(buffer.readBigInt64LE(0));
+ case TYPE.FLOAT64:
+ return buffer.readDoubleLE(0);
+ default:
+ throw new Error(`unsupported gguf scalar type ${type}`);
+ }
+ }
+}
+
+async function readArray(cursor, capture) {
+ const elementType = await cursor.readUint32();
+ const length = await cursor.readUint64();
+
+ if (elementType === TYPE.STRING) {
+ // Each element carries its own length prefix, so it must be walked rather
+ // than skipped wholesale. This is tokenizer.ggml.tokens, ~150k entries.
+ for (let index = 0; index < length; index += 1) await cursor.skipString();
+ return null;
+ }
+
+ const elementBytes = SCALAR_BYTES[elementType];
+ if (elementBytes === undefined) throw new Error(`unsupported gguf array type ${elementType}`);
+
+ if (!capture || length > MAX_CAPTURED_ARRAY_LENGTH) {
+ cursor.skip(length * elementBytes);
+ return null;
+ }
+
+ const values = [];
+ for (let index = 0; index < length; index += 1) values.push(await cursor.readScalar(elementType));
+ return values;
+}
+
+async function readValue(cursor, type, capture) {
+ if (type === TYPE.STRING) {
+ if (capture) return cursor.readString();
+ await cursor.skipString();
+ return null;
+ }
+ if (type === TYPE.ARRAY) return readArray(cursor, capture);
+
+ const bytes = SCALAR_BYTES[type];
+ if (bytes === undefined) throw new Error(`unsupported gguf value type ${type}`);
+ if (capture) return cursor.readScalar(type);
+ cursor.skip(bytes);
+ return null;
+}
+
+function isWantedKey(key) {
+ if (key === "general.architecture") return true;
+ return WANTED_SUFFIXES.some((suffix) => key.endsWith(suffix));
+}
+
+function pick(entries, architecture, suffix) {
+ return entries.get(`${architecture}${suffix}`);
+}
+
+/**
+ * @param source {{ read: (offset: number, length: number) => Promise<Buffer>, size?: number }}
+ * @returns the architecture fields needed to price the KV cache, or null.
+ */
+async function readGgufMetadata(source) {
+ try {
+ const cursor = new HeaderCursor(source);
+
+ if ((await cursor.readUint32()) !== GGUF_MAGIC) return null;
+ const version = await cursor.readUint32();
+ if (version < MIN_SUPPORTED_VERSION || version > MAX_SUPPORTED_VERSION) return null;
+
+ await cursor.readUint64(); // tensor_count, unused
+ const kvCount = await cursor.readUint64();
+ if (kvCount > MAX_KV_COUNT) return null;
+
+ const entries = new Map();
+ let hasRecurrentState = false;
+
+ for (let index = 0; index < kvCount; index += 1) {
+ const key = await cursor.readString();
+ const type = await cursor.readUint32();
+ if (key.includes(".ssm.")) hasRecurrentState = true;
+
+ const value = await readValue(cursor, type, isWantedKey(key));
+ if (value !== null) entries.set(key, value);
+ }
+
+ const architecture = entries.get("general.architecture");
+ if (typeof architecture !== "string" || architecture.length === 0) return null;
+
+ return {
+ architecture,
+ blockCount: pick(entries, architecture, ".block_count"),
+ contextLength: pick(entries, architecture, ".context_length"),
+ embeddingLength: pick(entries, architecture, ".embedding_length"),
+ headCount: pick(entries, architecture, ".attention.head_count"),
+ headCountKv: pick(entries, architecture, ".attention.head_count_kv"),
+ keyLength: pick(entries, architecture, ".attention.key_length"),
+ valueLength: pick(entries, architecture, ".attention.value_length"),
+ hasRecurrentState,
+ };
+ } catch {
+ return null;
+ }
+}
+
+/** Convenience wrapper over a file on disk. */
+async function readGgufMetadataFromFile(filePath) {
+ let handle = null;
+ try {
+ handle = await fs.promises.open(filePath, "r");
+ const { size } = await handle.stat();
+ const file = handle;
+ return await readGgufMetadata({
+ size,
+ read: async (offset, length) => {
+ const buffer = Buffer.alloc(Math.min(length, Math.max(0, size - offset)));
+ if (buffer.length === 0) return buffer;
+ const { bytesRead } = await file.read(buffer, 0, buffer.length, offset);
+ return buffer.subarray(0, bytesRead);
+ },
+ });
+ } catch {
+ return null;
+ } finally {
+ await handle?.close().catch(() => {});
+ }
+}
+
+module.exports = { readGgufMetadata, readGgufMetadataFromFile };
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 2559d61c9..1cd585030 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -5186,7 +5186,15 @@ class IPCHandlers {
const result = await LocalReasoningService.processText(text, modelId, config);
return { success: true, text: result };
} catch (error) {
- return { success: false, error: error.message };
+ // code/details carry the machine-readable failure across to the
+ // renderer, which owns the translation keys. Flattening to a bare
+ // string is how llama.cpp JSON used to reach users (#2142).
+ return {
+ success: false,
+ error: error.message,
+ code: error.code,
+ details: error.details,
+ };
}
});
diff --git a/src/helpers/llamaContextPolicy.js b/src/helpers/llamaContextPolicy.js
new file mode 100644
index 000000000..931b9b833
--- /dev/null
+++ b/src/helpers/llamaContextPolicy.js
@@ -0,0 +1,245 @@
+/**
+ * Pure policy for sizing the llama-server context window.
+ *
+ * Replaces the flat SERVER_CONTEXT_SIZE = 16384 cap (#2142), which blocked
+ * every transcript over ~80 minutes on every bundled model. The cap itself was
+ * not gratuitous: it fixed #1203, where an unbounded context allocated a ~15 GB
+ * Metal-wired KV cache and froze a 16 GB Mac into a watchdog kernel panic. So
+ * the replacement is not a bigger constant but a per-machine, per-model budget.
+ *
+ * Everything here is pure arithmetic so the memory guarantee can be tested
+ * without a GPU, a model file, or a spawned process.
+ */
+
+// Latin text tokenizes at roughly 3.5-4.2 chars/token on the Qwen/Llama BPEs
+// used here (measured: 4.16 on a synthetic English transcript). 3 leaves
+// headroom, because under-estimating is the bug and over-estimating only costs
+// KV bytes that the ceiling already bounds.
+const LATIN_CHARS_PER_TOKEN = 3;
+
+// CJK runs about 1-1.5 chars/token. Charging 1 keeps the estimate on the safe
+// side for the ja / zh-CN / zh-TW locales the app ships.
+const CJK_RANGES = [
+ [0x1100, 0x11ff], // Hangul Jamo
+ [0x3000, 0x303f], // CJK symbols and punctuation
+ [0x3040, 0x30ff], // Hiragana + Katakana
+ [0x3400, 0x4dbf], // CJK Unified Ideographs Extension A
+ [0x4e00, 0x9fff], // CJK Unified Ideographs
+ [0xac00, 0xd7af], // Hangul Syllables
+ [0xf900, 0xfaff], // CJK Compatibility Ideographs
+ [0xff00, 0xffef], // Halfwidth and Fullwidth Forms
+ [0x20000, 0x3ffff], // CJK Unified Ideographs Extensions B and beyond
+];
+
+function isCjkCodePoint(codePoint) {
+ for (const [start, end] of CJK_RANGES) {
+ if (codePoint >= start && codePoint <= end) return true;
+ }
+ return false;
+}
+
+/**
+ * Cheap, deliberately high estimate of how many tokens a string will occupy.
+ *
+ * Only used to choose the context size a server STARTS at. Once a server is
+ * running, POST /tokenize returns the exact count in ~13 ms and supersedes
+ * this; the estimate just avoids paying a restart in the common case.
+ */
+function estimateTokens(text) {
+ if (typeof text !== "string" || text.length === 0) return 0;
+
+ let cjkCount = 0;
+ let otherCount = 0;
+ for (const character of text) {
+ if (isCjkCodePoint(character.codePointAt(0))) cjkCount += 1;
+ else otherCount += 1;
+ }
+
+ return cjkCount + Math.ceil(otherCount / LATIN_CHARS_PER_TOKEN);
+}
+
+// K and V are cached as f16 by default (llama.cpp --cache-type-k/-v).
+const KV_ELEMENT_BYTES = 2;
+
+function resolveHeadCountKv(metadata) {
+ const raw = metadata.headCountKv;
+ // GGUF permits a per-layer array. The widest layer is the safe reading.
+ if (Array.isArray(raw)) {
+ const widest = raw.reduce((max, value) => (value > max ? value : max), 0);
+ return widest > 0 ? widest : null;
+ }
+ return typeof raw === "number" && raw > 0 ? raw : null;
+}
+
+function resolveHeadWidths(metadata) {
+ const { keyLength, valueLength, embeddingLength, headCount } = metadata;
+ if (keyLength > 0 && valueLength > 0) return { keyLength, valueLength };
+
+ // llama.cpp's own fallback when the widths are not declared.
+ if (embeddingLength > 0 && headCount > 0) {
+ const width = embeddingLength / headCount;
+ if (Number.isInteger(width) && width > 0) return { keyLength: width, valueLength: width };
+ }
+ return null;
+}
+
+/**
+ * Bytes of KV cache each token of context costs, from the model's own
+ * architecture. Assumes every layer is full attention, which over-estimates
+ * sliding-window models (Gemma 3/4) and models that interleave full attention
+ * (Qwen3.5). Over-estimating shrinks the ceiling, so it errs safe.
+ *
+ * Returns null when the architecture cannot be read. Callers must treat null
+ * as "fall back to the historical 16384", never as "assume it is cheap".
+ */
+function kvBytesPerToken(metadata) {
+ if (!metadata || typeof metadata !== "object") return null;
+
+ const blockCount = metadata.blockCount;
+ if (!(blockCount > 0)) return null;
+
+ const headCountKv = resolveHeadCountKv(metadata);
+ if (headCountKv === null) return null;
+
+ const widths = resolveHeadWidths(metadata);
+ if (widths === null) return null;
+
+ return blockCount * headCountKv * (widths.keyLength + widths.valueLength) * KV_ELEMENT_BYTES;
+}
+
+// The historical cap (#1236). Now the floor and the universal fallback rather
+// than the ceiling, so no configuration that works today can get smaller.
+const BASELINE_CONTEXT_SIZE = 16384;
+
+// Half the machine, minus a fixed slice for the OS itself. Deliberately ~1.7x
+// tighter than llama.cpp's own --fit, which would permit ~9.6 GiB of wired
+// memory on the 16 GB Mac that panicked in #1203.
+const USABLE_MEMORY_FRACTION = 0.5;
+const FIXED_OS_RESERVE_BYTES = 2 * 1024 * 1024 * 1024;
+
+// Compute buffers also grow with context: measured 144 MiB total at 16384 and
+// 240 MiB at 65536 for a 9B, i.e. ~2 KiB/token plus a fixed base. Doubled for
+// margin, and the base sized for the 31B-class graphs.
+const COMPUTE_FIXED_BYTES = 384 * 1024 * 1024;
+const COMPUTE_BYTES_PER_TOKEN = 4096;
+
+// Hybrid models (Qwen3.5, LFM2) carry recurrent state that is not part of the
+// KV cache and does not scale with context.
+const RECURRENT_STATE_BYTES = 512 * 1024 * 1024;
+
+// A discrete GPU's VRAM is not system RAM and we do not probe it. Until a real
+// measurement exists, stay modest and let the existing GPU fallback ladder
+// handle anything that still will not fit.
+const UNVERIFIED_GPU_CEILING = 65536;
+
+// Contexts are aligned down to a round number so that near-identical requests
+// land on the same value and cannot cause a second restart.
+const CONTEXT_ALIGNMENT = 512;
+
+const MIB = 1024 * 1024;
+const MIN_CACHE_RAM_MIB = 512;
+const MAX_CACHE_RAM_MIB = 2048;
+
+function clamp(value, low, high) {
+ return Math.min(high, Math.max(low, value));
+}
+
+function fallbackCeiling(reason) {
+ return { ceiling: BASELINE_CONTEXT_SIZE, reason, cacheRamMiB: MIN_CACHE_RAM_MIB };
+}
+
+/**
+ * The largest context this machine may allocate for this model.
+ *
+ * Every path that cannot answer confidently returns BASELINE_CONTEXT_SIZE, so
+ * the worst case of this whole mechanism is the behaviour that ships today.
+ */
+function resolveContextCeiling({
+ totalMemoryBytes,
+ weightsBytes = 0,
+ draftWeightsBytes = 0,
+ kvBytesPerToken,
+ hasRecurrentState = false,
+ trainedContextTokens = 0,
+ discreteGpuUnverified = false,
+}) {
+ if (!Number.isFinite(totalMemoryBytes) || totalMemoryBytes <= 0) {
+ return fallbackCeiling("no-memory-info");
+ }
+ if (!Number.isFinite(kvBytesPerToken) || kvBytesPerToken <= 0) {
+ return fallbackCeiling("unknown-kv-cost");
+ }
+
+ const budgetBytes = totalMemoryBytes * USABLE_MEMORY_FRACTION - FIXED_OS_RESERVE_BYTES;
+ const kvBudgetBytes =
+ budgetBytes -
+ weightsBytes -
+ draftWeightsBytes -
+ COMPUTE_FIXED_BYTES -
+ (hasRecurrentState ? RECURRENT_STATE_BYTES : 0);
+
+ if (kvBudgetBytes <= 0) return fallbackCeiling("floor");
+
+ const cacheRamMiB = clamp(
+ Math.floor(kvBudgetBytes / 4 / MIB),
+ MIN_CACHE_RAM_MIB,
+ MAX_CACHE_RAM_MIB
+ );
+
+ const perTokenBytes = kvBytesPerToken + COMPUTE_BYTES_PER_TOKEN;
+ const affordable =
+ Math.floor(Math.floor(kvBudgetBytes / perTokenBytes) / CONTEXT_ALIGNMENT) * CONTEXT_ALIGNMENT;
+
+ let hardMax = trainedContextTokens > 0 ? trainedContextTokens : Number.POSITIVE_INFINITY;
+ let hardMaxReason = "trained-context-bound";
+ if (discreteGpuUnverified && UNVERIFIED_GPU_CEILING < hardMax) {
+ hardMax = UNVERIFIED_GPU_CEILING;
+ hardMaxReason = "gpu-cap";
+ }
+
+ const ceiling = Math.min(affordable, hardMax);
+ if (ceiling <= BASELINE_CONTEXT_SIZE) return fallbackCeiling("floor");
+
+ return {
+ ceiling,
+ reason: ceiling === hardMax ? hardMaxReason : "memory-bound",
+ cacheRamMiB,
+ };
+}
+
+// Quantizing the chosen context bounds how many distinct restarts a session can
+// cause: at most one per rung per model, however many differently-sized
+// requests arrive.
+const CONTEXT_STEPS = [16384, 32768, 65536, 131072, 262144];
+
+/**
+ * The context size to start (or grow) the server at for a request needing
+ * `needed` tokens, given a caller floor and the machine's ceiling.
+ *
+ * The ceiling always wins: it is the memory guarantee, and it is not
+ * necessarily on a rung.
+ */
+function resolveContextSize({ needed = 0, floor = 0, ceiling = BASELINE_CONTEXT_SIZE }) {
+ const target = Math.max(needed || 0, floor || 0, BASELINE_CONTEXT_SIZE);
+ const rung = CONTEXT_STEPS.find((step) => step >= target) ?? Number.POSITIVE_INFINITY;
+ const bounded = Math.min(rung, Math.max(ceiling, BASELINE_CONTEXT_SIZE));
+ return Math.max(bounded, BASELINE_CONTEXT_SIZE);
+}
+
+module.exports = {
+ LATIN_CHARS_PER_TOKEN,
+ KV_ELEMENT_BYTES,
+ estimateTokens,
+ kvBytesPerToken,
+ resolveContextCeiling,
+ resolveContextSize,
+ CONTEXT_STEPS,
+ BASELINE_CONTEXT_SIZE,
+ USABLE_MEMORY_FRACTION,
+ FIXED_OS_RESERVE_BYTES,
+ COMPUTE_FIXED_BYTES,
+ COMPUTE_BYTES_PER_TOKEN,
+ RECURRENT_STATE_BYTES,
+ UNVERIFIED_GPU_CEILING,
+ CONTEXT_ALIGNMENT,
+};
diff --git a/src/helpers/llamaServer.js b/src/helpers/llamaServer.js
index c319cc23c..83fc519a5 100644
--- a/src/helpers/llamaServer.js
+++ b/src/helpers/llamaServer.js
@@ -9,6 +9,7 @@ const { getSafeTempDir } = require("./safeTempDir");
const { app } = require("electron");
const sidecarPidFile = require("./sidecarPidFile");
const { BIN_SUBDIR: LLAMA_VULKAN_BIN_SUBDIR } = require("./llamaVulkanManager");
+const { BASELINE_CONTEXT_SIZE } = require("./llamaContextPolicy");
// Range kept clear of cliBridge (8200-8219) to avoid port-bind collisions.
const PORT_RANGE_START = 8221;
@@ -21,6 +22,51 @@ const STARTUP_POLL_INTERVAL_MS = 500;
const HEALTH_CHECK_FAILURE_THRESHOLD = 3;
const IDLE_TIMEOUT_MS = 5 * 60 * 1000;
const DEFAULT_CONTEXT_SIZE = 4096;
+const PREFLIGHT_TIMEOUT_MS = 10000;
+
+/**
+ * llama-server reports a prompt that overflows the context as a 400 whose body
+ * is a JSON blob. Surfacing that verbatim is how customers ended up reading
+ * llama.cpp internals in a toast (#2142), so it becomes a typed error carrying
+ * the two numbers callers actually need.
+ */
+/** Same argv with --ctx-size rewritten; used by the context step-down rung. */
+function withContextSize(args, contextSize) {
+ const index = args.indexOf("--ctx-size");
+ if (index === -1) return args;
+ const copy = [...args];
+ copy[index + 1] = String(contextSize);
+ return copy;
+}
+
+function buildResponseError(statusCode, body) {
+ if (statusCode === 400) {
+ try {
+ const parsed = JSON.parse(body)?.error;
+ const isOverflow =
+ parsed?.type === "exceed_context_size_error" ||
+ /exceeds the available context size/i.test(parsed?.message || "");
+
+ if (isOverflow) {
+ const neededTokens = Number(parsed.n_prompt_tokens) || null;
+ const maxContextTokens = Number(parsed.n_ctx) || null;
+ const error = new Error(
+ neededTokens && maxContextTokens
+ ? `The request needs ${neededTokens} tokens of context but the model is running with ${maxContextTokens}.`
+ : "The request is longer than the context this model is running with."
+ );
+ error.code = "CONTEXT_TOO_LARGE";
+ error.neededTokens = neededTokens;
+ error.maxContextTokens = maxContextTokens;
+ return error;
+ }
+ } catch {
+ // Not JSON, or not the shape we expect: fall through to the generic error.
+ }
+ }
+
+ return new Error(`llama-server returned status ${statusCode}: ${body}`);
+}
class LlamaServerManager {
constructor() {
@@ -32,6 +78,13 @@ class LlamaServerManager {
// the start() restart check); activeDraftModelPath is the one that actually loaded.
this.draftModelPath = null;
this.activeDraftModelPath = null;
+ // The context that actually loaded, which the GPU ladder may step down
+ // below the requested one.
+ this.activeContextSize = null;
+ // The context the running server was started with. Grows on demand and is
+ // only reset by stop(), so a short request cannot shrink a window a long
+ // one just paid to open. See #2142.
+ this.contextSize = null;
this.startupPromise = null;
this.healthCheckInterval = null;
this.healthCheckFailures = 0;
@@ -124,7 +177,13 @@ class LlamaServerManager {
// A change in drafter presence for the same model must still restart the
// server so the new speculative-decoding flags take effect.
const requestedDraftPath = options.draftModelPath || null;
- if (this.ready && this.modelPath === modelPath && this.draftModelPath === requestedDraftPath)
+ const requestedContextSize = options.contextSize || DEFAULT_CONTEXT_SIZE;
+ if (
+ this.ready &&
+ this.modelPath === modelPath &&
+ this.draftModelPath === requestedDraftPath &&
+ requestedContextSize <= (this.contextSize || 0)
+ )
return;
if (this.process) {
@@ -134,39 +193,55 @@ class LlamaServerManager {
this.startupPromise = this._doStart(modelPath, options);
try {
await this.startupPromise;
+ // The ladder may have stepped the context down to get a GPU rung to
+ // start; record what loaded, not what was asked for.
+ this.contextSize = this.activeContextSize ?? requestedContextSize;
} finally {
this.startupPromise = null;
}
}
- async _doStart(modelPath, options = {}) {
- const binaryPaths = this.getServerBinaryPaths();
- if (Object.keys(binaryPaths).length === 0) throw new Error("llama-server binary not found");
- if (!fs.existsSync(modelPath)) throw new Error(`Model file not found: ${modelPath}`);
-
- this.port = await this.findAvailablePort();
- this.modelPath = modelPath;
- // Store the REQUESTED drafter so start() compares against a stable value across
- // identical requests; activeDraftModelPath tracks what actually loaded (see ctor).
- this.draftModelPath = options.draftModelPath || null;
- this.activeDraftModelPath = null;
-
- const baseArgs = [
+ _buildBaseArgs(modelPath, port, options = {}) {
+ const args = [
"--model",
modelPath,
"--host",
"127.0.0.1",
"--port",
- String(this.port),
+ String(port),
"--threads",
String(options.threads || 4),
// Unset, this defaults to the model's full trained context (128K+),
// whose KV cache can exceed total RAM with --fit disabled. See #1203.
+ // Sized per request against a memory budget by llamaContextPolicy.
"--ctx-size",
String(options.contextSize || DEFAULT_CONTEXT_SIZE),
"--jinja",
];
+ // llama-server otherwise reserves 8192 MiB of host RAM for the prompt
+ // cache, on top of the KV cache, which would undo the memory budget. Cap
+ // it rather than disabling it: 0 forces a full re-prefill every request.
+ if (options.cacheRamMiB) args.push("--cache-ram", String(options.cacheRamMiB));
+
+ return args;
+ }
+
+ async _doStart(modelPath, options = {}) {
+ const binaryPaths = this.getServerBinaryPaths();
+ if (Object.keys(binaryPaths).length === 0) throw new Error("llama-server binary not found");
+ if (!fs.existsSync(modelPath)) throw new Error(`Model file not found: ${modelPath}`);
+
+ this.port = await this.findAvailablePort();
+ this.modelPath = modelPath;
+ // Store the REQUESTED drafter so start() compares against a stable value across
+ // identical requests; activeDraftModelPath tracks what actually loaded (see ctor).
+ this.draftModelPath = options.draftModelPath || null;
+ this.activeDraftModelPath = null;
+
+ this.activeContextSize = options.contextSize || DEFAULT_CONTEXT_SIZE;
+ const baseArgs = this._buildBaseArgs(modelPath, this.port, options);
+
// Draft flags stay separate from baseArgs so the fallback ladder can retry without
// them when a stale (pre-b9763) binary rejects the MTP args at parse time.
const draftArgs = options.draftModelPath
@@ -209,6 +284,7 @@ class LlamaServerManager {
const gpuArgs = [...baseArgs, "--n-gpu-layers", String(options.gpuLayers ?? 99)];
const cpuArgs = baseArgs;
const hasDraft = draftArgs.length > 0;
+ const requestedContext = options.contextSize || DEFAULT_CONTEXT_SIZE;
// Degrade ladder: GPU+MTP, then GPU alone (a live GPU beats speculation), then
// CPU+MTP (the bundled pin normally accepts the flags), then plain CPU. The
@@ -234,6 +310,25 @@ class LlamaServerManager {
timeout: VULKAN_STARTUP_TIMEOUT_MS,
attemptMsg: "Attempting Vulkan backend startup",
},
+ // A context sized against system RAM can still be too big for a
+ // discrete GPU's own memory, which the app does not probe. Halving it is
+ // far cheaper than the 10-30x cost of dropping to CPU, so try that
+ // first. Omitted entirely when there is nothing to step down to, so a
+ // baseline start keeps today's exact ladder.
+ ...(requestedContext > BASELINE_CONTEXT_SIZE && binaryPaths.vulkan
+ ? [
+ {
+ backend: "vulkan",
+ name: "Vulkan (reduced context)",
+ binary: binaryPaths.vulkan,
+ args: withContextSize(gpuArgs, BASELINE_CONTEXT_SIZE),
+ mtp: false,
+ contextSize: BASELINE_CONTEXT_SIZE,
+ timeout: VULKAN_STARTUP_TIMEOUT_MS,
+ attemptMsg: "Attempting Vulkan backend startup with a reduced context",
+ },
+ ]
+ : []),
{
backend: "cpu",
name: "CPU",
@@ -272,6 +367,10 @@ class LlamaServerManager {
);
this.activeBackend = rung.backend;
this.activeDraftModelPath = rung.mtp ? this.draftModelPath : null;
+ // Without this the preflight would believe the full context is
+ // available and send a prompt the server cannot take.
+ this.activeContextSize = rung.contextSize ?? requestedContext;
+ this.contextSize = this.activeContextSize;
return;
} catch (err) {
lastError = err;
@@ -523,6 +622,90 @@ class LlamaServerManager {
}
}
+ /**
+ * Small JSON round-trip against the running server.
+ *
+ * Resolves null on any failure rather than rejecting: these calls exist to
+ * make a decision more precise, so a broken one must degrade to the estimate
+ * and never fail the user's request.
+ */
+ _requestJson(path, body = null, timeoutMs = PREFLIGHT_TIMEOUT_MS) {
+ return new Promise((resolve) => {
+ const payload = body === null ? null : JSON.stringify(body);
+ const req = http.request(
+ {
+ hostname: "127.0.0.1",
+ port: this.port,
+ path,
+ method: payload === null ? "GET" : "POST",
+ headers: payload
+ ? {
+ "Content-Type": "application/json",
+ "Content-Length": Buffer.byteLength(payload),
+ }
+ : {},
+ timeout: timeoutMs,
+ },
+ (res) => {
+ let data = "";
+ res.on("data", (chunk) => {
+ data += chunk;
+ });
+ res.on("end", () => {
+ if (res.statusCode !== 200) return resolve(null);
+ try {
+ resolve(JSON.parse(data));
+ } catch {
+ resolve(null);
+ }
+ });
+ }
+ );
+
+ req.on("error", () => resolve(null));
+ req.on("timeout", () => {
+ req.destroy();
+ resolve(null);
+ });
+ if (payload) req.write(payload);
+ req.end();
+ });
+ }
+
+ /**
+ * The context window actually in force, which is not always what we asked
+ * for: a concurrent caller can join an in-flight start that used a smaller
+ * size. Returns null when it cannot be read.
+ */
+ async usableContextSize() {
+ if (!this.ready || !this.process) return null;
+ const props = await this._requestJson("/props");
+ const contextSize = props?.default_generation_settings?.n_ctx;
+ return Number.isFinite(contextSize) && contextSize > 0 ? contextSize : null;
+ }
+
+ /**
+ * Exact token count for the prompt as the server will actually see it, chat
+ * template included. Roughly 15 ms for an 85 KB prompt, which is cheap
+ * enough to make the difference between a right-sized window and a 400.
+ */
+ async countPromptTokens(messages, options = {}) {
+ if (!this.ready || !this.process) return null;
+
+ const payload = { messages };
+ // Must mirror inference() exactly, or we would price a different prompt
+ // than the one we go on to send.
+ if (options.disableThinking !== false) {
+ payload.chat_template_kwargs = { enable_thinking: false };
+ }
+
+ const templated = await this._requestJson("/apply-template", payload);
+ if (typeof templated?.prompt !== "string") return null;
+
+ const tokenized = await this._requestJson("/tokenize", { content: templated.prompt });
+ return Array.isArray(tokenized?.tokens) ? tokenized.tokens.length : null;
+ }
+
async inference(messages, options = {}) {
if (!this.ready || !this.process) {
throw new Error("llama-server is not running");
@@ -572,7 +755,7 @@ class LlamaServerManager {
});
if (res.statusCode !== 200) {
- reject(new Error(`llama-server returned status ${res.statusCode}: ${data}`));
+ reject(buildResponseError(res.statusCode, data));
return;
}
@@ -614,6 +797,7 @@ class LlamaServerManager {
if (!this.process) {
this.ready = false;
+ this.contextSize = null;
return;
}
@@ -651,6 +835,8 @@ class LlamaServerManager {
this.draftModelPath = null;
this.activeDraftModelPath = null;
this.activeBackend = null;
+ this.contextSize = null;
+ this.activeContextSize = null;
}
getStatus() {
diff --git a/src/helpers/modelManagerBridge.js b/src/helpers/modelManagerBridge.js
index c6eed6a4d..d80d4ebab 100644
--- a/src/helpers/modelManagerBridge.js
+++ b/src/helpers/modelManagerBridge.js
@@ -12,13 +12,42 @@ const {
const modelRegistryData = require("../models/modelRegistryData.json");
const LlamaServerManager = require("./llamaServer");
const debugLogger = require("./debugLogger");
+const { readGgufMetadataFromFile } = require("./ggufMetadata");
+const {
+ estimateTokens,
+ kvBytesPerToken,
+ resolveContextCeiling,
+ resolveContextSize,
+ BASELINE_CONTEXT_SIZE,
+} = require("./llamaContextPolicy");
const MIN_FILE_SIZE = 1_000_000; // 1MB minimum for valid model files
-// Bounds the KV cache — registry contextLength is the full trained context
-// (128K+), which can exceed total RAM (#1203). Uniform for all start paths:
-// start() won't restart a ready server when only options change.
-const SERVER_CONTEXT_SIZE = 16384;
+// The context every server still STARTS at. Requests that need more grow it
+// on demand, bounded by a per-machine, per-model ceiling (llamaContextPolicy).
+// Keeping the starting value at the historical constant means prewarm, chat
+// and every short request behave exactly as they did before #2142.
+const SERVER_CONTEXT_SIZE = BASELINE_CONTEXT_SIZE;
+
+// Slack on top of the output reservation, for tokenizer drift and llama.cpp's
+// own bookkeeping.
+const CONTEXT_RESERVE_TOKENS = 512;
+
+// Context a request must leave free for its own answer. Derived in one place
+// so the size the window is chosen for and the size it is checked against can
+// never drift apart.
+const reserveFor = (maxTokens) => maxTokens + CONTEXT_RESERVE_TOKENS;
+
+// Only measure the prompt exactly when the estimate lands within this factor
+// of the running window. Below that the answer cannot change, and dictation
+// should not pay two HTTP round trips to learn nothing.
+const PREFLIGHT_MEASURE_FACTOR = 2;
+
+// The smallest answer still worth generating when the prompt has crowded the
+// window. Matches the historical floor of calculateMaxTokens, so a request
+// that gets trimmed to this is no smaller than the smallest one the app has
+// ever made. Below it, refusing beats handing back a sentence and a half.
+const MIN_OUTPUT_TOKENS = 512;
function getLocalProviders() {
return modelRegistryData.localProviders || [];
@@ -205,13 +234,66 @@ class ModelManager {
};
}
- async serverStartOptions(modelInfo) {
- const options = this.serverOptions(modelInfo);
+ async serverStartOptions(modelInfo, overrides = {}) {
+ const options = { ...this.serverOptions(modelInfo), ...overrides };
const draftPath = await this.resolveDraftPath(modelInfo.model);
if (draftPath) options.draftModelPath = draftPath;
return options;
}
+ // Seam so the ceiling can be exercised for any machine size in tests.
+ _systemMemoryBytes() {
+ return require("os").totalmem();
+ }
+
+ /**
+ * The largest context this machine may give this model, and the prompt-cache
+ * bound that goes with it. Memoized per model file: it reads a few MiB off
+ * the front of the GGUF, so it must not sit on the hot path.
+ */
+ async contextCeiling(modelInfo, modelPath) {
+ if (this._contextCeilingCache?.modelPath === modelPath) {
+ return this._contextCeilingCache.value;
+ }
+
+ const metadata = await readGgufMetadataFromFile(modelPath);
+ const sizeOf = async (filePath) => {
+ if (!filePath) return 0;
+ try {
+ return (await fsPromises.stat(filePath)).size;
+ } catch {
+ return 0;
+ }
+ };
+
+ const value = resolveContextCeiling({
+ totalMemoryBytes: this._systemMemoryBytes(),
+ weightsBytes: await sizeOf(modelPath),
+ draftWeightsBytes: await sizeOf(await this.resolveDraftPath(modelInfo.model)),
+ kvBytesPerToken: kvBytesPerToken(metadata),
+ hasRecurrentState: metadata?.hasRecurrentState,
+ // The GGUF is the file llama.cpp actually loads, so it outranks the
+ // registry, which is wrong for at least the qwen2.5-7b entries.
+ trainedContextTokens: metadata?.contextLength || modelInfo.model.contextLength || 0,
+ // Anywhere but Apple Silicon the GPU may have its own memory that we do
+ // not probe, so the system-RAM budget alone is not a safe bound. Stay
+ // modest and let the GPU fallback ladder handle the rest. Intel Macs
+ // count: a discrete Radeon has a few GB of its own, and the darwin start
+ // path has no reduced-context rung to catch an over-sized window.
+ discreteGpuUnverified: process.platform !== "darwin" || process.arch !== "arm64",
+ });
+
+ debugLogger.info("Resolved llama-server context ceiling", {
+ model: modelInfo.model.id,
+ architecture: metadata?.architecture ?? null,
+ kvBytesPerToken: kvBytesPerToken(metadata),
+ ...value,
+ });
+
+ this._contextCeilingCache = { modelPath, value };
+ return value;
+ }
+
getReservedDownloadBytes() {
let reserved = 0;
for (const [modelId, bytes] of this.downloadReservations) {
@@ -450,6 +532,106 @@ class ModelManager {
}
}
+ /**
+ * Make sure the prompt actually fits the running window, growing it once if
+ * it does not, trading output room for the prompt if it still does not, and
+ * refusing only when no usable answer is left.
+ *
+ * The estimate above only picks a starting size. This is where the decision
+ * becomes exact, because a wrong guess here is the bug: llama-server answers
+ * an overlong prompt with a 400 whose JSON body used to reach the user.
+ *
+ * Returns the output allowance the request should actually be sent with.
+ */
+ async _ensurePromptFits({
+ modelInfo,
+ messages,
+ options,
+ estimatedTokens,
+ maxTokens,
+ contextFloor,
+ ceilingFor,
+ startServer,
+ }) {
+ const reserveTokens = reserveFor(maxTokens);
+ const running = this.serverManager.contextSize || contextFloor;
+
+ // Comfortably inside the window: measuring cannot change the outcome, and
+ // every dictation would otherwise pay for two HTTP round trips.
+ if ((estimatedTokens + reserveTokens) * PREFLIGHT_MEASURE_FACTOR <= running) return maxTokens;
+
+ const usable = (await this.serverManager.usableContextSize()) ?? running;
+ const exactTokens = await this.serverManager.countPromptTokens(messages, {
+ disableThinking: options.disableThinking,
+ });
+
+ // A measurement we could not take must never fail the request; the
+ // estimate already sized the window, and an overflow would still surface
+ // as a typed error from llama-server itself.
+ if (exactTokens === null) {
+ debugLogger.warn("Could not measure prompt tokens; proceeding on the estimate", {
+ model: modelInfo.model.id,
+ estimatedTokens,
+ });
+ return maxTokens;
+ }
+
+ const neededTokens = exactTokens + reserveTokens;
+ if (neededTokens <= usable) return maxTokens;
+
+ const { ceiling: maxContext } = await ceilingFor();
+ const grown = resolveContextSize({
+ needed: neededTokens,
+ floor: contextFloor,
+ ceiling: maxContext,
+ });
+
+ if (grown > usable) await startServer(grown);
+
+ const nowUsable =
+ (await this.serverManager.usableContextSize()) ?? this.serverManager.contextSize ?? grown;
+ if (neededTokens <= nowUsable) return maxTokens;
+
+ // The window is as big as this machine allows and the prompt still does not
+ // leave room for the answer we wanted. But llama-server rejects only a
+ // prompt that fills the window on its own (`n_tokens >= n_ctx`) —
+ // overrunning during generation just ends the reply with finish_reason
+ // "length" — so a shorter answer is still a real answer, and refusing one
+ // the user could have had is the regression this preflight would otherwise
+ // introduce (#2142).
+ //
+ // No `- CONTEXT_RESERVE_TOKENS` here: that slack absorbs drift in the
+ // *estimate* that sized the window, while this is an exact count of the
+ // exact prompt. The MIN_OUTPUT_TOKENS floor below already keeps the prompt
+ // clear of the edge by the same margin, so charging it twice would only
+ // shorten the answer.
+ const affordableOutput = nowUsable - exactTokens;
+ if (!options.requireCompleteOutput && affordableOutput >= MIN_OUTPUT_TOKENS) {
+ debugLogger.info("Trimming the output allowance to fit the context window", {
+ model: modelInfo.model.id,
+ requested: maxTokens,
+ granted: affordableOutput,
+ promptTokens: exactTokens,
+ contextSize: nowUsable,
+ });
+ return affordableOutput;
+ }
+
+ // Either nothing usable is left, or the caller cannot accept a partial
+ // answer at all (a selection edit replaces the user's own text, so half of
+ // one is worse than none).
+ throw new ModelError(
+ `This content needs about ${neededTokens} tokens of context, but ${modelInfo.model.name} can only use ${nowUsable} on this computer.`,
+ "CONTEXT_TOO_LARGE",
+ {
+ modelId: modelInfo.model.id,
+ modelName: modelInfo.model.name,
+ neededTokens,
+ maxContextTokens: nowUsable,
+ }
+ );
+ }
+
async runInference(modelId, prompt, options = {}) {
this.ensureInitialized();
const startTime = Date.now();
@@ -490,28 +672,147 @@ class ModelManager {
);
}
- // Start/restart server if needed or if model changed
- if (!this.serverManager.ready || this.currentServerModelId !== modelId) {
+ // Build messages for chat completion
+ const messages = [
+ { role: "system", content: options.systemPrompt || "" },
+ { role: "user", content: prompt },
+ ];
+
+ const maxTokens = options.maxTokens ?? 512;
+ // Whatever the prompt needs, plus room for the reply. The output reserve
+ // is part of the context budget, not on top of it.
+ const reserveTokens = reserveFor(maxTokens);
+ const estimatedTokens = estimateTokens(options.systemPrompt || "") + estimateTokens(prompt);
+ // A caller may ask for a larger floor (selection editing does); it can
+ // raise the floor but never breach the ceiling.
+ const contextFloor = Math.max(SERVER_CONTEXT_SIZE, options.contextSize || 0);
+
+ let ceiling = null;
+ const ceilingFor = async () => {
+ if (ceiling === null) ceiling = await this.contextCeiling(modelInfo, modelPath);
+ return ceiling;
+ };
+
+ let targetContextSize = contextFloor;
+ // A raised floor has to clear the ceiling too, or a caller asking for a
+ // window the machine cannot afford reopens #1203. The baseline floor needs
+ // no check, which is what keeps the GGUF read off the short-request path.
+ if (contextFloor > SERVER_CONTEXT_SIZE || estimatedTokens + reserveTokens > contextFloor) {
+ const { ceiling: maxContext } = await ceilingFor();
+ targetContextSize = resolveContextSize({
+ needed: estimatedTokens + reserveTokens,
+ floor: contextFloor,
+ ceiling: maxContext,
+ });
+ }
+
+ // The smallest window this machine has already refused for this model,
+ // whether by failing to start or by quietly loading something smaller.
+ // Retrying it would cost a second model load to learn the same thing.
+ let smallestRefusedContextSize = Infinity;
+ const refuse = (size) => {
+ smallestRefusedContextSize = Math.min(smallestRefusedContextSize, size);
+ };
+
+ const startServer = async (contextSize) => {
+ if (this.serverManager.ready && contextSize >= smallestRefusedContextSize) return;
+
+ const { cacheRamMiB } = ceiling ?? {};
+ // Only a live server already serving THIS model has a window worth
+ // coming back to. contextSize outlives a crash — llamaServer clears it
+ // in stop() alone, not when a process dies or a health check trips — and
+ // another model's window says nothing about this one.
+ const restorableContextSize =
+ this.serverManager.ready &&
+ this.serverManager.process &&
+ this.currentServerModelId === modelId
+ ? this.serverManager.contextSize
+ : null;
+ const startAt = async (size) =>
+ this.serverManager.start(
+ modelPath,
+ await this.serverStartOptions(modelInfo, { contextSize: size, cacheRamMiB })
+ );
+ // llama.cpp's startup dump is the diagnostic, so it goes to the log here
+ // and nowhere else: details crosses IPC to the renderer, which neither
+ // reads it nor should carry a stderr blob and absolute model paths.
+ const unavailable = (cause) => {
+ debugLogger.error("llama-server could not be started", {
+ model: modelId,
+ contextSize,
+ error: cause.message,
+ });
+ return new ModelError(
+ `${modelInfo.model.name} could not be started on this computer.`,
+ "LOCAL_SERVER_UNAVAILABLE",
+ { modelId, modelName: modelInfo.model.name }
+ );
+ };
+
+ try {
+ await startAt(contextSize);
+ // The GPU ladder can load a smaller window than it was asked for
+ // without ever throwing. Recording that is what stops the preflight
+ // asking for the same window again and paying a second model load to
+ // be stepped down again.
+ if (this.serverManager.contextSize < contextSize) refuse(contextSize);
+ } catch (error) {
+ refuse(contextSize);
+ // start() stops the running server before it spawns, so a grow that
+ // dies has already taken a working window with it. Come back at the
+ // one that worked rather than leaving the user with no local model.
+ if (!restorableContextSize || restorableContextSize >= contextSize) {
+ throw unavailable(error);
+ }
+
+ debugLogger.warn("llama-server failed to grow; restoring the previous context", {
+ model: modelId,
+ attempted: contextSize,
+ restoring: restorableContextSize,
+ error: error.message,
+ });
+ try {
+ await startAt(restorableContextSize);
+ } catch (restoreError) {
+ throw unavailable(restoreError);
+ }
+ }
+ this.currentServerModelId = modelId;
+ };
+
+ // Start/restart the server if needed, if the model changed, or if this
+ // request needs a bigger window than the running one has.
+ if (
+ !this.serverManager.ready ||
+ this.currentServerModelId !== modelId ||
+ targetContextSize > (this.serverManager.contextSize || SERVER_CONTEXT_SIZE)
+ ) {
debugLogger.logReasoning("INFERENCE_STARTING_SERVER", {
currentModel: this.currentServerModelId,
requestedModel: modelId,
serverReady: this.serverManager.ready,
+ contextSize: targetContextSize,
});
- await this.serverManager.start(modelPath, await this.serverStartOptions(modelInfo));
- this.currentServerModelId = modelId;
+ await startServer(targetContextSize);
debugLogger.logReasoning("INFERENCE_SERVER_STARTED", {
port: this.serverManager.port,
model: modelId,
+ contextSize: this.serverManager.contextSize,
});
}
- // Build messages for chat completion
- const messages = [
- { role: "system", content: options.systemPrompt || "" },
- { role: "user", content: prompt },
- ];
+ const grantedMaxTokens = await this._ensurePromptFits({
+ modelInfo,
+ messages,
+ options,
+ estimatedTokens,
+ maxTokens,
+ contextFloor,
+ ceilingFor,
+ startServer,
+ });
debugLogger.logReasoning("INFERENCE_SENDING_REQUEST", {
messageCount: messages.length,
@@ -522,7 +823,9 @@ class ModelManager {
try {
const result = await this.serverManager.inference(messages, {
temperature: options.temperature ?? 0.7,
- max_tokens: options.maxTokens ?? 512,
+ // The preflight may have traded output room for prompt room; sending
+ // the original value would put the request back over the window.
+ max_tokens: grantedMaxTokens,
disableThinking: options.disableThinking,
requireCompleteOutput: options.requireCompleteOutput,
});
@@ -541,6 +844,16 @@ class ModelManager {
totalTimeMs: totalTime,
error: error.message,
});
+ // A typed failure (a context overflow, say) must keep its identity, or
+ // the renderer cannot translate it and the user sees raw server text.
+ if (error.code === "CONTEXT_TOO_LARGE") {
+ throw new ModelError(error.message, "CONTEXT_TOO_LARGE", {
+ modelId,
+ modelName: modelInfo.model.name,
+ neededTokens: error.neededTokens ?? null,
+ maxContextTokens: error.maxContextTokens ?? null,
+ });
+ }
throw new ModelError(`Inference failed: ${error.message}`, "INFERENCE_FAILED", {
error: error.message,
});
diff --git a/src/locales/ar/translation.json b/src/locales/ar/translation.json
index 62a3146d9..106c319dd 100644
--- a/src/locales/ar/translation.json
+++ b/src/locales/ar/translation.json
@@ -330,6 +330,11 @@
"liquidai_lfm2_5_350m_q8_0": "نموذج خفيف للغاية للتنظيف السريع للإملاء",
"liquidai_lfm2_5_230m_q8_0": "نموذج صغير جدًا، تنظيف فوري على الأجهزة المتواضعة"
}
+ },
+ "errors": {
+ "contextTooLarge": "هذا أطول من أن يقرأه {{model}} على هذا الكمبيوتر. عدد الرموز المطلوبة من السياق: نحو {{needed}}، لكن ما يتسع في الذاكرة: {{max}} فقط. جرّب نموذجًا سحابيًا، أو نفّذ الإجراء على تحديد أقصر.",
+ "contextTooLargeGeneric": "هذا أطول من أن يقرأه {{model}} على هذا الكمبيوتر. جرّب نموذجًا سحابيًا، أو نفّذ الإجراء على تحديد أقصر.",
+ "localServerUnavailable": "تعذّر تشغيل {{model}} على هذا الكمبيوتر. حاول مرة أخرى، أو بدّل إلى نموذج سحابي."
}
},
"reasoning": {
diff --git a/src/locales/de/translation.json b/src/locales/de/translation.json
index f748faf8c..ce4830220 100644
--- a/src/locales/de/translation.json
+++ b/src/locales/de/translation.json
@@ -2741,6 +2741,11 @@
"liquidai_lfm2_5_350m_q8_0": "Ultraleichtes Modell für schnelle Diktatbereinigung",
"liquidai_lfm2_5_230m_q8_0": "Winziges Modell, sofortige Bereinigung auf bescheidener Hardware"
}
+ },
+ "errors": {
+ "contextTooLarge": "Das ist zu lang, als dass {{model}} es auf diesem Computer lesen könnte. Es werden etwa {{needed}} Token Kontext benötigt, aber nur {{max}} passen in den Speicher. Versuchen Sie ein Cloud-Modell oder führen Sie die Aktion für eine kürzere Auswahl aus.",
+ "contextTooLargeGeneric": "Das ist zu lang, als dass {{model}} es auf diesem Computer lesen könnte. Versuchen Sie ein Cloud-Modell oder führen Sie die Aktion für eine kürzere Auswahl aus.",
+ "localServerUnavailable": "{{model}} konnte auf diesem Computer nicht gestartet werden. Versuchen Sie es erneut oder wechseln Sie zu einem Cloud-Modell."
}
},
"insights": {
diff --git a/src/locales/en/translation.json b/src/locales/en/translation.json
index 684e05c2c..b0208adf2 100644
--- a/src/locales/en/translation.json
+++ b/src/locales/en/translation.json
@@ -2741,6 +2741,11 @@
"liquidai_lfm2_5_350m_q8_0": "Ultra-light model for fast dictation cleanup",
"liquidai_lfm2_5_230m_q8_0": "Tiny model, instant cleanup on modest hardware"
}
+ },
+ "errors": {
+ "contextTooLarge": "This is too long for {{model}} to read on this computer. It needs about {{needed}} tokens of context, but only {{max}} fit in memory. Try a cloud model, or run the action on a shorter selection.",
+ "contextTooLargeGeneric": "This is too long for {{model}} to read on this computer. Try a cloud model, or run the action on a shorter selection.",
+ "localServerUnavailable": "{{model}} couldn't be started on this computer. Try again, or switch to a cloud model."
}
},
"sidebar": {
diff --git a/src/locales/es/translation.json b/src/locales/es/translation.json
index 3867ff59d..052ac522d 100644
--- a/src/locales/es/translation.json
+++ b/src/locales/es/translation.json
@@ -2701,6 +2701,11 @@
"segmentation": "Detección de límites del habla para la identificación de hablantes",
"embedding_en": "Huella vocal para la identificación del hablante"
}
+ },
+ "errors": {
+ "contextTooLarge": "Esto es demasiado largo para que {{model}} lo lea en este equipo. Necesita unos {{needed}} tokens de contexto, pero solo caben {{max}} en la memoria. Prueba con un modelo en la nube o ejecuta la acción sobre una selección más corta.",
+ "contextTooLargeGeneric": "Esto es demasiado largo para que {{model}} lo lea en este equipo. Prueba con un modelo en la nube o ejecuta la acción sobre una selección más corta.",
+ "localServerUnavailable": "No se pudo iniciar {{model}} en este ordenador. Inténtalo de nuevo o cambia a un modelo en la nube."
}
},
"referral": {
diff --git a/src/locales/fr/translation.json b/src/locales/fr/translation.json
index 78c3af08f..492b11c4d 100644
--- a/src/locales/fr/translation.json
+++ b/src/locales/fr/translation.json
@@ -2741,6 +2741,11 @@
"liquidai_lfm2_5_350m_q8_0": "Modèle ultraléger pour un nettoyage rapide de la dictée",
"liquidai_lfm2_5_230m_q8_0": "Modèle minuscule, nettoyage instantané sur du matériel modeste"
}
+ },
+ "errors": {
+ "contextTooLarge": "C’est trop long pour que {{model}} puisse le lire sur cet ordinateur. Il faut environ {{needed}} jetons de contexte, mais seuls {{max}} tiennent en mémoire. Essayez un modèle cloud ou exécutez l’action sur une sélection plus courte.",
+ "contextTooLargeGeneric": "C’est trop long pour que {{model}} puisse le lire sur cet ordinateur. Essayez un modèle cloud ou exécutez l’action sur une sélection plus courte.",
+ "localServerUnavailable": "Impossible de démarrer {{model}} sur cet ordinateur. Réessayez ou passez à un modèle cloud."
}
},
"insights": {
diff --git a/src/locales/it/translation.json b/src/locales/it/translation.json
index 1524d2e90..3765c88bc 100644
--- a/src/locales/it/translation.json
+++ b/src/locales/it/translation.json
@@ -2648,6 +2648,11 @@
"segmentation": "Rilevamento dei confini del parlato per l'identificazione dei parlanti",
"embedding_en": "Impronta vocale per l'identificazione dei parlanti"
}
+ },
+ "errors": {
+ "contextTooLarge": "È troppo lungo perché {{model}} possa leggerlo su questo computer. Servono circa {{needed}} token di contesto, ma in memoria ne entrano solo {{max}}. Prova un modello cloud oppure esegui l’azione su una selezione più breve.",
+ "contextTooLargeGeneric": "È troppo lungo perché {{model}} possa leggerlo su questo computer. Prova un modello cloud oppure esegui l’azione su una selezione più breve.",
+ "localServerUnavailable": "Impossibile avviare {{model}} su questo computer. Riprova o passa a un modello cloud."
}
},
"referral": {
diff --git a/src/locales/ja/translation.json b/src/locales/ja/translation.json
index 7d14c0dfd..a185584de 100644
--- a/src/locales/ja/translation.json
+++ b/src/locales/ja/translation.json
@@ -2648,6 +2648,11 @@
"segmentation": "話者分離のための発話区間検出",
"embedding_en": "話者識別のための声紋認識"
}
+ },
+ "errors": {
+ "contextTooLarge": "このコンピューターでは長すぎて {{model}} が読み取れません。約 {{needed}} トークンのコンテキストが必要ですが、メモリに収まるのは {{max}} までです。クラウドモデルを使うか、選択範囲を短くして実行してください。",
+ "contextTooLargeGeneric": "このコンピューターでは長すぎて {{model}} が読み取れません。クラウドモデルを使うか、選択範囲を短くして実行してください。",
+ "localServerUnavailable": "このコンピューターでは {{model}} を起動できませんでした。もう一度試すか、クラウドモデルに切り替えてください。"
}
},
"notes": {
diff --git a/src/locales/pt/translation.json b/src/locales/pt/translation.json
index 2f1c675d6..8ec33df24 100644
--- a/src/locales/pt/translation.json
+++ b/src/locales/pt/translation.json
@@ -2627,6 +2627,11 @@
"segmentation": "Detecção de limites de fala para identificação de falantes",
"embedding_en": "Impressão vocal para identificação de falantes"
}
+ },
+ "errors": {
+ "contextTooLarge": "Isto é longo demais para o {{model}} ler neste computador. São necessários cerca de {{needed}} tokens de contexto, mas apenas {{max}} cabem na memória. Experimente um modelo na nuvem ou execute a ação sobre uma seleção mais curta.",
+ "contextTooLargeGeneric": "Isto é longo demais para o {{model}} ler neste computador. Experimente um modelo na nuvem ou execute a ação sobre uma seleção mais curta.",
+ "localServerUnavailable": "Não foi possível iniciar o {{model}} neste computador. Tente novamente ou mude para um modelo na nuvem."
}
},
"upgradePrompt": {
diff --git a/src/locales/ru/translation.json b/src/locales/ru/translation.json
index a1439a7f7..5da457e04 100644
--- a/src/locales/ru/translation.json
+++ b/src/locales/ru/translation.json
@@ -2664,6 +2664,11 @@
"segmentation": "Обнаружение границ речи для идентификации говорящих",
"embedding_en": "Голосовой отпечаток для идентификации говорящих"
}
+ },
+ "errors": {
+ "contextTooLarge": "Это слишком длинно, чтобы {{model}} могла прочитать это на данном компьютере. Требуется около {{needed}} токенов контекста, но в память помещается только {{max}}. Попробуйте облачную модель или выполните действие для более короткого фрагмента.",
+ "contextTooLargeGeneric": "Это слишком длинно, чтобы {{model}} могла прочитать это на данном компьютере. Попробуйте облачную модель или выполните действие для более короткого фрагмента.",
+ "localServerUnavailable": "Не удалось запустить {{model}} на этом компьютере. Попробуйте снова или переключитесь на облачную модель."
}
},
"referral": {
diff --git a/src/locales/zh-CN/translation.json b/src/locales/zh-CN/translation.json
index a1155aa10..0490d7f2b 100644
--- a/src/locales/zh-CN/translation.json
+++ b/src/locales/zh-CN/translation.json
@@ -2648,6 +2648,11 @@
"segmentation": "用于说话人分离的语音边界检测",
"embedding_en": "用于说话人识别的声纹提取"
}
+ },
+ "errors": {
+ "contextTooLarge": "内容过长,{{model}} 无法在这台电脑上读取。大约需要 {{needed}} 个上下文 token,但内存中只能容纳 {{max}} 个。请改用云端模型,或对更短的选区执行该操作。",
+ "contextTooLargeGeneric": "内容过长,{{model}} 无法在这台电脑上读取。请改用云端模型,或对更短的选区执行该操作。",
+ "localServerUnavailable": "无法在这台电脑上启动 {{model}}。请重试,或改用云端模型。"
}
},
"referral": {
diff --git a/src/locales/zh-TW/translation.json b/src/locales/zh-TW/translation.json
index 07b096efb..4033732d8 100644
--- a/src/locales/zh-TW/translation.json
+++ b/src/locales/zh-TW/translation.json
@@ -2648,6 +2648,11 @@
"segmentation": "用於說話者分離的語音邊界偵測",
"embedding_en": "用於說話者辨識的聲紋提取"
}
+ },
+ "errors": {
+ "contextTooLarge": "內容過長,{{model}} 無法在這台電腦上讀取。大約需要 {{needed}} 個上下文 token,但記憶體只能容納 {{max}} 個。請改用雲端模型,或對更短的選取範圍執行該動作。",
+ "contextTooLargeGeneric": "內容過長,{{model}} 無法在這台電腦上讀取。請改用雲端模型,或對更短的選取範圍執行該動作。",
+ "localServerUnavailable": "無法在這台電腦上啟動 {{model}}。請重試,或改用雲端模型。"
}
},
"referral": {
diff --git a/src/services/ai/inferenceProviders/gemini.ts b/src/services/ai/inferenceProviders/gemini.ts
index 7bc4556b8..4ea83b6a1 100644
--- a/src/services/ai/inferenceProviders/gemini.ts
+++ b/src/services/ai/inferenceProviders/gemini.ts
@@ -38,17 +38,20 @@ export const geminiProvider: InferenceProvider = {
const generationConfig: GeminiGenerationConfig = {
temperature: config.temperature ?? (config.systemPrompt ? 0.3 : 0),
- maxOutputTokens:
- config.maxTokens ||
- Math.max(
- 2000,
- ctx.calculateMaxTokens(
- text.length,
- TOKEN_LIMITS.MIN_TOKENS_GEMINI,
- TOKEN_LIMITS.MAX_TOKENS_GEMINI,
- TOKEN_LIMITS.TOKEN_MULTIPLIER
- )
- ),
+ // A caller's budget raises this ceiling, never lowers it: note formatting
+ // pins maxTokens so the local path can price it against the context
+ // window, and letting that short-circuit Gemini's own allowance halved
+ // long summaries on the provider with the most room to give (#2142).
+ maxOutputTokens: Math.max(
+ config.maxTokens || 0,
+ 2000,
+ ctx.calculateMaxTokens(
+ text.length,
+ TOKEN_LIMITS.MIN_TOKENS_GEMINI,
+ TOKEN_LIMITS.MAX_TOKENS_GEMINI,
+ TOKEN_LIMITS.TOKEN_MULTIPLIER
+ )
+ ),
};
if (config.disableThinking === true && getCloudModel(model)?.supportsThinking) {
diff --git a/src/services/ai/inferenceProviders/local.ts b/src/services/ai/inferenceProviders/local.ts
index b42549204..49a0184ec 100644
--- a/src/services/ai/inferenceProviders/local.ts
+++ b/src/services/ai/inferenceProviders/local.ts
@@ -1,6 +1,7 @@
import type { InferenceProvider } from "./types";
import { wrapCleanupTranscript } from "../../../config/prompts";
import logger from "../../../utils/logger";
+import { buildLocalInferenceError } from "../../../utils/localInferenceError";
export const localProvider: InferenceProvider = {
id: "local",
@@ -25,7 +26,7 @@ export const localProvider: InferenceProvider = {
if (!result.success) {
logger.logReasoning("LOCAL_ERROR", { model, processingTimeMs, error: result.error });
- throw new Error(result.error);
+ throw buildLocalInferenceError(result);
}
logger.logReasoning("LOCAL_SUCCESS", {
diff --git a/src/services/localReasoningBridge.js b/src/services/localReasoningBridge.js
index b60935c5b..d3e930b11 100644
--- a/src/services/localReasoningBridge.js
+++ b/src/services/localReasoningBridge.js
@@ -40,6 +40,10 @@ class LocalReasoningService {
systemPrompt: config.systemPrompt || "",
disableThinking: config.disableThinking !== false,
requireCompleteOutput: config.requireCompleteOutput,
+ // A minimum context window for this request. Rebuilding this object
+ // field-by-field is what silently orphaned it before #2142: it was
+ // declared, written by selection editing, and never forwarded.
+ contextSize: config.contextSize,
};
debugLogger.logReasoning("LOCAL_BRIDGE_INFERENCE", {
diff --git a/src/stores/actionProcessingStore.ts b/src/stores/actionProcessingStore.ts
index 472152cbd..b347ecaa2 100644
--- a/src/stores/actionProcessingStore.ts
+++ b/src/stores/actionProcessingStore.ts
@@ -8,6 +8,20 @@ import { buildNoteFormattingOverrides } from "../helpers/noteFormattingOverrides
import { tagActionItemOwners, type MentionPerson } from "../utils/mentionMarkdown";
import type { ActionItem } from "../types/electron";
+/**
+ * Output budget for a formatted note.
+ *
+ * Without an explicit value this inherited the generic 2048-token default from
+ * calculateMaxTokens — roughly 1,500 words — so summaries of long meetings were
+ * cut off and saved anyway, with nothing to say they were incomplete (#2142).
+ *
+ * Deliberately not paired with requireCompleteOutput: unlike a selection edit,
+ * where a partial replacement corrupts the user's own text, a clipped summary
+ * is still worth keeping. The context preflight counts this reservation, so
+ * asking for more output room can grow the window rather than squeeze it.
+ */
+export const NOTE_OUTPUT_MAX_TOKENS = 4096;
+
export type ActionProcessingStatus = "idle" | "processing" | "success";
export interface NoteActionState {
@@ -18,6 +32,9 @@ export interface NoteActionState {
export interface ActionErrorEvent {
noteId: number;
message: string;
+ /** Set when the failure has a translatable form; the toast prefers it. */
+ messageKey?: string;
+ messageParams?: Record<string, string | number>;
}
interface ActionProcessingStoreState {
@@ -164,6 +181,7 @@ export function runBackgroundAction(
);
const enhanced = await reasoningService.processText(noteContent, modelId, null, {
systemPrompt,
+ maxTokens: NOTE_OUTPUT_MAX_TOKENS,
temperature: 0.3,
disableThinking: settings.noteFormattingDisableThinking,
...providerOverrides,
@@ -202,7 +220,11 @@ export function runBackgroundAction(
processingFlags.set(noteId, false);
clearNoteState(noteId);
const message = err instanceof Error ? err.message : labels.actionFailed;
- pushErrorEvent({ noteId, message });
+ const { messageKey, messageParams } = (err ?? {}) as {
+ messageKey?: string;
+ messageParams?: Record<string, string | number>;
+ };
+ pushErrorEvent({ noteId, message, messageKey, messageParams });
} finally {
cancelledFlags.delete(noteId);
}
diff --git a/src/types/electron.ts b/src/types/electron.ts
index eddb072f5..65e992c36 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -1841,7 +1841,13 @@ declare global {
modelId: string,
agentName: string | null,
config: any
- ) => Promise<{ success: boolean; text?: string; error?: string }>;
+ ) => Promise<{
+ success: boolean;
+ text?: string;
+ error?: string;
+ code?: string;
+ details?: Record<string, unknown>;
+ }>;
checkLocalReasoningAvailable: () => Promise<boolean>;
// Anthropic reasoning
diff --git a/src/utils/localInferenceError.ts b/src/utils/localInferenceError.ts
new file mode 100644
index 000000000..05ff11437
--- /dev/null
+++ b/src/utils/localInferenceError.ts
@@ -0,0 +1,61 @@
+/**
+ * Turns a failed local-inference IPC result into an Error the UI can render.
+ *
+ * The main process reports a machine-readable `code` and `details`; the
+ * translation keys live here, on the renderer side, so both surfaces that show
+ * these failures — the notes toast and the dictation cleanup toast — get the
+ * same wording for free.
+ *
+ * Before #2142 the raw llama.cpp 400 body was shown to the user verbatim.
+ */
+
+export interface LocalInferenceFailure {
+ error?: string;
+ code?: string;
+ details?: Record<string, unknown> | null;
+}
+
+export interface LocalInferenceError extends Error {
+ code?: string;
+ messageKey?: string;
+ messageParams?: Record<string, string | number>;
+}
+
+const asPositiveInteger = (value: unknown): number | null =>
+ typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.round(value) : null;
+
+function contextTooLarge(
+ details: Record<string, unknown>
+): Pick<LocalInferenceError, "messageKey" | "messageParams"> {
+ const model = typeof details.modelName === "string" ? details.modelName : "";
+ const needed = asPositiveInteger(details.neededTokens);
+ const max = asPositiveInteger(details.maxContextTokens);
+
+ if (needed === null || max === null) {
+ return {
+ messageKey: "models.errors.contextTooLargeGeneric",
+ messageParams: { model },
+ };
+ }
+ return {
+ messageKey: "models.errors.contextTooLarge",
+ messageParams: { model, needed, max },
+ };
+}
+
+export function buildLocalInferenceError(result: LocalInferenceFailure): LocalInferenceError {
+ const error: LocalInferenceError = new Error(result.error || "Local inference failed");
+ if (result.code) error.code = result.code;
+
+ const details = result.details ?? {};
+ if (result.code === "CONTEXT_TOO_LARGE") {
+ Object.assign(error, contextTooLarge(details));
+ } else if (result.code === "LOCAL_SERVER_UNAVAILABLE") {
+ // The llama.cpp startup dump stays in the main-process log; only the model
+ // name crosses, so there is nothing here that could reach the user raw.
+ error.messageKey = "models.errors.localServerUnavailable";
+ error.messageParams = { model: typeof details.modelName === "string" ? details.modelName : "" };
+ }
+
+ return error;
+}
diff --git a/test/helpers/gemmaMtpDrafter.test.js b/test/helpers/gemmaMtpDrafter.test.js
index f6e10b232..8790e4840 100644
--- a/test/helpers/gemmaMtpDrafter.test.js
+++ b/test/helpers/gemmaMtpDrafter.test.js
@@ -123,6 +123,9 @@ test("deleteModel removes the drafter alongside the main file, ignoring a missin
});
});
+// Context size is the third restart trigger; it is covered in
+// test/helpers/llamaServerContext.test.js. Every start() here passes the same
+// (default) context, so this file isolates the model/drafter dimension.
test("llama-server start restarts only when model or drafter presence changes", async () => {
const LlamaServerManager = require("../../src/helpers/llamaServer.js");
const manager = new LlamaServerManager();
diff --git a/test/helpers/ggufMetadata.test.js b/test/helpers/ggufMetadata.test.js
new file mode 100644
index 000000000..2adfe71ed
--- /dev/null
+++ b/test/helpers/ggufMetadata.test.js
@@ -0,0 +1,158 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+const { readGgufMetadata } = require("../../src/helpers/ggufMetadata");
+const { TYPE, buildGguf, bufferReader, LLAMA_3_2_3B_ENTRIES } = require("./harness/ggufFixtures");
+
+const read = (entries, options) => readGgufMetadata(bufferReader(buildGguf(entries, options)));
+
+test("reads the architecture fields that determine KV cost", async () => {
+ const metadata = await read(LLAMA_3_2_3B_ENTRIES);
+
+ assert.equal(metadata.architecture, "llama");
+ assert.equal(metadata.blockCount, 28);
+ assert.equal(metadata.headCountKv, 8);
+ assert.equal(metadata.keyLength, 128);
+ assert.equal(metadata.valueLength, 128);
+ assert.equal(metadata.embeddingLength, 3072);
+ assert.equal(metadata.headCount, 24);
+ assert.equal(metadata.contextLength, 131072);
+});
+
+test("prefers the model's own context length over anything declared elsewhere", async () => {
+ // The registry says 128000 for qwen2.5-7b, which is not what the GGUF says.
+ // The GGUF is the file llama.cpp actually loads, so it wins.
+ const metadata = await read([
+ { key: "general.architecture", type: TYPE.STRING, value: "qwen2" },
+ { key: "qwen2.block_count", type: TYPE.UINT32, value: 28 },
+ { key: "qwen2.context_length", type: TYPE.UINT32, value: 131072 },
+ { key: "qwen2.attention.head_count_kv", type: TYPE.UINT32, value: 4 },
+ { key: "qwen2.attention.key_length", type: TYPE.UINT32, value: 128 },
+ { key: "qwen2.attention.value_length", type: TYPE.UINT32, value: 128 },
+ ]);
+
+ assert.equal(metadata.contextLength, 131072);
+});
+
+test("reads a per-layer head_count_kv array", async () => {
+ const metadata = await read([
+ { key: "general.architecture", type: TYPE.STRING, value: "gemma3" },
+ { key: "gemma3.block_count", type: TYPE.UINT32, value: 4 },
+ {
+ key: "gemma3.attention.head_count_kv",
+ type: TYPE.ARRAY,
+ elementType: TYPE.UINT32,
+ values: [2, 8, 2, 2],
+ },
+ { key: "gemma3.attention.key_length", type: TYPE.UINT32, value: 256 },
+ { key: "gemma3.attention.value_length", type: TYPE.UINT32, value: 256 },
+ ]);
+
+ assert.deepEqual(metadata.headCountKv, [2, 8, 2, 2]);
+});
+
+test("flags recurrent state, which costs memory the KV formula does not capture", async () => {
+ const metadata = await read([
+ { key: "general.architecture", type: TYPE.STRING, value: "qwen35" },
+ { key: "qwen35.block_count", type: TYPE.UINT32, value: 33 },
+ { key: "qwen35.attention.head_count_kv", type: TYPE.UINT32, value: 4 },
+ { key: "qwen35.attention.key_length", type: TYPE.UINT32, value: 256 },
+ { key: "qwen35.attention.value_length", type: TYPE.UINT32, value: 256 },
+ { key: "qwen35.ssm.state_size", type: TYPE.UINT32, value: 128 },
+ ]);
+
+ assert.equal(metadata.hasRecurrentState, true);
+});
+
+test("walks past every scalar type without losing its place", async () => {
+ // A real GGUF interleaves dozens of unrelated keys of every type before the
+ // ones we want. Mis-sizing any of them desynchronises the whole parse.
+ const metadata = await read([
+ { key: "general.architecture", type: TYPE.STRING, value: "llama" },
+ { key: "noise.u8", type: TYPE.UINT8, value: 7 },
+ { key: "noise.i8", type: TYPE.INT8, value: 7 },
+ { key: "noise.u16", type: TYPE.UINT16, value: 7 },
+ { key: "noise.i16", type: TYPE.INT16, value: 7 },
+ { key: "noise.u32", type: TYPE.UINT32, value: 7 },
+ { key: "noise.i32", type: TYPE.INT32, value: 7 },
+ { key: "noise.f32", type: TYPE.FLOAT32, value: 7.5 },
+ { key: "noise.bool", type: TYPE.BOOL, value: true },
+ { key: "noise.string", type: TYPE.STRING, value: "a longer value" },
+ { key: "noise.u64", type: TYPE.UINT64, value: 7 },
+ { key: "noise.i64", type: TYPE.INT64, value: 7 },
+ { key: "noise.f64", type: TYPE.FLOAT64, value: 7.5 },
+ ...LLAMA_3_2_3B_ENTRIES.slice(1),
+ ]);
+
+ assert.equal(metadata.blockCount, 28);
+ assert.equal(metadata.headCountKv, 8);
+});
+
+test("skips the tokenizer arrays that dominate a real header", async () => {
+ // tokenizer.ggml.tokens is a string array with 150k+ entries and accounts for
+ // nearly all of a real 10 MiB metadata section. It must be stepped over, not
+ // materialised.
+ const tokens = Array.from({ length: 5000 }, (_, index) => `token_${index}`);
+ const metadata = await read([
+ { key: "general.architecture", type: TYPE.STRING, value: "llama" },
+ { key: "tokenizer.ggml.tokens", type: TYPE.ARRAY, elementType: TYPE.STRING, values: tokens },
+ {
+ key: "tokenizer.ggml.scores",
+ type: TYPE.ARRAY,
+ elementType: TYPE.FLOAT32,
+ values: tokens.map(() => 0),
+ },
+ ...LLAMA_3_2_3B_ENTRIES.slice(1),
+ ]);
+
+ assert.equal(metadata.blockCount, 28);
+ assert.equal(metadata.headCountKv, 8);
+});
+
+test("returns null instead of throwing on anything malformed", async () => {
+ // A null must route the caller to the historical 16384. Throwing here would
+ // fail the user's dictation instead.
+ const good = buildGguf(LLAMA_3_2_3B_ENTRIES);
+
+ const cases = {
+ "bad magic": bufferReader(buildGguf(LLAMA_3_2_3B_ENTRIES, { magic: 0xdeadbeef })),
+ "version 1": bufferReader(buildGguf(LLAMA_3_2_3B_ENTRIES, { version: 1 })),
+ "truncated mid-header": bufferReader(good.subarray(0, 60)),
+ "empty file": bufferReader(Buffer.alloc(0)),
+ "header only": bufferReader(good.subarray(0, 8)),
+ "implausible kv count": bufferReader(buildGguf(LLAMA_3_2_3B_ENTRIES, { kvCount: 99999 })),
+ "unknown value type": bufferReader(
+ buildGguf([{ key: "general.architecture", type: TYPE.STRING, value: "llama" }]).fill(
+ 0x7f,
+ // clobber the first value's type tag
+ 4 + 4 + 8 + 8 + 8 + "general.architecture".length,
+ 4 + 4 + 8 + 8 + 8 + "general.architecture".length + 4
+ )
+ ),
+ };
+
+ for (const [label, reader] of Object.entries(cases)) {
+ const metadata = await readGgufMetadata(reader);
+ assert.equal(metadata, null, `${label} should read as null`);
+ }
+});
+
+test("returns null when a read fails outright", async () => {
+ const metadata = await readGgufMetadata({
+ size: 1024,
+ read: async () => {
+ throw new Error("EIO");
+ },
+ });
+
+ assert.equal(metadata, null);
+});
+
+test("returns null when the architecture keys are simply absent", async () => {
+ const metadata = await read([
+ { key: "general.name", type: TYPE.STRING, value: "some model" },
+ { key: "general.file_type", type: TYPE.UINT32, value: 15 },
+ ]);
+
+ assert.equal(metadata, null);
+});
diff --git a/test/helpers/harness/ggufFixtures.js b/test/helpers/harness/ggufFixtures.js
new file mode 100644
index 000000000..a6097fbaa
--- /dev/null
+++ b/test/helpers/harness/ggufFixtures.js
@@ -0,0 +1,118 @@
+/**
+ * Builds GGUF headers in memory so the metadata reader can be tested without
+ * downloading a multi-gigabyte model. Only the header is produced: the reader
+ * never looks at tensor data.
+ */
+const GGUF_MAGIC = 0x46554747; // "GGUF" little-endian
+
+const TYPE = {
+ UINT8: 0,
+ INT8: 1,
+ UINT16: 2,
+ INT16: 3,
+ UINT32: 4,
+ INT32: 5,
+ FLOAT32: 6,
+ BOOL: 7,
+ STRING: 8,
+ ARRAY: 9,
+ UINT64: 10,
+ INT64: 11,
+ FLOAT64: 12,
+};
+
+const u32 = (value) => {
+ const b = Buffer.alloc(4);
+ b.writeUInt32LE(value);
+ return b;
+};
+
+const u64 = (value) => {
+ const b = Buffer.alloc(8);
+ b.writeBigUInt64LE(BigInt(value));
+ return b;
+};
+
+const ggufString = (value) => {
+ const bytes = Buffer.from(value, "utf8");
+ return Buffer.concat([u64(bytes.length), bytes]);
+};
+
+function encodeScalar(type, value) {
+ switch (type) {
+ case TYPE.UINT8:
+ case TYPE.INT8:
+ return Buffer.from([value & 0xff]);
+ case TYPE.UINT16:
+ case TYPE.INT16: {
+ const b = Buffer.alloc(2);
+ b.writeUInt16LE(value);
+ return b;
+ }
+ case TYPE.UINT32:
+ case TYPE.INT32:
+ return u32(value);
+ case TYPE.FLOAT32: {
+ const b = Buffer.alloc(4);
+ b.writeFloatLE(value);
+ return b;
+ }
+ case TYPE.BOOL:
+ return Buffer.from([value ? 1 : 0]);
+ case TYPE.STRING:
+ return ggufString(value);
+ case TYPE.UINT64:
+ case TYPE.INT64:
+ return u64(value);
+ case TYPE.FLOAT64: {
+ const b = Buffer.alloc(8);
+ b.writeDoubleLE(value);
+ return b;
+ }
+ default:
+ throw new Error(`unsupported scalar type ${type}`);
+ }
+}
+
+function encodeValue(entry) {
+ if (entry.type === TYPE.ARRAY) {
+ const elements = entry.values.map((value) => encodeScalar(entry.elementType, value));
+ return Buffer.concat([u32(entry.elementType), u64(entry.values.length), ...elements]);
+ }
+ return encodeScalar(entry.type, entry.value);
+}
+
+/**
+ * @param entries {Array<{key: string, type: number, value?: any,
+ * elementType?: number, values?: any[]}>}
+ */
+function buildGguf(entries, { version = 3, magic = GGUF_MAGIC, kvCount = null } = {}) {
+ const parts = [u32(magic), u32(version), u64(0) /* tensor_count */];
+ parts.push(u64(kvCount === null ? entries.length : kvCount));
+ for (const entry of entries) {
+ parts.push(ggufString(entry.key), u32(entry.type), encodeValue(entry));
+ }
+ return Buffer.concat(parts);
+}
+
+/** A reader over an in-memory buffer, matching the shape readGgufMetadata expects. */
+function bufferReader(buffer) {
+ return {
+ size: buffer.length,
+ read: async (offset, length) => buffer.subarray(offset, offset + length),
+ };
+}
+
+// The #1203 model, as its GGUF declares itself.
+const LLAMA_3_2_3B_ENTRIES = [
+ { key: "general.architecture", type: TYPE.STRING, value: "llama" },
+ { key: "llama.block_count", type: TYPE.UINT32, value: 28 },
+ { key: "llama.context_length", type: TYPE.UINT32, value: 131072 },
+ { key: "llama.embedding_length", type: TYPE.UINT32, value: 3072 },
+ { key: "llama.attention.head_count", type: TYPE.UINT32, value: 24 },
+ { key: "llama.attention.head_count_kv", type: TYPE.UINT32, value: 8 },
+ { key: "llama.attention.key_length", type: TYPE.UINT32, value: 128 },
+ { key: "llama.attention.value_length", type: TYPE.UINT32, value: 128 },
+];
+
+module.exports = { TYPE, buildGguf, bufferReader, LLAMA_3_2_3B_ENTRIES };
diff --git a/test/helpers/llamaContextPolicy.test.js b/test/helpers/llamaContextPolicy.test.js
new file mode 100644
index 000000000..c2e2688b4
--- /dev/null
+++ b/test/helpers/llamaContextPolicy.test.js
@@ -0,0 +1,350 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+const policy = require("../../src/helpers/llamaContextPolicy");
+
+// --- estimateTokens ------------------------------------------------------
+//
+// Used before any llama-server exists, to pick the context size the server
+// starts at. Once a server is up, POST /tokenize gives an exact count in ~13ms
+// and supersedes this. The estimate must therefore be cheap and biased HIGH:
+// over-estimating costs KV bytes (bounded by the ceiling), under-estimating
+// costs a restart, and under-estimating badly is the bug being fixed (#2142).
+
+test("estimateTokens counts Latin text at three characters per token", () => {
+ assert.equal(policy.estimateTokens("abcdef"), 2);
+ assert.equal(policy.estimateTokens("abcd"), 2); // rounds up, never down
+ assert.equal(policy.estimateTokens(""), 0);
+});
+
+test("estimateTokens over-estimates a real transcript rather than under-estimating it", () => {
+ // Measured against the bundled llama-server (b9763) POST /tokenize with
+ // Qwen3.5-9B: an 85,171-character synthetic meeting transcript tokenized to
+ // 20,473 tokens (4.16 chars/token). A chars/4 constant would sit right on
+ // that line and under-estimate real transcripts; chars/3 leaves headroom.
+ const transcript = "a".repeat(85171);
+ const estimate = policy.estimateTokens(transcript);
+
+ assert.ok(estimate > 20473, `expected the estimate to exceed the real count, got ${estimate}`);
+ assert.ok(estimate < 20473 * 2, `expected the estimate to stay within 2x, got ${estimate}`);
+});
+
+test("estimateTokens counts CJK at one token per codepoint", () => {
+ // ja, zh-CN and zh-TW all ship. CJK tokenizes at roughly 1-1.5 chars/token,
+ // so a global chars/3 constant under-counts a Japanese transcript ~3x and
+ // would silently reintroduce #2142 for those users.
+ assert.equal(policy.estimateTokens("会議の議事録"), 6);
+ assert.equal(policy.estimateTokens("ひらがな"), 4);
+ assert.equal(policy.estimateTokens("한국어"), 3);
+});
+
+test("estimateTokens charges CJK strictly more than Latin for the same length", () => {
+ const latin = policy.estimateTokens("x".repeat(300));
+ const cjk = policy.estimateTokens("会".repeat(300));
+
+ assert.ok(cjk > latin, `expected CJK (${cjk}) to cost more than Latin (${latin})`);
+});
+
+test("estimateTokens sums mixed scripts instead of picking one rate", () => {
+ assert.equal(policy.estimateTokens("会議abcdef"), 2 + 2);
+});
+
+test("estimateTokens treats absent or non-string input as zero", () => {
+ assert.equal(policy.estimateTokens(null), 0);
+ assert.equal(policy.estimateTokens(undefined), 0);
+ assert.equal(policy.estimateTokens(12345), 0);
+});
+
+// --- kvBytesPerToken -----------------------------------------------------
+//
+// The whole point of the fix. KV cost is exactly linear in context size, but
+// varies ~12x across the bundled registry and is ANTI-correlated with model
+// file size, so no ceiling based on RAM tiers, file size or parameter count is
+// sound. It has to come from the model's own architecture.
+
+test("kvBytesPerToken reproduces the measured cost of the #1203 model", () => {
+ // Llama-3.2-3B-Instruct-Q4_K_M: 28 blocks, 8 KV heads, 128-wide K and V.
+ // 28 * 8 * (128+128) * 2 bytes (f16) = 114688 = 112 KiB/token, which is what
+ // made its full 131072 context a ~15 GB KV cache on a 16 GB Mac.
+ const bytes = policy.kvBytesPerToken({
+ blockCount: 28,
+ headCountKv: 8,
+ keyLength: 128,
+ valueLength: 128,
+ });
+
+ assert.equal(bytes, 114688);
+ assert.equal(bytes * 131072, 15032385536); // ~15.0 GB, the #1203 allocation
+});
+
+test("kvBytesPerToken is anti-correlated with file size across the registry", () => {
+ // Qwen3.5-9B is a 5.9 GB file; Llama-3.2-3B is a 2.0 GB file. The smaller
+ // model costs far more per token. This is the fact that rules out every
+ // size-based or RAM-tier heuristic.
+ const qwen9b = policy.kvBytesPerToken({
+ blockCount: 33,
+ headCountKv: 4,
+ keyLength: 256,
+ valueLength: 256,
+ });
+ const llama3b = policy.kvBytesPerToken({
+ blockCount: 28,
+ headCountKv: 8,
+ keyLength: 128,
+ valueLength: 128,
+ });
+
+ assert.equal(qwen9b, 135168);
+ assert.ok(llama3b < qwen9b, "the 3B is cheaper per token than the 9B by this bound");
+ // The bound is deliberately conservative for Qwen3.5: it interleaves full
+ // attention every 4th layer, so the measured cost is 32768 B/token (512 MiB
+ // at 16384 ctx). Over-estimating shrinks the ceiling, which is the safe way
+ // to be wrong.
+ assert.ok(qwen9b > 32768, "full-attention bound must not undercut the measured cost");
+});
+
+test("kvBytesPerToken takes the largest layer when head_count_kv varies per layer", () => {
+ // GGUF allows a per-layer array. The max is the safe reading.
+ const bytes = policy.kvBytesPerToken({
+ blockCount: 4,
+ headCountKv: [2, 8, 2, 2],
+ keyLength: 128,
+ valueLength: 128,
+ });
+
+ assert.equal(bytes, 4 * 8 * 256 * 2);
+});
+
+test("kvBytesPerToken derives head width from the embedding when it is not declared", () => {
+ // key_length/value_length are optional; llama.cpp falls back to
+ // embedding_length / head_count.
+ const bytes = policy.kvBytesPerToken({
+ blockCount: 28,
+ headCountKv: 8,
+ embeddingLength: 3072,
+ headCount: 24, // 3072 / 24 = 128
+ });
+
+ assert.equal(bytes, 114688);
+});
+
+test("kvBytesPerToken returns null rather than guessing when the architecture is unreadable", () => {
+ // A null here must fall back to today's 16384 ceiling, never to a bigger
+ // number derived from incomplete data.
+ assert.equal(policy.kvBytesPerToken(null), null);
+ assert.equal(policy.kvBytesPerToken({}), null);
+ assert.equal(policy.kvBytesPerToken({ blockCount: 28 }), null);
+ assert.equal(policy.kvBytesPerToken({ blockCount: 28, headCountKv: 8 }), null);
+ assert.equal(policy.kvBytesPerToken({ blockCount: 0, headCountKv: 8, keyLength: 128 }), null);
+});
+
+// --- resolveContextCeiling -----------------------------------------------
+//
+// The largest context this machine may allocate for this model. This is the
+// function that has to keep the #1203 promise: it is the only thing standing
+// between a long transcript and a wired KV cache big enough to freeze a Mac.
+
+const GIB = 1024 * 1024 * 1024;
+
+const LLAMA_3_2_3B = {
+ // 28 blocks, 8 KV heads, 128-wide -> 114688 B/token. The #1203 model.
+ kvBytesPerToken: 114688,
+ weightsBytes: 2168958976,
+ trainedContextTokens: 131072,
+};
+
+const QWEN_3_5_9B = {
+ kvBytesPerToken: 135168,
+ weightsBytes: 6169341984,
+ trainedContextTokens: 262144,
+ hasRecurrentState: true,
+};
+
+test("regression #1203: a 16 GB Mac running Llama-3.2-3B stays far below the allocation that panicked", () => {
+ const { ceiling } = policy.resolveContextCeiling({
+ totalMemoryBytes: 16 * GIB,
+ ...LLAMA_3_2_3B,
+ });
+
+ assert.equal(ceiling, 32256);
+
+ // The actual guarantee, stated as memory rather than as a magic number: the
+ // resident footprint must stay well under half the machine, so macOS keeps
+ // room it can compress or swap. #1203 allocated ~15 GB of 16 GB as WIRED
+ // Metal memory, which can do neither, and watchdogd missed its check-ins.
+ const footprint =
+ LLAMA_3_2_3B.weightsBytes + ceiling * LLAMA_3_2_3B.kvBytesPerToken + policy.COMPUTE_FIXED_BYTES;
+ assert.ok(
+ footprint < 0.4 * 16 * GIB,
+ `footprint ${footprint} must stay under 40% of RAM (${0.4 * 16 * GIB})`
+ );
+
+ // And it must still be a real improvement, or the bug is not fixed.
+ assert.ok(ceiling > policy.BASELINE_CONTEXT_SIZE, "must beat today's flat 16384");
+});
+
+test("the ceiling tracks KV cost, not model size, on identical hardware", () => {
+ // Same machine, same weights, only the per-token cost differs by 12x. A
+ // RAM-tier or file-size heuristic would return the same answer for both.
+ const cheap = policy.resolveContextCeiling({
+ totalMemoryBytes: 48 * GIB,
+ kvBytesPerToken: 32768,
+ weightsBytes: 5 * GIB,
+ trainedContextTokens: 262144,
+ });
+ const expensive = policy.resolveContextCeiling({
+ totalMemoryBytes: 48 * GIB,
+ kvBytesPerToken: 393216, // Gemma-3-12B
+ weightsBytes: 5 * GIB,
+ trainedContextTokens: 131072,
+ });
+
+ assert.ok(
+ cheap.ceiling > expensive.ceiling * 3,
+ `expected a wide split, got ${cheap.ceiling} vs ${expensive.ceiling}`
+ );
+});
+
+test("the reported 48 GB machine can hold the transcript that fails today", () => {
+ const { ceiling } = policy.resolveContextCeiling({
+ totalMemoryBytes: 48 * GIB,
+ ...QWEN_3_5_9B,
+ });
+
+ assert.equal(ceiling, 118272);
+ // The customer's prompt measured 20,514 tokens against the bundled server.
+ assert.ok(ceiling > 20514 + 2048, "the failing request must now fit with output headroom");
+});
+
+test("an unreadable architecture falls back to today's limit, never to a larger guess", () => {
+ const { ceiling, reason } = policy.resolveContextCeiling({
+ totalMemoryBytes: 48 * GIB,
+ kvBytesPerToken: null,
+ weightsBytes: 2168958976,
+ trainedContextTokens: 131072,
+ });
+
+ assert.equal(ceiling, policy.BASELINE_CONTEXT_SIZE);
+ assert.equal(reason, "unknown-kv-cost");
+});
+
+test("unusable memory readings fall back to today's limit", () => {
+ for (const totalMemoryBytes of [0, Number.NaN, undefined, null, -1]) {
+ const { ceiling, reason } = policy.resolveContextCeiling({
+ totalMemoryBytes,
+ ...LLAMA_3_2_3B,
+ });
+ assert.equal(ceiling, policy.BASELINE_CONTEXT_SIZE, `total=${totalMemoryBytes}`);
+ assert.equal(reason, "no-memory-info");
+ }
+});
+
+test("a machine too small for any headroom keeps exactly today's behaviour", () => {
+ // 8 GB Mac, 2 GB of weights: the budget goes negative. This must be the
+ // historical 16384 rather than something smaller, so the change can never
+ // make a shipping configuration worse.
+ const { ceiling, reason } = policy.resolveContextCeiling({
+ totalMemoryBytes: 8 * GIB,
+ ...LLAMA_3_2_3B,
+ });
+
+ assert.equal(ceiling, policy.BASELINE_CONTEXT_SIZE);
+ assert.equal(reason, "floor");
+});
+
+test("the ceiling never exceeds what the model was trained for", () => {
+ const { ceiling, reason } = policy.resolveContextCeiling({
+ totalMemoryBytes: 512 * GIB,
+ kvBytesPerToken: 32768,
+ weightsBytes: 5 * GIB,
+ trainedContextTokens: 32768,
+ });
+
+ assert.equal(ceiling, 32768);
+ assert.equal(reason, "trained-context-bound");
+});
+
+test("an unmeasured discrete GPU is capped even when system RAM looks generous", () => {
+ const { ceiling } = policy.resolveContextCeiling({
+ totalMemoryBytes: 128 * GIB,
+ kvBytesPerToken: 32768,
+ weightsBytes: 3 * GIB,
+ trainedContextTokens: 262144,
+ discreteGpuUnverified: true,
+ });
+
+ assert.ok(ceiling <= 65536, `expected the unverified-GPU cap, got ${ceiling}`);
+});
+
+test("the ceiling moves monotonically with each input", () => {
+ const base = { totalMemoryBytes: 32 * GIB, ...LLAMA_3_2_3B };
+ const at = (overrides) => policy.resolveContextCeiling({ ...base, ...overrides }).ceiling;
+
+ assert.ok(at({ totalMemoryBytes: 64 * GIB }) >= at({}), "more RAM never shrinks the ceiling");
+ assert.ok(at({ weightsBytes: 6 * GIB }) <= at({}), "bigger weights never grow the ceiling");
+ assert.ok(at({ kvBytesPerToken: 393216 }) <= at({}), "costlier KV never grows the ceiling");
+ assert.ok(at({ hasRecurrentState: true }) <= at({}), "recurrent state never grows the ceiling");
+ assert.ok(at({ draftWeightsBytes: 2 * GIB }) <= at({}), "a drafter never grows the ceiling");
+});
+
+test("the prompt cache is bounded so it cannot eat the budget we just computed", () => {
+ // llama-server defaults --cache-ram to 8192 MiB of host RAM, on top of the
+ // KV cache and 32 context checkpoints per slot. Unbounded, it would undo the
+ // budget. Cap it rather than disabling it: 0 forces a full re-prefill on
+ // every request.
+ for (const totalMemoryBytes of [8 * GIB, 16 * GIB, 48 * GIB, 128 * GIB]) {
+ const { cacheRamMiB } = policy.resolveContextCeiling({
+ totalMemoryBytes,
+ ...LLAMA_3_2_3B,
+ });
+ assert.ok(
+ cacheRamMiB >= 512 && cacheRamMiB <= 2048,
+ `cacheRamMiB ${cacheRamMiB} out of range at ${totalMemoryBytes}`
+ );
+ }
+});
+
+// --- resolveContextSize --------------------------------------------------
+//
+// Snapping to a ladder is what keeps restarts bounded. Sizing the context to
+// the exact request would restart the server for a 20,600-token note and again
+// for a 21,100-token one; snapping puts both on the same rung.
+
+test("resolveContextSize keeps short requests on the baseline rung", () => {
+ assert.equal(policy.resolveContextSize({ needed: 800, ceiling: 131072 }), 16384);
+ assert.equal(policy.resolveContextSize({ needed: 0, ceiling: 131072 }), 16384);
+ assert.equal(policy.resolveContextSize({ ceiling: 131072 }), 16384);
+});
+
+test("resolveContextSize grows to the smallest rung that fits the request", () => {
+ // The reported failure: 20,514 prompt tokens plus output and template slack.
+ assert.equal(policy.resolveContextSize({ needed: 23074, ceiling: 131072 }), 32768);
+ assert.equal(policy.resolveContextSize({ needed: 16385, ceiling: 131072 }), 32768);
+ assert.equal(policy.resolveContextSize({ needed: 32768, ceiling: 131072 }), 32768);
+ assert.equal(policy.resolveContextSize({ needed: 32769, ceiling: 131072 }), 65536);
+});
+
+test("resolveContextSize puts near-identical requests on the same rung", () => {
+ const first = policy.resolveContextSize({ needed: 20600, ceiling: 131072 });
+ const second = policy.resolveContextSize({ needed: 21100, ceiling: 131072 });
+
+ assert.equal(first, second, "a second similar note must not cost a second restart");
+});
+
+test("resolveContextSize never returns more than the ceiling allows", () => {
+ // The #1203 machine: the ceiling is not on a rung, and it wins outright.
+ assert.equal(policy.resolveContextSize({ needed: 40000, ceiling: 32256 }), 32256);
+ assert.equal(policy.resolveContextSize({ needed: 1000000, ceiling: 32256 }), 32256);
+ assert.equal(policy.resolveContextSize({ needed: 100000, ceiling: 65536 }), 65536);
+});
+
+test("resolveContextSize honours a caller floor without breaching the ceiling", () => {
+ // Selection editing asks for a larger window up front (audioManager.js).
+ assert.equal(policy.resolveContextSize({ needed: 500, floor: 32768, ceiling: 131072 }), 32768);
+ // ...but a caller cannot talk its way past the memory guard.
+ assert.equal(policy.resolveContextSize({ needed: 500, floor: 262144, ceiling: 32256 }), 32256);
+});
+
+test("resolveContextSize never drops below the baseline even with a tiny ceiling", () => {
+ assert.equal(policy.resolveContextSize({ needed: 100, ceiling: 4096 }), 16384);
+});
diff --git a/test/helpers/llamaServerContext.test.js b/test/helpers/llamaServerContext.test.js
new file mode 100644
index 000000000..d17f974b8
--- /dev/null
+++ b/test/helpers/llamaServerContext.test.js
@@ -0,0 +1,349 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const http = require("node:http");
+
+const LlamaServerManager = require("../../src/helpers/llamaServer.js");
+
+// A manager whose spawn is stubbed at the _doStart boundary, recording the
+// options each start was asked for.
+function makeManager() {
+ const manager = new LlamaServerManager();
+ const starts = [];
+ manager._doStart = async (modelPath, options = {}) => {
+ manager.ready = true;
+ manager.modelPath = modelPath;
+ manager.draftModelPath = options.draftModelPath || null;
+ starts.push({ modelPath, contextSize: options.contextSize });
+ };
+ return { manager, starts };
+}
+
+// --- grow-only restart ---------------------------------------------------
+//
+// Before #2142 the context was a single constant, so start() could ignore
+// options entirely once a server was ready. Now that the context varies per
+// request, it has to restart to grow — but only to grow, or a long note
+// followed by a short dictation would restart the server twice.
+
+test("a larger context restarts the server", async () => {
+ const { manager, starts } = makeManager();
+
+ await manager.start("/models/main.gguf", { contextSize: 16384 });
+ assert.equal(starts.length, 1);
+
+ await manager.start("/models/main.gguf", { contextSize: 32768 });
+ assert.equal(starts.length, 2);
+ assert.equal(manager.contextSize, 32768);
+});
+
+test("an equal or smaller context reuses the running server", async () => {
+ const { manager, starts } = makeManager();
+
+ await manager.start("/models/main.gguf", { contextSize: 32768 });
+ assert.equal(starts.length, 1);
+
+ // Exactly the same → no restart.
+ await manager.start("/models/main.gguf", { contextSize: 32768 });
+ assert.equal(starts.length, 1);
+
+ // Smaller → no restart, and the larger window is kept. This is what stops a
+ // short dictation right after a long note from costing a second restart.
+ await manager.start("/models/main.gguf", { contextSize: 16384 });
+ assert.equal(starts.length, 1);
+ assert.equal(manager.contextSize, 32768);
+});
+
+test("growing the context still restarts when the drafter is unchanged", async () => {
+ const { manager, starts } = makeManager();
+
+ await manager.start("/models/main.gguf", { contextSize: 16384, draftModelPath: "/d.gguf" });
+ await manager.start("/models/main.gguf", { contextSize: 65536, draftModelPath: "/d.gguf" });
+
+ assert.equal(starts.length, 2);
+ assert.equal(manager.contextSize, 65536);
+});
+
+test("a context grown for one model does not carry over to another", async () => {
+ const { manager, starts } = makeManager();
+
+ await manager.start("/models/big.gguf", { contextSize: 65536 });
+ await manager.start("/models/other.gguf", { contextSize: 16384 });
+
+ assert.equal(starts.length, 2);
+ assert.equal(manager.contextSize, 16384, "the new model starts at its own size");
+});
+
+test("stopping the server forgets the grown context", async () => {
+ const { manager } = makeManager();
+ manager._killCurrentProcess = async () => {};
+ manager.process = {};
+
+ await manager.start("/models/main.gguf", { contextSize: 65536 });
+ assert.equal(manager.contextSize, 65536);
+
+ await manager.stop();
+ assert.equal(manager.contextSize, null);
+});
+
+// --- spawn arguments -----------------------------------------------------
+
+test("every GPU ladder rung inherits the requested context and cache bounds", async () => {
+ const manager = new LlamaServerManager();
+ const seen = [];
+ manager._buildEnv = () => ({});
+ manager._killCurrentProcess = async () => {};
+ manager.findAvailablePort = async () => 8221;
+ manager.draftModelPath = null;
+ manager._startWithBinary = async (binary, args) => {
+ seen.push(args);
+ throw new Error("stub fail");
+ };
+
+ const baseArgs = manager._buildBaseArgs("/models/main.gguf", 8221, {
+ contextSize: 32768,
+ cacheRamMiB: 1024,
+ });
+ await assert.rejects(() =>
+ manager._startWithGpuFallback({ vulkan: "/bin/vulkan", cpu: "/bin/cpu" }, baseArgs, {}, [])
+ );
+
+ assert.ok(seen.length >= 2, "expected the ladder to try more than one rung");
+ for (const args of seen) {
+ assert.equal(args[args.indexOf("--ctx-size") + 1], "32768");
+ // llama-server defaults --cache-ram to 8192 MiB of host RAM on top of the
+ // KV cache. Left unbounded it would undo the memory budget entirely.
+ assert.equal(args[args.indexOf("--cache-ram") + 1], "1024");
+ }
+});
+
+// --- context-overflow errors --------------------------------------------
+
+test("a context overflow is reported as a typed error, not as llama.cpp JSON", async () => {
+ const overflowBody = JSON.stringify({
+ error: {
+ code: 400,
+ message: "request (20514 tokens) exceeds the available context size (16384 tokens)",
+ type: "exceed_context_size_error",
+ n_prompt_tokens: 20514,
+ n_ctx: 16384,
+ },
+ });
+
+ const server = http.createServer((req, res) => {
+ res.writeHead(400, { "Content-Type": "application/json" });
+ res.end(overflowBody);
+ });
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+ const manager = new LlamaServerManager();
+ manager.ready = true;
+ manager.process = {};
+ manager.port = server.address().port;
+
+ try {
+ await assert.rejects(
+ () => manager.inference([{ role: "user", content: "hi" }], {}),
+ (error) => {
+ assert.equal(error.code, "CONTEXT_TOO_LARGE");
+ assert.equal(error.neededTokens, 20514);
+ assert.equal(error.maxContextTokens, 16384);
+ // The whole point: a customer must never be shown a JSON blob.
+ assert.ok(!error.message.includes("{"), `raw JSON leaked: ${error.message}`);
+ return true;
+ }
+ );
+ } finally {
+ manager.clearIdleTimer();
+ await new Promise((resolve) => server.close(resolve));
+ }
+});
+
+test("other llama-server failures keep their existing shape", async () => {
+ const server = http.createServer((req, res) => {
+ res.writeHead(500, { "Content-Type": "text/plain" });
+ res.end("internal error");
+ });
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+ const manager = new LlamaServerManager();
+ manager.ready = true;
+ manager.process = {};
+ manager.port = server.address().port;
+
+ try {
+ await assert.rejects(
+ () => manager.inference([{ role: "user", content: "hi" }], {}),
+ (error) => {
+ assert.equal(error.code, undefined);
+ assert.match(error.message, /llama-server returned status 500/);
+ return true;
+ }
+ );
+ } finally {
+ manager.clearIdleTimer();
+ await new Promise((resolve) => server.close(resolve));
+ }
+});
+
+// --- measuring the running server ----------------------------------------
+//
+// The estimate only picks the size the server starts at. These two calls are
+// what make the decision exact: /props reports the window actually in force
+// (which can differ from what we asked for), and /apply-template + /tokenize
+// price the real prompt, chat template and all.
+
+function withStubServer(routes) {
+ // A Map rather than indexing the object: req.url is request-controlled, so a
+ // plain lookup answers /toString or /constructor with an inherited
+ // Object.prototype method and then calls it.
+ const routeTable = new Map(Object.entries(routes));
+ return async (run) => {
+ const server = http.createServer((req, res) => {
+ const handler = routeTable.get(req.url);
+ if (typeof handler !== "function") {
+ res.writeHead(404).end();
+ return;
+ }
+ let body = "";
+ req.on("data", (chunk) => {
+ body += chunk;
+ });
+ req.on("end", () => {
+ const { status = 200, payload } = handler(body);
+ res.writeHead(status, { "Content-Type": "application/json" });
+ res.end(JSON.stringify(payload));
+ });
+ });
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+
+ const manager = new LlamaServerManager();
+ manager.ready = true;
+ manager.process = {};
+ manager.port = server.address().port;
+ try {
+ await run(manager);
+ } finally {
+ manager.clearIdleTimer();
+ await new Promise((resolve) => server.close(resolve));
+ }
+ };
+}
+
+test("usableContextSize reports the window the server actually has", async () => {
+ await withStubServer({
+ "/props": () => ({ payload: { default_generation_settings: { n_ctx: 32768 } } }),
+ })(async (manager) => {
+ assert.equal(await manager.usableContextSize(), 32768);
+ });
+});
+
+test("usableContextSize returns null rather than guessing when props is unreadable", async () => {
+ // A broken measurement must degrade to the estimate, never fail the request.
+ await withStubServer({ "/props": () => ({ status: 500, payload: {} }) })(async (manager) => {
+ assert.equal(await manager.usableContextSize(), null);
+ });
+ await withStubServer({})(async (manager) => {
+ assert.equal(await manager.usableContextSize(), null);
+ });
+});
+
+test("countPromptTokens prices the rendered template, not just the message text", async () => {
+ let templatedWith = null;
+ let tokenized = null;
+
+ await withStubServer({
+ "/apply-template": (body) => {
+ templatedWith = JSON.parse(body);
+ // The real server renders role tags and, with thinking disabled, an
+ // empty <think></think> block. All of it costs context.
+ return { payload: { prompt: "<|im_start|>system\nSYS<|im_end|>\nhello" } };
+ },
+ "/tokenize": (body) => {
+ tokenized = JSON.parse(body);
+ return { payload: { tokens: new Array(20514).fill(0) } };
+ },
+ })(async (manager) => {
+ const messages = [
+ { role: "system", content: "SYS" },
+ { role: "user", content: "hello" },
+ ];
+ assert.equal(await manager.countPromptTokens(messages, { disableThinking: true }), 20514);
+ assert.deepEqual(templatedWith.messages, messages);
+ assert.deepEqual(templatedWith.chat_template_kwargs, { enable_thinking: false });
+ assert.equal(tokenized.content, "<|im_start|>system\nSYS<|im_end|>\nhello");
+ });
+});
+
+test("countPromptTokens returns null when the server cannot measure", async () => {
+ await withStubServer({ "/apply-template": () => ({ status: 500, payload: {} }) })(
+ async (manager) => {
+ assert.equal(await manager.countPromptTokens([{ role: "user", content: "hi" }], {}), null);
+ }
+ );
+});
+
+// --- Vulkan context step-down --------------------------------------------
+//
+// Sizing the context to the request is bounded by SYSTEM RAM, but a discrete
+// GPU has its own memory that the app does not probe. A machine with plenty of
+// RAM and a small card can therefore now ask for a context that will not fit
+// in VRAM — a failure this change introduces. Dropping straight to CPU costs
+// 10-30x; halving the context is far cheaper, so try that first.
+
+function makeLadder(shouldFail, { contextSize = 32768 } = {}) {
+ const manager = new LlamaServerManager();
+ const attempts = [];
+ manager._buildEnv = () => ({});
+ manager._killCurrentProcess = async () => {};
+ manager.findAvailablePort = async () => 8221;
+ manager.draftModelPath = null;
+ manager._startWithBinary = async (binary, args) => {
+ const attempt = { binary, ctx: Number(args[args.indexOf("--ctx-size") + 1]) };
+ attempts.push(attempt);
+ if (shouldFail(attempt)) throw new Error(`stub fail: ${binary} ctx=${attempt.ctx}`);
+ };
+ const baseArgs = manager._buildBaseArgs("/models/main.gguf", 8221, { contextSize });
+ return { manager, attempts, baseArgs, options: { contextSize } };
+}
+
+const BINARIES = { vulkan: "/bin/vulkan", cpu: "/bin/cpu" };
+
+test("a Vulkan start that fails at a grown context retries smaller before dropping to CPU", async () => {
+ // Vulkan only succeeds at the baseline context.
+ const { manager, attempts, baseArgs, options } = makeLadder(
+ (attempt) => attempt.binary === "/bin/vulkan" && attempt.ctx > 16384
+ );
+
+ await manager._startWithGpuFallback(BINARIES, baseArgs, options, []);
+
+ assert.deepEqual(attempts, [
+ { binary: "/bin/vulkan", ctx: 32768 },
+ { binary: "/bin/vulkan", ctx: 16384 },
+ ]);
+ assert.equal(manager.activeBackend, "vulkan", "a live GPU beats a CPU fallback");
+});
+
+test("a baseline-context start keeps today's exact ladder, with no extra rung", async () => {
+ // The step-down must not add an attempt when there is nothing to step down
+ // to, or every existing failure path gets slower.
+ const { manager, attempts, baseArgs, options } = makeLadder(() => true, {
+ contextSize: 16384,
+ });
+
+ await assert.rejects(() => manager._startWithGpuFallback(BINARIES, baseArgs, options, []));
+
+ assert.deepEqual(attempts, [
+ { binary: "/bin/vulkan", ctx: 16384 },
+ { binary: "/bin/cpu", ctx: 16384 },
+ ]);
+});
+
+test("a context step-down is recorded, so the server reports what it actually has", async () => {
+ const { manager, baseArgs, options } = makeLadder(
+ (attempt) => attempt.binary === "/bin/vulkan" && attempt.ctx > 16384
+ );
+
+ await manager._startWithGpuFallback(BINARIES, baseArgs, options, []);
+
+ // Otherwise the preflight would believe the request still fits and send a
+ // prompt the server cannot take.
+ assert.equal(manager.contextSize, 16384);
+});
diff --git a/test/helpers/localReasoningBridgeChain.test.js b/test/helpers/localReasoningBridgeChain.test.js
index 85eee1c77..13d7b80c3 100644
--- a/test/helpers/localReasoningBridgeChain.test.js
+++ b/test/helpers/localReasoningBridgeChain.test.js
@@ -7,6 +7,7 @@ const os = require("node:os");
const path = require("node:path");
const modelRegistryData = require("../../src/models/modelRegistryData.json");
+const { buildGguf, LLAMA_3_2_3B_ENTRIES } = require("./harness/ggufFixtures");
// Drives the primary local LLM path end to end — the IPC bridge
// (localReasoningBridge) → modelManagerBridge.runInference →
@@ -52,7 +53,11 @@ function loadChain() {
}
}
-const OVER_MIN_FILE_SIZE = Buffer.alloc(1_000_001, 1);
+// A real GGUF header padded past the minimum-size check, so the context
+// ceiling is computed from the model's own architecture rather than falling
+// back to the baseline the moment a caller asks for a bigger window.
+const MODEL_FILE = Buffer.concat([buildGguf(LLAMA_3_2_3B_ENTRIES), Buffer.alloc(1_000_001, 1)]);
+const ROOMY_MACHINE_BYTES = 48 * 1024 * 1024 * 1024;
// Stands up the stub server plus a bridge whose model manager already
// believes llama-server is running that model on the stub's port.
@@ -78,7 +83,8 @@ async function setupChain(t, respond) {
const model = modelRegistryData.localProviders[0].models[0];
modelManager.ensureInitialized();
await fs.mkdir(modelManager.modelsDir, { recursive: true });
- await fs.writeFile(path.join(modelManager.modelsDir, model.fileName), OVER_MIN_FILE_SIZE);
+ await fs.writeFile(path.join(modelManager.modelsDir, model.fileName), MODEL_FILE);
+ modelManager._systemMemoryBytes = () => ROOMY_MACHINE_BYTES;
const serverManager = modelManager.serverManager;
serverManager.cachedServerBinaryPaths = { default: "/stub/llama-server" };
@@ -88,7 +94,7 @@ async function setupChain(t, respond) {
modelManager.currentServerModelId = model.id;
t.after(() => serverManager.clearIdleTimer());
- return { bridge, modelId: model.id, requests };
+ return { bridge, modelId: model.id, requests, serverManager };
}
const completion = (finishReason, content) => ({
@@ -118,3 +124,30 @@ test("an explicit temperature of 0 reaches llama-server instead of the 0.7 defau
assert.equal(requests.length, 1);
assert.equal(requests[0].temperature, 0);
});
+
+test("a caller's contextSize reaches the server start (regression: it was dropped)", async (t) => {
+ // ReasoningConfig.contextSize was declared, written by selection editing,
+ // and then rebuilt away in this bridge, so it never reached anything. The
+ // field only means something now that the server can grow, so the wiring
+ // needs a guard that fails if anyone rebuilds the config object again.
+ // It raises the floor only as far as the machine's ceiling allows, which is
+ // why this harness has to present a machine and a model that can afford it.
+ const { bridge, modelId, serverManager } = await setupChain(t, () => completion("stop", "ok"));
+
+ const started = [];
+ serverManager._doStart = async (modelPath, options = {}) => {
+ started.push(options.contextSize);
+ serverManager.ready = true;
+ serverManager.process = {};
+ };
+ serverManager.stop = async () => {
+ serverManager.ready = false;
+ serverManager.process = null;
+ serverManager.contextSize = null;
+ };
+ serverManager.contextSize = 16384;
+
+ await bridge.processText("short text", modelId, { contextSize: 32768 });
+
+ assert.deepEqual(started, [32768]);
+});
diff --git a/test/helpers/modelManagerContextSizing.test.js b/test/helpers/modelManagerContextSizing.test.js
new file mode 100644
index 000000000..9289946cc
--- /dev/null
+++ b/test/helpers/modelManagerContextSizing.test.js
@@ -0,0 +1,479 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const Module = require("node:module");
+const fs = require("node:fs/promises");
+const http = require("node:http");
+const os = require("node:os");
+const path = require("node:path");
+
+const modelRegistryData = require("../../src/models/modelRegistryData.json");
+const { buildGguf, LLAMA_3_2_3B_ENTRIES } = require("./harness/ggufFixtures");
+
+// Drives modelManagerBridge.runInference against a stub llama-server, to pin
+// how the context window is chosen for a request (#2142). The model file on
+// disk is a REAL GGUF header padded to a plausible size, so the ceiling is
+// computed through the production path rather than from a fixture object.
+
+const originalLoad = Module._load;
+const CHAIN_MODULES = [
+ "../../src/helpers/modelManagerBridge.js",
+ "../../src/helpers/modelDirUtils.js",
+ "../../src/helpers/llamaServer.js",
+].map((relative) => require.resolve(relative));
+let electronHome = os.tmpdir();
+
+function loadModelManager() {
+ for (const modulePath of CHAIN_MODULES) delete require.cache[modulePath];
+ Module._load = function loadWithMocks(request, parent, isMain) {
+ if (request === "electron") {
+ return {
+ app: {
+ isReady: () => true,
+ getAppPath: () => process.cwd(),
+ getPath: (name) => (name === "home" ? electronHome : path.join(electronHome, name)),
+ },
+ net: {},
+ };
+ }
+ return originalLoad.call(this, request, parent, isMain);
+ };
+ try {
+ require("../../src/helpers/modelDirUtils.js");
+ return require("../../src/helpers/modelManagerBridge.js").default;
+ } finally {
+ Module._load = originalLoad;
+ }
+}
+
+const GIB = 1024 * 1024 * 1024;
+const SHORT_PROMPT = "Clean up this sentence.";
+// ~20k estimated tokens, the size that fails on main today.
+const LONG_PROMPT = "word ".repeat(12000);
+
+async function setup(t, { tokenCount = null, totalMemoryBytes = 48 * GIB } = {}) {
+ const home = await fs.mkdtemp(path.join(os.tmpdir(), "openwhispr-ctx-sizing-"));
+ electronHome = home;
+ t.after(() => fs.rm(home, { recursive: true, force: true }));
+
+ const calls = { tokenize: 0, props: 0, completions: 0, template: 0 };
+ // The completion body is where the output allowance actually lands, so tests
+ // about it have to read the wire rather than the config object.
+ let completionBody = null;
+ const server = http.createServer((req, res) => {
+ let raw = "";
+ req.on("data", (chunk) => (raw += chunk));
+ req.on("end", () => {
+ let status = 200;
+ let payload;
+ if (req.url === "/props") {
+ calls.props += 1;
+ payload = { default_generation_settings: { n_ctx: serverManager.contextSize } };
+ } else if (req.url === "/apply-template") {
+ calls.template += 1;
+ payload = { prompt: "RENDERED" };
+ } else if (req.url === "/tokenize") {
+ calls.tokenize += 1;
+ if (tokenCount === null) {
+ status = 500;
+ payload = {};
+ } else {
+ payload = { tokens: new Array(tokenCount).fill(0) };
+ }
+ } else {
+ calls.completions += 1;
+ completionBody = JSON.parse(raw);
+ payload = { choices: [{ finish_reason: "stop", message: { content: "done" } }] };
+ }
+ res.writeHead(status, { "Content-Type": "application/json" });
+ res.end(JSON.stringify(payload));
+ });
+ });
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+ t.after(() => new Promise((resolve) => server.close(resolve)));
+
+ const modelManager = loadModelManager();
+ const model = modelRegistryData.localProviders[0].models[0];
+ modelManager.ensureInitialized();
+ await fs.mkdir(modelManager.modelsDir, { recursive: true });
+
+ // A valid GGUF header, padded past the minimum-size check.
+ const header = buildGguf(LLAMA_3_2_3B_ENTRIES);
+ await fs.writeFile(
+ path.join(modelManager.modelsDir, model.fileName),
+ Buffer.concat([header, Buffer.alloc(1_000_001, 1)])
+ );
+
+ const serverManager = modelManager.serverManager;
+ serverManager.cachedServerBinaryPaths = { default: "/stub/llama-server" };
+ serverManager.ready = true;
+ serverManager.process = {};
+ serverManager.port = server.address().port;
+ serverManager.contextSize = 16384;
+ modelManager.currentServerModelId = model.id;
+ modelManager._systemMemoryBytes = () => totalMemoryBytes;
+ t.after(() => serverManager.clearIdleTimer());
+
+ // Stand in for the spawn. The real _doStart leaves the manager ready on a
+ // live port, so the stub must too, or the preflight would see a dead server.
+ const restarts = [];
+ serverManager._doStart = async (modelPath, options = {}) => {
+ restarts.push(options.contextSize);
+ serverManager.ready = true;
+ serverManager.process = {};
+ serverManager.port = server.address().port;
+ };
+ // Avoid signalling a pid that does not belong to us; stop() itself is
+ // covered in llamaServerContext.test.js.
+ serverManager.stop = async () => {
+ serverManager.ready = false;
+ serverManager.process = null;
+ serverManager.contextSize = null;
+ };
+
+ return {
+ modelManager,
+ modelId: model.id,
+ calls,
+ restarts,
+ serverManager,
+ completionBody: () => completionBody,
+ };
+}
+
+test("a short request costs no measurement and no restart", async (t) => {
+ // The dictation-cleanup invariant. Every dictation goes through this path;
+ // adding a round trip or a restart here would be a far worse regression than
+ // the bug being fixed.
+ const { modelManager, modelId, calls, restarts, completionBody } = await setup(t, {
+ tokenCount: 40,
+ });
+
+ const result = await modelManager.runInference(modelId, SHORT_PROMPT, {
+ systemPrompt: "Clean up dictation.",
+ maxTokens: 900,
+ });
+
+ assert.equal(result, "done");
+ assert.equal(restarts.length, 0, "a short request must not restart the server");
+ assert.equal(calls.tokenize, 0, "a short request must not pay for tokenization");
+ assert.equal(calls.completions, 1);
+ // A request that comfortably fits must reach the model with the allowance
+ // the caller asked for. Without this, trimming everything to a stub would
+ // still pass every other test in this file.
+ assert.equal(completionBody().max_tokens, 900);
+});
+
+test("a long request grows the context once and then succeeds", async (t) => {
+ const { modelManager, modelId, calls, restarts, serverManager } = await setup(t, {
+ tokenCount: 20514, // the customer's measured prompt
+ });
+
+ const result = await modelManager.runInference(modelId, LONG_PROMPT, {
+ systemPrompt: "Write meeting notes.",
+ });
+
+ assert.equal(result, "done");
+ assert.deepEqual(restarts, [32768], "exactly one restart, onto the rung that fits");
+ assert.equal(serverManager.contextSize, 32768);
+ assert.equal(calls.completions, 1);
+});
+
+test("a request too large for the machine fails before it reaches the model", async (t) => {
+ const { modelManager, modelId, calls } = await setup(t, { tokenCount: 200000 });
+
+ await assert.rejects(
+ () => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),
+ (error) => {
+ assert.equal(error.code, "CONTEXT_TOO_LARGE");
+ assert.ok(error.details.neededTokens >= 200000);
+ assert.ok(error.details.maxContextTokens > 0);
+ assert.ok(!error.message.includes("{"), `raw JSON leaked: ${error.message}`);
+ return true;
+ }
+ );
+
+ // Sending it anyway would burn minutes of prefill to earn a 400.
+ assert.equal(calls.completions, 0, "must not send a request that cannot fit");
+});
+
+test("a broken measurement falls back to the estimate instead of failing the request", async (t) => {
+ const { modelManager, modelId, calls } = await setup(t, { tokenCount: null }); // /tokenize 500s
+
+ const result = await modelManager.runInference(modelId, LONG_PROMPT, {
+ systemPrompt: "Write notes.",
+ });
+
+ assert.equal(result, "done");
+ assert.ok(calls.tokenize > 0, "it should have tried to measure");
+ assert.equal(calls.completions, 1, "and still completed on the estimate");
+});
+
+test("a short request after a long one reuses the grown context", async (t) => {
+ const { modelManager, modelId, restarts } = await setup(t, { tokenCount: 20514 });
+
+ await modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." });
+ assert.deepEqual(restarts, [32768]);
+
+ await modelManager.runInference(modelId, SHORT_PROMPT, { systemPrompt: "Clean up." });
+ assert.deepEqual(restarts, [32768], "the short request must not shrink or restart the server");
+});
+
+test("a small machine refuses what a large one accepts, for the same request", async (t) => {
+ // The #1203 guarantee, expressed end to end: the ceiling is a property of
+ // the machine, so the same transcript is allowed on 48 GB and refused on 8.
+ const small = await setup(t, { tokenCount: 60000, totalMemoryBytes: 8 * GIB });
+ await assert.rejects(
+ () => small.modelManager.runInference(small.modelId, LONG_PROMPT, { systemPrompt: "Notes." }),
+ (error) => error.code === "CONTEXT_TOO_LARGE"
+ );
+ assert.equal(small.calls.completions, 0);
+});
+
+test("a running server whose context is unknown is not restarted for a short request", async (t) => {
+ // A ready server always has at least the starting context. Treating an
+ // unknown value as zero restarted every warm server on its next request.
+ const { modelManager, modelId, restarts, serverManager } = await setup(t, { tokenCount: 40 });
+ serverManager.contextSize = null;
+
+ await modelManager.runInference(modelId, SHORT_PROMPT, { systemPrompt: "Clean up." });
+
+ assert.deepEqual(restarts, []);
+});
+
+test("a caller's context floor cannot breach the machine's ceiling", async (t) => {
+ // ReasoningConfig.contextSize raises the floor a request starts at, but the
+ // ceiling is the memory guarantee: honouring a floor the machine cannot
+ // afford is exactly the unbounded KV cache that panicked #1203.
+ //
+ // 12 GiB puts the ceiling strictly between the baseline and the floor, so
+ // the three possible outcomes — clamp to the ceiling, collapse to the
+ // baseline, obey the floor — are told apart. The assertions stay relational
+ // because the exact ceiling is the policy's arithmetic, not this test's, and
+ // because it must hold on Linux CI too (below the 65536 unverified-GPU cap,
+ // so memory-bound on either platform).
+ const { modelManager, modelId, restarts, serverManager } = await setup(t, {
+ tokenCount: 40,
+ totalMemoryBytes: 12 * GIB,
+ });
+
+ await modelManager.runInference(modelId, SHORT_PROMPT, {
+ systemPrompt: "Edit this selection.",
+ contextSize: 131072,
+ });
+
+ assert.equal(restarts.length, 1, "the raised floor should grow the window once");
+ const [started] = restarts;
+ assert.ok(started > 16384, `should grow past the baseline, started at ${started}`);
+ assert.ok(started < 131072, `must never reach the caller's floor, started at ${started}`);
+ assert.equal(serverManager.contextSize, started);
+});
+
+test("a prompt that fits gets a smaller answer rather than a refusal", async (t) => {
+ // llama-server only rejects a prompt that overflows the window on its own,
+ // so charging the full output reservation up front refused notes that work
+ // today. Shrink the answer to what is left instead (#2142).
+ const { modelManager, modelId, calls, completionBody } = await setup(t, {
+ tokenCount: 13000,
+ totalMemoryBytes: 8 * GIB, // ceiling collapses to the 16384 floor
+ });
+
+ const result = await modelManager.runInference(modelId, LONG_PROMPT, {
+ systemPrompt: "Write meeting notes.",
+ maxTokens: 4096,
+ });
+
+ assert.equal(result, "done");
+ assert.equal(calls.completions, 1, "the request must still be sent");
+ // Every token the window has left, not a token less: the prompt was
+ // measured exactly, and llama.cpp stops generation at the limit rather than
+ // rejecting it, so slack here would only shorten the note for nothing.
+ assert.equal(completionBody().max_tokens, 16384 - 13000);
+});
+
+test("a prompt leaving no room for a usable answer is still refused", async (t) => {
+ const { modelManager, modelId, calls } = await setup(t, {
+ tokenCount: 16000,
+ totalMemoryBytes: 8 * GIB,
+ });
+
+ await assert.rejects(
+ () => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),
+ (error) => error.code === "CONTEXT_TOO_LARGE"
+ );
+ assert.equal(calls.completions, 0, "must not send a request that cannot answer");
+});
+
+test("a selection edit is refused rather than given a clipped answer", async (t) => {
+ // requireCompleteOutput means a partial replacement corrupts the user's own
+ // text, so trading output room for a smaller answer is not an option there.
+ const { modelManager, modelId, calls } = await setup(t, {
+ tokenCount: 13000,
+ totalMemoryBytes: 8 * GIB,
+ });
+
+ await assert.rejects(
+ () =>
+ modelManager.runInference(modelId, LONG_PROMPT, {
+ systemPrompt: "Rewrite the selection.",
+ maxTokens: 8192,
+ requireCompleteOutput: true,
+ }),
+ (error) => error.code === "CONTEXT_TOO_LARGE"
+ );
+ assert.equal(calls.completions, 0);
+});
+
+test("a grow that fails comes back at the window that was working", async (t) => {
+ // start() stops the running server before it spawns the bigger one, so a
+ // grow that dies takes a working window with it. Restoring it keeps local
+ // inference alive for the next request instead of until the next launch.
+ const { modelManager, modelId, serverManager } = await setup(t, { tokenCount: 20514 });
+ const port = serverManager.port;
+ const attempts = [];
+ serverManager._doStart = async (modelPath, options = {}) => {
+ attempts.push(options.contextSize);
+ if (options.contextSize > 16384) {
+ throw new Error(
+ "llama-server process died during startup (signal: SIGKILL)\n" +
+ "Process output: ggml_metal: failed to allocate buffer of size 4096.00 MiB"
+ );
+ }
+ serverManager.ready = true;
+ serverManager.process = {};
+ serverManager.port = port;
+ };
+
+ await assert.rejects(
+ () => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),
+ (error) => error.code === "CONTEXT_TOO_LARGE"
+ );
+
+ assert.deepEqual(attempts, [32768, 16384], "one failed grow, then back to what worked");
+ assert.equal(serverManager.ready, true, "the user must not be left without a server");
+});
+
+test("a server that cannot come back reports a typed failure, not llama.cpp stderr", async (t) => {
+ const { modelManager, modelId, serverManager } = await setup(t, { tokenCount: 20514 });
+ serverManager._doStart = async () => {
+ throw new Error(
+ "llama-server process died during startup (signal: SIGKILL)\n" +
+ "Process output: ggml_metal_graph_compute: command buffer 0 failed with status 5"
+ );
+ };
+
+ await assert.rejects(
+ () => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),
+ (error) => {
+ assert.equal(error.code, "LOCAL_SERVER_UNAVAILABLE");
+ assert.ok(
+ !error.message.includes("Process output"),
+ `server output leaked: ${error.message}`
+ );
+ // details crosses IPC to the renderer, so the startup dump must not ride
+ // along in there either — it belongs in the main-process log.
+ assert.ok(
+ !JSON.stringify(error.details).includes("Process output"),
+ `server output leaked through details: ${JSON.stringify(error.details)}`
+ );
+ return true;
+ }
+ );
+});
+
+test("an answer too small to be worth generating is refused rather than stubbed", async (t) => {
+ // 16100 of a 16384 window leaves 284 tokens, a few sentences for a whole
+ // meeting. Below the floor the honest answer is that it does not fit, not a
+ // stub saved as the note.
+ const { modelManager, modelId, calls } = await setup(t, {
+ tokenCount: 16100,
+ totalMemoryBytes: 8 * GIB,
+ });
+
+ await assert.rejects(
+ () =>
+ modelManager.runInference(modelId, LONG_PROMPT, {
+ systemPrompt: "Write notes.",
+ maxTokens: 4096,
+ }),
+ (error) => error.code === "CONTEXT_TOO_LARGE"
+ );
+ assert.equal(calls.completions, 0);
+});
+
+test("a start that steps the context down is not retried at the size that stepped down", async (t) => {
+ // The GPU ladder can load a smaller window than it was asked for without
+ // ever throwing. Treating that as success left the preflight to ask for the
+ // same window again, paying a second full model load to be stepped down
+ // again — the wasted restart the memo exists to prevent.
+ const { modelManager, modelId, serverManager } = await setup(t, { tokenCount: 20514 });
+ const port = serverManager.port;
+ const attempts = [];
+ serverManager._doStart = async (modelPath, options = {}) => {
+ attempts.push(options.contextSize);
+ serverManager.ready = true;
+ serverManager.process = {};
+ serverManager.port = port;
+ // What _startWithGpuFallback records when the reduced-context rung wins.
+ serverManager.activeContextSize = 16384;
+ };
+
+ await assert.rejects(
+ () => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),
+ (error) => error.code === "CONTEXT_TOO_LARGE"
+ );
+
+ assert.deepEqual(attempts, [32768], "the step-down must be remembered, not re-attempted");
+});
+
+test("a server that already died is not restored to a window it no longer has", async (t) => {
+ // llamaServer clears contextSize only in stop(); a crash or a failed health
+ // check leaves it set while ready goes false. Reading it blind made a failed
+ // start pay for a second one that was never going to help.
+ const { modelManager, modelId, serverManager } = await setup(t, { tokenCount: 20514 });
+ serverManager.ready = false;
+ serverManager.process = null;
+
+ const attempts = [];
+ serverManager._doStart = async (modelPath, options = {}) => {
+ attempts.push(options.contextSize);
+ throw new Error("llama-server process died during startup (signal: SIGKILL)");
+ };
+
+ await assert.rejects(
+ () => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),
+ (error) => error.code === "LOCAL_SERVER_UNAVAILABLE"
+ );
+
+ assert.deepEqual(attempts, [32768], "no second attempt at a window that was already gone");
+});
+
+test("only Apple Silicon is treated as memory the GPU already shares", async (t) => {
+ // discreteGpuUnverified reads darwin as unified memory, but an Intel Mac's
+ // Radeon has its own few GB. Sizing against system RAM there hands the card
+ // a context it cannot hold — and the darwin start path has no reduced-context
+ // rung to fall back on.
+ const real = { platform: process.platform, arch: process.arch };
+ const pretend = (platform, arch) => {
+ Object.defineProperty(process, "platform", { value: platform, configurable: true });
+ Object.defineProperty(process, "arch", { value: arch, configurable: true });
+ };
+ t.after(() => pretend(real.platform, real.arch));
+
+ // Both managers are built before any pretending, so only the ceiling
+ // decision sees the faked platform.
+ const roomy = { tokenCount: 40, totalMemoryBytes: 64 * GIB };
+ const apple = await setup(t, roomy);
+ const intel = await setup(t, roomy);
+ const model = modelRegistryData.localProviders[0].models[0];
+ const ceilingFor = (manager) =>
+ manager.contextCeiling({ model }, path.join(manager.modelsDir, model.fileName));
+
+ pretend("darwin", "arm64");
+ const unified = await ceilingFor(apple.modelManager);
+ pretend("darwin", "x64");
+ const discrete = await ceilingFor(intel.modelManager);
+
+ assert.notEqual(unified.reason, "gpu-cap", "unified memory needs no unverified-GPU cap");
+ assert.equal(discrete.reason, "gpu-cap", "an Intel Mac's GPU memory is not system memory");
+ assert.ok(discrete.ceiling < unified.ceiling);
+});
diff --git a/test/locales/translationCoverage.test.js b/test/locales/translationCoverage.test.js
index dcd54be81..181b6d7f1 100644
--- a/test/locales/translationCoverage.test.js
+++ b/test/locales/translationCoverage.test.js
@@ -10,6 +10,9 @@ const LOCALES = path.join(SRC, "locales");
const NAMESPACES = ["translation", "prompts"];
const PLURAL_SUFFIX = /_(zero|one|two|few|many|other)$/;
const T_CALL = /\bt\(\s*(['"`])([A-Za-z0-9_.-]+)\1/g;
+// messageKey values are passed to t() as a variable, so the T_CALL scan
+// cannot see them and a typo would render as the raw key string.
+const MESSAGE_KEY = /\bmessageKey\s*[:=]\s*(['"`])([A-Za-z0-9_.-]+)\1/g;
const INTERPOLATION = /\{\{\s*([\w.]+)/g;
const ARABIC_PLURAL_CATEGORIES = ["zero", "one", "two", "few", "many", "other"];
const ARABIC_PLURAL_BASES = [
@@ -181,6 +184,23 @@ test("interpolation variables match en in every language", () => {
}
});
+test("every messageKey literal resolves in en", () => {
+ const keys = new Set();
+ for (const namespace of NAMESPACES) {
+ for (const key of flatten(load("en", namespace)).keys()) keys.add(key);
+ }
+
+ const missing = [];
+ for (const file of sourceFiles(SRC)) {
+ const source = fs.readFileSync(file, "utf8");
+ for (const [, , key] of source.matchAll(MESSAGE_KEY)) {
+ if (!keys.has(stripPlural(key))) missing.push(`${key} (${path.relative(SRC, file)})`);
+ }
+ }
+
+ assert.deepEqual(missing, [], `unresolved messageKey values:\n${missing.join("\n")}`);
+});
+
test("Arabic defines every CLDR plural category for each counted message", () => {
const ar = flatten(load("ar", "translation"));
for (const base of ARABIC_PLURAL_BASES) {
diff --git a/test/services/geminiProviderResponse.test.js b/test/services/geminiProviderResponse.test.js
index 0c768f564..c69f01a1f 100644
--- a/test/services/geminiProviderResponse.test.js
+++ b/test/services/geminiProviderResponse.test.js
@@ -88,3 +88,33 @@ test("Gemini provider preserves complete-output truncation errors before extract
/Model output was truncated before the selection edit completed/
);
});
+
+test("a caller's output budget cannot shrink the one Gemini would have computed", async (t) => {
+ // Note formatting pins maxTokens so the local path can price it against the
+ // context window. On Gemini that short-circuited a computed allowance of up
+ // to 8192 and halved long summaries — on the provider with the most room.
+ const originalFetch = globalThis.fetch;
+ t.after(() => {
+ globalThis.fetch = originalFetch;
+ });
+
+ let requestBody = null;
+ globalThis.fetch = async (_url, init) => {
+ requestBody = JSON.parse(init.body);
+ return new Response(JSON.stringify(createGeminiResponse([{ text: "ok" }])), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ });
+ };
+
+ const { geminiProvider } = await load();
+ await geminiProvider.call({
+ text: "a long meeting transcript",
+ model: "gemini-2.5-flash",
+ agentName: null,
+ config: { systemPrompt: "Write meeting notes.", maxTokens: 4096 },
+ ctx: { ...providerContext, calculateMaxTokens: () => 8192 },
+ });
+
+ assert.equal(requestBody.generationConfig.maxOutputTokens, 8192);
+});
diff --git a/test/stores/actionProcessingNoteOutput.test.js b/test/stores/actionProcessingNoteOutput.test.js
new file mode 100644
index 000000000..c6f73f081
--- /dev/null
+++ b/test/stores/actionProcessingNoteOutput.test.js
@@ -0,0 +1,98 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { createRendererServer, installBrowserGlobals } = require("../lib/rendererTestHarness");
+
+// Note formatting had no explicit output budget, so it inherited the generic
+// 2048-token default from calculateMaxTokens. Summaries of long meetings were
+// cut off at that ceiling and saved anyway, with no error and nothing in the
+// UI to say the notes were incomplete (#2142).
+
+const ACTION = { id: 1, name: "Generate Notes", prompt: "Summarize the meeting." };
+const LABELS = { noModel: "no model", noEndpoint: "no endpoint", actionFailed: "failed" };
+
+async function loadStore(t) {
+ const updates = [];
+ installBrowserGlobals(t, {
+ window: {
+ electronAPI: {
+ updateNote: async (noteId, payload) => {
+ updates.push({ noteId, payload });
+ return { success: true };
+ },
+ },
+ },
+ });
+
+ const calls = [];
+ const vite = await createRendererServer(t, {
+ cachePrefix: "openwhispr-action-output-test-",
+ mockModules: {
+ "/services/ReasoningService": `
+ export default {
+ processText: async (text, model, agentName, config) => {
+ globalThis.__processTextCalls.push({ text, model, config });
+ return "# Notes\\n- decided things";
+ },
+ };
+ `,
+ "/utils/generateTitle": `export const generateNoteTitle = async () => undefined;`,
+ },
+ });
+ globalThis.__processTextCalls = calls;
+ t.after(() => {
+ delete globalThis.__processTextCalls;
+ });
+
+ const store = await vite.ssrLoadModule("/stores/actionProcessingStore.ts");
+ return { store, calls, updates };
+}
+
+async function waitFor(predicate, label) {
+ for (let attempt = 0; attempt < 200; attempt += 1) {
+ if (predicate()) return;
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+ throw new Error(`timed out waiting for ${label}`);
+}
+
+test("note formatting asks for enough output tokens to hold a long meeting summary", async (t) => {
+ const { store, calls, updates } = await loadStore(t);
+
+ store.runBackgroundAction(
+ 7,
+ "## Meeting Transcript\n" + "Alice: we agreed to ship on Friday.\n".repeat(500),
+ "hash",
+ ACTION,
+ { modelId: "gpt-4.1", isCloudMode: true, isMeetingNote: true },
+ LABELS
+ );
+
+ await waitFor(() => updates.length > 0, "the note to be written");
+
+ assert.equal(calls.length, 1);
+ // 2048 is roughly 1,500 words — short of a structured summary of a long
+ // meeting, which is exactly the case this feature exists for.
+ assert.ok(
+ calls[0].config.maxTokens >= 4096,
+ `expected a real output budget, got ${calls[0].config.maxTokens}`
+ );
+});
+
+test("a truncated summary is still saved rather than discarded", async (t) => {
+ // Deliberate: unlike a selection edit, where a partial replacement would
+ // corrupt the user's own text, a clipped summary is still worth keeping.
+ // So note formatting must NOT set requireCompleteOutput.
+ const { store, calls, updates } = await loadStore(t);
+
+ store.runBackgroundAction(
+ 8,
+ "some notes",
+ "hash",
+ ACTION,
+ { modelId: "gpt-4.1", isCloudMode: true, isMeetingNote: true },
+ LABELS
+ );
+
+ await waitFor(() => updates.length > 0, "the note to be written");
+ assert.notEqual(calls[0].config.requireCompleteOutput, true);
+});
diff --git a/test/utils/localInferenceError.test.js b/test/utils/localInferenceError.test.js
new file mode 100644
index 000000000..15376cd50
--- /dev/null
+++ b/test/utils/localInferenceError.test.js
@@ -0,0 +1,88 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+let errors;
+
+test.before(async () => {
+ errors = await import("../../src/utils/localInferenceError.ts");
+});
+
+// The local path is the only one that can fail because the machine is too
+// small for the request. Before #2142 that surfaced as a raw llama.cpp JSON
+// body in a toast; this is where it becomes something a person can act on.
+
+test("a context overflow becomes a translatable message naming the model and the numbers", () => {
+ const error = errors.buildLocalInferenceError({
+ error: "This content needs about 21538 tokens of context, but Qwen3.5 9B can only use 16384.",
+ code: "CONTEXT_TOO_LARGE",
+ details: {
+ modelName: "Qwen3.5 9B",
+ neededTokens: 21538,
+ maxContextTokens: 16384,
+ },
+ });
+
+ assert.equal(error.code, "CONTEXT_TOO_LARGE");
+ assert.equal(error.messageKey, "models.errors.contextTooLarge");
+ assert.deepEqual(error.messageParams, { model: "Qwen3.5 9B", needed: 21538, max: 16384 });
+});
+
+test("a context overflow without numbers still gets a translatable message", () => {
+ // The reactive path: llama-server rejected the prompt and we only know which
+ // model it was.
+ const error = errors.buildLocalInferenceError({
+ error: "The request is longer than the context this model is running with.",
+ code: "CONTEXT_TOO_LARGE",
+ details: { modelName: "Gemma 3 4B" },
+ });
+
+ assert.equal(error.messageKey, "models.errors.contextTooLargeGeneric");
+ assert.deepEqual(error.messageParams, { model: "Gemma 3 4B" });
+});
+
+test("unrelated failures keep their message and gain no translation", () => {
+ const error = errors.buildLocalInferenceError({
+ error: "Model qwen3-8b is not downloaded or is corrupted",
+ code: "MODEL_NOT_DOWNLOADED",
+ });
+
+ assert.equal(error.message, "Model qwen3-8b is not downloaded or is corrupted");
+ assert.equal(error.messageKey, undefined);
+ assert.equal(error.code, "MODEL_NOT_DOWNLOADED");
+});
+
+test("a failure with no information at all still produces a usable Error", () => {
+ const error = errors.buildLocalInferenceError({});
+
+ assert.ok(error instanceof Error);
+ assert.ok(error.message.length > 0);
+});
+
+test("no llama.cpp JSON can reach the message", () => {
+ // The literal regression: a JSON body used to be concatenated into the
+ // message and shown to the customer.
+ const error = errors.buildLocalInferenceError({
+ error: 'llama-server returned status 400: {"error":{"code":400,"n_ctx":16384}}',
+ code: "CONTEXT_TOO_LARGE",
+ details: { modelName: "Llama 3.2 3B", neededTokens: 20000, maxContextTokens: 16384 },
+ });
+
+ // The raw text is kept for logs, but the UI renders the key, which has no
+ // JSON in it.
+ assert.equal(error.messageKey, "models.errors.contextTooLarge");
+});
+
+test("a server that will not start becomes a translatable message, not llama.cpp output", () => {
+ const error = errors.buildLocalInferenceError({
+ error: "Qwen3.5 9B could not be started on this computer.",
+ code: "LOCAL_SERVER_UNAVAILABLE",
+ details: {
+ modelName: "Qwen3.5 9B",
+ error: "llama-server process died during startup\nProcess output: ggml_metal: failed",
+ },
+ });
+
+ assert.equal(error.messageKey, "models.errors.localServerUnavailable");
+ assert.deepEqual(error.messageParams, { model: "Qwen3.5 9B" });
+ assert.ok(!error.message.includes("Process output"));
+});