Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(llama): size the local context to the request instead of capping it at 16384 (#2155)
Chad Piha4335972b96a6
diff --git a/src/components/notes/BackgroundActionToastListener.tsx b/src/components/notes/BackgroundActionToastListener.tsxindex f61ac34a6..c8bb090a1 100644--- a/src/components/notes/BackgroundActionToastListener.tsx+++ b/src/components/notes/BackgroundActionToastListener.tsx@@ -18,7 +18,7 @@ export default function BackgroundActionToastListener() {for (const event of consumeErrorEvents()) {toast({title: t("notes.enhance.title"),- description: event.message,+ description: event.messageKey ? t(event.messageKey, event.messageParams) : event.message,variant: "destructive",});}diff --git a/src/helpers/ggufMetadata.js b/src/helpers/ggufMetadata.jsnew file mode 100644index 000000000..b245b54d1--- /dev/null+++ b/src/helpers/ggufMetadata.js@@ -0,0 +1,282 @@+/**+ * Minimal GGUF metadata reader.+ *+ * Only the header is parsed — the magic, version, counts, and the metadata+ * key/value section. Tensor descriptors and tensor data are never touched, so+ * this reads a few megabytes off the front of a multi-gigabyte file.+ *+ * It exists to answer one question: how many bytes of KV cache does one token+ * of context cost for this model? That number varies ~12x across the bundled+ * registry and is anti-correlated with file size, so it cannot be guessed from+ * anything the model registry records (#2142). The GGUF is also the file+ * llama.cpp actually loads, which makes it the honest source for the trained+ * context length as well.+ *+ * Every failure returns null. Callers must read null as "fall back to the+ * historical context size", never as "assume this model is cheap".+ */+const fs = require("fs");++const GGUF_MAGIC = 0x46554747; // "GGUF" little-endian+const MIN_SUPPORTED_VERSION = 2; // v1 used 32-bit lengths; no bundled model uses it+const MAX_SUPPORTED_VERSION = 3;++const READ_CHUNK_BYTES = 1024 * 1024;+// A real header is ~10 MiB, dominated by the tokenizer token array. These are+// guards against a corrupt file steering us into an unbounded allocation.+const MAX_METADATA_BYTES = 64 * 1024 * 1024;+const MAX_KV_COUNT = 4096;+const MAX_STRING_BYTES = 64 * 1024 * 1024;+const MAX_CAPTURED_ARRAY_LENGTH = 1024;++const TYPE = {+ UINT8: 0,+ INT8: 1,+ UINT16: 2,+ INT16: 3,+ UINT32: 4,+ INT32: 5,+ FLOAT32: 6,+ BOOL: 7,+ STRING: 8,+ ARRAY: 9,+ UINT64: 10,+ INT64: 11,+ FLOAT64: 12,+};++const SCALAR_BYTES = {+ [TYPE.UINT8]: 1,+ [TYPE.INT8]: 1,+ [TYPE.UINT16]: 2,+ [TYPE.INT16]: 2,+ [TYPE.UINT32]: 4,+ [TYPE.INT32]: 4,+ [TYPE.FLOAT32]: 4,+ [TYPE.BOOL]: 1,+ [TYPE.UINT64]: 8,+ [TYPE.INT64]: 8,+ [TYPE.FLOAT64]: 8,+};++// Suffixes whose values decide KV cost. Matched against the architecture+// prefix the file declares, so this works for any arch without an allow-list.+const WANTED_SUFFIXES = [+ ".block_count",+ ".context_length",+ ".embedding_length",+ ".attention.head_count",+ ".attention.head_count_kv",+ ".attention.key_length",+ ".attention.value_length",+];++/** Forward-only cursor over the front of the file, buffered in chunks. */+class HeaderCursor {+ constructor(source) {+ this.source = source;+ this.chunk = Buffer.alloc(0);+ this.chunkStart = 0;+ this.position = 0;+ }++ async ensure(length) {+ if (this.position + length > MAX_METADATA_BYTES) {+ throw new Error("gguf metadata section is implausibly large");+ }+ const withinChunk =+ this.position >= this.chunkStart &&+ this.position + length <= this.chunkStart + this.chunk.length;+ if (withinChunk) return;++ const buffer = await this.source.read(this.position, Math.max(length, READ_CHUNK_BYTES));+ if (!buffer || buffer.length < length) throw new Error("unexpected end of gguf header");+ this.chunk = buffer;+ this.chunkStart = this.position;+ }++ async take(length) {+ await this.ensure(length);+ const start = this.position - this.chunkStart;+ this.position += length;+ return this.chunk.subarray(start, start + length);+ }++ /** Advances without reading, so huge arrays cost nothing. */+ skip(length) {+ if (length < 0) throw new Error("negative skip in gguf header");+ this.position += length;+ }++ async readUint32() {+ return (await this.take(4)).readUInt32LE(0);+ }++ async readUint64() {+ const value = (await this.take(8)).readBigUInt64LE(0);+ if (value > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error("gguf length out of range");+ return Number(value);+ }++ async readString() {+ const length = await this.readUint64();+ if (length > MAX_STRING_BYTES) throw new Error("gguf string is implausibly long");+ return (await this.take(length)).toString("utf8");+ }++ async skipString() {+ const length = await this.readUint64();+ if (length > MAX_STRING_BYTES) throw new Error("gguf string is implausibly long");+ this.skip(length);+ }++ async readScalar(type) {+ const buffer = await this.take(SCALAR_BYTES[type]);+ switch (type) {+ case TYPE.UINT8:+ return buffer.readUInt8(0);+ case TYPE.INT8:+ return buffer.readInt8(0);+ case TYPE.UINT16:+ return buffer.readUInt16LE(0);+ case TYPE.INT16:+ return buffer.readInt16LE(0);+ case TYPE.UINT32:+ return buffer.readUInt32LE(0);+ case TYPE.INT32:+ return buffer.readInt32LE(0);+ case TYPE.FLOAT32:+ return buffer.readFloatLE(0);+ case TYPE.BOOL:+ return buffer.readUInt8(0) !== 0;+ case TYPE.UINT64:+ return Number(buffer.readBigUInt64LE(0));+ case TYPE.INT64:+ return Number(buffer.readBigInt64LE(0));+ case TYPE.FLOAT64:+ return buffer.readDoubleLE(0);+ default:+ throw new Error(`unsupported gguf scalar type ${type}`);+ }+ }+}++async function readArray(cursor, capture) {+ const elementType = await cursor.readUint32();+ const length = await cursor.readUint64();++ if (elementType === TYPE.STRING) {+ // Each element carries its own length prefix, so it must be walked rather+ // than skipped wholesale. This is tokenizer.ggml.tokens, ~150k entries.+ for (let index = 0; index < length; index += 1) await cursor.skipString();+ return null;+ }++ const elementBytes = SCALAR_BYTES[elementType];+ if (elementBytes === undefined) throw new Error(`unsupported gguf array type ${elementType}`);++ if (!capture || length > MAX_CAPTURED_ARRAY_LENGTH) {+ cursor.skip(length * elementBytes);+ return null;+ }++ const values = [];+ for (let index = 0; index < length; index += 1) values.push(await cursor.readScalar(elementType));+ return values;+}++async function readValue(cursor, type, capture) {+ if (type === TYPE.STRING) {+ if (capture) return cursor.readString();+ await cursor.skipString();+ return null;+ }+ if (type === TYPE.ARRAY) return readArray(cursor, capture);++ const bytes = SCALAR_BYTES[type];+ if (bytes === undefined) throw new Error(`unsupported gguf value type ${type}`);+ if (capture) return cursor.readScalar(type);+ cursor.skip(bytes);+ return null;+}++function isWantedKey(key) {+ if (key === "general.architecture") return true;+ return WANTED_SUFFIXES.some((suffix) => key.endsWith(suffix));+}++function pick(entries, architecture, suffix) {+ return entries.get(`${architecture}${suffix}`);+}++/**+ * @param source {{ read: (offset: number, length: number) => Promise<Buffer>, size?: number }}+ * @returns the architecture fields needed to price the KV cache, or null.+ */+async function readGgufMetadata(source) {+ try {+ const cursor = new HeaderCursor(source);++ if ((await cursor.readUint32()) !== GGUF_MAGIC) return null;+ const version = await cursor.readUint32();+ if (version < MIN_SUPPORTED_VERSION || version > MAX_SUPPORTED_VERSION) return null;++ await cursor.readUint64(); // tensor_count, unused+ const kvCount = await cursor.readUint64();+ if (kvCount > MAX_KV_COUNT) return null;++ const entries = new Map();+ let hasRecurrentState = false;++ for (let index = 0; index < kvCount; index += 1) {+ const key = await cursor.readString();+ const type = await cursor.readUint32();+ if (key.includes(".ssm.")) hasRecurrentState = true;++ const value = await readValue(cursor, type, isWantedKey(key));+ if (value !== null) entries.set(key, value);+ }++ const architecture = entries.get("general.architecture");+ if (typeof architecture !== "string" || architecture.length === 0) return null;++ return {+ architecture,+ blockCount: pick(entries, architecture, ".block_count"),+ contextLength: pick(entries, architecture, ".context_length"),+ embeddingLength: pick(entries, architecture, ".embedding_length"),+ headCount: pick(entries, architecture, ".attention.head_count"),+ headCountKv: pick(entries, architecture, ".attention.head_count_kv"),+ keyLength: pick(entries, architecture, ".attention.key_length"),+ valueLength: pick(entries, architecture, ".attention.value_length"),+ hasRecurrentState,+ };+ } catch {+ return null;+ }+}++/** Convenience wrapper over a file on disk. */+async function readGgufMetadataFromFile(filePath) {+ let handle = null;+ try {+ handle = await fs.promises.open(filePath, "r");+ const { size } = await handle.stat();+ const file = handle;+ return await readGgufMetadata({+ size,+ read: async (offset, length) => {+ const buffer = Buffer.alloc(Math.min(length, Math.max(0, size - offset)));+ if (buffer.length === 0) return buffer;+ const { bytesRead } = await file.read(buffer, 0, buffer.length, offset);+ return buffer.subarray(0, bytesRead);+ },+ });+ } catch {+ return null;+ } finally {+ await handle?.close().catch(() => {});+ }+}++module.exports = { readGgufMetadata, readGgufMetadataFromFile };diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 2559d61c9..1cd585030 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -5186,7 +5186,15 @@ class IPCHandlers {const result = await LocalReasoningService.processText(text, modelId, config);return { success: true, text: result };} catch (error) {- return { success: false, error: error.message };+ // code/details carry the machine-readable failure across to the+ // renderer, which owns the translation keys. Flattening to a bare+ // string is how llama.cpp JSON used to reach users (#2142).+ return {+ success: false,+ error: error.message,+ code: error.code,+ details: error.details,+ };}});diff --git a/src/helpers/llamaContextPolicy.js b/src/helpers/llamaContextPolicy.jsnew file mode 100644index 000000000..931b9b833--- /dev/null+++ b/src/helpers/llamaContextPolicy.js@@ -0,0 +1,245 @@+/**+ * Pure policy for sizing the llama-server context window.+ *+ * Replaces the flat SERVER_CONTEXT_SIZE = 16384 cap (#2142), which blocked+ * every transcript over ~80 minutes on every bundled model. The cap itself was+ * not gratuitous: it fixed #1203, where an unbounded context allocated a ~15 GB+ * Metal-wired KV cache and froze a 16 GB Mac into a watchdog kernel panic. So+ * the replacement is not a bigger constant but a per-machine, per-model budget.+ *+ * Everything here is pure arithmetic so the memory guarantee can be tested+ * without a GPU, a model file, or a spawned process.+ */++// Latin text tokenizes at roughly 3.5-4.2 chars/token on the Qwen/Llama BPEs+// used here (measured: 4.16 on a synthetic English transcript). 3 leaves+// headroom, because under-estimating is the bug and over-estimating only costs+// KV bytes that the ceiling already bounds.+const LATIN_CHARS_PER_TOKEN = 3;++// CJK runs about 1-1.5 chars/token. Charging 1 keeps the estimate on the safe+// side for the ja / zh-CN / zh-TW locales the app ships.+const CJK_RANGES = [+ [0x1100, 0x11ff], // Hangul Jamo+ [0x3000, 0x303f], // CJK symbols and punctuation+ [0x3040, 0x30ff], // Hiragana + Katakana+ [0x3400, 0x4dbf], // CJK Unified Ideographs Extension A+ [0x4e00, 0x9fff], // CJK Unified Ideographs+ [0xac00, 0xd7af], // Hangul Syllables+ [0xf900, 0xfaff], // CJK Compatibility Ideographs+ [0xff00, 0xffef], // Halfwidth and Fullwidth Forms+ [0x20000, 0x3ffff], // CJK Unified Ideographs Extensions B and beyond+];++function isCjkCodePoint(codePoint) {+ for (const [start, end] of CJK_RANGES) {+ if (codePoint >= start && codePoint <= end) return true;+ }+ return false;+}++/**+ * Cheap, deliberately high estimate of how many tokens a string will occupy.+ *+ * Only used to choose the context size a server STARTS at. Once a server is+ * running, POST /tokenize returns the exact count in ~13 ms and supersedes+ * this; the estimate just avoids paying a restart in the common case.+ */+function estimateTokens(text) {+ if (typeof text !== "string" || text.length === 0) return 0;++ let cjkCount = 0;+ let otherCount = 0;+ for (const character of text) {+ if (isCjkCodePoint(character.codePointAt(0))) cjkCount += 1;+ else otherCount += 1;+ }++ return cjkCount + Math.ceil(otherCount / LATIN_CHARS_PER_TOKEN);+}++// K and V are cached as f16 by default (llama.cpp --cache-type-k/-v).+const KV_ELEMENT_BYTES = 2;++function resolveHeadCountKv(metadata) {+ const raw = metadata.headCountKv;+ // GGUF permits a per-layer array. The widest layer is the safe reading.+ if (Array.isArray(raw)) {+ const widest = raw.reduce((max, value) => (value > max ? value : max), 0);+ return widest > 0 ? widest : null;+ }+ return typeof raw === "number" && raw > 0 ? raw : null;+}++function resolveHeadWidths(metadata) {+ const { keyLength, valueLength, embeddingLength, headCount } = metadata;+ if (keyLength > 0 && valueLength > 0) return { keyLength, valueLength };++ // llama.cpp's own fallback when the widths are not declared.+ if (embeddingLength > 0 && headCount > 0) {+ const width = embeddingLength / headCount;+ if (Number.isInteger(width) && width > 0) return { keyLength: width, valueLength: width };+ }+ return null;+}++/**+ * Bytes of KV cache each token of context costs, from the model's own+ * architecture. Assumes every layer is full attention, which over-estimates+ * sliding-window models (Gemma 3/4) and models that interleave full attention+ * (Qwen3.5). Over-estimating shrinks the ceiling, so it errs safe.+ *+ * Returns null when the architecture cannot be read. Callers must treat null+ * as "fall back to the historical 16384", never as "assume it is cheap".+ */+function kvBytesPerToken(metadata) {+ if (!metadata || typeof metadata !== "object") return null;++ const blockCount = metadata.blockCount;+ if (!(blockCount > 0)) return null;++ const headCountKv = resolveHeadCountKv(metadata);+ if (headCountKv === null) return null;++ const widths = resolveHeadWidths(metadata);+ if (widths === null) return null;++ return blockCount * headCountKv * (widths.keyLength + widths.valueLength) * KV_ELEMENT_BYTES;+}++// The historical cap (#1236). Now the floor and the universal fallback rather+// than the ceiling, so no configuration that works today can get smaller.+const BASELINE_CONTEXT_SIZE = 16384;++// Half the machine, minus a fixed slice for the OS itself. Deliberately ~1.7x+// tighter than llama.cpp's own --fit, which would permit ~9.6 GiB of wired+// memory on the 16 GB Mac that panicked in #1203.+const USABLE_MEMORY_FRACTION = 0.5;+const FIXED_OS_RESERVE_BYTES = 2 * 1024 * 1024 * 1024;++// Compute buffers also grow with context: measured 144 MiB total at 16384 and+// 240 MiB at 65536 for a 9B, i.e. ~2 KiB/token plus a fixed base. Doubled for+// margin, and the base sized for the 31B-class graphs.+const COMPUTE_FIXED_BYTES = 384 * 1024 * 1024;+const COMPUTE_BYTES_PER_TOKEN = 4096;++// Hybrid models (Qwen3.5, LFM2) carry recurrent state that is not part of the+// KV cache and does not scale with context.+const RECURRENT_STATE_BYTES = 512 * 1024 * 1024;++// A discrete GPU's VRAM is not system RAM and we do not probe it. Until a real+// measurement exists, stay modest and let the existing GPU fallback ladder+// handle anything that still will not fit.+const UNVERIFIED_GPU_CEILING = 65536;++// Contexts are aligned down to a round number so that near-identical requests+// land on the same value and cannot cause a second restart.+const CONTEXT_ALIGNMENT = 512;++const MIB = 1024 * 1024;+const MIN_CACHE_RAM_MIB = 512;+const MAX_CACHE_RAM_MIB = 2048;++function clamp(value, low, high) {+ return Math.min(high, Math.max(low, value));+}++function fallbackCeiling(reason) {+ return { ceiling: BASELINE_CONTEXT_SIZE, reason, cacheRamMiB: MIN_CACHE_RAM_MIB };+}++/**+ * The largest context this machine may allocate for this model.+ *+ * Every path that cannot answer confidently returns BASELINE_CONTEXT_SIZE, so+ * the worst case of this whole mechanism is the behaviour that ships today.+ */+function resolveContextCeiling({+ totalMemoryBytes,+ weightsBytes = 0,+ draftWeightsBytes = 0,+ kvBytesPerToken,+ hasRecurrentState = false,+ trainedContextTokens = 0,+ discreteGpuUnverified = false,+}) {+ if (!Number.isFinite(totalMemoryBytes) || totalMemoryBytes <= 0) {+ return fallbackCeiling("no-memory-info");+ }+ if (!Number.isFinite(kvBytesPerToken) || kvBytesPerToken <= 0) {+ return fallbackCeiling("unknown-kv-cost");+ }++ const budgetBytes = totalMemoryBytes * USABLE_MEMORY_FRACTION - FIXED_OS_RESERVE_BYTES;+ const kvBudgetBytes =+ budgetBytes -+ weightsBytes -+ draftWeightsBytes -+ COMPUTE_FIXED_BYTES -+ (hasRecurrentState ? RECURRENT_STATE_BYTES : 0);++ if (kvBudgetBytes <= 0) return fallbackCeiling("floor");++ const cacheRamMiB = clamp(+ Math.floor(kvBudgetBytes / 4 / MIB),+ MIN_CACHE_RAM_MIB,+ MAX_CACHE_RAM_MIB+ );++ const perTokenBytes = kvBytesPerToken + COMPUTE_BYTES_PER_TOKEN;+ const affordable =+ Math.floor(Math.floor(kvBudgetBytes / perTokenBytes) / CONTEXT_ALIGNMENT) * CONTEXT_ALIGNMENT;++ let hardMax = trainedContextTokens > 0 ? trainedContextTokens : Number.POSITIVE_INFINITY;+ let hardMaxReason = "trained-context-bound";+ if (discreteGpuUnverified && UNVERIFIED_GPU_CEILING < hardMax) {+ hardMax = UNVERIFIED_GPU_CEILING;+ hardMaxReason = "gpu-cap";+ }++ const ceiling = Math.min(affordable, hardMax);+ if (ceiling <= BASELINE_CONTEXT_SIZE) return fallbackCeiling("floor");++ return {+ ceiling,+ reason: ceiling === hardMax ? hardMaxReason : "memory-bound",+ cacheRamMiB,+ };+}++// Quantizing the chosen context bounds how many distinct restarts a session can+// cause: at most one per rung per model, however many differently-sized+// requests arrive.+const CONTEXT_STEPS = [16384, 32768, 65536, 131072, 262144];++/**+ * The context size to start (or grow) the server at for a request needing+ * `needed` tokens, given a caller floor and the machine's ceiling.+ *+ * The ceiling always wins: it is the memory guarantee, and it is not+ * necessarily on a rung.+ */+function resolveContextSize({ needed = 0, floor = 0, ceiling = BASELINE_CONTEXT_SIZE }) {+ const target = Math.max(needed || 0, floor || 0, BASELINE_CONTEXT_SIZE);+ const rung = CONTEXT_STEPS.find((step) => step >= target) ?? Number.POSITIVE_INFINITY;+ const bounded = Math.min(rung, Math.max(ceiling, BASELINE_CONTEXT_SIZE));+ return Math.max(bounded, BASELINE_CONTEXT_SIZE);+}++module.exports = {+ LATIN_CHARS_PER_TOKEN,+ KV_ELEMENT_BYTES,+ estimateTokens,+ kvBytesPerToken,+ resolveContextCeiling,+ resolveContextSize,+ CONTEXT_STEPS,+ BASELINE_CONTEXT_SIZE,+ USABLE_MEMORY_FRACTION,+ FIXED_OS_RESERVE_BYTES,+ COMPUTE_FIXED_BYTES,+ COMPUTE_BYTES_PER_TOKEN,+ RECURRENT_STATE_BYTES,+ UNVERIFIED_GPU_CEILING,+ CONTEXT_ALIGNMENT,+};diff --git a/src/helpers/llamaServer.js b/src/helpers/llamaServer.jsindex c319cc23c..83fc519a5 100644--- a/src/helpers/llamaServer.js+++ b/src/helpers/llamaServer.js@@ -9,6 +9,7 @@ const { getSafeTempDir } = require("./safeTempDir");const { app } = require("electron");const sidecarPidFile = require("./sidecarPidFile");const { BIN_SUBDIR: LLAMA_VULKAN_BIN_SUBDIR } = require("./llamaVulkanManager");+const { BASELINE_CONTEXT_SIZE } = require("./llamaContextPolicy");// Range kept clear of cliBridge (8200-8219) to avoid port-bind collisions.const PORT_RANGE_START = 8221;@@ -21,6 +22,51 @@ const STARTUP_POLL_INTERVAL_MS = 500;const HEALTH_CHECK_FAILURE_THRESHOLD = 3;const IDLE_TIMEOUT_MS = 5 * 60 * 1000;const DEFAULT_CONTEXT_SIZE = 4096;+const PREFLIGHT_TIMEOUT_MS = 10000;++/**+ * llama-server reports a prompt that overflows the context as a 400 whose body+ * is a JSON blob. Surfacing that verbatim is how customers ended up reading+ * llama.cpp internals in a toast (#2142), so it becomes a typed error carrying+ * the two numbers callers actually need.+ */+/** Same argv with --ctx-size rewritten; used by the context step-down rung. */+function withContextSize(args, contextSize) {+ const index = args.indexOf("--ctx-size");+ if (index === -1) return args;+ const copy = [...args];+ copy[index + 1] = String(contextSize);+ return copy;+}++function buildResponseError(statusCode, body) {+ if (statusCode === 400) {+ try {+ const parsed = JSON.parse(body)?.error;+ const isOverflow =+ parsed?.type === "exceed_context_size_error" ||+ /exceeds the available context size/i.test(parsed?.message || "");++ if (isOverflow) {+ const neededTokens = Number(parsed.n_prompt_tokens) || null;+ const maxContextTokens = Number(parsed.n_ctx) || null;+ const error = new Error(+ neededTokens && maxContextTokens+ ? `The request needs ${neededTokens} tokens of context but the model is running with ${maxContextTokens}.`+ : "The request is longer than the context this model is running with."+ );+ error.code = "CONTEXT_TOO_LARGE";+ error.neededTokens = neededTokens;+ error.maxContextTokens = maxContextTokens;+ return error;+ }+ } catch {+ // Not JSON, or not the shape we expect: fall through to the generic error.+ }+ }++ return new Error(`llama-server returned status ${statusCode}: ${body}`);+}class LlamaServerManager {constructor() {@@ -32,6 +78,13 @@ class LlamaServerManager {// the start() restart check); activeDraftModelPath is the one that actually loaded.this.draftModelPath = null;this.activeDraftModelPath = null;+ // The context that actually loaded, which the GPU ladder may step down+ // below the requested one.+ this.activeContextSize = null;+ // The context the running server was started with. Grows on demand and is+ // only reset by stop(), so a short request cannot shrink a window a long+ // one just paid to open. See #2142.+ this.contextSize = null;this.startupPromise = null;this.healthCheckInterval = null;this.healthCheckFailures = 0;@@ -124,7 +177,13 @@ class LlamaServerManager {// A change in drafter presence for the same model must still restart the// server so the new speculative-decoding flags take effect.const requestedDraftPath = options.draftModelPath || null;- if (this.ready && this.modelPath === modelPath && this.draftModelPath === requestedDraftPath)+ const requestedContextSize = options.contextSize || DEFAULT_CONTEXT_SIZE;+ if (+ this.ready &&+ this.modelPath === modelPath &&+ this.draftModelPath === requestedDraftPath &&+ requestedContextSize <= (this.contextSize || 0)+ )return;if (this.process) {@@ -134,39 +193,55 @@ class LlamaServerManager {this.startupPromise = this._doStart(modelPath, options);try {await this.startupPromise;+ // The ladder may have stepped the context down to get a GPU rung to+ // start; record what loaded, not what was asked for.+ this.contextSize = this.activeContextSize ?? requestedContextSize;} finally {this.startupPromise = null;}}- async _doStart(modelPath, options = {}) {- const binaryPaths = this.getServerBinaryPaths();- if (Object.keys(binaryPaths).length === 0) throw new Error("llama-server binary not found");- if (!fs.existsSync(modelPath)) throw new Error(`Model file not found: ${modelPath}`);-- this.port = await this.findAvailablePort();- this.modelPath = modelPath;- // Store the REQUESTED drafter so start() compares against a stable value across- // identical requests; activeDraftModelPath tracks what actually loaded (see ctor).- this.draftModelPath = options.draftModelPath || null;- this.activeDraftModelPath = null;-- const baseArgs = [+ _buildBaseArgs(modelPath, port, options = {}) {+ const args = ["--model",modelPath,"--host","127.0.0.1","--port",- String(this.port),+ String(port),"--threads",String(options.threads || 4),// Unset, this defaults to the model's full trained context (128K+),// whose KV cache can exceed total RAM with --fit disabled. See #1203.+ // Sized per request against a memory budget by llamaContextPolicy."--ctx-size",String(options.contextSize || DEFAULT_CONTEXT_SIZE),"--jinja",];+ // llama-server otherwise reserves 8192 MiB of host RAM for the prompt+ // cache, on top of the KV cache, which would undo the memory budget. Cap+ // it rather than disabling it: 0 forces a full re-prefill every request.+ if (options.cacheRamMiB) args.push("--cache-ram", String(options.cacheRamMiB));++ return args;+ }++ async _doStart(modelPath, options = {}) {+ const binaryPaths = this.getServerBinaryPaths();+ if (Object.keys(binaryPaths).length === 0) throw new Error("llama-server binary not found");+ if (!fs.existsSync(modelPath)) throw new Error(`Model file not found: ${modelPath}`);++ this.port = await this.findAvailablePort();+ this.modelPath = modelPath;+ // Store the REQUESTED drafter so start() compares against a stable value across+ // identical requests; activeDraftModelPath tracks what actually loaded (see ctor).+ this.draftModelPath = options.draftModelPath || null;+ this.activeDraftModelPath = null;++ this.activeContextSize = options.contextSize || DEFAULT_CONTEXT_SIZE;+ const baseArgs = this._buildBaseArgs(modelPath, this.port, options);+// Draft flags stay separate from baseArgs so the fallback ladder can retry without// them when a stale (pre-b9763) binary rejects the MTP args at parse time.const draftArgs = options.draftModelPath@@ -209,6 +284,7 @@ class LlamaServerManager {const gpuArgs = [...baseArgs, "--n-gpu-layers", String(options.gpuLayers ?? 99)];const cpuArgs = baseArgs;const hasDraft = draftArgs.length > 0;+ const requestedContext = options.contextSize || DEFAULT_CONTEXT_SIZE;// Degrade ladder: GPU+MTP, then GPU alone (a live GPU beats speculation), then// CPU+MTP (the bundled pin normally accepts the flags), then plain CPU. The@@ -234,6 +310,25 @@ class LlamaServerManager {timeout: VULKAN_STARTUP_TIMEOUT_MS,attemptMsg: "Attempting Vulkan backend startup",},+ // A context sized against system RAM can still be too big for a+ // discrete GPU's own memory, which the app does not probe. Halving it is+ // far cheaper than the 10-30x cost of dropping to CPU, so try that+ // first. Omitted entirely when there is nothing to step down to, so a+ // baseline start keeps today's exact ladder.+ ...(requestedContext > BASELINE_CONTEXT_SIZE && binaryPaths.vulkan+ ? [+ {+ backend: "vulkan",+ name: "Vulkan (reduced context)",+ binary: binaryPaths.vulkan,+ args: withContextSize(gpuArgs, BASELINE_CONTEXT_SIZE),+ mtp: false,+ contextSize: BASELINE_CONTEXT_SIZE,+ timeout: VULKAN_STARTUP_TIMEOUT_MS,+ attemptMsg: "Attempting Vulkan backend startup with a reduced context",+ },+ ]+ : []),{backend: "cpu",name: "CPU",@@ -272,6 +367,10 @@ class LlamaServerManager {);this.activeBackend = rung.backend;this.activeDraftModelPath = rung.mtp ? this.draftModelPath : null;+ // Without this the preflight would believe the full context is+ // available and send a prompt the server cannot take.+ this.activeContextSize = rung.contextSize ?? requestedContext;+ this.contextSize = this.activeContextSize;return;} catch (err) {lastError = err;@@ -523,6 +622,90 @@ class LlamaServerManager {}}+ /**+ * Small JSON round-trip against the running server.+ *+ * Resolves null on any failure rather than rejecting: these calls exist to+ * make a decision more precise, so a broken one must degrade to the estimate+ * and never fail the user's request.+ */+ _requestJson(path, body = null, timeoutMs = PREFLIGHT_TIMEOUT_MS) {+ return new Promise((resolve) => {+ const payload = body === null ? null : JSON.stringify(body);+ const req = http.request(+ {+ hostname: "127.0.0.1",+ port: this.port,+ path,+ method: payload === null ? "GET" : "POST",+ headers: payload+ ? {+ "Content-Type": "application/json",+ "Content-Length": Buffer.byteLength(payload),+ }+ : {},+ timeout: timeoutMs,+ },+ (res) => {+ let data = "";+ res.on("data", (chunk) => {+ data += chunk;+ });+ res.on("end", () => {+ if (res.statusCode !== 200) return resolve(null);+ try {+ resolve(JSON.parse(data));+ } catch {+ resolve(null);+ }+ });+ }+ );++ req.on("error", () => resolve(null));+ req.on("timeout", () => {+ req.destroy();+ resolve(null);+ });+ if (payload) req.write(payload);+ req.end();+ });+ }++ /**+ * The context window actually in force, which is not always what we asked+ * for: a concurrent caller can join an in-flight start that used a smaller+ * size. Returns null when it cannot be read.+ */+ async usableContextSize() {+ if (!this.ready || !this.process) return null;+ const props = await this._requestJson("/props");+ const contextSize = props?.default_generation_settings?.n_ctx;+ return Number.isFinite(contextSize) && contextSize > 0 ? contextSize : null;+ }++ /**+ * Exact token count for the prompt as the server will actually see it, chat+ * template included. Roughly 15 ms for an 85 KB prompt, which is cheap+ * enough to make the difference between a right-sized window and a 400.+ */+ async countPromptTokens(messages, options = {}) {+ if (!this.ready || !this.process) return null;++ const payload = { messages };+ // Must mirror inference() exactly, or we would price a different prompt+ // than the one we go on to send.+ if (options.disableThinking !== false) {+ payload.chat_template_kwargs = { enable_thinking: false };+ }++ const templated = await this._requestJson("/apply-template", payload);+ if (typeof templated?.prompt !== "string") return null;++ const tokenized = await this._requestJson("/tokenize", { content: templated.prompt });+ return Array.isArray(tokenized?.tokens) ? tokenized.tokens.length : null;+ }+async inference(messages, options = {}) {if (!this.ready || !this.process) {throw new Error("llama-server is not running");@@ -572,7 +755,7 @@ class LlamaServerManager {});if (res.statusCode !== 200) {- reject(new Error(`llama-server returned status ${res.statusCode}: ${data}`));+ reject(buildResponseError(res.statusCode, data));return;}@@ -614,6 +797,7 @@ class LlamaServerManager {if (!this.process) {this.ready = false;+ this.contextSize = null;return;}@@ -651,6 +835,8 @@ class LlamaServerManager {this.draftModelPath = null;this.activeDraftModelPath = null;this.activeBackend = null;+ this.contextSize = null;+ this.activeContextSize = null;}getStatus() {diff --git a/src/helpers/modelManagerBridge.js b/src/helpers/modelManagerBridge.jsindex c6eed6a4d..d80d4ebab 100644--- a/src/helpers/modelManagerBridge.js+++ b/src/helpers/modelManagerBridge.js@@ -12,13 +12,42 @@ const {const modelRegistryData = require("../models/modelRegistryData.json");const LlamaServerManager = require("./llamaServer");const debugLogger = require("./debugLogger");+const { readGgufMetadataFromFile } = require("./ggufMetadata");+const {+ estimateTokens,+ kvBytesPerToken,+ resolveContextCeiling,+ resolveContextSize,+ BASELINE_CONTEXT_SIZE,+} = require("./llamaContextPolicy");const MIN_FILE_SIZE = 1_000_000; // 1MB minimum for valid model files-// Bounds the KV cache — registry contextLength is the full trained context-// (128K+), which can exceed total RAM (#1203). Uniform for all start paths:-// start() won't restart a ready server when only options change.-const SERVER_CONTEXT_SIZE = 16384;+// The context every server still STARTS at. Requests that need more grow it+// on demand, bounded by a per-machine, per-model ceiling (llamaContextPolicy).+// Keeping the starting value at the historical constant means prewarm, chat+// and every short request behave exactly as they did before #2142.+const SERVER_CONTEXT_SIZE = BASELINE_CONTEXT_SIZE;++// Slack on top of the output reservation, for tokenizer drift and llama.cpp's+// own bookkeeping.+const CONTEXT_RESERVE_TOKENS = 512;++// Context a request must leave free for its own answer. Derived in one place+// so the size the window is chosen for and the size it is checked against can+// never drift apart.+const reserveFor = (maxTokens) => maxTokens + CONTEXT_RESERVE_TOKENS;++// Only measure the prompt exactly when the estimate lands within this factor+// of the running window. Below that the answer cannot change, and dictation+// should not pay two HTTP round trips to learn nothing.+const PREFLIGHT_MEASURE_FACTOR = 2;++// The smallest answer still worth generating when the prompt has crowded the+// window. Matches the historical floor of calculateMaxTokens, so a request+// that gets trimmed to this is no smaller than the smallest one the app has+// ever made. Below it, refusing beats handing back a sentence and a half.+const MIN_OUTPUT_TOKENS = 512;function getLocalProviders() {return modelRegistryData.localProviders || [];@@ -205,13 +234,66 @@ class ModelManager {};}- async serverStartOptions(modelInfo) {- const options = this.serverOptions(modelInfo);+ async serverStartOptions(modelInfo, overrides = {}) {+ const options = { ...this.serverOptions(modelInfo), ...overrides };const draftPath = await this.resolveDraftPath(modelInfo.model);if (draftPath) options.draftModelPath = draftPath;return options;}+ // Seam so the ceiling can be exercised for any machine size in tests.+ _systemMemoryBytes() {+ return require("os").totalmem();+ }++ /**+ * The largest context this machine may give this model, and the prompt-cache+ * bound that goes with it. Memoized per model file: it reads a few MiB off+ * the front of the GGUF, so it must not sit on the hot path.+ */+ async contextCeiling(modelInfo, modelPath) {+ if (this._contextCeilingCache?.modelPath === modelPath) {+ return this._contextCeilingCache.value;+ }++ const metadata = await readGgufMetadataFromFile(modelPath);+ const sizeOf = async (filePath) => {+ if (!filePath) return 0;+ try {+ return (await fsPromises.stat(filePath)).size;+ } catch {+ return 0;+ }+ };++ const value = resolveContextCeiling({+ totalMemoryBytes: this._systemMemoryBytes(),+ weightsBytes: await sizeOf(modelPath),+ draftWeightsBytes: await sizeOf(await this.resolveDraftPath(modelInfo.model)),+ kvBytesPerToken: kvBytesPerToken(metadata),+ hasRecurrentState: metadata?.hasRecurrentState,+ // The GGUF is the file llama.cpp actually loads, so it outranks the+ // registry, which is wrong for at least the qwen2.5-7b entries.+ trainedContextTokens: metadata?.contextLength || modelInfo.model.contextLength || 0,+ // Anywhere but Apple Silicon the GPU may have its own memory that we do+ // not probe, so the system-RAM budget alone is not a safe bound. Stay+ // modest and let the GPU fallback ladder handle the rest. Intel Macs+ // count: a discrete Radeon has a few GB of its own, and the darwin start+ // path has no reduced-context rung to catch an over-sized window.+ discreteGpuUnverified: process.platform !== "darwin" || process.arch !== "arm64",+ });++ debugLogger.info("Resolved llama-server context ceiling", {+ model: modelInfo.model.id,+ architecture: metadata?.architecture ?? null,+ kvBytesPerToken: kvBytesPerToken(metadata),+ ...value,+ });++ this._contextCeilingCache = { modelPath, value };+ return value;+ }+getReservedDownloadBytes() {let reserved = 0;for (const [modelId, bytes] of this.downloadReservations) {@@ -450,6 +532,106 @@ class ModelManager {}}+ /**+ * Make sure the prompt actually fits the running window, growing it once if+ * it does not, trading output room for the prompt if it still does not, and+ * refusing only when no usable answer is left.+ *+ * The estimate above only picks a starting size. This is where the decision+ * becomes exact, because a wrong guess here is the bug: llama-server answers+ * an overlong prompt with a 400 whose JSON body used to reach the user.+ *+ * Returns the output allowance the request should actually be sent with.+ */+ async _ensurePromptFits({+ modelInfo,+ messages,+ options,+ estimatedTokens,+ maxTokens,+ contextFloor,+ ceilingFor,+ startServer,+ }) {+ const reserveTokens = reserveFor(maxTokens);+ const running = this.serverManager.contextSize || contextFloor;++ // Comfortably inside the window: measuring cannot change the outcome, and+ // every dictation would otherwise pay for two HTTP round trips.+ if ((estimatedTokens + reserveTokens) * PREFLIGHT_MEASURE_FACTOR <= running) return maxTokens;++ const usable = (await this.serverManager.usableContextSize()) ?? running;+ const exactTokens = await this.serverManager.countPromptTokens(messages, {+ disableThinking: options.disableThinking,+ });++ // A measurement we could not take must never fail the request; the+ // estimate already sized the window, and an overflow would still surface+ // as a typed error from llama-server itself.+ if (exactTokens === null) {+ debugLogger.warn("Could not measure prompt tokens; proceeding on the estimate", {+ model: modelInfo.model.id,+ estimatedTokens,+ });+ return maxTokens;+ }++ const neededTokens = exactTokens + reserveTokens;+ if (neededTokens <= usable) return maxTokens;++ const { ceiling: maxContext } = await ceilingFor();+ const grown = resolveContextSize({+ needed: neededTokens,+ floor: contextFloor,+ ceiling: maxContext,+ });++ if (grown > usable) await startServer(grown);++ const nowUsable =+ (await this.serverManager.usableContextSize()) ?? this.serverManager.contextSize ?? grown;+ if (neededTokens <= nowUsable) return maxTokens;++ // The window is as big as this machine allows and the prompt still does not+ // leave room for the answer we wanted. But llama-server rejects only a+ // prompt that fills the window on its own (`n_tokens >= n_ctx`) —+ // overrunning during generation just ends the reply with finish_reason+ // "length" — so a shorter answer is still a real answer, and refusing one+ // the user could have had is the regression this preflight would otherwise+ // introduce (#2142).+ //+ // No `- CONTEXT_RESERVE_TOKENS` here: that slack absorbs drift in the+ // *estimate* that sized the window, while this is an exact count of the+ // exact prompt. The MIN_OUTPUT_TOKENS floor below already keeps the prompt+ // clear of the edge by the same margin, so charging it twice would only+ // shorten the answer.+ const affordableOutput = nowUsable - exactTokens;+ if (!options.requireCompleteOutput && affordableOutput >= MIN_OUTPUT_TOKENS) {+ debugLogger.info("Trimming the output allowance to fit the context window", {+ model: modelInfo.model.id,+ requested: maxTokens,+ granted: affordableOutput,+ promptTokens: exactTokens,+ contextSize: nowUsable,+ });+ return affordableOutput;+ }++ // Either nothing usable is left, or the caller cannot accept a partial+ // answer at all (a selection edit replaces the user's own text, so half of+ // one is worse than none).+ throw new ModelError(+ `This content needs about ${neededTokens} tokens of context, but ${modelInfo.model.name} can only use ${nowUsable} on this computer.`,+ "CONTEXT_TOO_LARGE",+ {+ modelId: modelInfo.model.id,+ modelName: modelInfo.model.name,+ neededTokens,+ maxContextTokens: nowUsable,+ }+ );+ }+async runInference(modelId, prompt, options = {}) {this.ensureInitialized();const startTime = Date.now();@@ -490,28 +672,147 @@ class ModelManager {);}- // Start/restart server if needed or if model changed- if (!this.serverManager.ready || this.currentServerModelId !== modelId) {+ // Build messages for chat completion+ const messages = [+ { role: "system", content: options.systemPrompt || "" },+ { role: "user", content: prompt },+ ];++ const maxTokens = options.maxTokens ?? 512;+ // Whatever the prompt needs, plus room for the reply. The output reserve+ // is part of the context budget, not on top of it.+ const reserveTokens = reserveFor(maxTokens);+ const estimatedTokens = estimateTokens(options.systemPrompt || "") + estimateTokens(prompt);+ // A caller may ask for a larger floor (selection editing does); it can+ // raise the floor but never breach the ceiling.+ const contextFloor = Math.max(SERVER_CONTEXT_SIZE, options.contextSize || 0);++ let ceiling = null;+ const ceilingFor = async () => {+ if (ceiling === null) ceiling = await this.contextCeiling(modelInfo, modelPath);+ return ceiling;+ };++ let targetContextSize = contextFloor;+ // A raised floor has to clear the ceiling too, or a caller asking for a+ // window the machine cannot afford reopens #1203. The baseline floor needs+ // no check, which is what keeps the GGUF read off the short-request path.+ if (contextFloor > SERVER_CONTEXT_SIZE || estimatedTokens + reserveTokens > contextFloor) {+ const { ceiling: maxContext } = await ceilingFor();+ targetContextSize = resolveContextSize({+ needed: estimatedTokens + reserveTokens,+ floor: contextFloor,+ ceiling: maxContext,+ });+ }++ // The smallest window this machine has already refused for this model,+ // whether by failing to start or by quietly loading something smaller.+ // Retrying it would cost a second model load to learn the same thing.+ let smallestRefusedContextSize = Infinity;+ const refuse = (size) => {+ smallestRefusedContextSize = Math.min(smallestRefusedContextSize, size);+ };++ const startServer = async (contextSize) => {+ if (this.serverManager.ready && contextSize >= smallestRefusedContextSize) return;++ const { cacheRamMiB } = ceiling ?? {};+ // Only a live server already serving THIS model has a window worth+ // coming back to. contextSize outlives a crash — llamaServer clears it+ // in stop() alone, not when a process dies or a health check trips — and+ // another model's window says nothing about this one.+ const restorableContextSize =+ this.serverManager.ready &&+ this.serverManager.process &&+ this.currentServerModelId === modelId+ ? this.serverManager.contextSize+ : null;+ const startAt = async (size) =>+ this.serverManager.start(+ modelPath,+ await this.serverStartOptions(modelInfo, { contextSize: size, cacheRamMiB })+ );+ // llama.cpp's startup dump is the diagnostic, so it goes to the log here+ // and nowhere else: details crosses IPC to the renderer, which neither+ // reads it nor should carry a stderr blob and absolute model paths.+ const unavailable = (cause) => {+ debugLogger.error("llama-server could not be started", {+ model: modelId,+ contextSize,+ error: cause.message,+ });+ return new ModelError(+ `${modelInfo.model.name} could not be started on this computer.`,+ "LOCAL_SERVER_UNAVAILABLE",+ { modelId, modelName: modelInfo.model.name }+ );+ };++ try {+ await startAt(contextSize);+ // The GPU ladder can load a smaller window than it was asked for+ // without ever throwing. Recording that is what stops the preflight+ // asking for the same window again and paying a second model load to+ // be stepped down again.+ if (this.serverManager.contextSize < contextSize) refuse(contextSize);+ } catch (error) {+ refuse(contextSize);+ // start() stops the running server before it spawns, so a grow that+ // dies has already taken a working window with it. Come back at the+ // one that worked rather than leaving the user with no local model.+ if (!restorableContextSize || restorableContextSize >= contextSize) {+ throw unavailable(error);+ }++ debugLogger.warn("llama-server failed to grow; restoring the previous context", {+ model: modelId,+ attempted: contextSize,+ restoring: restorableContextSize,+ error: error.message,+ });+ try {+ await startAt(restorableContextSize);+ } catch (restoreError) {+ throw unavailable(restoreError);+ }+ }+ this.currentServerModelId = modelId;+ };++ // Start/restart the server if needed, if the model changed, or if this+ // request needs a bigger window than the running one has.+ if (+ !this.serverManager.ready ||+ this.currentServerModelId !== modelId ||+ targetContextSize > (this.serverManager.contextSize || SERVER_CONTEXT_SIZE)+ ) {debugLogger.logReasoning("INFERENCE_STARTING_SERVER", {currentModel: this.currentServerModelId,requestedModel: modelId,serverReady: this.serverManager.ready,+ contextSize: targetContextSize,});- await this.serverManager.start(modelPath, await this.serverStartOptions(modelInfo));- this.currentServerModelId = modelId;+ await startServer(targetContextSize);debugLogger.logReasoning("INFERENCE_SERVER_STARTED", {port: this.serverManager.port,model: modelId,+ contextSize: this.serverManager.contextSize,});}- // Build messages for chat completion- const messages = [- { role: "system", content: options.systemPrompt || "" },- { role: "user", content: prompt },- ];+ const grantedMaxTokens = await this._ensurePromptFits({+ modelInfo,+ messages,+ options,+ estimatedTokens,+ maxTokens,+ contextFloor,+ ceilingFor,+ startServer,+ });debugLogger.logReasoning("INFERENCE_SENDING_REQUEST", {messageCount: messages.length,@@ -522,7 +823,9 @@ class ModelManager {try {const result = await this.serverManager.inference(messages, {temperature: options.temperature ?? 0.7,- max_tokens: options.maxTokens ?? 512,+ // The preflight may have traded output room for prompt room; sending+ // the original value would put the request back over the window.+ max_tokens: grantedMaxTokens,disableThinking: options.disableThinking,requireCompleteOutput: options.requireCompleteOutput,});@@ -541,6 +844,16 @@ class ModelManager {totalTimeMs: totalTime,error: error.message,});+ // A typed failure (a context overflow, say) must keep its identity, or+ // the renderer cannot translate it and the user sees raw server text.+ if (error.code === "CONTEXT_TOO_LARGE") {+ throw new ModelError(error.message, "CONTEXT_TOO_LARGE", {+ modelId,+ modelName: modelInfo.model.name,+ neededTokens: error.neededTokens ?? null,+ maxContextTokens: error.maxContextTokens ?? null,+ });+ }throw new ModelError(`Inference failed: ${error.message}`, "INFERENCE_FAILED", {error: error.message,});diff --git a/src/locales/ar/translation.json b/src/locales/ar/translation.jsonindex 62a3146d9..106c319dd 100644--- a/src/locales/ar/translation.json+++ b/src/locales/ar/translation.json@@ -330,6 +330,11 @@"liquidai_lfm2_5_350m_q8_0": "نموذج خفيف للغاية للتنظيف السريع للإملاء","liquidai_lfm2_5_230m_q8_0": "نموذج صغير جدًا، تنظيف فوري على الأجهزة المتواضعة"}+ },+ "errors": {+ "contextTooLarge": "هذا أطول من أن يقرأه {{model}} على هذا الكمبيوتر. عدد الرموز المطلوبة من السياق: نحو {{needed}}، لكن ما يتسع في الذاكرة: {{max}} فقط. جرّب نموذجًا سحابيًا، أو نفّذ الإجراء على تحديد أقصر.",+ "contextTooLargeGeneric": "هذا أطول من أن يقرأه {{model}} على هذا الكمبيوتر. جرّب نموذجًا سحابيًا، أو نفّذ الإجراء على تحديد أقصر.",+ "localServerUnavailable": "تعذّر تشغيل {{model}} على هذا الكمبيوتر. حاول مرة أخرى، أو بدّل إلى نموذج سحابي."}},"reasoning": {diff --git a/src/locales/de/translation.json b/src/locales/de/translation.jsonindex f748faf8c..ce4830220 100644--- a/src/locales/de/translation.json+++ b/src/locales/de/translation.json@@ -2741,6 +2741,11 @@"liquidai_lfm2_5_350m_q8_0": "Ultraleichtes Modell für schnelle Diktatbereinigung","liquidai_lfm2_5_230m_q8_0": "Winziges Modell, sofortige Bereinigung auf bescheidener Hardware"}+ },+ "errors": {+ "contextTooLarge": "Das ist zu lang, als dass {{model}} es auf diesem Computer lesen könnte. Es werden etwa {{needed}} Token Kontext benötigt, aber nur {{max}} passen in den Speicher. Versuchen Sie ein Cloud-Modell oder führen Sie die Aktion für eine kürzere Auswahl aus.",+ "contextTooLargeGeneric": "Das ist zu lang, als dass {{model}} es auf diesem Computer lesen könnte. Versuchen Sie ein Cloud-Modell oder führen Sie die Aktion für eine kürzere Auswahl aus.",+ "localServerUnavailable": "{{model}} konnte auf diesem Computer nicht gestartet werden. Versuchen Sie es erneut oder wechseln Sie zu einem Cloud-Modell."}},"insights": {diff --git a/src/locales/en/translation.json b/src/locales/en/translation.jsonindex 684e05c2c..b0208adf2 100644--- a/src/locales/en/translation.json+++ b/src/locales/en/translation.json@@ -2741,6 +2741,11 @@"liquidai_lfm2_5_350m_q8_0": "Ultra-light model for fast dictation cleanup","liquidai_lfm2_5_230m_q8_0": "Tiny model, instant cleanup on modest hardware"}+ },+ "errors": {+ "contextTooLarge": "This is too long for {{model}} to read on this computer. It needs about {{needed}} tokens of context, but only {{max}} fit in memory. Try a cloud model, or run the action on a shorter selection.",+ "contextTooLargeGeneric": "This is too long for {{model}} to read on this computer. Try a cloud model, or run the action on a shorter selection.",+ "localServerUnavailable": "{{model}} couldn't be started on this computer. Try again, or switch to a cloud model."}},"sidebar": {diff --git a/src/locales/es/translation.json b/src/locales/es/translation.jsonindex 3867ff59d..052ac522d 100644--- a/src/locales/es/translation.json+++ b/src/locales/es/translation.json@@ -2701,6 +2701,11 @@"segmentation": "Detección de límites del habla para la identificación de hablantes","embedding_en": "Huella vocal para la identificación del hablante"}+ },+ "errors": {+ "contextTooLarge": "Esto es demasiado largo para que {{model}} lo lea en este equipo. Necesita unos {{needed}} tokens de contexto, pero solo caben {{max}} en la memoria. Prueba con un modelo en la nube o ejecuta la acción sobre una selección más corta.",+ "contextTooLargeGeneric": "Esto es demasiado largo para que {{model}} lo lea en este equipo. Prueba con un modelo en la nube o ejecuta la acción sobre una selección más corta.",+ "localServerUnavailable": "No se pudo iniciar {{model}} en este ordenador. Inténtalo de nuevo o cambia a un modelo en la nube."}},"referral": {diff --git a/src/locales/fr/translation.json b/src/locales/fr/translation.jsonindex 78c3af08f..492b11c4d 100644--- a/src/locales/fr/translation.json+++ b/src/locales/fr/translation.json@@ -2741,6 +2741,11 @@"liquidai_lfm2_5_350m_q8_0": "Modèle ultraléger pour un nettoyage rapide de la dictée","liquidai_lfm2_5_230m_q8_0": "Modèle minuscule, nettoyage instantané sur du matériel modeste"}+ },+ "errors": {+ "contextTooLarge": "C’est trop long pour que {{model}} puisse le lire sur cet ordinateur. Il faut environ {{needed}} jetons de contexte, mais seuls {{max}} tiennent en mémoire. Essayez un modèle cloud ou exécutez l’action sur une sélection plus courte.",+ "contextTooLargeGeneric": "C’est trop long pour que {{model}} puisse le lire sur cet ordinateur. Essayez un modèle cloud ou exécutez l’action sur une sélection plus courte.",+ "localServerUnavailable": "Impossible de démarrer {{model}} sur cet ordinateur. Réessayez ou passez à un modèle cloud."}},"insights": {diff --git a/src/locales/it/translation.json b/src/locales/it/translation.jsonindex 1524d2e90..3765c88bc 100644--- a/src/locales/it/translation.json+++ b/src/locales/it/translation.json@@ -2648,6 +2648,11 @@"segmentation": "Rilevamento dei confini del parlato per l'identificazione dei parlanti","embedding_en": "Impronta vocale per l'identificazione dei parlanti"}+ },+ "errors": {+ "contextTooLarge": "È troppo lungo perché {{model}} possa leggerlo su questo computer. Servono circa {{needed}} token di contesto, ma in memoria ne entrano solo {{max}}. Prova un modello cloud oppure esegui l’azione su una selezione più breve.",+ "contextTooLargeGeneric": "È troppo lungo perché {{model}} possa leggerlo su questo computer. Prova un modello cloud oppure esegui l’azione su una selezione più breve.",+ "localServerUnavailable": "Impossibile avviare {{model}} su questo computer. Riprova o passa a un modello cloud."}},"referral": {diff --git a/src/locales/ja/translation.json b/src/locales/ja/translation.jsonindex 7d14c0dfd..a185584de 100644--- a/src/locales/ja/translation.json+++ b/src/locales/ja/translation.json@@ -2648,6 +2648,11 @@"segmentation": "話者分離のための発話区間検出","embedding_en": "話者識別のための声紋認識"}+ },+ "errors": {+ "contextTooLarge": "このコンピューターでは長すぎて {{model}} が読み取れません。約 {{needed}} トークンのコンテキストが必要ですが、メモリに収まるのは {{max}} までです。クラウドモデルを使うか、選択範囲を短くして実行してください。",+ "contextTooLargeGeneric": "このコンピューターでは長すぎて {{model}} が読み取れません。クラウドモデルを使うか、選択範囲を短くして実行してください。",+ "localServerUnavailable": "このコンピューターでは {{model}} を起動できませんでした。もう一度試すか、クラウドモデルに切り替えてください。"}},"notes": {diff --git a/src/locales/pt/translation.json b/src/locales/pt/translation.jsonindex 2f1c675d6..8ec33df24 100644--- a/src/locales/pt/translation.json+++ b/src/locales/pt/translation.json@@ -2627,6 +2627,11 @@"segmentation": "Detecção de limites de fala para identificação de falantes","embedding_en": "Impressão vocal para identificação de falantes"}+ },+ "errors": {+ "contextTooLarge": "Isto é longo demais para o {{model}} ler neste computador. São necessários cerca de {{needed}} tokens de contexto, mas apenas {{max}} cabem na memória. Experimente um modelo na nuvem ou execute a ação sobre uma seleção mais curta.",+ "contextTooLargeGeneric": "Isto é longo demais para o {{model}} ler neste computador. Experimente um modelo na nuvem ou execute a ação sobre uma seleção mais curta.",+ "localServerUnavailable": "Não foi possível iniciar o {{model}} neste computador. Tente novamente ou mude para um modelo na nuvem."}},"upgradePrompt": {diff --git a/src/locales/ru/translation.json b/src/locales/ru/translation.jsonindex a1439a7f7..5da457e04 100644--- a/src/locales/ru/translation.json+++ b/src/locales/ru/translation.json@@ -2664,6 +2664,11 @@"segmentation": "Обнаружение границ речи для идентификации говорящих","embedding_en": "Голосовой отпечаток для идентификации говорящих"}+ },+ "errors": {+ "contextTooLarge": "Это слишком длинно, чтобы {{model}} могла прочитать это на данном компьютере. Требуется около {{needed}} токенов контекста, но в память помещается только {{max}}. Попробуйте облачную модель или выполните действие для более короткого фрагмента.",+ "contextTooLargeGeneric": "Это слишком длинно, чтобы {{model}} могла прочитать это на данном компьютере. Попробуйте облачную модель или выполните действие для более короткого фрагмента.",+ "localServerUnavailable": "Не удалось запустить {{model}} на этом компьютере. Попробуйте снова или переключитесь на облачную модель."}},"referral": {diff --git a/src/locales/zh-CN/translation.json b/src/locales/zh-CN/translation.jsonindex a1155aa10..0490d7f2b 100644--- a/src/locales/zh-CN/translation.json+++ b/src/locales/zh-CN/translation.json@@ -2648,6 +2648,11 @@"segmentation": "用于说话人分离的语音边界检测","embedding_en": "用于说话人识别的声纹提取"}+ },+ "errors": {+ "contextTooLarge": "内容过长,{{model}} 无法在这台电脑上读取。大约需要 {{needed}} 个上下文 token,但内存中只能容纳 {{max}} 个。请改用云端模型,或对更短的选区执行该操作。",+ "contextTooLargeGeneric": "内容过长,{{model}} 无法在这台电脑上读取。请改用云端模型,或对更短的选区执行该操作。",+ "localServerUnavailable": "无法在这台电脑上启动 {{model}}。请重试,或改用云端模型。"}},"referral": {diff --git a/src/locales/zh-TW/translation.json b/src/locales/zh-TW/translation.jsonindex 07b096efb..4033732d8 100644--- a/src/locales/zh-TW/translation.json+++ b/src/locales/zh-TW/translation.json@@ -2648,6 +2648,11 @@"segmentation": "用於說話者分離的語音邊界偵測","embedding_en": "用於說話者辨識的聲紋提取"}+ },+ "errors": {+ "contextTooLarge": "內容過長,{{model}} 無法在這台電腦上讀取。大約需要 {{needed}} 個上下文 token,但記憶體只能容納 {{max}} 個。請改用雲端模型,或對更短的選取範圍執行該動作。",+ "contextTooLargeGeneric": "內容過長,{{model}} 無法在這台電腦上讀取。請改用雲端模型,或對更短的選取範圍執行該動作。",+ "localServerUnavailable": "無法在這台電腦上啟動 {{model}}。請重試,或改用雲端模型。"}},"referral": {diff --git a/src/services/ai/inferenceProviders/gemini.ts b/src/services/ai/inferenceProviders/gemini.tsindex 7bc4556b8..4ea83b6a1 100644--- a/src/services/ai/inferenceProviders/gemini.ts+++ b/src/services/ai/inferenceProviders/gemini.ts@@ -38,17 +38,20 @@ export const geminiProvider: InferenceProvider = {const generationConfig: GeminiGenerationConfig = {temperature: config.temperature ?? (config.systemPrompt ? 0.3 : 0),- maxOutputTokens:- config.maxTokens ||- Math.max(- 2000,- ctx.calculateMaxTokens(- text.length,- TOKEN_LIMITS.MIN_TOKENS_GEMINI,- TOKEN_LIMITS.MAX_TOKENS_GEMINI,- TOKEN_LIMITS.TOKEN_MULTIPLIER- )- ),+ // A caller's budget raises this ceiling, never lowers it: note formatting+ // pins maxTokens so the local path can price it against the context+ // window, and letting that short-circuit Gemini's own allowance halved+ // long summaries on the provider with the most room to give (#2142).+ maxOutputTokens: Math.max(+ config.maxTokens || 0,+ 2000,+ ctx.calculateMaxTokens(+ text.length,+ TOKEN_LIMITS.MIN_TOKENS_GEMINI,+ TOKEN_LIMITS.MAX_TOKENS_GEMINI,+ TOKEN_LIMITS.TOKEN_MULTIPLIER+ )+ ),};if (config.disableThinking === true && getCloudModel(model)?.supportsThinking) {diff --git a/src/services/ai/inferenceProviders/local.ts b/src/services/ai/inferenceProviders/local.tsindex b42549204..49a0184ec 100644--- a/src/services/ai/inferenceProviders/local.ts+++ b/src/services/ai/inferenceProviders/local.ts@@ -1,6 +1,7 @@import type { InferenceProvider } from "./types";import { wrapCleanupTranscript } from "../../../config/prompts";import logger from "../../../utils/logger";+import { buildLocalInferenceError } from "../../../utils/localInferenceError";export const localProvider: InferenceProvider = {id: "local",@@ -25,7 +26,7 @@ export const localProvider: InferenceProvider = {if (!result.success) {logger.logReasoning("LOCAL_ERROR", { model, processingTimeMs, error: result.error });- throw new Error(result.error);+ throw buildLocalInferenceError(result);}logger.logReasoning("LOCAL_SUCCESS", {diff --git a/src/services/localReasoningBridge.js b/src/services/localReasoningBridge.jsindex b60935c5b..d3e930b11 100644--- a/src/services/localReasoningBridge.js+++ b/src/services/localReasoningBridge.js@@ -40,6 +40,10 @@ class LocalReasoningService {systemPrompt: config.systemPrompt || "",disableThinking: config.disableThinking !== false,requireCompleteOutput: config.requireCompleteOutput,+ // A minimum context window for this request. Rebuilding this object+ // field-by-field is what silently orphaned it before #2142: it was+ // declared, written by selection editing, and never forwarded.+ contextSize: config.contextSize,};debugLogger.logReasoning("LOCAL_BRIDGE_INFERENCE", {diff --git a/src/stores/actionProcessingStore.ts b/src/stores/actionProcessingStore.tsindex 472152cbd..b347ecaa2 100644--- a/src/stores/actionProcessingStore.ts+++ b/src/stores/actionProcessingStore.ts@@ -8,6 +8,20 @@ import { buildNoteFormattingOverrides } from "../helpers/noteFormattingOverridesimport { tagActionItemOwners, type MentionPerson } from "../utils/mentionMarkdown";import type { ActionItem } from "../types/electron";+/**+ * Output budget for a formatted note.+ *+ * Without an explicit value this inherited the generic 2048-token default from+ * calculateMaxTokens — roughly 1,500 words — so summaries of long meetings were+ * cut off and saved anyway, with nothing to say they were incomplete (#2142).+ *+ * Deliberately not paired with requireCompleteOutput: unlike a selection edit,+ * where a partial replacement corrupts the user's own text, a clipped summary+ * is still worth keeping. The context preflight counts this reservation, so+ * asking for more output room can grow the window rather than squeeze it.+ */+export const NOTE_OUTPUT_MAX_TOKENS = 4096;+export type ActionProcessingStatus = "idle" | "processing" | "success";export interface NoteActionState {@@ -18,6 +32,9 @@ export interface NoteActionState {export interface ActionErrorEvent {noteId: number;message: string;+ /** Set when the failure has a translatable form; the toast prefers it. */+ messageKey?: string;+ messageParams?: Record<string, string | number>;}interface ActionProcessingStoreState {@@ -164,6 +181,7 @@ export function runBackgroundAction();const enhanced = await reasoningService.processText(noteContent, modelId, null, {systemPrompt,+ maxTokens: NOTE_OUTPUT_MAX_TOKENS,temperature: 0.3,disableThinking: settings.noteFormattingDisableThinking,...providerOverrides,@@ -202,7 +220,11 @@ export function runBackgroundAction(processingFlags.set(noteId, false);clearNoteState(noteId);const message = err instanceof Error ? err.message : labels.actionFailed;- pushErrorEvent({ noteId, message });+ const { messageKey, messageParams } = (err ?? {}) as {+ messageKey?: string;+ messageParams?: Record<string, string | number>;+ };+ pushErrorEvent({ noteId, message, messageKey, messageParams });} finally {cancelledFlags.delete(noteId);}diff --git a/src/types/electron.ts b/src/types/electron.tsindex eddb072f5..65e992c36 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -1841,7 +1841,13 @@ declare global {modelId: string,agentName: string | null,config: any- ) => Promise<{ success: boolean; text?: string; error?: string }>;+ ) => Promise<{+ success: boolean;+ text?: string;+ error?: string;+ code?: string;+ details?: Record<string, unknown>;+ }>;checkLocalReasoningAvailable: () => Promise<boolean>;// Anthropic reasoningdiff --git a/src/utils/localInferenceError.ts b/src/utils/localInferenceError.tsnew file mode 100644index 000000000..05ff11437--- /dev/null+++ b/src/utils/localInferenceError.ts@@ -0,0 +1,61 @@+/**+ * Turns a failed local-inference IPC result into an Error the UI can render.+ *+ * The main process reports a machine-readable `code` and `details`; the+ * translation keys live here, on the renderer side, so both surfaces that show+ * these failures — the notes toast and the dictation cleanup toast — get the+ * same wording for free.+ *+ * Before #2142 the raw llama.cpp 400 body was shown to the user verbatim.+ */++export interface LocalInferenceFailure {+ error?: string;+ code?: string;+ details?: Record<string, unknown> | null;+}++export interface LocalInferenceError extends Error {+ code?: string;+ messageKey?: string;+ messageParams?: Record<string, string | number>;+}++const asPositiveInteger = (value: unknown): number | null =>+ typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.round(value) : null;++function contextTooLarge(+ details: Record<string, unknown>+): Pick<LocalInferenceError, "messageKey" | "messageParams"> {+ const model = typeof details.modelName === "string" ? details.modelName : "";+ const needed = asPositiveInteger(details.neededTokens);+ const max = asPositiveInteger(details.maxContextTokens);++ if (needed === null || max === null) {+ return {+ messageKey: "models.errors.contextTooLargeGeneric",+ messageParams: { model },+ };+ }+ return {+ messageKey: "models.errors.contextTooLarge",+ messageParams: { model, needed, max },+ };+}++export function buildLocalInferenceError(result: LocalInferenceFailure): LocalInferenceError {+ const error: LocalInferenceError = new Error(result.error || "Local inference failed");+ if (result.code) error.code = result.code;++ const details = result.details ?? {};+ if (result.code === "CONTEXT_TOO_LARGE") {+ Object.assign(error, contextTooLarge(details));+ } else if (result.code === "LOCAL_SERVER_UNAVAILABLE") {+ // The llama.cpp startup dump stays in the main-process log; only the model+ // name crosses, so there is nothing here that could reach the user raw.+ error.messageKey = "models.errors.localServerUnavailable";+ error.messageParams = { model: typeof details.modelName === "string" ? details.modelName : "" };+ }++ return error;+}diff --git a/test/helpers/gemmaMtpDrafter.test.js b/test/helpers/gemmaMtpDrafter.test.jsindex f6e10b232..8790e4840 100644--- a/test/helpers/gemmaMtpDrafter.test.js+++ b/test/helpers/gemmaMtpDrafter.test.js@@ -123,6 +123,9 @@ test("deleteModel removes the drafter alongside the main file, ignoring a missin});});+// Context size is the third restart trigger; it is covered in+// test/helpers/llamaServerContext.test.js. Every start() here passes the same+// (default) context, so this file isolates the model/drafter dimension.test("llama-server start restarts only when model or drafter presence changes", async () => {const LlamaServerManager = require("../../src/helpers/llamaServer.js");const manager = new LlamaServerManager();diff --git a/test/helpers/ggufMetadata.test.js b/test/helpers/ggufMetadata.test.jsnew file mode 100644index 000000000..2adfe71ed--- /dev/null+++ b/test/helpers/ggufMetadata.test.js@@ -0,0 +1,158 @@+const test = require("node:test");+const assert = require("node:assert/strict");++const { readGgufMetadata } = require("../../src/helpers/ggufMetadata");+const { TYPE, buildGguf, bufferReader, LLAMA_3_2_3B_ENTRIES } = require("./harness/ggufFixtures");++const read = (entries, options) => readGgufMetadata(bufferReader(buildGguf(entries, options)));++test("reads the architecture fields that determine KV cost", async () => {+ const metadata = await read(LLAMA_3_2_3B_ENTRIES);++ assert.equal(metadata.architecture, "llama");+ assert.equal(metadata.blockCount, 28);+ assert.equal(metadata.headCountKv, 8);+ assert.equal(metadata.keyLength, 128);+ assert.equal(metadata.valueLength, 128);+ assert.equal(metadata.embeddingLength, 3072);+ assert.equal(metadata.headCount, 24);+ assert.equal(metadata.contextLength, 131072);+});++test("prefers the model's own context length over anything declared elsewhere", async () => {+ // The registry says 128000 for qwen2.5-7b, which is not what the GGUF says.+ // The GGUF is the file llama.cpp actually loads, so it wins.+ const metadata = await read([+ { key: "general.architecture", type: TYPE.STRING, value: "qwen2" },+ { key: "qwen2.block_count", type: TYPE.UINT32, value: 28 },+ { key: "qwen2.context_length", type: TYPE.UINT32, value: 131072 },+ { key: "qwen2.attention.head_count_kv", type: TYPE.UINT32, value: 4 },+ { key: "qwen2.attention.key_length", type: TYPE.UINT32, value: 128 },+ { key: "qwen2.attention.value_length", type: TYPE.UINT32, value: 128 },+ ]);++ assert.equal(metadata.contextLength, 131072);+});++test("reads a per-layer head_count_kv array", async () => {+ const metadata = await read([+ { key: "general.architecture", type: TYPE.STRING, value: "gemma3" },+ { key: "gemma3.block_count", type: TYPE.UINT32, value: 4 },+ {+ key: "gemma3.attention.head_count_kv",+ type: TYPE.ARRAY,+ elementType: TYPE.UINT32,+ values: [2, 8, 2, 2],+ },+ { key: "gemma3.attention.key_length", type: TYPE.UINT32, value: 256 },+ { key: "gemma3.attention.value_length", type: TYPE.UINT32, value: 256 },+ ]);++ assert.deepEqual(metadata.headCountKv, [2, 8, 2, 2]);+});++test("flags recurrent state, which costs memory the KV formula does not capture", async () => {+ const metadata = await read([+ { key: "general.architecture", type: TYPE.STRING, value: "qwen35" },+ { key: "qwen35.block_count", type: TYPE.UINT32, value: 33 },+ { key: "qwen35.attention.head_count_kv", type: TYPE.UINT32, value: 4 },+ { key: "qwen35.attention.key_length", type: TYPE.UINT32, value: 256 },+ { key: "qwen35.attention.value_length", type: TYPE.UINT32, value: 256 },+ { key: "qwen35.ssm.state_size", type: TYPE.UINT32, value: 128 },+ ]);++ assert.equal(metadata.hasRecurrentState, true);+});++test("walks past every scalar type without losing its place", async () => {+ // A real GGUF interleaves dozens of unrelated keys of every type before the+ // ones we want. Mis-sizing any of them desynchronises the whole parse.+ const metadata = await read([+ { key: "general.architecture", type: TYPE.STRING, value: "llama" },+ { key: "noise.u8", type: TYPE.UINT8, value: 7 },+ { key: "noise.i8", type: TYPE.INT8, value: 7 },+ { key: "noise.u16", type: TYPE.UINT16, value: 7 },+ { key: "noise.i16", type: TYPE.INT16, value: 7 },+ { key: "noise.u32", type: TYPE.UINT32, value: 7 },+ { key: "noise.i32", type: TYPE.INT32, value: 7 },+ { key: "noise.f32", type: TYPE.FLOAT32, value: 7.5 },+ { key: "noise.bool", type: TYPE.BOOL, value: true },+ { key: "noise.string", type: TYPE.STRING, value: "a longer value" },+ { key: "noise.u64", type: TYPE.UINT64, value: 7 },+ { key: "noise.i64", type: TYPE.INT64, value: 7 },+ { key: "noise.f64", type: TYPE.FLOAT64, value: 7.5 },+ ...LLAMA_3_2_3B_ENTRIES.slice(1),+ ]);++ assert.equal(metadata.blockCount, 28);+ assert.equal(metadata.headCountKv, 8);+});++test("skips the tokenizer arrays that dominate a real header", async () => {+ // tokenizer.ggml.tokens is a string array with 150k+ entries and accounts for+ // nearly all of a real 10 MiB metadata section. It must be stepped over, not+ // materialised.+ const tokens = Array.from({ length: 5000 }, (_, index) => `token_${index}`);+ const metadata = await read([+ { key: "general.architecture", type: TYPE.STRING, value: "llama" },+ { key: "tokenizer.ggml.tokens", type: TYPE.ARRAY, elementType: TYPE.STRING, values: tokens },+ {+ key: "tokenizer.ggml.scores",+ type: TYPE.ARRAY,+ elementType: TYPE.FLOAT32,+ values: tokens.map(() => 0),+ },+ ...LLAMA_3_2_3B_ENTRIES.slice(1),+ ]);++ assert.equal(metadata.blockCount, 28);+ assert.equal(metadata.headCountKv, 8);+});++test("returns null instead of throwing on anything malformed", async () => {+ // A null must route the caller to the historical 16384. Throwing here would+ // fail the user's dictation instead.+ const good = buildGguf(LLAMA_3_2_3B_ENTRIES);++ const cases = {+ "bad magic": bufferReader(buildGguf(LLAMA_3_2_3B_ENTRIES, { magic: 0xdeadbeef })),+ "version 1": bufferReader(buildGguf(LLAMA_3_2_3B_ENTRIES, { version: 1 })),+ "truncated mid-header": bufferReader(good.subarray(0, 60)),+ "empty file": bufferReader(Buffer.alloc(0)),+ "header only": bufferReader(good.subarray(0, 8)),+ "implausible kv count": bufferReader(buildGguf(LLAMA_3_2_3B_ENTRIES, { kvCount: 99999 })),+ "unknown value type": bufferReader(+ buildGguf([{ key: "general.architecture", type: TYPE.STRING, value: "llama" }]).fill(+ 0x7f,+ // clobber the first value's type tag+ 4 + 4 + 8 + 8 + 8 + "general.architecture".length,+ 4 + 4 + 8 + 8 + 8 + "general.architecture".length + 4+ )+ ),+ };++ for (const [label, reader] of Object.entries(cases)) {+ const metadata = await readGgufMetadata(reader);+ assert.equal(metadata, null, `${label} should read as null`);+ }+});++test("returns null when a read fails outright", async () => {+ const metadata = await readGgufMetadata({+ size: 1024,+ read: async () => {+ throw new Error("EIO");+ },+ });++ assert.equal(metadata, null);+});++test("returns null when the architecture keys are simply absent", async () => {+ const metadata = await read([+ { key: "general.name", type: TYPE.STRING, value: "some model" },+ { key: "general.file_type", type: TYPE.UINT32, value: 15 },+ ]);++ assert.equal(metadata, null);+});diff --git a/test/helpers/harness/ggufFixtures.js b/test/helpers/harness/ggufFixtures.jsnew file mode 100644index 000000000..a6097fbaa--- /dev/null+++ b/test/helpers/harness/ggufFixtures.js@@ -0,0 +1,118 @@+/**+ * Builds GGUF headers in memory so the metadata reader can be tested without+ * downloading a multi-gigabyte model. Only the header is produced: the reader+ * never looks at tensor data.+ */+const GGUF_MAGIC = 0x46554747; // "GGUF" little-endian++const TYPE = {+ UINT8: 0,+ INT8: 1,+ UINT16: 2,+ INT16: 3,+ UINT32: 4,+ INT32: 5,+ FLOAT32: 6,+ BOOL: 7,+ STRING: 8,+ ARRAY: 9,+ UINT64: 10,+ INT64: 11,+ FLOAT64: 12,+};++const u32 = (value) => {+ const b = Buffer.alloc(4);+ b.writeUInt32LE(value);+ return b;+};++const u64 = (value) => {+ const b = Buffer.alloc(8);+ b.writeBigUInt64LE(BigInt(value));+ return b;+};++const ggufString = (value) => {+ const bytes = Buffer.from(value, "utf8");+ return Buffer.concat([u64(bytes.length), bytes]);+};++function encodeScalar(type, value) {+ switch (type) {+ case TYPE.UINT8:+ case TYPE.INT8:+ return Buffer.from([value & 0xff]);+ case TYPE.UINT16:+ case TYPE.INT16: {+ const b = Buffer.alloc(2);+ b.writeUInt16LE(value);+ return b;+ }+ case TYPE.UINT32:+ case TYPE.INT32:+ return u32(value);+ case TYPE.FLOAT32: {+ const b = Buffer.alloc(4);+ b.writeFloatLE(value);+ return b;+ }+ case TYPE.BOOL:+ return Buffer.from([value ? 1 : 0]);+ case TYPE.STRING:+ return ggufString(value);+ case TYPE.UINT64:+ case TYPE.INT64:+ return u64(value);+ case TYPE.FLOAT64: {+ const b = Buffer.alloc(8);+ b.writeDoubleLE(value);+ return b;+ }+ default:+ throw new Error(`unsupported scalar type ${type}`);+ }+}++function encodeValue(entry) {+ if (entry.type === TYPE.ARRAY) {+ const elements = entry.values.map((value) => encodeScalar(entry.elementType, value));+ return Buffer.concat([u32(entry.elementType), u64(entry.values.length), ...elements]);+ }+ return encodeScalar(entry.type, entry.value);+}++/**+ * @param entries {Array<{key: string, type: number, value?: any,+ * elementType?: number, values?: any[]}>}+ */+function buildGguf(entries, { version = 3, magic = GGUF_MAGIC, kvCount = null } = {}) {+ const parts = [u32(magic), u32(version), u64(0) /* tensor_count */];+ parts.push(u64(kvCount === null ? entries.length : kvCount));+ for (const entry of entries) {+ parts.push(ggufString(entry.key), u32(entry.type), encodeValue(entry));+ }+ return Buffer.concat(parts);+}++/** A reader over an in-memory buffer, matching the shape readGgufMetadata expects. */+function bufferReader(buffer) {+ return {+ size: buffer.length,+ read: async (offset, length) => buffer.subarray(offset, offset + length),+ };+}++// The #1203 model, as its GGUF declares itself.+const LLAMA_3_2_3B_ENTRIES = [+ { key: "general.architecture", type: TYPE.STRING, value: "llama" },+ { key: "llama.block_count", type: TYPE.UINT32, value: 28 },+ { key: "llama.context_length", type: TYPE.UINT32, value: 131072 },+ { key: "llama.embedding_length", type: TYPE.UINT32, value: 3072 },+ { key: "llama.attention.head_count", type: TYPE.UINT32, value: 24 },+ { key: "llama.attention.head_count_kv", type: TYPE.UINT32, value: 8 },+ { key: "llama.attention.key_length", type: TYPE.UINT32, value: 128 },+ { key: "llama.attention.value_length", type: TYPE.UINT32, value: 128 },+];++module.exports = { TYPE, buildGguf, bufferReader, LLAMA_3_2_3B_ENTRIES };diff --git a/test/helpers/llamaContextPolicy.test.js b/test/helpers/llamaContextPolicy.test.jsnew file mode 100644index 000000000..c2e2688b4--- /dev/null+++ b/test/helpers/llamaContextPolicy.test.js@@ -0,0 +1,350 @@+const test = require("node:test");+const assert = require("node:assert/strict");++const policy = require("../../src/helpers/llamaContextPolicy");++// --- estimateTokens ------------------------------------------------------+//+// Used before any llama-server exists, to pick the context size the server+// starts at. Once a server is up, POST /tokenize gives an exact count in ~13ms+// and supersedes this. The estimate must therefore be cheap and biased HIGH:+// over-estimating costs KV bytes (bounded by the ceiling), under-estimating+// costs a restart, and under-estimating badly is the bug being fixed (#2142).++test("estimateTokens counts Latin text at three characters per token", () => {+ assert.equal(policy.estimateTokens("abcdef"), 2);+ assert.equal(policy.estimateTokens("abcd"), 2); // rounds up, never down+ assert.equal(policy.estimateTokens(""), 0);+});++test("estimateTokens over-estimates a real transcript rather than under-estimating it", () => {+ // Measured against the bundled llama-server (b9763) POST /tokenize with+ // Qwen3.5-9B: an 85,171-character synthetic meeting transcript tokenized to+ // 20,473 tokens (4.16 chars/token). A chars/4 constant would sit right on+ // that line and under-estimate real transcripts; chars/3 leaves headroom.+ const transcript = "a".repeat(85171);+ const estimate = policy.estimateTokens(transcript);++ assert.ok(estimate > 20473, `expected the estimate to exceed the real count, got ${estimate}`);+ assert.ok(estimate < 20473 * 2, `expected the estimate to stay within 2x, got ${estimate}`);+});++test("estimateTokens counts CJK at one token per codepoint", () => {+ // ja, zh-CN and zh-TW all ship. CJK tokenizes at roughly 1-1.5 chars/token,+ // so a global chars/3 constant under-counts a Japanese transcript ~3x and+ // would silently reintroduce #2142 for those users.+ assert.equal(policy.estimateTokens("会議の議事録"), 6);+ assert.equal(policy.estimateTokens("ひらがな"), 4);+ assert.equal(policy.estimateTokens("한국어"), 3);+});++test("estimateTokens charges CJK strictly more than Latin for the same length", () => {+ const latin = policy.estimateTokens("x".repeat(300));+ const cjk = policy.estimateTokens("会".repeat(300));++ assert.ok(cjk > latin, `expected CJK (${cjk}) to cost more than Latin (${latin})`);+});++test("estimateTokens sums mixed scripts instead of picking one rate", () => {+ assert.equal(policy.estimateTokens("会議abcdef"), 2 + 2);+});++test("estimateTokens treats absent or non-string input as zero", () => {+ assert.equal(policy.estimateTokens(null), 0);+ assert.equal(policy.estimateTokens(undefined), 0);+ assert.equal(policy.estimateTokens(12345), 0);+});++// --- kvBytesPerToken -----------------------------------------------------+//+// The whole point of the fix. KV cost is exactly linear in context size, but+// varies ~12x across the bundled registry and is ANTI-correlated with model+// file size, so no ceiling based on RAM tiers, file size or parameter count is+// sound. It has to come from the model's own architecture.++test("kvBytesPerToken reproduces the measured cost of the #1203 model", () => {+ // Llama-3.2-3B-Instruct-Q4_K_M: 28 blocks, 8 KV heads, 128-wide K and V.+ // 28 * 8 * (128+128) * 2 bytes (f16) = 114688 = 112 KiB/token, which is what+ // made its full 131072 context a ~15 GB KV cache on a 16 GB Mac.+ const bytes = policy.kvBytesPerToken({+ blockCount: 28,+ headCountKv: 8,+ keyLength: 128,+ valueLength: 128,+ });++ assert.equal(bytes, 114688);+ assert.equal(bytes * 131072, 15032385536); // ~15.0 GB, the #1203 allocation+});++test("kvBytesPerToken is anti-correlated with file size across the registry", () => {+ // Qwen3.5-9B is a 5.9 GB file; Llama-3.2-3B is a 2.0 GB file. The smaller+ // model costs far more per token. This is the fact that rules out every+ // size-based or RAM-tier heuristic.+ const qwen9b = policy.kvBytesPerToken({+ blockCount: 33,+ headCountKv: 4,+ keyLength: 256,+ valueLength: 256,+ });+ const llama3b = policy.kvBytesPerToken({+ blockCount: 28,+ headCountKv: 8,+ keyLength: 128,+ valueLength: 128,+ });++ assert.equal(qwen9b, 135168);+ assert.ok(llama3b < qwen9b, "the 3B is cheaper per token than the 9B by this bound");+ // The bound is deliberately conservative for Qwen3.5: it interleaves full+ // attention every 4th layer, so the measured cost is 32768 B/token (512 MiB+ // at 16384 ctx). Over-estimating shrinks the ceiling, which is the safe way+ // to be wrong.+ assert.ok(qwen9b > 32768, "full-attention bound must not undercut the measured cost");+});++test("kvBytesPerToken takes the largest layer when head_count_kv varies per layer", () => {+ // GGUF allows a per-layer array. The max is the safe reading.+ const bytes = policy.kvBytesPerToken({+ blockCount: 4,+ headCountKv: [2, 8, 2, 2],+ keyLength: 128,+ valueLength: 128,+ });++ assert.equal(bytes, 4 * 8 * 256 * 2);+});++test("kvBytesPerToken derives head width from the embedding when it is not declared", () => {+ // key_length/value_length are optional; llama.cpp falls back to+ // embedding_length / head_count.+ const bytes = policy.kvBytesPerToken({+ blockCount: 28,+ headCountKv: 8,+ embeddingLength: 3072,+ headCount: 24, // 3072 / 24 = 128+ });++ assert.equal(bytes, 114688);+});++test("kvBytesPerToken returns null rather than guessing when the architecture is unreadable", () => {+ // A null here must fall back to today's 16384 ceiling, never to a bigger+ // number derived from incomplete data.+ assert.equal(policy.kvBytesPerToken(null), null);+ assert.equal(policy.kvBytesPerToken({}), null);+ assert.equal(policy.kvBytesPerToken({ blockCount: 28 }), null);+ assert.equal(policy.kvBytesPerToken({ blockCount: 28, headCountKv: 8 }), null);+ assert.equal(policy.kvBytesPerToken({ blockCount: 0, headCountKv: 8, keyLength: 128 }), null);+});++// --- resolveContextCeiling -----------------------------------------------+//+// The largest context this machine may allocate for this model. This is the+// function that has to keep the #1203 promise: it is the only thing standing+// between a long transcript and a wired KV cache big enough to freeze a Mac.++const GIB = 1024 * 1024 * 1024;++const LLAMA_3_2_3B = {+ // 28 blocks, 8 KV heads, 128-wide -> 114688 B/token. The #1203 model.+ kvBytesPerToken: 114688,+ weightsBytes: 2168958976,+ trainedContextTokens: 131072,+};++const QWEN_3_5_9B = {+ kvBytesPerToken: 135168,+ weightsBytes: 6169341984,+ trainedContextTokens: 262144,+ hasRecurrentState: true,+};++test("regression #1203: a 16 GB Mac running Llama-3.2-3B stays far below the allocation that panicked", () => {+ const { ceiling } = policy.resolveContextCeiling({+ totalMemoryBytes: 16 * GIB,+ ...LLAMA_3_2_3B,+ });++ assert.equal(ceiling, 32256);++ // The actual guarantee, stated as memory rather than as a magic number: the+ // resident footprint must stay well under half the machine, so macOS keeps+ // room it can compress or swap. #1203 allocated ~15 GB of 16 GB as WIRED+ // Metal memory, which can do neither, and watchdogd missed its check-ins.+ const footprint =+ LLAMA_3_2_3B.weightsBytes + ceiling * LLAMA_3_2_3B.kvBytesPerToken + policy.COMPUTE_FIXED_BYTES;+ assert.ok(+ footprint < 0.4 * 16 * GIB,+ `footprint ${footprint} must stay under 40% of RAM (${0.4 * 16 * GIB})`+ );++ // And it must still be a real improvement, or the bug is not fixed.+ assert.ok(ceiling > policy.BASELINE_CONTEXT_SIZE, "must beat today's flat 16384");+});++test("the ceiling tracks KV cost, not model size, on identical hardware", () => {+ // Same machine, same weights, only the per-token cost differs by 12x. A+ // RAM-tier or file-size heuristic would return the same answer for both.+ const cheap = policy.resolveContextCeiling({+ totalMemoryBytes: 48 * GIB,+ kvBytesPerToken: 32768,+ weightsBytes: 5 * GIB,+ trainedContextTokens: 262144,+ });+ const expensive = policy.resolveContextCeiling({+ totalMemoryBytes: 48 * GIB,+ kvBytesPerToken: 393216, // Gemma-3-12B+ weightsBytes: 5 * GIB,+ trainedContextTokens: 131072,+ });++ assert.ok(+ cheap.ceiling > expensive.ceiling * 3,+ `expected a wide split, got ${cheap.ceiling} vs ${expensive.ceiling}`+ );+});++test("the reported 48 GB machine can hold the transcript that fails today", () => {+ const { ceiling } = policy.resolveContextCeiling({+ totalMemoryBytes: 48 * GIB,+ ...QWEN_3_5_9B,+ });++ assert.equal(ceiling, 118272);+ // The customer's prompt measured 20,514 tokens against the bundled server.+ assert.ok(ceiling > 20514 + 2048, "the failing request must now fit with output headroom");+});++test("an unreadable architecture falls back to today's limit, never to a larger guess", () => {+ const { ceiling, reason } = policy.resolveContextCeiling({+ totalMemoryBytes: 48 * GIB,+ kvBytesPerToken: null,+ weightsBytes: 2168958976,+ trainedContextTokens: 131072,+ });++ assert.equal(ceiling, policy.BASELINE_CONTEXT_SIZE);+ assert.equal(reason, "unknown-kv-cost");+});++test("unusable memory readings fall back to today's limit", () => {+ for (const totalMemoryBytes of [0, Number.NaN, undefined, null, -1]) {+ const { ceiling, reason } = policy.resolveContextCeiling({+ totalMemoryBytes,+ ...LLAMA_3_2_3B,+ });+ assert.equal(ceiling, policy.BASELINE_CONTEXT_SIZE, `total=${totalMemoryBytes}`);+ assert.equal(reason, "no-memory-info");+ }+});++test("a machine too small for any headroom keeps exactly today's behaviour", () => {+ // 8 GB Mac, 2 GB of weights: the budget goes negative. This must be the+ // historical 16384 rather than something smaller, so the change can never+ // make a shipping configuration worse.+ const { ceiling, reason } = policy.resolveContextCeiling({+ totalMemoryBytes: 8 * GIB,+ ...LLAMA_3_2_3B,+ });++ assert.equal(ceiling, policy.BASELINE_CONTEXT_SIZE);+ assert.equal(reason, "floor");+});++test("the ceiling never exceeds what the model was trained for", () => {+ const { ceiling, reason } = policy.resolveContextCeiling({+ totalMemoryBytes: 512 * GIB,+ kvBytesPerToken: 32768,+ weightsBytes: 5 * GIB,+ trainedContextTokens: 32768,+ });++ assert.equal(ceiling, 32768);+ assert.equal(reason, "trained-context-bound");+});++test("an unmeasured discrete GPU is capped even when system RAM looks generous", () => {+ const { ceiling } = policy.resolveContextCeiling({+ totalMemoryBytes: 128 * GIB,+ kvBytesPerToken: 32768,+ weightsBytes: 3 * GIB,+ trainedContextTokens: 262144,+ discreteGpuUnverified: true,+ });++ assert.ok(ceiling <= 65536, `expected the unverified-GPU cap, got ${ceiling}`);+});++test("the ceiling moves monotonically with each input", () => {+ const base = { totalMemoryBytes: 32 * GIB, ...LLAMA_3_2_3B };+ const at = (overrides) => policy.resolveContextCeiling({ ...base, ...overrides }).ceiling;++ assert.ok(at({ totalMemoryBytes: 64 * GIB }) >= at({}), "more RAM never shrinks the ceiling");+ assert.ok(at({ weightsBytes: 6 * GIB }) <= at({}), "bigger weights never grow the ceiling");+ assert.ok(at({ kvBytesPerToken: 393216 }) <= at({}), "costlier KV never grows the ceiling");+ assert.ok(at({ hasRecurrentState: true }) <= at({}), "recurrent state never grows the ceiling");+ assert.ok(at({ draftWeightsBytes: 2 * GIB }) <= at({}), "a drafter never grows the ceiling");+});++test("the prompt cache is bounded so it cannot eat the budget we just computed", () => {+ // llama-server defaults --cache-ram to 8192 MiB of host RAM, on top of the+ // KV cache and 32 context checkpoints per slot. Unbounded, it would undo the+ // budget. Cap it rather than disabling it: 0 forces a full re-prefill on+ // every request.+ for (const totalMemoryBytes of [8 * GIB, 16 * GIB, 48 * GIB, 128 * GIB]) {+ const { cacheRamMiB } = policy.resolveContextCeiling({+ totalMemoryBytes,+ ...LLAMA_3_2_3B,+ });+ assert.ok(+ cacheRamMiB >= 512 && cacheRamMiB <= 2048,+ `cacheRamMiB ${cacheRamMiB} out of range at ${totalMemoryBytes}`+ );+ }+});++// --- resolveContextSize --------------------------------------------------+//+// Snapping to a ladder is what keeps restarts bounded. Sizing the context to+// the exact request would restart the server for a 20,600-token note and again+// for a 21,100-token one; snapping puts both on the same rung.++test("resolveContextSize keeps short requests on the baseline rung", () => {+ assert.equal(policy.resolveContextSize({ needed: 800, ceiling: 131072 }), 16384);+ assert.equal(policy.resolveContextSize({ needed: 0, ceiling: 131072 }), 16384);+ assert.equal(policy.resolveContextSize({ ceiling: 131072 }), 16384);+});++test("resolveContextSize grows to the smallest rung that fits the request", () => {+ // The reported failure: 20,514 prompt tokens plus output and template slack.+ assert.equal(policy.resolveContextSize({ needed: 23074, ceiling: 131072 }), 32768);+ assert.equal(policy.resolveContextSize({ needed: 16385, ceiling: 131072 }), 32768);+ assert.equal(policy.resolveContextSize({ needed: 32768, ceiling: 131072 }), 32768);+ assert.equal(policy.resolveContextSize({ needed: 32769, ceiling: 131072 }), 65536);+});++test("resolveContextSize puts near-identical requests on the same rung", () => {+ const first = policy.resolveContextSize({ needed: 20600, ceiling: 131072 });+ const second = policy.resolveContextSize({ needed: 21100, ceiling: 131072 });++ assert.equal(first, second, "a second similar note must not cost a second restart");+});++test("resolveContextSize never returns more than the ceiling allows", () => {+ // The #1203 machine: the ceiling is not on a rung, and it wins outright.+ assert.equal(policy.resolveContextSize({ needed: 40000, ceiling: 32256 }), 32256);+ assert.equal(policy.resolveContextSize({ needed: 1000000, ceiling: 32256 }), 32256);+ assert.equal(policy.resolveContextSize({ needed: 100000, ceiling: 65536 }), 65536);+});++test("resolveContextSize honours a caller floor without breaching the ceiling", () => {+ // Selection editing asks for a larger window up front (audioManager.js).+ assert.equal(policy.resolveContextSize({ needed: 500, floor: 32768, ceiling: 131072 }), 32768);+ // ...but a caller cannot talk its way past the memory guard.+ assert.equal(policy.resolveContextSize({ needed: 500, floor: 262144, ceiling: 32256 }), 32256);+});++test("resolveContextSize never drops below the baseline even with a tiny ceiling", () => {+ assert.equal(policy.resolveContextSize({ needed: 100, ceiling: 4096 }), 16384);+});diff --git a/test/helpers/llamaServerContext.test.js b/test/helpers/llamaServerContext.test.jsnew file mode 100644index 000000000..d17f974b8--- /dev/null+++ b/test/helpers/llamaServerContext.test.js@@ -0,0 +1,349 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const http = require("node:http");++const LlamaServerManager = require("../../src/helpers/llamaServer.js");++// A manager whose spawn is stubbed at the _doStart boundary, recording the+// options each start was asked for.+function makeManager() {+ const manager = new LlamaServerManager();+ const starts = [];+ manager._doStart = async (modelPath, options = {}) => {+ manager.ready = true;+ manager.modelPath = modelPath;+ manager.draftModelPath = options.draftModelPath || null;+ starts.push({ modelPath, contextSize: options.contextSize });+ };+ return { manager, starts };+}++// --- grow-only restart ---------------------------------------------------+//+// Before #2142 the context was a single constant, so start() could ignore+// options entirely once a server was ready. Now that the context varies per+// request, it has to restart to grow — but only to grow, or a long note+// followed by a short dictation would restart the server twice.++test("a larger context restarts the server", async () => {+ const { manager, starts } = makeManager();++ await manager.start("/models/main.gguf", { contextSize: 16384 });+ assert.equal(starts.length, 1);++ await manager.start("/models/main.gguf", { contextSize: 32768 });+ assert.equal(starts.length, 2);+ assert.equal(manager.contextSize, 32768);+});++test("an equal or smaller context reuses the running server", async () => {+ const { manager, starts } = makeManager();++ await manager.start("/models/main.gguf", { contextSize: 32768 });+ assert.equal(starts.length, 1);++ // Exactly the same → no restart.+ await manager.start("/models/main.gguf", { contextSize: 32768 });+ assert.equal(starts.length, 1);++ // Smaller → no restart, and the larger window is kept. This is what stops a+ // short dictation right after a long note from costing a second restart.+ await manager.start("/models/main.gguf", { contextSize: 16384 });+ assert.equal(starts.length, 1);+ assert.equal(manager.contextSize, 32768);+});++test("growing the context still restarts when the drafter is unchanged", async () => {+ const { manager, starts } = makeManager();++ await manager.start("/models/main.gguf", { contextSize: 16384, draftModelPath: "/d.gguf" });+ await manager.start("/models/main.gguf", { contextSize: 65536, draftModelPath: "/d.gguf" });++ assert.equal(starts.length, 2);+ assert.equal(manager.contextSize, 65536);+});++test("a context grown for one model does not carry over to another", async () => {+ const { manager, starts } = makeManager();++ await manager.start("/models/big.gguf", { contextSize: 65536 });+ await manager.start("/models/other.gguf", { contextSize: 16384 });++ assert.equal(starts.length, 2);+ assert.equal(manager.contextSize, 16384, "the new model starts at its own size");+});++test("stopping the server forgets the grown context", async () => {+ const { manager } = makeManager();+ manager._killCurrentProcess = async () => {};+ manager.process = {};++ await manager.start("/models/main.gguf", { contextSize: 65536 });+ assert.equal(manager.contextSize, 65536);++ await manager.stop();+ assert.equal(manager.contextSize, null);+});++// --- spawn arguments -----------------------------------------------------++test("every GPU ladder rung inherits the requested context and cache bounds", async () => {+ const manager = new LlamaServerManager();+ const seen = [];+ manager._buildEnv = () => ({});+ manager._killCurrentProcess = async () => {};+ manager.findAvailablePort = async () => 8221;+ manager.draftModelPath = null;+ manager._startWithBinary = async (binary, args) => {+ seen.push(args);+ throw new Error("stub fail");+ };++ const baseArgs = manager._buildBaseArgs("/models/main.gguf", 8221, {+ contextSize: 32768,+ cacheRamMiB: 1024,+ });+ await assert.rejects(() =>+ manager._startWithGpuFallback({ vulkan: "/bin/vulkan", cpu: "/bin/cpu" }, baseArgs, {}, [])+ );++ assert.ok(seen.length >= 2, "expected the ladder to try more than one rung");+ for (const args of seen) {+ assert.equal(args[args.indexOf("--ctx-size") + 1], "32768");+ // llama-server defaults --cache-ram to 8192 MiB of host RAM on top of the+ // KV cache. Left unbounded it would undo the memory budget entirely.+ assert.equal(args[args.indexOf("--cache-ram") + 1], "1024");+ }+});++// --- context-overflow errors --------------------------------------------++test("a context overflow is reported as a typed error, not as llama.cpp JSON", async () => {+ const overflowBody = JSON.stringify({+ error: {+ code: 400,+ message: "request (20514 tokens) exceeds the available context size (16384 tokens)",+ type: "exceed_context_size_error",+ n_prompt_tokens: 20514,+ n_ctx: 16384,+ },+ });++ const server = http.createServer((req, res) => {+ res.writeHead(400, { "Content-Type": "application/json" });+ res.end(overflowBody);+ });+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));+ const manager = new LlamaServerManager();+ manager.ready = true;+ manager.process = {};+ manager.port = server.address().port;++ try {+ await assert.rejects(+ () => manager.inference([{ role: "user", content: "hi" }], {}),+ (error) => {+ assert.equal(error.code, "CONTEXT_TOO_LARGE");+ assert.equal(error.neededTokens, 20514);+ assert.equal(error.maxContextTokens, 16384);+ // The whole point: a customer must never be shown a JSON blob.+ assert.ok(!error.message.includes("{"), `raw JSON leaked: ${error.message}`);+ return true;+ }+ );+ } finally {+ manager.clearIdleTimer();+ await new Promise((resolve) => server.close(resolve));+ }+});++test("other llama-server failures keep their existing shape", async () => {+ const server = http.createServer((req, res) => {+ res.writeHead(500, { "Content-Type": "text/plain" });+ res.end("internal error");+ });+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));+ const manager = new LlamaServerManager();+ manager.ready = true;+ manager.process = {};+ manager.port = server.address().port;++ try {+ await assert.rejects(+ () => manager.inference([{ role: "user", content: "hi" }], {}),+ (error) => {+ assert.equal(error.code, undefined);+ assert.match(error.message, /llama-server returned status 500/);+ return true;+ }+ );+ } finally {+ manager.clearIdleTimer();+ await new Promise((resolve) => server.close(resolve));+ }+});++// --- measuring the running server ----------------------------------------+//+// The estimate only picks the size the server starts at. These two calls are+// what make the decision exact: /props reports the window actually in force+// (which can differ from what we asked for), and /apply-template + /tokenize+// price the real prompt, chat template and all.++function withStubServer(routes) {+ // A Map rather than indexing the object: req.url is request-controlled, so a+ // plain lookup answers /toString or /constructor with an inherited+ // Object.prototype method and then calls it.+ const routeTable = new Map(Object.entries(routes));+ return async (run) => {+ const server = http.createServer((req, res) => {+ const handler = routeTable.get(req.url);+ if (typeof handler !== "function") {+ res.writeHead(404).end();+ return;+ }+ let body = "";+ req.on("data", (chunk) => {+ body += chunk;+ });+ req.on("end", () => {+ const { status = 200, payload } = handler(body);+ res.writeHead(status, { "Content-Type": "application/json" });+ res.end(JSON.stringify(payload));+ });+ });+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));++ const manager = new LlamaServerManager();+ manager.ready = true;+ manager.process = {};+ manager.port = server.address().port;+ try {+ await run(manager);+ } finally {+ manager.clearIdleTimer();+ await new Promise((resolve) => server.close(resolve));+ }+ };+}++test("usableContextSize reports the window the server actually has", async () => {+ await withStubServer({+ "/props": () => ({ payload: { default_generation_settings: { n_ctx: 32768 } } }),+ })(async (manager) => {+ assert.equal(await manager.usableContextSize(), 32768);+ });+});++test("usableContextSize returns null rather than guessing when props is unreadable", async () => {+ // A broken measurement must degrade to the estimate, never fail the request.+ await withStubServer({ "/props": () => ({ status: 500, payload: {} }) })(async (manager) => {+ assert.equal(await manager.usableContextSize(), null);+ });+ await withStubServer({})(async (manager) => {+ assert.equal(await manager.usableContextSize(), null);+ });+});++test("countPromptTokens prices the rendered template, not just the message text", async () => {+ let templatedWith = null;+ let tokenized = null;++ await withStubServer({+ "/apply-template": (body) => {+ templatedWith = JSON.parse(body);+ // The real server renders role tags and, with thinking disabled, an+ // empty <think></think> block. All of it costs context.+ return { payload: { prompt: "<|im_start|>system\nSYS<|im_end|>\nhello" } };+ },+ "/tokenize": (body) => {+ tokenized = JSON.parse(body);+ return { payload: { tokens: new Array(20514).fill(0) } };+ },+ })(async (manager) => {+ const messages = [+ { role: "system", content: "SYS" },+ { role: "user", content: "hello" },+ ];+ assert.equal(await manager.countPromptTokens(messages, { disableThinking: true }), 20514);+ assert.deepEqual(templatedWith.messages, messages);+ assert.deepEqual(templatedWith.chat_template_kwargs, { enable_thinking: false });+ assert.equal(tokenized.content, "<|im_start|>system\nSYS<|im_end|>\nhello");+ });+});++test("countPromptTokens returns null when the server cannot measure", async () => {+ await withStubServer({ "/apply-template": () => ({ status: 500, payload: {} }) })(+ async (manager) => {+ assert.equal(await manager.countPromptTokens([{ role: "user", content: "hi" }], {}), null);+ }+ );+});++// --- Vulkan context step-down --------------------------------------------+//+// Sizing the context to the request is bounded by SYSTEM RAM, but a discrete+// GPU has its own memory that the app does not probe. A machine with plenty of+// RAM and a small card can therefore now ask for a context that will not fit+// in VRAM — a failure this change introduces. Dropping straight to CPU costs+// 10-30x; halving the context is far cheaper, so try that first.++function makeLadder(shouldFail, { contextSize = 32768 } = {}) {+ const manager = new LlamaServerManager();+ const attempts = [];+ manager._buildEnv = () => ({});+ manager._killCurrentProcess = async () => {};+ manager.findAvailablePort = async () => 8221;+ manager.draftModelPath = null;+ manager._startWithBinary = async (binary, args) => {+ const attempt = { binary, ctx: Number(args[args.indexOf("--ctx-size") + 1]) };+ attempts.push(attempt);+ if (shouldFail(attempt)) throw new Error(`stub fail: ${binary} ctx=${attempt.ctx}`);+ };+ const baseArgs = manager._buildBaseArgs("/models/main.gguf", 8221, { contextSize });+ return { manager, attempts, baseArgs, options: { contextSize } };+}++const BINARIES = { vulkan: "/bin/vulkan", cpu: "/bin/cpu" };++test("a Vulkan start that fails at a grown context retries smaller before dropping to CPU", async () => {+ // Vulkan only succeeds at the baseline context.+ const { manager, attempts, baseArgs, options } = makeLadder(+ (attempt) => attempt.binary === "/bin/vulkan" && attempt.ctx > 16384+ );++ await manager._startWithGpuFallback(BINARIES, baseArgs, options, []);++ assert.deepEqual(attempts, [+ { binary: "/bin/vulkan", ctx: 32768 },+ { binary: "/bin/vulkan", ctx: 16384 },+ ]);+ assert.equal(manager.activeBackend, "vulkan", "a live GPU beats a CPU fallback");+});++test("a baseline-context start keeps today's exact ladder, with no extra rung", async () => {+ // The step-down must not add an attempt when there is nothing to step down+ // to, or every existing failure path gets slower.+ const { manager, attempts, baseArgs, options } = makeLadder(() => true, {+ contextSize: 16384,+ });++ await assert.rejects(() => manager._startWithGpuFallback(BINARIES, baseArgs, options, []));++ assert.deepEqual(attempts, [+ { binary: "/bin/vulkan", ctx: 16384 },+ { binary: "/bin/cpu", ctx: 16384 },+ ]);+});++test("a context step-down is recorded, so the server reports what it actually has", async () => {+ const { manager, baseArgs, options } = makeLadder(+ (attempt) => attempt.binary === "/bin/vulkan" && attempt.ctx > 16384+ );++ await manager._startWithGpuFallback(BINARIES, baseArgs, options, []);++ // Otherwise the preflight would believe the request still fits and send a+ // prompt the server cannot take.+ assert.equal(manager.contextSize, 16384);+});diff --git a/test/helpers/localReasoningBridgeChain.test.js b/test/helpers/localReasoningBridgeChain.test.jsindex 85eee1c77..13d7b80c3 100644--- a/test/helpers/localReasoningBridgeChain.test.js+++ b/test/helpers/localReasoningBridgeChain.test.js@@ -7,6 +7,7 @@ const os = require("node:os");const path = require("node:path");const modelRegistryData = require("../../src/models/modelRegistryData.json");+const { buildGguf, LLAMA_3_2_3B_ENTRIES } = require("./harness/ggufFixtures");// Drives the primary local LLM path end to end — the IPC bridge// (localReasoningBridge) → modelManagerBridge.runInference →@@ -52,7 +53,11 @@ function loadChain() {}}-const OVER_MIN_FILE_SIZE = Buffer.alloc(1_000_001, 1);+// A real GGUF header padded past the minimum-size check, so the context+// ceiling is computed from the model's own architecture rather than falling+// back to the baseline the moment a caller asks for a bigger window.+const MODEL_FILE = Buffer.concat([buildGguf(LLAMA_3_2_3B_ENTRIES), Buffer.alloc(1_000_001, 1)]);+const ROOMY_MACHINE_BYTES = 48 * 1024 * 1024 * 1024;// Stands up the stub server plus a bridge whose model manager already// believes llama-server is running that model on the stub's port.@@ -78,7 +83,8 @@ async function setupChain(t, respond) {const model = modelRegistryData.localProviders[0].models[0];modelManager.ensureInitialized();await fs.mkdir(modelManager.modelsDir, { recursive: true });- await fs.writeFile(path.join(modelManager.modelsDir, model.fileName), OVER_MIN_FILE_SIZE);+ await fs.writeFile(path.join(modelManager.modelsDir, model.fileName), MODEL_FILE);+ modelManager._systemMemoryBytes = () => ROOMY_MACHINE_BYTES;const serverManager = modelManager.serverManager;serverManager.cachedServerBinaryPaths = { default: "/stub/llama-server" };@@ -88,7 +94,7 @@ async function setupChain(t, respond) {modelManager.currentServerModelId = model.id;t.after(() => serverManager.clearIdleTimer());- return { bridge, modelId: model.id, requests };+ return { bridge, modelId: model.id, requests, serverManager };}const completion = (finishReason, content) => ({@@ -118,3 +124,30 @@ test("an explicit temperature of 0 reaches llama-server instead of the 0.7 defauassert.equal(requests.length, 1);assert.equal(requests[0].temperature, 0);});++test("a caller's contextSize reaches the server start (regression: it was dropped)", async (t) => {+ // ReasoningConfig.contextSize was declared, written by selection editing,+ // and then rebuilt away in this bridge, so it never reached anything. The+ // field only means something now that the server can grow, so the wiring+ // needs a guard that fails if anyone rebuilds the config object again.+ // It raises the floor only as far as the machine's ceiling allows, which is+ // why this harness has to present a machine and a model that can afford it.+ const { bridge, modelId, serverManager } = await setupChain(t, () => completion("stop", "ok"));++ const started = [];+ serverManager._doStart = async (modelPath, options = {}) => {+ started.push(options.contextSize);+ serverManager.ready = true;+ serverManager.process = {};+ };+ serverManager.stop = async () => {+ serverManager.ready = false;+ serverManager.process = null;+ serverManager.contextSize = null;+ };+ serverManager.contextSize = 16384;++ await bridge.processText("short text", modelId, { contextSize: 32768 });++ assert.deepEqual(started, [32768]);+});diff --git a/test/helpers/modelManagerContextSizing.test.js b/test/helpers/modelManagerContextSizing.test.jsnew file mode 100644index 000000000..9289946cc--- /dev/null+++ b/test/helpers/modelManagerContextSizing.test.js@@ -0,0 +1,479 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const Module = require("node:module");+const fs = require("node:fs/promises");+const http = require("node:http");+const os = require("node:os");+const path = require("node:path");++const modelRegistryData = require("../../src/models/modelRegistryData.json");+const { buildGguf, LLAMA_3_2_3B_ENTRIES } = require("./harness/ggufFixtures");++// Drives modelManagerBridge.runInference against a stub llama-server, to pin+// how the context window is chosen for a request (#2142). The model file on+// disk is a REAL GGUF header padded to a plausible size, so the ceiling is+// computed through the production path rather than from a fixture object.++const originalLoad = Module._load;+const CHAIN_MODULES = [+ "../../src/helpers/modelManagerBridge.js",+ "../../src/helpers/modelDirUtils.js",+ "../../src/helpers/llamaServer.js",+].map((relative) => require.resolve(relative));+let electronHome = os.tmpdir();++function loadModelManager() {+ for (const modulePath of CHAIN_MODULES) delete require.cache[modulePath];+ Module._load = function loadWithMocks(request, parent, isMain) {+ if (request === "electron") {+ return {+ app: {+ isReady: () => true,+ getAppPath: () => process.cwd(),+ getPath: (name) => (name === "home" ? electronHome : path.join(electronHome, name)),+ },+ net: {},+ };+ }+ return originalLoad.call(this, request, parent, isMain);+ };+ try {+ require("../../src/helpers/modelDirUtils.js");+ return require("../../src/helpers/modelManagerBridge.js").default;+ } finally {+ Module._load = originalLoad;+ }+}++const GIB = 1024 * 1024 * 1024;+const SHORT_PROMPT = "Clean up this sentence.";+// ~20k estimated tokens, the size that fails on main today.+const LONG_PROMPT = "word ".repeat(12000);++async function setup(t, { tokenCount = null, totalMemoryBytes = 48 * GIB } = {}) {+ const home = await fs.mkdtemp(path.join(os.tmpdir(), "openwhispr-ctx-sizing-"));+ electronHome = home;+ t.after(() => fs.rm(home, { recursive: true, force: true }));++ const calls = { tokenize: 0, props: 0, completions: 0, template: 0 };+ // The completion body is where the output allowance actually lands, so tests+ // about it have to read the wire rather than the config object.+ let completionBody = null;+ const server = http.createServer((req, res) => {+ let raw = "";+ req.on("data", (chunk) => (raw += chunk));+ req.on("end", () => {+ let status = 200;+ let payload;+ if (req.url === "/props") {+ calls.props += 1;+ payload = { default_generation_settings: { n_ctx: serverManager.contextSize } };+ } else if (req.url === "/apply-template") {+ calls.template += 1;+ payload = { prompt: "RENDERED" };+ } else if (req.url === "/tokenize") {+ calls.tokenize += 1;+ if (tokenCount === null) {+ status = 500;+ payload = {};+ } else {+ payload = { tokens: new Array(tokenCount).fill(0) };+ }+ } else {+ calls.completions += 1;+ completionBody = JSON.parse(raw);+ payload = { choices: [{ finish_reason: "stop", message: { content: "done" } }] };+ }+ res.writeHead(status, { "Content-Type": "application/json" });+ res.end(JSON.stringify(payload));+ });+ });+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));+ t.after(() => new Promise((resolve) => server.close(resolve)));++ const modelManager = loadModelManager();+ const model = modelRegistryData.localProviders[0].models[0];+ modelManager.ensureInitialized();+ await fs.mkdir(modelManager.modelsDir, { recursive: true });++ // A valid GGUF header, padded past the minimum-size check.+ const header = buildGguf(LLAMA_3_2_3B_ENTRIES);+ await fs.writeFile(+ path.join(modelManager.modelsDir, model.fileName),+ Buffer.concat([header, Buffer.alloc(1_000_001, 1)])+ );++ const serverManager = modelManager.serverManager;+ serverManager.cachedServerBinaryPaths = { default: "/stub/llama-server" };+ serverManager.ready = true;+ serverManager.process = {};+ serverManager.port = server.address().port;+ serverManager.contextSize = 16384;+ modelManager.currentServerModelId = model.id;+ modelManager._systemMemoryBytes = () => totalMemoryBytes;+ t.after(() => serverManager.clearIdleTimer());++ // Stand in for the spawn. The real _doStart leaves the manager ready on a+ // live port, so the stub must too, or the preflight would see a dead server.+ const restarts = [];+ serverManager._doStart = async (modelPath, options = {}) => {+ restarts.push(options.contextSize);+ serverManager.ready = true;+ serverManager.process = {};+ serverManager.port = server.address().port;+ };+ // Avoid signalling a pid that does not belong to us; stop() itself is+ // covered in llamaServerContext.test.js.+ serverManager.stop = async () => {+ serverManager.ready = false;+ serverManager.process = null;+ serverManager.contextSize = null;+ };++ return {+ modelManager,+ modelId: model.id,+ calls,+ restarts,+ serverManager,+ completionBody: () => completionBody,+ };+}++test("a short request costs no measurement and no restart", async (t) => {+ // The dictation-cleanup invariant. Every dictation goes through this path;+ // adding a round trip or a restart here would be a far worse regression than+ // the bug being fixed.+ const { modelManager, modelId, calls, restarts, completionBody } = await setup(t, {+ tokenCount: 40,+ });++ const result = await modelManager.runInference(modelId, SHORT_PROMPT, {+ systemPrompt: "Clean up dictation.",+ maxTokens: 900,+ });++ assert.equal(result, "done");+ assert.equal(restarts.length, 0, "a short request must not restart the server");+ assert.equal(calls.tokenize, 0, "a short request must not pay for tokenization");+ assert.equal(calls.completions, 1);+ // A request that comfortably fits must reach the model with the allowance+ // the caller asked for. Without this, trimming everything to a stub would+ // still pass every other test in this file.+ assert.equal(completionBody().max_tokens, 900);+});++test("a long request grows the context once and then succeeds", async (t) => {+ const { modelManager, modelId, calls, restarts, serverManager } = await setup(t, {+ tokenCount: 20514, // the customer's measured prompt+ });++ const result = await modelManager.runInference(modelId, LONG_PROMPT, {+ systemPrompt: "Write meeting notes.",+ });++ assert.equal(result, "done");+ assert.deepEqual(restarts, [32768], "exactly one restart, onto the rung that fits");+ assert.equal(serverManager.contextSize, 32768);+ assert.equal(calls.completions, 1);+});++test("a request too large for the machine fails before it reaches the model", async (t) => {+ const { modelManager, modelId, calls } = await setup(t, { tokenCount: 200000 });++ await assert.rejects(+ () => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),+ (error) => {+ assert.equal(error.code, "CONTEXT_TOO_LARGE");+ assert.ok(error.details.neededTokens >= 200000);+ assert.ok(error.details.maxContextTokens > 0);+ assert.ok(!error.message.includes("{"), `raw JSON leaked: ${error.message}`);+ return true;+ }+ );++ // Sending it anyway would burn minutes of prefill to earn a 400.+ assert.equal(calls.completions, 0, "must not send a request that cannot fit");+});++test("a broken measurement falls back to the estimate instead of failing the request", async (t) => {+ const { modelManager, modelId, calls } = await setup(t, { tokenCount: null }); // /tokenize 500s++ const result = await modelManager.runInference(modelId, LONG_PROMPT, {+ systemPrompt: "Write notes.",+ });++ assert.equal(result, "done");+ assert.ok(calls.tokenize > 0, "it should have tried to measure");+ assert.equal(calls.completions, 1, "and still completed on the estimate");+});++test("a short request after a long one reuses the grown context", async (t) => {+ const { modelManager, modelId, restarts } = await setup(t, { tokenCount: 20514 });++ await modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." });+ assert.deepEqual(restarts, [32768]);++ await modelManager.runInference(modelId, SHORT_PROMPT, { systemPrompt: "Clean up." });+ assert.deepEqual(restarts, [32768], "the short request must not shrink or restart the server");+});++test("a small machine refuses what a large one accepts, for the same request", async (t) => {+ // The #1203 guarantee, expressed end to end: the ceiling is a property of+ // the machine, so the same transcript is allowed on 48 GB and refused on 8.+ const small = await setup(t, { tokenCount: 60000, totalMemoryBytes: 8 * GIB });+ await assert.rejects(+ () => small.modelManager.runInference(small.modelId, LONG_PROMPT, { systemPrompt: "Notes." }),+ (error) => error.code === "CONTEXT_TOO_LARGE"+ );+ assert.equal(small.calls.completions, 0);+});++test("a running server whose context is unknown is not restarted for a short request", async (t) => {+ // A ready server always has at least the starting context. Treating an+ // unknown value as zero restarted every warm server on its next request.+ const { modelManager, modelId, restarts, serverManager } = await setup(t, { tokenCount: 40 });+ serverManager.contextSize = null;++ await modelManager.runInference(modelId, SHORT_PROMPT, { systemPrompt: "Clean up." });++ assert.deepEqual(restarts, []);+});++test("a caller's context floor cannot breach the machine's ceiling", async (t) => {+ // ReasoningConfig.contextSize raises the floor a request starts at, but the+ // ceiling is the memory guarantee: honouring a floor the machine cannot+ // afford is exactly the unbounded KV cache that panicked #1203.+ //+ // 12 GiB puts the ceiling strictly between the baseline and the floor, so+ // the three possible outcomes — clamp to the ceiling, collapse to the+ // baseline, obey the floor — are told apart. The assertions stay relational+ // because the exact ceiling is the policy's arithmetic, not this test's, and+ // because it must hold on Linux CI too (below the 65536 unverified-GPU cap,+ // so memory-bound on either platform).+ const { modelManager, modelId, restarts, serverManager } = await setup(t, {+ tokenCount: 40,+ totalMemoryBytes: 12 * GIB,+ });++ await modelManager.runInference(modelId, SHORT_PROMPT, {+ systemPrompt: "Edit this selection.",+ contextSize: 131072,+ });++ assert.equal(restarts.length, 1, "the raised floor should grow the window once");+ const [started] = restarts;+ assert.ok(started > 16384, `should grow past the baseline, started at ${started}`);+ assert.ok(started < 131072, `must never reach the caller's floor, started at ${started}`);+ assert.equal(serverManager.contextSize, started);+});++test("a prompt that fits gets a smaller answer rather than a refusal", async (t) => {+ // llama-server only rejects a prompt that overflows the window on its own,+ // so charging the full output reservation up front refused notes that work+ // today. Shrink the answer to what is left instead (#2142).+ const { modelManager, modelId, calls, completionBody } = await setup(t, {+ tokenCount: 13000,+ totalMemoryBytes: 8 * GIB, // ceiling collapses to the 16384 floor+ });++ const result = await modelManager.runInference(modelId, LONG_PROMPT, {+ systemPrompt: "Write meeting notes.",+ maxTokens: 4096,+ });++ assert.equal(result, "done");+ assert.equal(calls.completions, 1, "the request must still be sent");+ // Every token the window has left, not a token less: the prompt was+ // measured exactly, and llama.cpp stops generation at the limit rather than+ // rejecting it, so slack here would only shorten the note for nothing.+ assert.equal(completionBody().max_tokens, 16384 - 13000);+});++test("a prompt leaving no room for a usable answer is still refused", async (t) => {+ const { modelManager, modelId, calls } = await setup(t, {+ tokenCount: 16000,+ totalMemoryBytes: 8 * GIB,+ });++ await assert.rejects(+ () => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),+ (error) => error.code === "CONTEXT_TOO_LARGE"+ );+ assert.equal(calls.completions, 0, "must not send a request that cannot answer");+});++test("a selection edit is refused rather than given a clipped answer", async (t) => {+ // requireCompleteOutput means a partial replacement corrupts the user's own+ // text, so trading output room for a smaller answer is not an option there.+ const { modelManager, modelId, calls } = await setup(t, {+ tokenCount: 13000,+ totalMemoryBytes: 8 * GIB,+ });++ await assert.rejects(+ () =>+ modelManager.runInference(modelId, LONG_PROMPT, {+ systemPrompt: "Rewrite the selection.",+ maxTokens: 8192,+ requireCompleteOutput: true,+ }),+ (error) => error.code === "CONTEXT_TOO_LARGE"+ );+ assert.equal(calls.completions, 0);+});++test("a grow that fails comes back at the window that was working", async (t) => {+ // start() stops the running server before it spawns the bigger one, so a+ // grow that dies takes a working window with it. Restoring it keeps local+ // inference alive for the next request instead of until the next launch.+ const { modelManager, modelId, serverManager } = await setup(t, { tokenCount: 20514 });+ const port = serverManager.port;+ const attempts = [];+ serverManager._doStart = async (modelPath, options = {}) => {+ attempts.push(options.contextSize);+ if (options.contextSize > 16384) {+ throw new Error(+ "llama-server process died during startup (signal: SIGKILL)\n" ++ "Process output: ggml_metal: failed to allocate buffer of size 4096.00 MiB"+ );+ }+ serverManager.ready = true;+ serverManager.process = {};+ serverManager.port = port;+ };++ await assert.rejects(+ () => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),+ (error) => error.code === "CONTEXT_TOO_LARGE"+ );++ assert.deepEqual(attempts, [32768, 16384], "one failed grow, then back to what worked");+ assert.equal(serverManager.ready, true, "the user must not be left without a server");+});++test("a server that cannot come back reports a typed failure, not llama.cpp stderr", async (t) => {+ const { modelManager, modelId, serverManager } = await setup(t, { tokenCount: 20514 });+ serverManager._doStart = async () => {+ throw new Error(+ "llama-server process died during startup (signal: SIGKILL)\n" ++ "Process output: ggml_metal_graph_compute: command buffer 0 failed with status 5"+ );+ };++ await assert.rejects(+ () => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),+ (error) => {+ assert.equal(error.code, "LOCAL_SERVER_UNAVAILABLE");+ assert.ok(+ !error.message.includes("Process output"),+ `server output leaked: ${error.message}`+ );+ // details crosses IPC to the renderer, so the startup dump must not ride+ // along in there either — it belongs in the main-process log.+ assert.ok(+ !JSON.stringify(error.details).includes("Process output"),+ `server output leaked through details: ${JSON.stringify(error.details)}`+ );+ return true;+ }+ );+});++test("an answer too small to be worth generating is refused rather than stubbed", async (t) => {+ // 16100 of a 16384 window leaves 284 tokens, a few sentences for a whole+ // meeting. Below the floor the honest answer is that it does not fit, not a+ // stub saved as the note.+ const { modelManager, modelId, calls } = await setup(t, {+ tokenCount: 16100,+ totalMemoryBytes: 8 * GIB,+ });++ await assert.rejects(+ () =>+ modelManager.runInference(modelId, LONG_PROMPT, {+ systemPrompt: "Write notes.",+ maxTokens: 4096,+ }),+ (error) => error.code === "CONTEXT_TOO_LARGE"+ );+ assert.equal(calls.completions, 0);+});++test("a start that steps the context down is not retried at the size that stepped down", async (t) => {+ // The GPU ladder can load a smaller window than it was asked for without+ // ever throwing. Treating that as success left the preflight to ask for the+ // same window again, paying a second full model load to be stepped down+ // again — the wasted restart the memo exists to prevent.+ const { modelManager, modelId, serverManager } = await setup(t, { tokenCount: 20514 });+ const port = serverManager.port;+ const attempts = [];+ serverManager._doStart = async (modelPath, options = {}) => {+ attempts.push(options.contextSize);+ serverManager.ready = true;+ serverManager.process = {};+ serverManager.port = port;+ // What _startWithGpuFallback records when the reduced-context rung wins.+ serverManager.activeContextSize = 16384;+ };++ await assert.rejects(+ () => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),+ (error) => error.code === "CONTEXT_TOO_LARGE"+ );++ assert.deepEqual(attempts, [32768], "the step-down must be remembered, not re-attempted");+});++test("a server that already died is not restored to a window it no longer has", async (t) => {+ // llamaServer clears contextSize only in stop(); a crash or a failed health+ // check leaves it set while ready goes false. Reading it blind made a failed+ // start pay for a second one that was never going to help.+ const { modelManager, modelId, serverManager } = await setup(t, { tokenCount: 20514 });+ serverManager.ready = false;+ serverManager.process = null;++ const attempts = [];+ serverManager._doStart = async (modelPath, options = {}) => {+ attempts.push(options.contextSize);+ throw new Error("llama-server process died during startup (signal: SIGKILL)");+ };++ await assert.rejects(+ () => modelManager.runInference(modelId, LONG_PROMPT, { systemPrompt: "Write notes." }),+ (error) => error.code === "LOCAL_SERVER_UNAVAILABLE"+ );++ assert.deepEqual(attempts, [32768], "no second attempt at a window that was already gone");+});++test("only Apple Silicon is treated as memory the GPU already shares", async (t) => {+ // discreteGpuUnverified reads darwin as unified memory, but an Intel Mac's+ // Radeon has its own few GB. Sizing against system RAM there hands the card+ // a context it cannot hold — and the darwin start path has no reduced-context+ // rung to fall back on.+ const real = { platform: process.platform, arch: process.arch };+ const pretend = (platform, arch) => {+ Object.defineProperty(process, "platform", { value: platform, configurable: true });+ Object.defineProperty(process, "arch", { value: arch, configurable: true });+ };+ t.after(() => pretend(real.platform, real.arch));++ // Both managers are built before any pretending, so only the ceiling+ // decision sees the faked platform.+ const roomy = { tokenCount: 40, totalMemoryBytes: 64 * GIB };+ const apple = await setup(t, roomy);+ const intel = await setup(t, roomy);+ const model = modelRegistryData.localProviders[0].models[0];+ const ceilingFor = (manager) =>+ manager.contextCeiling({ model }, path.join(manager.modelsDir, model.fileName));++ pretend("darwin", "arm64");+ const unified = await ceilingFor(apple.modelManager);+ pretend("darwin", "x64");+ const discrete = await ceilingFor(intel.modelManager);++ assert.notEqual(unified.reason, "gpu-cap", "unified memory needs no unverified-GPU cap");+ assert.equal(discrete.reason, "gpu-cap", "an Intel Mac's GPU memory is not system memory");+ assert.ok(discrete.ceiling < unified.ceiling);+});diff --git a/test/locales/translationCoverage.test.js b/test/locales/translationCoverage.test.jsindex dcd54be81..181b6d7f1 100644--- a/test/locales/translationCoverage.test.js+++ b/test/locales/translationCoverage.test.js@@ -10,6 +10,9 @@ const LOCALES = path.join(SRC, "locales");const NAMESPACES = ["translation", "prompts"];const PLURAL_SUFFIX = /_(zero|one|two|few|many|other)$/;const T_CALL = /\bt\(\s*(['"`])([A-Za-z0-9_.-]+)\1/g;+// messageKey values are passed to t() as a variable, so the T_CALL scan+// cannot see them and a typo would render as the raw key string.+const MESSAGE_KEY = /\bmessageKey\s*[:=]\s*(['"`])([A-Za-z0-9_.-]+)\1/g;const INTERPOLATION = /\{\{\s*([\w.]+)/g;const ARABIC_PLURAL_CATEGORIES = ["zero", "one", "two", "few", "many", "other"];const ARABIC_PLURAL_BASES = [@@ -181,6 +184,23 @@ test("interpolation variables match en in every language", () => {}});+test("every messageKey literal resolves in en", () => {+ const keys = new Set();+ for (const namespace of NAMESPACES) {+ for (const key of flatten(load("en", namespace)).keys()) keys.add(key);+ }++ const missing = [];+ for (const file of sourceFiles(SRC)) {+ const source = fs.readFileSync(file, "utf8");+ for (const [, , key] of source.matchAll(MESSAGE_KEY)) {+ if (!keys.has(stripPlural(key))) missing.push(`${key} (${path.relative(SRC, file)})`);+ }+ }++ assert.deepEqual(missing, [], `unresolved messageKey values:\n${missing.join("\n")}`);+});+test("Arabic defines every CLDR plural category for each counted message", () => {const ar = flatten(load("ar", "translation"));for (const base of ARABIC_PLURAL_BASES) {diff --git a/test/services/geminiProviderResponse.test.js b/test/services/geminiProviderResponse.test.jsindex 0c768f564..c69f01a1f 100644--- a/test/services/geminiProviderResponse.test.js+++ b/test/services/geminiProviderResponse.test.js@@ -88,3 +88,33 @@ test("Gemini provider preserves complete-output truncation errors before extract/Model output was truncated before the selection edit completed/);});++test("a caller's output budget cannot shrink the one Gemini would have computed", async (t) => {+ // Note formatting pins maxTokens so the local path can price it against the+ // context window. On Gemini that short-circuited a computed allowance of up+ // to 8192 and halved long summaries — on the provider with the most room.+ const originalFetch = globalThis.fetch;+ t.after(() => {+ globalThis.fetch = originalFetch;+ });++ let requestBody = null;+ globalThis.fetch = async (_url, init) => {+ requestBody = JSON.parse(init.body);+ return new Response(JSON.stringify(createGeminiResponse([{ text: "ok" }])), {+ status: 200,+ headers: { "Content-Type": "application/json" },+ });+ };++ const { geminiProvider } = await load();+ await geminiProvider.call({+ text: "a long meeting transcript",+ model: "gemini-2.5-flash",+ agentName: null,+ config: { systemPrompt: "Write meeting notes.", maxTokens: 4096 },+ ctx: { ...providerContext, calculateMaxTokens: () => 8192 },+ });++ assert.equal(requestBody.generationConfig.maxOutputTokens, 8192);+});diff --git a/test/stores/actionProcessingNoteOutput.test.js b/test/stores/actionProcessingNoteOutput.test.jsnew file mode 100644index 000000000..c6f73f081--- /dev/null+++ b/test/stores/actionProcessingNoteOutput.test.js@@ -0,0 +1,98 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const { createRendererServer, installBrowserGlobals } = require("../lib/rendererTestHarness");++// Note formatting had no explicit output budget, so it inherited the generic+// 2048-token default from calculateMaxTokens. Summaries of long meetings were+// cut off at that ceiling and saved anyway, with no error and nothing in the+// UI to say the notes were incomplete (#2142).++const ACTION = { id: 1, name: "Generate Notes", prompt: "Summarize the meeting." };+const LABELS = { noModel: "no model", noEndpoint: "no endpoint", actionFailed: "failed" };++async function loadStore(t) {+ const updates = [];+ installBrowserGlobals(t, {+ window: {+ electronAPI: {+ updateNote: async (noteId, payload) => {+ updates.push({ noteId, payload });+ return { success: true };+ },+ },+ },+ });++ const calls = [];+ const vite = await createRendererServer(t, {+ cachePrefix: "openwhispr-action-output-test-",+ mockModules: {+ "/services/ReasoningService": `+ export default {+ processText: async (text, model, agentName, config) => {+ globalThis.__processTextCalls.push({ text, model, config });+ return "# Notes\\n- decided things";+ },+ };+ `,+ "/utils/generateTitle": `export const generateNoteTitle = async () => undefined;`,+ },+ });+ globalThis.__processTextCalls = calls;+ t.after(() => {+ delete globalThis.__processTextCalls;+ });++ const store = await vite.ssrLoadModule("/stores/actionProcessingStore.ts");+ return { store, calls, updates };+}++async function waitFor(predicate, label) {+ for (let attempt = 0; attempt < 200; attempt += 1) {+ if (predicate()) return;+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+ throw new Error(`timed out waiting for ${label}`);+}++test("note formatting asks for enough output tokens to hold a long meeting summary", async (t) => {+ const { store, calls, updates } = await loadStore(t);++ store.runBackgroundAction(+ 7,+ "## Meeting Transcript\n" + "Alice: we agreed to ship on Friday.\n".repeat(500),+ "hash",+ ACTION,+ { modelId: "gpt-4.1", isCloudMode: true, isMeetingNote: true },+ LABELS+ );++ await waitFor(() => updates.length > 0, "the note to be written");++ assert.equal(calls.length, 1);+ // 2048 is roughly 1,500 words — short of a structured summary of a long+ // meeting, which is exactly the case this feature exists for.+ assert.ok(+ calls[0].config.maxTokens >= 4096,+ `expected a real output budget, got ${calls[0].config.maxTokens}`+ );+});++test("a truncated summary is still saved rather than discarded", async (t) => {+ // Deliberate: unlike a selection edit, where a partial replacement would+ // corrupt the user's own text, a clipped summary is still worth keeping.+ // So note formatting must NOT set requireCompleteOutput.+ const { store, calls, updates } = await loadStore(t);++ store.runBackgroundAction(+ 8,+ "some notes",+ "hash",+ ACTION,+ { modelId: "gpt-4.1", isCloudMode: true, isMeetingNote: true },+ LABELS+ );++ await waitFor(() => updates.length > 0, "the note to be written");+ assert.notEqual(calls[0].config.requireCompleteOutput, true);+});diff --git a/test/utils/localInferenceError.test.js b/test/utils/localInferenceError.test.jsnew file mode 100644index 000000000..15376cd50--- /dev/null+++ b/test/utils/localInferenceError.test.js@@ -0,0 +1,88 @@+const test = require("node:test");+const assert = require("node:assert/strict");++let errors;++test.before(async () => {+ errors = await import("../../src/utils/localInferenceError.ts");+});++// The local path is the only one that can fail because the machine is too+// small for the request. Before #2142 that surfaced as a raw llama.cpp JSON+// body in a toast; this is where it becomes something a person can act on.++test("a context overflow becomes a translatable message naming the model and the numbers", () => {+ const error = errors.buildLocalInferenceError({+ error: "This content needs about 21538 tokens of context, but Qwen3.5 9B can only use 16384.",+ code: "CONTEXT_TOO_LARGE",+ details: {+ modelName: "Qwen3.5 9B",+ neededTokens: 21538,+ maxContextTokens: 16384,+ },+ });++ assert.equal(error.code, "CONTEXT_TOO_LARGE");+ assert.equal(error.messageKey, "models.errors.contextTooLarge");+ assert.deepEqual(error.messageParams, { model: "Qwen3.5 9B", needed: 21538, max: 16384 });+});++test("a context overflow without numbers still gets a translatable message", () => {+ // The reactive path: llama-server rejected the prompt and we only know which+ // model it was.+ const error = errors.buildLocalInferenceError({+ error: "The request is longer than the context this model is running with.",+ code: "CONTEXT_TOO_LARGE",+ details: { modelName: "Gemma 3 4B" },+ });++ assert.equal(error.messageKey, "models.errors.contextTooLargeGeneric");+ assert.deepEqual(error.messageParams, { model: "Gemma 3 4B" });+});++test("unrelated failures keep their message and gain no translation", () => {+ const error = errors.buildLocalInferenceError({+ error: "Model qwen3-8b is not downloaded or is corrupted",+ code: "MODEL_NOT_DOWNLOADED",+ });++ assert.equal(error.message, "Model qwen3-8b is not downloaded or is corrupted");+ assert.equal(error.messageKey, undefined);+ assert.equal(error.code, "MODEL_NOT_DOWNLOADED");+});++test("a failure with no information at all still produces a usable Error", () => {+ const error = errors.buildLocalInferenceError({});++ assert.ok(error instanceof Error);+ assert.ok(error.message.length > 0);+});++test("no llama.cpp JSON can reach the message", () => {+ // The literal regression: a JSON body used to be concatenated into the+ // message and shown to the customer.+ const error = errors.buildLocalInferenceError({+ error: 'llama-server returned status 400: {"error":{"code":400,"n_ctx":16384}}',+ code: "CONTEXT_TOO_LARGE",+ details: { modelName: "Llama 3.2 3B", neededTokens: 20000, maxContextTokens: 16384 },+ });++ // The raw text is kept for logs, but the UI renders the key, which has no+ // JSON in it.+ assert.equal(error.messageKey, "models.errors.contextTooLarge");+});++test("a server that will not start becomes a translatable message, not llama.cpp output", () => {+ const error = errors.buildLocalInferenceError({+ error: "Qwen3.5 9B could not be started on this computer.",+ code: "LOCAL_SERVER_UNAVAILABLE",+ details: {+ modelName: "Qwen3.5 9B",+ error: "llama-server process died during startup\nProcess output: ggml_metal: failed",+ },+ });++ assert.equal(error.messageKey, "models.errors.localServerUnavailable");+ assert.deepEqual(error.messageParams, { model: "Qwen3.5 9B" });+ assert.ok(!error.message.includes("Process output"));+});