Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(llama): bound llama-server context size to prevent OOM system freeze (#1236)
Gabriel Stein466145fe2c26
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 26ec371e7..049ff7dbb 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -2120,10 +2120,10 @@ class IPCHandlers {{ from: oldUuid, to: uuid },"gpu");- const modelPath = modelManager.serverManager.modelPath;+ const modelId = modelManager.currentServerModelId;await modelManager.serverManager.stop();- if (modelPath) {- await modelManager.serverManager.start(modelPath);+ if (modelId) {+ await modelManager.prewarmServer(modelId);}}}@@ -3661,7 +3661,7 @@ class IPCHandlers {const modelPath = require("path").join(modelManager.modelsDir, modelInfo.model.fileName);- await modelManager.serverManager.start(modelPath, { threads: 4 });+ await modelManager.serverManager.start(modelPath, modelManager.serverOptions(modelInfo));modelManager.currentServerModelId = modelId;this.environmentManager.saveAllKeysToEnvFile().catch(() => {});diff --git a/src/helpers/llamaServer.js b/src/helpers/llamaServer.jsindex 892a43886..f2a20a46c 100644--- a/src/helpers/llamaServer.js+++ b/src/helpers/llamaServer.js@@ -19,6 +19,7 @@ const HEALTH_CHECK_TIMEOUT_MS = 2000;const STARTUP_POLL_INTERVAL_MS = 500;const HEALTH_CHECK_FAILURE_THRESHOLD = 3;const IDLE_TIMEOUT_MS = 5 * 60 * 1000;+const DEFAULT_CONTEXT_SIZE = 4096;class LlamaServerManager {constructor() {@@ -137,11 +138,15 @@ class LlamaServerManager {String(this.port),"--threads",String(options.threads || 4),+ // Unset, this defaults to the model's full trained context (128K+),+ // whose KV cache can exceed total RAM with --fit disabled. See #1203.+ "--ctx-size",+ String(options.contextSize || DEFAULT_CONTEXT_SIZE),"--jinja",];if (process.platform === "darwin") {- const args = [...baseArgs, "--n-gpu-layers", "99"];+ const args = [...baseArgs, "--n-gpu-layers", String(options.gpuLayers ?? 99)];await this._startWithBinary(binaryPaths.default,args,@@ -163,7 +168,7 @@ class LlamaServerManager {}async _startWithGpuFallback(binaryPaths, baseArgs, options) {- const gpuArgs = [...baseArgs, "--n-gpu-layers", "99"];+ const gpuArgs = [...baseArgs, "--n-gpu-layers", String(options.gpuLayers ?? 99)];const cpuArgs = baseArgs;if (binaryPaths.vulkan) {diff --git a/src/helpers/modelManagerBridge.js b/src/helpers/modelManagerBridge.jsindex 5bd3e9d1d..883405758 100644--- a/src/helpers/modelManagerBridge.js+++ b/src/helpers/modelManagerBridge.js@@ -15,6 +15,11 @@ const debugLogger = require("./debugLogger");const MIN_FILE_SIZE = 1_000_000; // 1MB minimum for valid model files+// Bounds the KV cache — registry contextLength is the full trained context+// (128K+), which can exceed total RAM (#1203). Uniform for all start paths:+// start() won't restart a ready server when only options change.+const SERVER_CONTEXT_SIZE = 16384;+function getLocalProviders() {return modelRegistryData.localProviders || [];}@@ -167,6 +172,17 @@ class ModelManager {return null;}+ serverOptions(modelInfo) {+ return {+ contextSize: Math.min(+ SERVER_CONTEXT_SIZE,+ modelInfo.model.contextLength || SERVER_CONTEXT_SIZE+ ),+ threads: 4,+ gpuLayers: 99,+ };+ }+async downloadModel(modelId, onProgress) {this.ensureInitialized();const modelInfo = this.findModelById(modelId);@@ -366,11 +382,7 @@ class ModelManager {serverReady: this.serverManager.ready,});- await this.serverManager.start(modelPath, {- contextSize: options.contextSize || modelInfo.model.contextLength || 4096,- threads: options.threads || 4,- gpuLayers: 99,- });+ await this.serverManager.start(modelPath, this.serverOptions(modelInfo));this.currentServerModelId = modelId;debugLogger.logReasoning("INFERENCE_SERVER_STARTED", {@@ -440,11 +452,7 @@ class ModelManager {if (!this.serverManager.isAvailable()) return false;try {- await this.serverManager.start(modelPath, {- contextSize: modelInfo.model.contextLength || 4096,- threads: 4,- gpuLayers: 99,- });+ await this.serverManager.start(modelPath, this.serverOptions(modelInfo));this.currentServerModelId = modelId;debugLogger.info("llama-server pre-warmed", { modelId });return true;diff --git a/src/services/localReasoningBridge.js b/src/services/localReasoningBridge.jsindex 3dfe378de..cf9d7ec9f 100644--- a/src/services/localReasoningBridge.js+++ b/src/services/localReasoningBridge.js@@ -37,8 +37,6 @@ class LocalReasoningService {topK: config.topK || 40,topP: config.topP || 0.9,repeatPenalty: config.repeatPenalty || 1.1,- contextSize: config.contextSize || 4096,- threads: config.threads || 4,systemPrompt: config.systemPrompt || "",disableThinking: config.disableThinking !== false,};