Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(llama): bound llama-server context size to prevent OOM system freeze (#1236)

Gabriel Stein466145fe2c26
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 26ec371e7..049ff7dbb 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -2120,10 +2120,10 @@ class IPCHandlers {
{ from: oldUuid, to: uuid },
"gpu"
);
- const modelPath = modelManager.serverManager.modelPath;
+ const modelId = modelManager.currentServerModelId;
await modelManager.serverManager.stop();
- if (modelPath) {
- await modelManager.serverManager.start(modelPath);
+ if (modelId) {
+ await modelManager.prewarmServer(modelId);
}
}
}
@@ -3661,7 +3661,7 @@ class IPCHandlers {
const modelPath = require("path").join(modelManager.modelsDir, modelInfo.model.fileName);
- await modelManager.serverManager.start(modelPath, { threads: 4 });
+ await modelManager.serverManager.start(modelPath, modelManager.serverOptions(modelInfo));
modelManager.currentServerModelId = modelId;
this.environmentManager.saveAllKeysToEnvFile().catch(() => {});
diff --git a/src/helpers/llamaServer.js b/src/helpers/llamaServer.js
index 892a43886..f2a20a46c 100644
--- a/src/helpers/llamaServer.js
+++ b/src/helpers/llamaServer.js
@@ -19,6 +19,7 @@ const HEALTH_CHECK_TIMEOUT_MS = 2000;
const STARTUP_POLL_INTERVAL_MS = 500;
const HEALTH_CHECK_FAILURE_THRESHOLD = 3;
const IDLE_TIMEOUT_MS = 5 * 60 * 1000;
+const DEFAULT_CONTEXT_SIZE = 4096;
class LlamaServerManager {
constructor() {
@@ -137,11 +138,15 @@ class LlamaServerManager {
String(this.port),
"--threads",
String(options.threads || 4),
+ // Unset, this defaults to the model's full trained context (128K+),
+ // whose KV cache can exceed total RAM with --fit disabled. See #1203.
+ "--ctx-size",
+ String(options.contextSize || DEFAULT_CONTEXT_SIZE),
"--jinja",
];
if (process.platform === "darwin") {
- const args = [...baseArgs, "--n-gpu-layers", "99"];
+ const args = [...baseArgs, "--n-gpu-layers", String(options.gpuLayers ?? 99)];
await this._startWithBinary(
binaryPaths.default,
args,
@@ -163,7 +168,7 @@ class LlamaServerManager {
}
async _startWithGpuFallback(binaryPaths, baseArgs, options) {
- const gpuArgs = [...baseArgs, "--n-gpu-layers", "99"];
+ const gpuArgs = [...baseArgs, "--n-gpu-layers", String(options.gpuLayers ?? 99)];
const cpuArgs = baseArgs;
if (binaryPaths.vulkan) {
diff --git a/src/helpers/modelManagerBridge.js b/src/helpers/modelManagerBridge.js
index 5bd3e9d1d..883405758 100644
--- a/src/helpers/modelManagerBridge.js
+++ b/src/helpers/modelManagerBridge.js
@@ -15,6 +15,11 @@ const debugLogger = require("./debugLogger");
const MIN_FILE_SIZE = 1_000_000; // 1MB minimum for valid model files
+// Bounds the KV cache — registry contextLength is the full trained context
+// (128K+), which can exceed total RAM (#1203). Uniform for all start paths:
+// start() won't restart a ready server when only options change.
+const SERVER_CONTEXT_SIZE = 16384;
+
function getLocalProviders() {
return modelRegistryData.localProviders || [];
}
@@ -167,6 +172,17 @@ class ModelManager {
return null;
}
+ serverOptions(modelInfo) {
+ return {
+ contextSize: Math.min(
+ SERVER_CONTEXT_SIZE,
+ modelInfo.model.contextLength || SERVER_CONTEXT_SIZE
+ ),
+ threads: 4,
+ gpuLayers: 99,
+ };
+ }
+
async downloadModel(modelId, onProgress) {
this.ensureInitialized();
const modelInfo = this.findModelById(modelId);
@@ -366,11 +382,7 @@ class ModelManager {
serverReady: this.serverManager.ready,
});
- await this.serverManager.start(modelPath, {
- contextSize: options.contextSize || modelInfo.model.contextLength || 4096,
- threads: options.threads || 4,
- gpuLayers: 99,
- });
+ await this.serverManager.start(modelPath, this.serverOptions(modelInfo));
this.currentServerModelId = modelId;
debugLogger.logReasoning("INFERENCE_SERVER_STARTED", {
@@ -440,11 +452,7 @@ class ModelManager {
if (!this.serverManager.isAvailable()) return false;
try {
- await this.serverManager.start(modelPath, {
- contextSize: modelInfo.model.contextLength || 4096,
- threads: 4,
- gpuLayers: 99,
- });
+ await this.serverManager.start(modelPath, this.serverOptions(modelInfo));
this.currentServerModelId = modelId;
debugLogger.info("llama-server pre-warmed", { modelId });
return true;
diff --git a/src/services/localReasoningBridge.js b/src/services/localReasoningBridge.js
index 3dfe378de..cf9d7ec9f 100644
--- a/src/services/localReasoningBridge.js
+++ b/src/services/localReasoningBridge.js
@@ -37,8 +37,6 @@ class LocalReasoningService {
topK: config.topK || 40,
topP: config.topP || 0.9,
repeatPenalty: config.repeatPenalty || 1.1,
- contextSize: config.contextSize || 4096,
- threads: config.threads || 4,
systemPrompt: config.systemPrompt || "",
disableThinking: config.disableThinking !== false,
};