Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
const { spawn } = require("child_process");
const fs = require("fs");
const path = require("path");
const http = require("http");
const debugLogger = require("./debugLogger");
const { killProcess } = require("../utils/process");
const { isPortAvailable } = require("../utils/serverUtils");
const { getSafeTempDir } = require("./safeTempDir");
const { app } = require("electron");
const sidecarPidFile = require("./sidecarPidFile");
const { BIN_SUBDIR: LLAMA_VULKAN_BIN_SUBDIR } = require("./llamaVulkanManager");
const { BASELINE_CONTEXT_SIZE } = require("./llamaContextPolicy");
// Range kept clear of cliBridge (8200-8219) to avoid port-bind collisions.
const PORT_RANGE_START = 8221;
const PORT_RANGE_END = 8240;
const STARTUP_TIMEOUT_MS = 120000;
const VULKAN_STARTUP_TIMEOUT_MS = 120000;
const HEALTH_CHECK_INTERVAL_MS = 5000;
const HEALTH_CHECK_TIMEOUT_MS = 2000;
const STARTUP_POLL_INTERVAL_MS = 500;
const HEALTH_CHECK_FAILURE_THRESHOLD = 3;
const IDLE_TIMEOUT_MS = 5 * 60 * 1000;
const DEFAULT_CONTEXT_SIZE = 4096;
const PREFLIGHT_TIMEOUT_MS = 10000;
/**
* llama-server reports a prompt that overflows the context as a 400 whose body
* is a JSON blob. Surfacing that verbatim is how customers ended up reading
* llama.cpp internals in a toast (#2142), so it becomes a typed error carrying
* the two numbers callers actually need.
*/
/** Same argv with --ctx-size rewritten; used by the context step-down rung. */
function withContextSize(args, contextSize) {
const index = args.indexOf("--ctx-size");
if (index === -1) return args;
const copy = [...args];
copy[index + 1] = String(contextSize);
return copy;
}
function buildResponseError(statusCode, body) {
if (statusCode === 400) {
try {
const parsed = JSON.parse(body)?.error;
const isOverflow =
parsed?.type === "exceed_context_size_error" ||
/exceeds the available context size/i.test(parsed?.message || "");
if (isOverflow) {
const neededTokens = Number(parsed.n_prompt_tokens) || null;
const maxContextTokens = Number(parsed.n_ctx) || null;
const error = new Error(
neededTokens && maxContextTokens
? `The request needs ${neededTokens} tokens of context but the model is running with ${maxContextTokens}.`
: "The request is longer than the context this model is running with."
);
error.code = "CONTEXT_TOO_LARGE";
error.neededTokens = neededTokens;
error.maxContextTokens = maxContextTokens;
return error;
}
} catch {
// Not JSON, or not the shape we expect: fall through to the generic error.
}
}
return new Error(`llama-server returned status ${statusCode}: ${body}`);
}
class LlamaServerManager {
constructor() {
this.keepResident = false;
this.process = null;
this.port = null;
this.ready = false;
this.activeRequest = null;
this.modelPath = null;
// draftModelPath is the REQUESTED drafter (stable across identical requests, drives
// the start() restart check); activeDraftModelPath is the one that actually loaded.
this.draftModelPath = null;
this.activeDraftModelPath = null;
// The context that actually loaded, which the GPU ladder may step down
// below the requested one.
this.activeContextSize = null;
// The context the running server was started with. Grows on demand and is
// only reset by stop(), so a short request cannot shrink a window a long
// one just paid to open. See #2142.
this.contextSize = null;
this.startupPromise = null;
this.healthCheckInterval = null;
this.healthCheckFailures = 0;
this.cachedServerBinaryPaths = null;
this.activeBackend = null;
this.idleTimer = null;
}
getServerBinaryPaths() {
if (this.cachedServerBinaryPaths) return this.cachedServerBinaryPaths;
const platform = process.platform;
const arch = process.arch;
const platformArch = `${platform}-${arch}`;
const ext = platform === "win32" ? ".exe" : "";
const resolveBinary = (name) => {
const candidates = [];
if (process.resourcesPath) {
candidates.push(path.join(process.resourcesPath, "bin", name));
}
candidates.push(path.join(__dirname, "..", "..", "resources", "bin", name));
for (const candidate of candidates) {
try {
if (fs.existsSync(candidate)) {
fs.statSync(candidate);
return candidate;
}
} catch {
// Can't access binary
}
}
return null;
};
let paths;
if (platform === "darwin") {
const defaultBin =
resolveBinary(`llama-server-${platformArch}`) || resolveBinary(`llama-server${ext}`);
paths = defaultBin ? { default: defaultBin } : {};
} else {
const vulkanName = `llama-server-vulkan${ext}`;
let vulkanBin = null;
try {
// Installed by LlamaVulkanManager into its own pack directory
const vulkanPath = path.join(
app.getPath("userData"),
"bin",
LLAMA_VULKAN_BIN_SUBDIR,
vulkanName
);
if (fs.existsSync(vulkanPath)) vulkanBin = vulkanPath;
} catch {}
const cpuBin =
resolveBinary(`llama-server-${platformArch}-cpu${ext}`) ||
resolveBinary(`llama-server-${platformArch}${ext}`) ||
resolveBinary(`llama-server${ext}`);
paths = {};
if (vulkanBin) paths.vulkan = vulkanBin;
if (cpuBin) paths.cpu = cpuBin;
}
// Only cache hits: a miss may be transient (binary downloaded after
// startup), and caching it would require an app restart to recover.
if (Object.keys(paths).length > 0) {
this.cachedServerBinaryPaths = paths;
}
return paths;
}
isAvailable() {
const paths = this.getServerBinaryPaths();
return Object.keys(paths).length > 0;
}
async findAvailablePort() {
for (let port = PORT_RANGE_START; port <= PORT_RANGE_END; port++) {
if (await isPortAvailable(port)) return port;
}
throw new Error(`No available ports in range ${PORT_RANGE_START}-${PORT_RANGE_END}`);
}
async start(modelPath, options = {}) {
if (this.startupPromise) return this.startupPromise;
// A change in drafter presence for the same model must still restart the
// server so the new speculative-decoding flags take effect.
const requestedDraftPath = options.draftModelPath || null;
const requestedContextSize = options.contextSize || DEFAULT_CONTEXT_SIZE;
if (
this.ready &&
this.modelPath === modelPath &&
this.draftModelPath === requestedDraftPath &&
requestedContextSize <= (this.contextSize || 0)
) {
// Streaming chat reaches the port directly and never goes through
// inference(), so asking for the running server is its only activity.
this.resetIdleTimer();
return;
}
if (this.process) {
await this.stop();
}
// Residency may have been turned off while the old server stopped.
this.clearIdleTimer();
this.startupPromise = this._doStart(modelPath, options);
try {
await this.startupPromise;
// The ladder may have stepped the context down to get a GPU rung to
// start; record what loaded, not what was asked for.
this.contextSize = this.activeContextSize ?? requestedContextSize;
} finally {
this.startupPromise = null;
}
}
_buildBaseArgs(modelPath, port, options = {}) {
const args = [
"--model",
modelPath,
"--host",
"127.0.0.1",
"--port",
String(port),
"--threads",
String(options.threads || 4),
// Unset, this defaults to the model's full trained context (128K+),
// whose KV cache can exceed total RAM with --fit disabled. See #1203.
// Sized per request against a memory budget by llamaContextPolicy.
"--ctx-size",
String(options.contextSize || DEFAULT_CONTEXT_SIZE),
"--jinja",
];
// llama-server otherwise reserves 8192 MiB of host RAM for the prompt
// cache, on top of the KV cache, which would undo the memory budget. Cap
// it rather than disabling it: 0 forces a full re-prefill every request.
if (options.cacheRamMiB) args.push("--cache-ram", String(options.cacheRamMiB));
return args;
}
async _doStart(modelPath, options = {}) {
const binaryPaths = this.getServerBinaryPaths();
if (Object.keys(binaryPaths).length === 0) throw new Error("llama-server binary not found");
if (!fs.existsSync(modelPath)) throw new Error(`Model file not found: ${modelPath}`);
this.port = await this.findAvailablePort();
this.modelPath = modelPath;
// Store the REQUESTED drafter so start() compares against a stable value across
// identical requests; activeDraftModelPath tracks what actually loaded (see ctor).
this.draftModelPath = options.draftModelPath || null;
this.activeDraftModelPath = null;
this.activeContextSize = options.contextSize || DEFAULT_CONTEXT_SIZE;
const baseArgs = this._buildBaseArgs(modelPath, this.port, options);
// Draft flags stay separate from baseArgs so the fallback ladder can retry without
// them when a stale (pre-b9763) binary rejects the MTP args at parse time.
const draftArgs = options.draftModelPath
? [
"--model-draft",
options.draftModelPath,
"--spec-type",
"draft-mtp",
"--spec-draft-n-max",
"3",
]
: [];
if (process.platform === "darwin") {
// The metal binary is always the bundled pin, so it never rejects the draft flags.
const args = [...baseArgs, "--n-gpu-layers", String(options.gpuLayers ?? 99), ...draftArgs];
await this._startWithBinary(
binaryPaths.default,
args,
this._buildEnv(binaryPaths.default),
STARTUP_TIMEOUT_MS
);
this.activeBackend = "metal";
this.activeDraftModelPath = this.draftModelPath;
} else {
await this._startWithGpuFallback(binaryPaths, baseArgs, options, draftArgs);
}
this.startHealthCheck();
this.resetIdleTimer();
debugLogger.info("llama-server started successfully", {
port: this.port,
model: path.basename(modelPath),
backend: this.activeBackend,
mtp: this.activeDraftModelPath !== null,
});
}
async _startWithGpuFallback(binaryPaths, baseArgs, options, draftArgs = []) {
const gpuArgs = [...baseArgs, "--n-gpu-layers", String(options.gpuLayers ?? 99)];
const cpuArgs = baseArgs;
const hasDraft = draftArgs.length > 0;
const requestedContext = options.contextSize || DEFAULT_CONTEXT_SIZE;
// Degrade ladder: GPU+MTP, then GPU alone (a live GPU beats speculation), then
// CPU+MTP (the bundled pin normally accepts the flags), then plain CPU. The
// no-draft rungs collapse into their twins when no drafter is declared, so a
// drafterless start keeps today's exact single vulkan->cpu fallback.
const rungs = [
{
backend: "vulkan",
name: "Vulkan",
binary: binaryPaths.vulkan,
args: [...gpuArgs, ...draftArgs],
mtp: hasDraft,
timeout: VULKAN_STARTUP_TIMEOUT_MS,
attemptMsg: "Attempting Vulkan backend startup",
},
{
backend: "vulkan",
name: "Vulkan",
binary: binaryPaths.vulkan,
args: gpuArgs,
mtp: false,
noDraft: true,
timeout: VULKAN_STARTUP_TIMEOUT_MS,
attemptMsg: "Attempting Vulkan backend startup",
},
// A context sized against system RAM can still be too big for a
// discrete GPU's own memory, which the app does not probe. Halving it is
// far cheaper than the 10-30x cost of dropping to CPU, so try that
// first. Omitted entirely when there is nothing to step down to, so a
// baseline start keeps today's exact ladder.
...(requestedContext > BASELINE_CONTEXT_SIZE && binaryPaths.vulkan
? [
{
backend: "vulkan",
name: "Vulkan (reduced context)",
binary: binaryPaths.vulkan,
args: withContextSize(gpuArgs, BASELINE_CONTEXT_SIZE),
mtp: false,
contextSize: BASELINE_CONTEXT_SIZE,
timeout: VULKAN_STARTUP_TIMEOUT_MS,
attemptMsg: "Attempting Vulkan backend startup with a reduced context",
},
]
: []),
{
backend: "cpu",
name: "CPU",
binary: binaryPaths.cpu,
args: [...cpuArgs, ...draftArgs],
mtp: hasDraft,
timeout: STARTUP_TIMEOUT_MS,
attemptMsg: "Starting with CPU backend",
},
{
backend: "cpu",
name: "CPU",
binary: binaryPaths.cpu,
args: cpuArgs,
mtp: false,
noDraft: true,
timeout: STARTUP_TIMEOUT_MS,
attemptMsg: "Starting with CPU backend",
},
];
const ladder = rungs.filter((rung) => rung.binary && !(rung.noDraft && !hasDraft));
if (ladder.length === 0) throw new Error("No CPU llama-server binary available");
let lastError = null;
for (let i = 0; i < ladder.length; i++) {
const rung = ladder[i];
const next = ladder[i + 1];
try {
debugLogger.debug(rung.attemptMsg);
await this._startWithBinary(
rung.binary,
rung.args,
this._buildEnv(rung.binary),
rung.timeout
);
this.activeBackend = rung.backend;
this.activeDraftModelPath = rung.mtp ? this.draftModelPath : null;
// Without this the preflight would believe the full context is
// available and send a prompt the server cannot take.
this.activeContextSize = rung.contextSize ?? requestedContext;
this.contextSize = this.activeContextSize;
return;
} catch (err) {
lastError = err;
if (next) {
debugLogger.warn(`${rung.name} backend failed, falling back to ${next.name}`, {
error: err.message,
});
await this._killCurrentProcess();
this.port = await this.findAvailablePort();
}
}
}
throw lastError || new Error("No CPU llama-server binary available");
}
_buildEnv(binaryPath) {
const binDir = path.dirname(binaryPath);
const env = { ...process.env };
if (process.platform === "darwin") {
env.DYLD_LIBRARY_PATH = binDir + (env.DYLD_LIBRARY_PATH ? `:${env.DYLD_LIBRARY_PATH}` : "");
} else if (process.platform === "linux") {
env.LD_LIBRARY_PATH = binDir + (env.LD_LIBRARY_PATH ? `:${env.LD_LIBRARY_PATH}` : "");
} else if (process.platform === "win32") {
env.PATH = binDir + (env.PATH ? `;${env.PATH}` : "");
}
// Select GPU by UUID + PCI_BUS_ID order so the device is unambiguous. See #531.
env.CUDA_DEVICE_ORDER = "PCI_BUS_ID";
if (process.env.INTELLIGENCE_GPU_UUID) {
env.CUDA_VISIBLE_DEVICES = process.env.INTELLIGENCE_GPU_UUID;
}
// Disable llama.cpp auto-fit memory probing (adds ~70s to startup). Set via env
// so builds without --fit ignore it instead of erroring. See LLAMA_ARG_FIT.
env.LLAMA_ARG_FIT = process.env.LLAMA_ARG_FIT || "off";
return env;
}
_startWithBinary(binaryPath, args, env, timeoutMs) {
return new Promise((resolve, reject) => {
debugLogger.debug("Spawning llama-server", { binary: binaryPath, port: this.port, args });
this.process = spawn(binaryPath, args, {
stdio: ["ignore", "pipe", "pipe"],
windowsHide: true,
cwd: getSafeTempDir(),
env,
detached: process.platform !== "win32",
});
sidecarPidFile.write("llama", this.process.pid);
let stderrBuffer = "";
let exitCode = null;
let exitSignal = null;
let settled = false;
const settle = (fn) => {
if (settled) return;
settled = true;
fn();
};
this.process.stdout.on("data", (data) => {
debugLogger.debug("llama-server stdout", { data: data.toString().trim() });
});
this.process.stderr.on("data", (data) => {
stderrBuffer += data.toString();
debugLogger.debug("llama-server stderr", { data: data.toString().trim() });
});
this.process.on("error", (error) => {
debugLogger.error("llama-server process error", { error: error.message });
this.ready = false;
settle(() => reject(new Error(`Failed to spawn llama-server: ${error.message}`)));
});
this.process.on("close", (code, signal) => {
exitCode = code;
exitSignal = signal;
debugLogger.debug("llama-server process exited", { code, signal });
this.ready = false;
this.process = null;
this.stopHealthCheck();
sidecarPidFile.clear("llama");
});
const getProcessInfo = () => ({ stderr: stderrBuffer, exitCode, exitSignal });
const startTime = Date.now();
let pollCount = 0;
const poll = async () => {
if (settled) return;
if (!this.process || this.process.killed) {
const info = getProcessInfo();
const signal = info.exitSignal;
const diagParts = [];
if (signal) diagParts.push(`signal: ${signal}`);
else if (info.exitCode !== null && info.exitCode !== undefined)
diagParts.push(`exit code: ${info.exitCode}`);
const oomHint =
signal === "SIGKILL"
? " — the process was killed by the OS, likely due to insufficient memory. Try a smaller/more quantized model, or reduce the context size."
: "";
const stderr = info.stderr ? info.stderr.trim().slice(-800) : "";
const diagStr = diagParts.length ? ` (${diagParts.join(", ")})` : "";
settle(() =>
reject(
new Error(
`llama-server process died during startup${diagStr}${oomHint}${stderr ? `\nProcess output: ${stderr}` : ""}`
)
)
);
return;
}
pollCount++;
if (await this.checkHealth()) {
this.ready = true;
debugLogger.debug("llama-server ready", {
startupTimeMs: Date.now() - startTime,
pollCount,
});
settle(() => resolve());
return;
}
if (Date.now() - startTime >= timeoutMs) {
settle(() => reject(new Error(`llama-server failed to start within ${timeoutMs}ms`)));
return;
}
setTimeout(poll, STARTUP_POLL_INTERVAL_MS);
};
poll();
});
}
async _killCurrentProcess() {
if (!this.process) return;
this.stopHealthCheck();
try {
killProcess(this.process, "SIGTERM");
await new Promise((resolve) => {
const timeout = setTimeout(() => {
if (this.process) killProcess(this.process, "SIGKILL");
resolve();
}, 5000);
if (this.process) {
this.process.once("close", () => {
clearTimeout(timeout);
resolve();
});
} else {
clearTimeout(timeout);
resolve();
}
});
} catch (error) {
debugLogger.error("Error killing llama-server process", { error: error.message });
}
this.process = null;
this.ready = false;
}
checkHealth() {
return new Promise((resolve) => {
const req = http.request(
{
hostname: "127.0.0.1",
port: this.port,
path: "/health",
method: "GET",
timeout: HEALTH_CHECK_TIMEOUT_MS,
},
(res) => {
resolve(res.statusCode === 200);
res.resume();
}
);
req.on("error", () => resolve(false));
req.on("timeout", () => {
req.destroy();
resolve(false);
});
req.end();
});
}
startHealthCheck() {
this.stopHealthCheck();
this.healthCheckFailures = 0;
this.healthCheckInterval = setInterval(async () => {
try {
if (!this.process) {
this.stopHealthCheck();
return;
}
if (await this.checkHealth()) {
this.healthCheckFailures = 0;
} else {
this.healthCheckFailures++;
if (this.healthCheckFailures >= HEALTH_CHECK_FAILURE_THRESHOLD) {
debugLogger.warn("llama-server health check failed", {
consecutiveFailures: this.healthCheckFailures,
});
this.ready = false;
}
}
} catch (err) {
debugLogger.error("Health check error", { error: err.message });
}
}, HEALTH_CHECK_INTERVAL_MS);
}
stopHealthCheck() {
if (this.healthCheckInterval) {
clearInterval(this.healthCheckInterval);
this.healthCheckInterval = null;
}
}
// The "Keep model loaded" setting. Every window resends it on load and when
// its local-model or policy inputs change, so only a change may touch the timer.
setKeepResident(keepResident) {
if (this.keepResident === keepResident) return;
this.keepResident = keepResident;
if (keepResident) this.clearIdleTimer();
// A start in progress arms the timer itself once the server is ready.
else if (this.process && !this.startupPromise) this.resetIdleTimer();
}
resetIdleTimer() {
this.clearIdleTimer();
// Explicit stop, model/context changes and app shutdown still release it.
if (this.keepResident) return;
const timer = setTimeout(async () => {
// Streaming chat talks to the port directly, so only the server knows
// whether an answer that outlived the timeout is still being generated.
const slots = await this._requestJson("/slots");
if (this.idleTimer !== timer) return;
if (Array.isArray(slots) && slots.some((slot) => slot.is_processing)) {
this.resetIdleTimer();
return;
}
debugLogger.info("llama-server idle timeout reached, stopping to free VRAM", {
timeoutMs: IDLE_TIMEOUT_MS,
model: this.modelPath ? path.basename(this.modelPath) : null,
});
this.stop();
}, IDLE_TIMEOUT_MS);
// Freeing an idle server is housekeeping; it must never hold the process open.
timer.unref();
this.idleTimer = timer;
}
clearIdleTimer() {
if (this.idleTimer) {
clearTimeout(this.idleTimer);
this.idleTimer = null;
}
}
/**
* Small JSON round-trip against the running server.
*
* Resolves null on any failure rather than rejecting: these calls exist to
* make a decision more precise, so a broken one must degrade to the estimate
* and never fail the user's request.
*/
_requestJson(path, body = null, timeoutMs = PREFLIGHT_TIMEOUT_MS) {
return new Promise((resolve) => {
const payload = body === null ? null : JSON.stringify(body);
const req = http.request(
{
hostname: "127.0.0.1",
port: this.port,
path,
method: payload === null ? "GET" : "POST",
headers: payload
? {
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(payload),
}
: {},
timeout: timeoutMs,
},
(res) => {
let data = "";
res.on("data", (chunk) => {
data += chunk;
});
res.on("end", () => {
if (res.statusCode !== 200) return resolve(null);
try {
resolve(JSON.parse(data));
} catch {
resolve(null);
}
});
}
);
req.on("error", () => resolve(null));
req.on("timeout", () => {
req.destroy();
resolve(null);
});
if (payload) req.write(payload);
req.end();
});
}
/**
* The context window actually in force, which is not always what we asked
* for: a concurrent caller can join an in-flight start that used a smaller
* size. Returns null when it cannot be read.
*/
async usableContextSize() {
if (!this.ready || !this.process) return null;
const props = await this._requestJson("/props");
const contextSize = props?.default_generation_settings?.n_ctx;
return Number.isFinite(contextSize) && contextSize > 0 ? contextSize : null;
}
/**
* Exact token count for the prompt as the server will actually see it, chat
* template included. Roughly 15 ms for an 85 KB prompt, which is cheap
* enough to make the difference between a right-sized window and a 400.
*/
async countPromptTokens(messages, options = {}) {
if (!this.ready || !this.process) return null;
const payload = { messages };
// Must mirror inference() exactly, or we would price a different prompt
// than the one we go on to send.
if (options.disableThinking !== false) {
payload.chat_template_kwargs = { enable_thinking: false };
}
const templated = await this._requestJson("/apply-template", payload);
if (typeof templated?.prompt !== "string") return null;
const tokenized = await this._requestJson("/tokenize", { content: templated.prompt });
return Array.isArray(tokenized?.tokens) ? tokenized.tokens.length : null;
}
async inference(messages, options = {}) {
if (!this.ready || !this.process) {
throw new Error("llama-server is not running");
}
this.clearIdleTimer();
const requestBody = {
messages,
temperature: options.temperature ?? 0.7,
max_tokens: options.max_tokens ?? 512,
stream: false,
};
if (options.responseFormat) requestBody.response_format = options.responseFormat;
// Without this, Qwen chat templates think into `reasoning_content` first
// and can spend the whole budget there. Non-Qwen templates ignore it.
const suppressThinking = options.disableThinking !== false;
if (suppressThinking) {
requestBody.chat_template_kwargs = { enable_thinking: false };
}
const body = JSON.stringify(requestBody);
let req;
return new Promise((resolve, reject) => {
const startTime = Date.now();
req = http.request(
{
hostname: "127.0.0.1",
port: this.port,
path: "/v1/chat/completions",
method: "POST",
headers: {
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(body),
},
timeout: 300000,
},
(res) => {
let data = "";
res.on("data", (chunk) => {
data += chunk;
});
res.on("end", () => {
debugLogger.debug("llama-server inference completed", {
statusCode: res.statusCode,
elapsed: Date.now() - startTime,
});
if (res.statusCode !== 200) {
reject(buildResponseError(res.statusCode, data));
return;
}
try {
const response = JSON.parse(data);
const choice = response.choices?.[0];
const message = choice?.message;
const truncated = ["length", "max_tokens"].includes(choice?.finish_reason);
// A cut-off reply with no content spent its whole budget reasoning
// (#2187): there is no answer to return, whichever field the
// reasoning sits in. Lenient callers still accept a partial answer.
if (truncated && (options.requireCompleteOutput || !message?.content?.trim())) {
// The renderer maps the code to the cleanup toast's wording (#2091).
const error = new Error("Model output was truncated");
error.code = "OUTPUT_TRUNCATED";
reject(error);
return;
}
if (truncated) {
debugLogger.warn("llama-server reply was cut off at max_tokens", {
maxTokens: requestBody.max_tokens,
});
}
if (options.responseFormat) {
if (choice?.finish_reason !== "stop") {
reject(
Object.assign(new Error("Model completion could not be verified"), {
code: "OUTPUT_COMPLETION_UNVERIFIED",
})
);
return;
}
resolve(typeof message?.content === "string" ? message.content : "");
return;
}
// Some builds still route a suppressed-thinking answer into
// `reasoning_content` (#809). With thinking on, that field is the
// reasoning itself, which must never stand in for the answer.
const text =
message?.content || (suppressThinking && message?.reasoning_content) || "";
resolve(text.trim());
} catch (e) {
reject(new Error(`Failed to parse llama-server response: ${e.message}`));
}
});
}
);
req.on("error", (error) => {
reject(new Error(`llama-server request failed: ${error.message}`));
});
req.on("timeout", () => {
req.destroy();
reject(new Error("llama-server request timed out"));
});
this.activeRequest = req;
req.write(body);
req.end();
}).finally(() => {
if (this.activeRequest === req) this.activeRequest = null;
this.resetIdleTimer();
});
}
// Closing the connection is what makes llama-server stop generating.
cancelInference() {
this.activeRequest?.destroy(new Error("cancelled"));
}
async stop() {
this.clearIdleTimer();
this.stopHealthCheck();
if (!this.process) {
this.ready = false;
this.contextSize = null;
return;
}
debugLogger.debug("Stopping llama-server");
try {
killProcess(this.process, "SIGTERM");
await new Promise((resolve) => {
const timeout = setTimeout(() => {
if (this.process) {
killProcess(this.process, "SIGKILL");
}
resolve();
}, 5000);
if (this.process) {
this.process.once("close", () => {
clearTimeout(timeout);
resolve();
});
} else {
clearTimeout(timeout);
resolve();
}
});
} catch (error) {
debugLogger.error("Error stopping llama-server", { error: error.message });
}
this.process = null;
this.ready = false;
this.port = null;
this.modelPath = null;
this.draftModelPath = null;
this.activeDraftModelPath = null;
this.activeBackend = null;
this.contextSize = null;
this.activeContextSize = null;
}
getStatus() {
return {
available: this.isAvailable(),
running: this.ready && this.process !== null,
port: this.port,
modelPath: this.modelPath,
modelName: this.modelPath ? path.basename(this.modelPath, ".gguf") : null,
backend: this.activeBackend,
gpuAccelerated: this.activeBackend === "vulkan" || this.activeBackend === "metal",
};
}
resetGpuDetection() {
this.activeBackend = null;
this.cachedServerBinaryPaths = null;
}
}
module.exports = LlamaServerManager;