Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

16.5 KB
Raw Blame
const test = require("node:test");
const assert = require("node:assert/strict");
const http = require("node:http");

const LlamaServerManager = require("../../src/helpers/llamaServer.js");

// A manager whose spawn is stubbed at the _doStart boundary, recording the
// options each start was asked for.
function makeManager() {
  const manager = new LlamaServerManager();
  const starts = [];
  manager._doStart = async (modelPath, options = {}) => {
    manager.ready = true;
    manager.modelPath = modelPath;
    manager.draftModelPath = options.draftModelPath || null;
    starts.push({ modelPath, contextSize: options.contextSize });
  };
  return { manager, starts };
}

// --- grow-only restart ---------------------------------------------------
//
// Before #2142 the context was a single constant, so start() could ignore
// options entirely once a server was ready. Now that the context varies per
// request, it has to restart to grow — but only to grow, or a long note
// followed by a short dictation would restart the server twice.

test("a larger context restarts the server", async () => {
  const { manager, starts } = makeManager();

  await manager.start("/models/main.gguf", { contextSize: 16384 });
  assert.equal(starts.length, 1);

  await manager.start("/models/main.gguf", { contextSize: 32768 });
  assert.equal(starts.length, 2);
  assert.equal(manager.contextSize, 32768);
});

test("an equal or smaller context reuses the running server", async () => {
  const { manager, starts } = makeManager();

  await manager.start("/models/main.gguf", { contextSize: 32768 });
  assert.equal(starts.length, 1);

  // Exactly the same → no restart.
  await manager.start("/models/main.gguf", { contextSize: 32768 });
  assert.equal(starts.length, 1);

  // Smaller → no restart, and the larger window is kept. This is what stops a
  // short dictation right after a long note from costing a second restart.
  await manager.start("/models/main.gguf", { contextSize: 16384 });
  assert.equal(starts.length, 1);
  assert.equal(manager.contextSize, 32768);
});

test("growing the context still restarts when the drafter is unchanged", async () => {
  const { manager, starts } = makeManager();

  await manager.start("/models/main.gguf", { contextSize: 16384, draftModelPath: "/d.gguf" });
  await manager.start("/models/main.gguf", { contextSize: 65536, draftModelPath: "/d.gguf" });

  assert.equal(starts.length, 2);
  assert.equal(manager.contextSize, 65536);
});

test("a context grown for one model does not carry over to another", async () => {
  const { manager, starts } = makeManager();

  await manager.start("/models/big.gguf", { contextSize: 65536 });
  await manager.start("/models/other.gguf", { contextSize: 16384 });

  assert.equal(starts.length, 2);
  assert.equal(manager.contextSize, 16384, "the new model starts at its own size");
});

test("stopping the server forgets the grown context", async () => {
  const { manager } = makeManager();
  manager._killCurrentProcess = async () => {};
  manager.process = {};

  await manager.start("/models/main.gguf", { contextSize: 65536 });
  assert.equal(manager.contextSize, 65536);

  await manager.stop();
  assert.equal(manager.contextSize, null);
});

// --- spawn arguments -----------------------------------------------------

test("every GPU ladder rung inherits the requested context and cache bounds", async () => {
  const manager = new LlamaServerManager();
  const seen = [];
  manager._buildEnv = () => ({});
  manager._killCurrentProcess = async () => {};
  manager.findAvailablePort = async () => 8221;
  manager.draftModelPath = null;
  manager._startWithBinary = async (binary, args) => {
    seen.push(args);
    throw new Error("stub fail");
  };

  const baseArgs = manager._buildBaseArgs("/models/main.gguf", 8221, {
    contextSize: 32768,
    cacheRamMiB: 1024,
  });
  await assert.rejects(() =>
    manager._startWithGpuFallback({ vulkan: "/bin/vulkan", cpu: "/bin/cpu" }, baseArgs, {}, [])
  );

  assert.ok(seen.length >= 2, "expected the ladder to try more than one rung");
  for (const args of seen) {
    assert.equal(args[args.indexOf("--ctx-size") + 1], "32768");
    // llama-server defaults --cache-ram to 8192 MiB of host RAM on top of the
    // KV cache. Left unbounded it would undo the memory budget entirely.
    assert.equal(args[args.indexOf("--cache-ram") + 1], "1024");
  }
});

// --- context-overflow errors --------------------------------------------

test("a context overflow is reported as a typed error, not as llama.cpp JSON", async () => {
  const overflowBody = JSON.stringify({
    error: {
      code: 400,
      message: "request (20514 tokens) exceeds the available context size (16384 tokens)",
      type: "exceed_context_size_error",
      n_prompt_tokens: 20514,
      n_ctx: 16384,
    },
  });

  const server = http.createServer((req, res) => {
    res.writeHead(400, { "Content-Type": "application/json" });
    res.end(overflowBody);
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
  const manager = new LlamaServerManager();
  manager.ready = true;
  manager.process = {};
  manager.port = server.address().port;

  try {
    await assert.rejects(
      () => manager.inference([{ role: "user", content: "hi" }], {}),
      (error) => {
        assert.equal(error.code, "CONTEXT_TOO_LARGE");
        assert.equal(error.neededTokens, 20514);
        assert.equal(error.maxContextTokens, 16384);
        // The whole point: a customer must never be shown a JSON blob.
        assert.ok(!error.message.includes("{"), `raw JSON leaked: ${error.message}`);
        return true;
      }
    );
  } finally {
    manager.clearIdleTimer();
    await new Promise((resolve) => server.close(resolve));
  }
});

test("other llama-server failures keep their existing shape", async () => {
  const server = http.createServer((req, res) => {
    res.writeHead(500, { "Content-Type": "text/plain" });
    res.end("internal error");
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
  const manager = new LlamaServerManager();
  manager.ready = true;
  manager.process = {};
  manager.port = server.address().port;

  try {
    await assert.rejects(
      () => manager.inference([{ role: "user", content: "hi" }], {}),
      (error) => {
        assert.equal(error.code, undefined);
        assert.match(error.message, /llama-server returned status 500/);
        return true;
      }
    );
  } finally {
    manager.clearIdleTimer();
    await new Promise((resolve) => server.close(resolve));
  }
});

// --- measuring the running server ----------------------------------------
//
// The estimate only picks the size the server starts at. These two calls are
// what make the decision exact: /props reports the window actually in force
// (which can differ from what we asked for), and /apply-template + /tokenize
// price the real prompt, chat template and all.

function withStubServer(routes) {
  // A Map rather than indexing the object: req.url is request-controlled, so a
  // plain lookup answers /toString or /constructor with an inherited
  // Object.prototype method and then calls it.
  const routeTable = new Map(Object.entries(routes));
  return async (run) => {
    const server = http.createServer((req, res) => {
      const handler = routeTable.get(req.url);
      if (typeof handler !== "function") {
        res.writeHead(404).end();
        return;
      }
      let body = "";
      req.on("data", (chunk) => {
        body += chunk;
      });
      req.on("end", () => {
        const { status = 200, payload } = handler(body);
        res.writeHead(status, { "Content-Type": "application/json" });
        res.end(JSON.stringify(payload));
      });
    });
    await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

    const manager = new LlamaServerManager();
    manager.ready = true;
    manager.process = {};
    manager.port = server.address().port;
    try {
      await run(manager);
    } finally {
      manager.clearIdleTimer();
      await new Promise((resolve) => server.close(resolve));
    }
  };
}

test("usableContextSize reports the window the server actually has", async () => {
  await withStubServer({
    "/props": () => ({ payload: { default_generation_settings: { n_ctx: 32768 } } }),
  })(async (manager) => {
    assert.equal(await manager.usableContextSize(), 32768);
  });
});

test("usableContextSize returns null rather than guessing when props is unreadable", async () => {
  // A broken measurement must degrade to the estimate, never fail the request.
  await withStubServer({ "/props": () => ({ status: 500, payload: {} }) })(async (manager) => {
    assert.equal(await manager.usableContextSize(), null);
  });
  await withStubServer({})(async (manager) => {
    assert.equal(await manager.usableContextSize(), null);
  });
});

test("countPromptTokens prices the rendered template, not just the message text", async () => {
  let templatedWith = null;
  let tokenized = null;

  await withStubServer({
    "/apply-template": (body) => {
      templatedWith = JSON.parse(body);
      // The real server renders role tags and, with thinking disabled, an
      // empty <think></think> block. All of it costs context.
      return { payload: { prompt: "<|im_start|>system\nSYS<|im_end|>\nhello" } };
    },
    "/tokenize": (body) => {
      tokenized = JSON.parse(body);
      return { payload: { tokens: new Array(20514).fill(0) } };
    },
  })(async (manager) => {
    const messages = [
      { role: "system", content: "SYS" },
      { role: "user", content: "hello" },
    ];
    assert.equal(await manager.countPromptTokens(messages, { disableThinking: true }), 20514);
    assert.deepEqual(templatedWith.messages, messages);
    assert.deepEqual(templatedWith.chat_template_kwargs, { enable_thinking: false });
    assert.equal(tokenized.content, "<|im_start|>system\nSYS<|im_end|>\nhello");
  });
});

test("countPromptTokens returns null when the server cannot measure", async () => {
  await withStubServer({ "/apply-template": () => ({ status: 500, payload: {} }) })(
    async (manager) => {
      assert.equal(await manager.countPromptTokens([{ role: "user", content: "hi" }], {}), null);
    }
  );
});

// --- reading the reply ---------------------------------------------------
//
// With thinking on, `reasoning_content` is the reasoning, not the answer (#2187).

const MESSAGES = [{ role: "user", content: "hi" }];

// Stubs one chat completion and records the request bodies it receives.
function withReply(message, finishReason = "stop") {
  const requests = [];
  const run = withStubServer({
    "/v1/chat/completions": (body) => {
      requests.push(JSON.parse(body));
      return { payload: { choices: [{ message, finish_reason: finishReason }] } };
    },
  });
  return { run, requests };
}

const TRUNCATED = { code: "OUTPUT_TRUNCATED" };

test("reasoning never stands in for the answer when thinking was requested", async () => {
  const reasoningOnly = { content: "", reasoning_content: "Thinking Process:\n\n1. Analyze" };

  // Cut off mid-reasoning: there is no answer to return, so every caller gets
  // the truncation error rather than an empty string it would have to explain.
  const truncated = withReply(reasoningOnly, "length");
  await truncated.run(async (manager) => {
    await assert.rejects(manager.inference(MESSAGES, { disableThinking: false }), TRUNCATED);
    const strict = { disableThinking: false, requireCompleteOutput: true };
    await assert.rejects(manager.inference(MESSAGES, strict), TRUNCATED);
  });
  assert.equal(truncated.requests[0].chat_template_kwargs, undefined);

  // Finished with no answer: reaches strict callers such as dictation cleanup too.
  const finished = withReply(reasoningOnly, "stop");
  await finished.run(async (manager) => {
    const options = { disableThinking: false, requireCompleteOutput: true };
    assert.equal(await manager.inference(MESSAGES, options), "");
  });
});

test("a cut-off reply with no content is truncated even when thinking was suppressed", async () => {
  // gpt-oss ignores enable_thinking and reasons into reasoning_content anyway;
  // a budget spent entirely there is a cut-off reply, not a misrouted answer.
  const analysisOnly = withReply({ content: "", reasoning_content: "analysis..." }, "length");
  await analysisOnly.run(async (manager) => {
    await assert.rejects(manager.inference(MESSAGES, { disableThinking: true }), TRUNCATED);
  });
});

test("a cut-off reply that has content is still returned to lenient callers", async () => {
  // Only requireCompleteOutput callers reject a partial answer (#2130).
  const partial = withReply({ content: "Cleaned te", reasoning_content: "" }, "length");
  await partial.run(async (manager) => {
    assert.equal(await manager.inference(MESSAGES, { disableThinking: true }), "Cleaned te");
    await assert.rejects(manager.inference(MESSAGES, { requireCompleteOutput: true }), TRUNCATED);
  });
});

test("an answer routed into reasoning_content with thinking suppressed is still returned", async () => {
  // Some llama-server builds do this despite enable_thinking: false (#809).
  const misrouted = withReply({ content: "", reasoning_content: "Cleaned text." });
  await misrouted.run(async (manager) => {
    assert.equal(await manager.inference(MESSAGES, { disableThinking: true }), "Cleaned text.");
    assert.equal(await manager.inference(MESSAGES, {}), "Cleaned text.");
  });
  assert.deepEqual(misrouted.requests[0].chat_template_kwargs, { enable_thinking: false });
});

test("content is returned over reasoning_content whether or not thinking was on", async () => {
  const both = withReply({ content: "Cleaned text.", reasoning_content: "Thinking Process:" });
  await both.run(async (manager) => {
    assert.equal(await manager.inference(MESSAGES, { disableThinking: false }), "Cleaned text.");
    assert.equal(await manager.inference(MESSAGES, {}), "Cleaned text.");
  });
});

// --- Vulkan context step-down --------------------------------------------
//
// Sizing the context to the request is bounded by SYSTEM RAM, but a discrete
// GPU has its own memory that the app does not probe. A machine with plenty of
// RAM and a small card can therefore now ask for a context that will not fit
// in VRAM — a failure this change introduces. Dropping straight to CPU costs
// 10-30x; halving the context is far cheaper, so try that first.

function makeLadder(shouldFail, { contextSize = 32768 } = {}) {
  const manager = new LlamaServerManager();
  const attempts = [];
  manager._buildEnv = () => ({});
  manager._killCurrentProcess = async () => {};
  manager.findAvailablePort = async () => 8221;
  manager.draftModelPath = null;
  manager._startWithBinary = async (binary, args) => {
    const attempt = { binary, ctx: Number(args[args.indexOf("--ctx-size") + 1]) };
    attempts.push(attempt);
    if (shouldFail(attempt)) throw new Error(`stub fail: ${binary} ctx=${attempt.ctx}`);
  };
  const baseArgs = manager._buildBaseArgs("/models/main.gguf", 8221, { contextSize });
  return { manager, attempts, baseArgs, options: { contextSize } };
}

const BINARIES = { vulkan: "/bin/vulkan", cpu: "/bin/cpu" };

test("a Vulkan start that fails at a grown context retries smaller before dropping to CPU", async () => {
  // Vulkan only succeeds at the baseline context.
  const { manager, attempts, baseArgs, options } = makeLadder(
    (attempt) => attempt.binary === "/bin/vulkan" && attempt.ctx > 16384
  );

  await manager._startWithGpuFallback(BINARIES, baseArgs, options, []);

  assert.deepEqual(attempts, [
    { binary: "/bin/vulkan", ctx: 32768 },
    { binary: "/bin/vulkan", ctx: 16384 },
  ]);
  assert.equal(manager.activeBackend, "vulkan", "a live GPU beats a CPU fallback");
});

test("a baseline-context start keeps today's exact ladder, with no extra rung", async () => {
  // The step-down must not add an attempt when there is nothing to step down
  // to, or every existing failure path gets slower.
  const { manager, attempts, baseArgs, options } = makeLadder(() => true, {
    contextSize: 16384,
  });

  await assert.rejects(() => manager._startWithGpuFallback(BINARIES, baseArgs, options, []));

  assert.deepEqual(attempts, [
    { binary: "/bin/vulkan", ctx: 16384 },
    { binary: "/bin/cpu", ctx: 16384 },
  ]);
});

test("a context step-down is recorded, so the server reports what it actually has", async () => {
  const { manager, baseArgs, options } = makeLadder(
    (attempt) => attempt.binary === "/bin/vulkan" && attempt.ctx > 16384
  );

  await manager._startWithGpuFallback(BINARIES, baseArgs, options, []);

  // Otherwise the preflight would believe the request still fits and send a
  // prompt the server cannot take.
  assert.equal(manager.contextSize, 16384);
});