Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
import {
getLlamaServerStatus,
startLlamaServer,
stopLlamaServer,
relaunchLlamaServerWithTuning,
relaunchLlamaServerWithAlias,
captureLlamaServerConfig,
restoreLlamaServerConfig,
installLlamaServer,
getLlamaServerUpdateStatus,
upgradeLlamaServer,
_resetLlamaServerStateForTests,
_setLlamaKeepLoadedOverrideForTests,
LLAMA_APP,
} from './llamaServerManager.js';
import * as processEnv from '../lib/processEnv.js';
import * as commandExistsModule from '../lib/commandExists.js';
import * as childProcess from '../lib/childProcess.js';
import * as platform from '../lib/platform.js';
import * as openAiModelsProbe from '../lib/openAiModelsProbe.js';
import * as pm2Module from './pm2.js';
import * as bufferedSpawnModule from '../lib/bufferedSpawn.js';
import * as streamingSpawnModule from '../lib/streamingSpawn.js';
import { PORTS } from '../lib/ports.js';
import { EventEmitter } from 'events';
import { mkdir, mkdtemp, rm, symlink, writeFile } from 'fs/promises';
import { tmpdir } from 'os';
import { delimiter, join } from 'path';
function fakeSpawnProcess() {
const child = new EventEmitter();
child.stdout = new EventEmitter();
child.stderr = new EventEmitter();
return child;
}
// A real child emits 'exit' and then 'close' (once stdio has flushed). Mirror
// both so the mocks stay honest for either listener: `brew install` is awaited
// on 'exit', while the `brew link` step waits for 'close' to capture output.
function endProcess(child, code) {
child.emit('exit', code);
child.emit('close', code);
}
const FAST_TIMING = {
startupWaitTimeout: 50,
startupPollDelay: 0,
relaunchReadyTimeout: 50,
relaunchPollDelay: 0,
pm2ReadRetryDelay: 0,
};
// Pinned to darwin so the Homebrew paths are exercised identically on every
// developer's OS — otherwise the same assertions would run against winget on a
// Windows checkout. The winget half pins `platform: 'win32'` the same way.
const resetForTest = (overrides = {}) => {
_resetLlamaServerStateForTests({ ...FAST_TIMING, platform: 'darwin', ...overrides });
};
const brewInfoJson = ({ installedVersion = 'build-100', latestVersion = '0.3.0', outdated = true, pinned = false, linkedKeg = 'build-100' } = {}) => JSON.stringify({
formulae: [{
name: 'llama.cpp',
versions: { stable: latestVersion },
installed: [{ version: installedVersion }],
outdated,
pinned,
linked_keg: linkedKeg,
}],
});
describe('llamaServerManager', () => {
// startLlamaServer refuses to spawn for a GGUF that is not on disk, so the
// lifecycle tests need real files to point at.
let modelDir;
let modelPath;
let draftPath;
let projectorPath;
let homebrewBinaryPath;
let homebrewPrefix;
let pm2State = null;
let execPm2Calls = [];
// Failed PM2 reads, on demand. `getAppStatusStrict` answers `null` for a read
// that FAILED — distinct from a successful read that found no process — and
// several paths must not mistake that for "not PortOS's". Set `failures` to
// the number of reads to fail, or `Infinity` for a PM2 that never answers
// again; `count` is what a retry assertion reads.
let pm2Reads = { failures: 0, count: 0 };
beforeAll(async () => {
modelDir = await mkdtemp(join(tmpdir(), 'portos-llama-'));
modelPath = join(modelDir, 'model.gguf');
draftPath = join(modelDir, 'draft.gguf');
projectorPath = join(modelDir, 'mmproj.gguf');
await writeFile(modelPath, 'gguf');
await writeFile(draftPath, 'gguf');
await writeFile(projectorPath, 'gguf');
const homebrewRoot = join(modelDir, 'homebrew');
const cellarRoot = join(homebrewRoot, 'Cellar', 'llama.cpp', 'build-100');
homebrewPrefix = join(homebrewRoot, 'opt', 'llama.cpp');
homebrewBinaryPath = join(homebrewRoot, 'bin', 'llama-server');
await mkdir(join(cellarRoot, 'bin'), { recursive: true });
await mkdir(join(homebrewRoot, 'bin'), { recursive: true });
await mkdir(join(homebrewRoot, 'opt'), { recursive: true });
await writeFile(join(cellarRoot, 'bin', 'llama-server'), 'binary');
await symlink(cellarRoot, homebrewPrefix);
await symlink(join(homebrewPrefix, 'bin', 'llama-server'), homebrewBinaryPath);
});
afterAll(async () => {
await rm(modelDir, { recursive: true, force: true });
});
beforeEach(() => {
// Zero retry delay: the paths that re-read an unreadable PM2 are asserted
// here, and a suite must not sit through the production backoff to see them.
resetForTest();
vi.restoreAllMocks();
pm2State = null;
execPm2Calls = [];
pm2Reads = { failures: 0, count: 0 };
// The host may have an unrelated listener on the requested port (8080 is
// especially common), so lifecycle tests pin the port-discovery result.
vi.spyOn(platform, 'isPortInUse').mockResolvedValue(false);
// Same reason, one layer up: `startLlamaServer` refuses to spawn when an
// OpenAI-compatible server already ANSWERS on the endpoint, and it probes
// over the real network. A developer with llama.cpp actually running on
// PortOS's own extension port failed five of these for reasons that have
// nothing to do with the code under test, so pin the probe too. Tests that
// need a reachable endpoint re-mock this with `{ reachable: true }`.
vi.spyOn(openAiModelsProbe, 'probeOpenAiModels')
.mockImplementation(async () => ({ reachable: pm2State?.status === 'online', models: [] }));
// `createDaemonWatcher` reads the saved boot list on every status read, and
// the dump is a real file on the developer's machine — pin it so these tests
// are about this code, not their PM2 state. (Its twin in
// mtplxServerManager.test.js pins it for the same reason.)
vi.spyOn(pm2Module, 'getSavedProcessNames').mockResolvedValue([]);
vi.spyOn(pm2Module, 'execPm2').mockImplementation(async (args) => {
execPm2Calls.push(args);
const action = args[0];
if (action === 'start') {
const nameIdx = args.indexOf('--name');
const name = nameIdx !== -1 ? args[nameIdx + 1] : args[1];
const dashIdx = args.indexOf('--');
const procArgs = dashIdx !== -1 ? args.slice(dashIdx + 1) : [];
pm2State = { name, status: 'online', pid: 12345, args: procArgs };
return { stdout: '', stderr: '' };
}
if (action === 'delete') {
pm2State = null;
return { stdout: '', stderr: '' };
}
if (action === 'logs') {
return { stdout: '', stderr: '' };
}
return { stdout: '', stderr: '' };
});
vi.spyOn(pm2Module, 'getAppStatus').mockImplementation(async (name) => {
if (pm2State && pm2State.name === name) return pm2State;
return { name, status: 'not_found', pm2_env: null };
});
vi.spyOn(pm2Module, 'getAppStatusStrict').mockImplementation(async (name) => {
pm2Reads.count += 1;
if (pm2Reads.failures > 0) {
// `Infinity - 1` is still `Infinity`, so a permanent outage needs no
// special case here.
pm2Reads.failures -= 1;
return null;
}
if (pm2State && pm2State.name === name) return pm2State;
return { name, status: 'not_found', pm2_env: null };
});
});
afterEach(() => {
_resetLlamaServerStateForTests();
vi.restoreAllMocks();
});
it('reports installed: false when binary is not found on PATH', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(null);
const status = await getLlamaServerStatus();
expect(status.installed).toBe(false);
expect(status.running).toBe(false);
expect(status.managed).toBe(false);
});
it('reports installed: true when binary is found on PATH', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/opt/homebrew/bin/llama-server');
const execProbe = vi.spyOn(commandExistsModule, 'commandExists').mockResolvedValue(true);
const status = await getLlamaServerStatus();
expect(status.installed).toBe(true);
// Regression: the binary must never be executed to answer
// "installed?". llama.cpp initializes its ggml/Metal backends at launch, so
// a cold run right after `brew link` blew past commandExists' 5s bound and
// reported an installed, working binary as missing.
expect(execProbe).not.toHaveBeenCalled();
});
it('reports the binary version and Homebrew update metadata', async () => {
const binaryPath = homebrewBinaryPath;
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(binaryPath);
const buffered = vi.spyOn(bufferedSpawnModule, 'bufferedSpawn').mockImplementation(async (command, args) => ({
success: true,
code: 0,
stdout: command === 'brew' && args[0] === '--prefix'
? `${homebrewPrefix}\n`
: command === 'brew' ? brewInfoJson() : 'version: 0.1.1-dev (build 100, commit abc123)',
stderr: '',
timedOut: false,
}));
const status = await getLlamaServerUpdateStatus();
expect(status).toMatchObject({
version: '0.1.1-dev',
latestVersion: '0.3.0',
updateAvailable: true,
canUpgrade: true,
});
expect(buffered).toHaveBeenCalledWith(binaryPath, ['--version'], { timeoutMs: 5000, shell: false });
expect(buffered).toHaveBeenCalledWith(
'brew',
['info', '--json=v2', '--formula', 'llama.cpp'],
{ timeoutMs: 15_000, shell: false },
);
});
it('does not offer an automatic update for a pinned Homebrew formula', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(homebrewBinaryPath);
vi.spyOn(bufferedSpawnModule, 'bufferedSpawn').mockImplementation(async (command, args) => ({
success: true,
code: 0,
stdout: command === 'brew' && args[0] === '--prefix'
? `${homebrewPrefix}\n`
: command === 'brew' ? brewInfoJson({ pinned: true }) : 'version: 0.1.1-dev',
stderr: '',
timedOut: false,
}));
await expect(getLlamaServerUpdateStatus()).resolves.toMatchObject({
updateAvailable: true,
canUpgrade: false,
latestVersion: '0.3.0',
});
});
it('keeps a custom llama-server build updateable by manual means only', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
vi.spyOn(bufferedSpawnModule, 'bufferedSpawn').mockImplementation(async (command) => ({
success: command !== 'brew',
code: command === 'brew' ? 1 : 0,
stdout: command === 'brew' ? '' : 'version: custom-build',
stderr: command === 'brew' ? 'Error: No available formula' : '',
timedOut: false,
}));
await expect(getLlamaServerUpdateStatus()).resolves.toMatchObject({
version: 'custom-build',
latestVersion: null,
updateAvailable: false,
canUpgrade: false,
});
});
it('keeps the Homebrew update manual when a source build is first on PATH', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
vi.spyOn(bufferedSpawnModule, 'bufferedSpawn').mockImplementation(async (command, args) => ({
success: true,
code: 0,
stdout: command === 'brew' && args[0] === '--prefix'
? `${homebrewPrefix}\n`
: command === 'brew' ? brewInfoJson() : 'version: custom-build',
stderr: '',
timedOut: false,
}));
await expect(getLlamaServerUpdateStatus()).resolves.toMatchObject({
version: 'custom-build',
latestVersion: '0.3.0',
updateAvailable: true,
canUpgrade: false,
});
});
it('rejects start when binary is missing', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(null);
await expect(startLlamaServer({ model: modelPath })).rejects.toThrow(
/llama-server binary was not found/i
);
});
it('spawns llama-server with draftModel and specType arguments under PM2', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
const result = await startLlamaServer({
model: modelPath,
draftModel: draftPath,
specType: 'draft-dflash',
port: 8080,
host: '127.0.0.1',
alias: 'dflash',
});
expect(result.success).toBe(true);
expect(result.pid).toBe(12345);
const startCall = execPm2Calls.find((c) => c[0] === 'start');
expect(startCall[1]).toBe('/usr/local/bin/llama-server');
expect(startCall.slice(2, 8)).toEqual([
'--name', LLAMA_APP,
'--interpreter', 'none',
'--no-autorestart',
'--',
]);
expect(startCall.slice(8)).toEqual([
'-m', modelPath,
'--model-draft', draftPath,
'--spec-type', 'draft-dflash',
'--port', '8080',
'--host', '127.0.0.1',
'--ctx-size', '32768',
'-ngl', '99',
'--parallel', '1',
'--alias', 'dflash',
]);
const status = await getLlamaServerStatus();
expect(status.running).toBe(true);
expect(status.managed).toBe(true);
expect(status.pid).toBe(12345);
expect(status.config.parallel).toBe(1);
});
// The whole point of #7611: a vision-capable GGUF loads text-only unless
// llama.cpp is handed its projector sidecar. Both halves are asserted here —
// the flag lands when a projector is set, and the line is UNCHANGED when one
// is not, because every existing launch must keep producing the same argv.
it('puts --mmproj on the launch line for a projector, and nothing at all without one', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await startLlamaServer({ model: modelPath, specType: 'none', projector: projectorPath });
const withProjector = execPm2Calls.find((c) => c[0] === 'start').slice(8);
expect(withProjector.slice(0, 4)).toEqual(['-m', modelPath, '--mmproj', projectorPath]);
const running = await getLlamaServerStatus();
expect(running.config.projector).toBe(projectorPath);
await stopLlamaServer();
execPm2Calls = [];
await startLlamaServer({ model: modelPath, specType: 'none' });
const withoutProjector = execPm2Calls.find((c) => c[0] === 'start').slice(8);
expect(withoutProjector).not.toContain('--mmproj');
expect(withoutProjector).toEqual(withProjector.filter((a) => a !== '--mmproj' && a !== projectorPath));
expect((await getLlamaServerStatus()).config.projector).toBeNull();
});
// Vision is orthogonal to speculative decoding — no spec type turns it on or
// off — so the projector survives the resolution that drops an unused drafter.
it('keeps the projector when an ngram spec type drops the drafter', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await startLlamaServer({
model: modelPath,
draftModel: draftPath,
projector: projectorPath,
specType: 'ngram-map-k',
});
const startCall = execPm2Calls.find((c) => c[0] === 'start');
expect(startCall).not.toContain('--model-draft');
expect(startCall[startCall.indexOf('--mmproj') + 1]).toBe(projectorPath);
});
it('starts an ngram spec type with no drafter model at all', async () => {
// `ngram-*` implementations draft from the tokens already in context, so a
// drafter GGUF is not just optional — there is nothing to download.
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await startLlamaServer({ model: modelPath, specType: 'ngram-map-k' });
const startCall = execPm2Calls.find((c) => c[0] === 'start');
expect(startCall).not.toContain('--model-draft');
expect(startCall.slice(startCall.indexOf('--spec-type'), startCall.indexOf('--spec-type') + 2))
.toEqual(['--spec-type', 'ngram-map-k']);
});
it('passes a combined drafter + ngram spec type through as one comma-separated flag', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await startLlamaServer({ model: modelPath, draftModel: draftPath, specType: 'draft-dflash, ngram-map-k' });
const startCall = execPm2Calls.find((c) => c[0] === 'start');
expect(startCall[startCall.indexOf('--spec-type') + 1]).toBe('draft-dflash,ngram-map-k');
});
it('drops only the drafter-based half of a combined spec type when no drafter is set', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await startLlamaServer({ model: modelPath, specType: 'draft-dflash,ngram-map-k' });
const startCall = execPm2Calls.find((c) => c[0] === 'start');
expect(startCall[startCall.indexOf('--spec-type') + 1]).toBe('ngram-map-k');
const status = await getLlamaServerStatus();
// The card reports what is running, not what was asked for.
expect(status.config.specType).toBe('ngram-map-k');
});
it('ignores a preset drafter path when only ngram spec types were requested', async () => {
// The launcher card seeds BOTH fields from a preset, so switching Spec Type
// to an ngram implementation leaves the drafter path behind — loading it
// would cost VRAM the run can't use, and a preset GGUF that was never
// downloaded would fail the existence check and block Start outright.
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await startLlamaServer({ model: modelPath, draftModel: draftPath, specType: 'ngram-map-k' });
const startCall = execPm2Calls.find((c) => c[0] === 'start');
expect(startCall).not.toContain('--model-draft');
expect(startCall[startCall.indexOf('--spec-type') + 1]).toBe('ngram-map-k');
const status = await getLlamaServerStatus();
expect(status.config.draftModel).toBeNull();
});
it('keeps a drafter when no spec type is set at all — llama.cpp drafts off --model-draft alone', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await startLlamaServer({ model: modelPath, draftModel: draftPath, specType: '' });
const startCall = execPm2Calls.find((c) => c[0] === 'start');
expect(startCall[startCall.indexOf('--model-draft') + 1]).toBe(draftPath);
expect(startCall).not.toContain('--spec-type');
});
it('omits --spec-type entirely when the only requested type needs an absent drafter', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await startLlamaServer({ model: modelPath, specType: 'draft-dspark' });
const startCall = execPm2Calls.find((c) => c[0] === 'start');
expect(startCall).not.toContain('--spec-type');
});
it('honours an explicit parallel slot count', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await startLlamaServer({ model: modelPath, parallel: 4 });
const startCall = execPm2Calls.find((c) => c[0] === 'start');
expect(startCall.slice(startCall.indexOf('--parallel'), startCall.indexOf('--parallel') + 2))
.toEqual(['--parallel', '4']);
});
it('uses PortOS\'s extension port when no port is supplied', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
const result = await startLlamaServer({ model: modelPath });
expect(result.endpoint).toBe(`http://127.0.0.1:${PORTS.LLAMA_SERVER}/v1`);
const startCall = execPm2Calls.find((c) => c[0] === 'start');
expect(startCall).toContain('--port');
expect(startCall).toContain(String(PORTS.LLAMA_SERVER));
});
it('rejects before spawning when the requested port is occupied by another process', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
vi.spyOn(platform, 'isPortInUse').mockResolvedValue(true);
await expect(startLlamaServer({ model: modelPath, port: 49876 })).rejects.toMatchObject({
code: 'LLAMA_SERVER_PORT_IN_USE',
status: 409,
message: expect.stringContaining('Choose a different port'),
});
expect(execPm2Calls.filter((c) => c[0] === 'start')).toHaveLength(0);
});
it('refuses to start when the GGUF the launch line names is not on disk', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await expect(startLlamaServer({ model: join(modelDir, 'absent.gguf') })).rejects.toThrow(
/base model was not found/i
);
await expect(startLlamaServer({ model: modelPath, draftModel: join(modelDir, 'absent.gguf') })).rejects.toThrow(
/drafter model was not found/i
);
// Same gate for the projector: an `--mmproj` llama-server cannot open is a
// failed launch, so it is refused here rather than in a server log (#7611).
await expect(startLlamaServer({ model: modelPath, projector: join(modelDir, 'absent.gguf') })).rejects.toThrow(
/vision projector was not found/i
);
// The weights are a separate multi-gigabyte download; spawning anyway just
// buries that in a server log.
expect(execPm2Calls.filter((c) => c[0] === 'start')).toHaveLength(0);
});
it('reports a failure — not a PID — when llama-server exits during startup', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
vi.spyOn(pm2Module, 'execPm2').mockImplementation(async (args) => {
if (args[0] === 'logs') return { stdout: '', stderr: 'error: unknown spec type' };
return { stdout: '', stderr: '' };
});
vi.spyOn(pm2Module, 'getAppStatusStrict').mockResolvedValue({ name: LLAMA_APP, status: 'errored' });
await expect(startLlamaServer({ model: modelPath, specType: 'draft-nope' })).rejects.toThrow(
/llama-server exited immediately/i
);
});
it('stops managed process cleanly', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await startLlamaServer({ model: modelPath });
const stopResult = await stopLlamaServer();
expect(stopResult.success).toBe(true);
const status = await getLlamaServerStatus();
expect(status.managed).toBe(false);
});
it('recovers launch configuration from PM2 args after server restarts', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
pm2State = {
name: LLAMA_APP,
status: 'online',
pid: 98765,
args: ['-m', modelPath, '--model-draft', draftPath, '--mmproj', projectorPath, '--port', '8090', '--host', '127.0.0.1', '--parallel', '4'],
};
const status = await getLlamaServerStatus();
expect(status.managed).toBe(true);
expect(status.pid).toBe(98765);
expect(status.port).toBe(8090);
expect(status.endpoint).toBe('http://127.0.0.1:8090/v1');
expect(status.config?.model).toBe(modelPath);
expect(status.config?.draftModel).toBe(draftPath);
// Recovered from argv too, so the status card describes a running vision
// launch correctly — and so a relaunch replays the same line rather than
// silently demoting the daemon to text-only.
expect(status.config?.projector).toBe(projectorPath);
expect(status.config?.parallel).toBe(4);
});
it('surfaces unknown/degraded state when PM2 read fails', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/opt/homebrew/bin/llama-server');
vi.spyOn(pm2Module, 'getAppStatusStrict').mockResolvedValue(null);
const status = await getLlamaServerStatus();
expect(status.managed).toBeNull();
expect(status.lastExitError).toBe('Failed to read PM2 status');
});
// `managed: null` exists to say "could not tell", but nulling `config` on the
// same failed read handed every caller guarding on `!managed ||
// !config?.model` the same answer as "somebody else started it".
it('keeps the last known launch line when the PM2 read fails', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await startLlamaServer({ model: modelPath, port: PORTS.LLAMA_SERVER });
pm2Reads.failures = Infinity;
const status = await getLlamaServerStatus();
expect(status.managed).toBeNull();
expect(status.config?.model).toBe(modelPath);
});
// A read that SUCCEEDED and found nothing is real evidence: there is no
// launch line, and reporting a stale one would be a lie in the other
// direction.
it('reports no launch line when the PM2 read succeeds and finds nothing', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
await startLlamaServer({ model: modelPath, port: PORTS.LLAMA_SERVER });
pm2State = null;
const status = await getLlamaServerStatus();
expect(status.managed).toBe(false);
expect(status.config).toBeNull();
});
it('propagates error when stopping PM2 process fails', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
pm2State = { name: LLAMA_APP, status: 'online', pid: 12345 };
vi.spyOn(pm2Module, 'execPm2').mockImplementation(async (args) => {
if (args[0] === 'delete') throw new Error('PM2 daemon down');
return { stdout: '', stderr: '' };
});
await expect(stopLlamaServer()).rejects.toThrow(/Failed to stop llama-server: PM2 daemon down/);
});
it('installs llama.cpp via Homebrew when brew is available', async () => {
vi.spyOn(commandExistsModule, 'commandExists').mockImplementation(async (cmd) => true);
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/opt/homebrew/bin/llama-server');
const fakeChild = new EventEmitter();
fakeChild.stdout = new EventEmitter();
fakeChild.stderr = new EventEmitter();
vi.spyOn(childProcess, 'spawn').mockImplementation(() => {
setTimeout(() => endProcess(fakeChild, 0), 10);
return fakeChild;
});
const result = await installLlamaServer();
expect(result.success).toBe(true);
});
it('rejects install when Homebrew is missing', async () => {
vi.spyOn(commandExistsModule, 'commandExists').mockResolvedValue(false);
await expect(installLlamaServer()).rejects.toThrow(/Homebrew was not found/i);
});
it('links an already-installed-but-unlinked keg after `brew install` exits 0', async () => {
// brew is present; llama-server is NOT on PATH until after the link step.
vi.spyOn(commandExistsModule, 'commandExists').mockImplementation(async (cmd) => cmd === 'brew');
const findSpy = vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(null);
const installChild = fakeSpawnProcess();
const linkChild = fakeSpawnProcess();
const spawnCalls = [];
vi.spyOn(childProcess, 'spawn').mockImplementation((cmd, args) => {
spawnCalls.push({ cmd, args });
const child = args[0] === 'install' ? installChild : linkChild;
// The binary shows up on PATH because `brew link` ran — key the mock off
// that spawn, not a wall-clock timer. A timer races the install child's
// own exit handler, and on a loaded runner it can fire first, making the
// binary look already-linked so the link step never happens (#4642).
if (args[0] === 'link') findSpy.mockReturnValue('/opt/homebrew/bin/llama-server');
setTimeout(() => endProcess(child, 0), 10);
return child;
});
const result = await installLlamaServer();
expect(result.success).toBe(true);
expect(spawnCalls).toEqual([
{ cmd: 'brew', args: ['install', 'llama.cpp'] },
{ cmd: 'brew', args: ['link', '--overwrite', 'llama.cpp'] },
]);
});
it('surfaces brew link output when the link step fails', async () => {
vi.spyOn(commandExistsModule, 'commandExists').mockImplementation(async (cmd) => cmd === 'brew');
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(null);
vi.spyOn(childProcess, 'spawn').mockImplementation((cmd, args) => {
const child = fakeSpawnProcess();
setTimeout(() => {
if (args[0] === 'link') {
child.stderr.emit('data', Buffer.from('Error: Could not symlink bin/llama-server'));
endProcess(child, 1);
} else {
endProcess(child, 0);
}
}, 10);
return child;
});
await expect(installLlamaServer()).rejects.toThrow(/Could not symlink bin\/llama-server/);
});
it('rejects instead of hanging when the link spawn throws synchronously', async () => {
// The exit listener is async and lives inside a Promise executor, so an
// unguarded throw would escape as an unhandled rejection while the install
// request never settled.
vi.spyOn(commandExistsModule, 'commandExists').mockImplementation(async (cmd) => cmd === 'brew');
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(null);
vi.spyOn(childProcess, 'spawn').mockImplementation((cmd, args) => {
if (args[0] === 'link') throw new Error('spawn EACCES');
const child = fakeSpawnProcess();
setTimeout(() => endProcess(child, 0), 10);
return child;
});
await expect(installLlamaServer()).rejects.toThrow(/Failed to verify the llama\.cpp install: spawn EACCES/);
});
it('rejects with a `brew link` hint when linking does not resolve the binary', async () => {
vi.spyOn(commandExistsModule, 'commandExists').mockImplementation(async (cmd) => cmd === 'brew');
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(null);
vi.spyOn(childProcess, 'spawn').mockImplementation(() => {
const child = fakeSpawnProcess();
setTimeout(() => endProcess(child, 0), 10);
return child;
});
await expect(installLlamaServer()).rejects.toThrow(/brew link --overwrite llama\.cpp/i);
});
describe('upgradeLlamaServer', () => {
const stubBrewInfo = (options) => vi.spyOn(bufferedSpawnModule, 'bufferedSpawn').mockImplementation(async (_command, args) => ({
success: true,
code: 0,
stdout: args[0] === '--prefix' ? `${homebrewPrefix}\n` : brewInfoJson(options),
stderr: '',
timedOut: false,
}));
it('updates an idle installation through Homebrew without starting it', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(homebrewBinaryPath);
stubBrewInfo();
const progress = vi.fn();
const stream = vi.spyOn(streamingSpawnModule, 'runStreamingCommand').mockImplementation(async (_cmd, _args, onLine) => {
onLine('Updated llama.cpp');
return { success: true };
});
const result = await upgradeLlamaServer({ onProgress: progress });
expect(result).toMatchObject({ success: true });
expect(stream).toHaveBeenCalledWith(
'brew',
['upgrade', 'llama.cpp'],
expect.any(Function),
{ timeoutMs: 30 * 60 * 1000 },
);
expect(progress).toHaveBeenCalledWith({ event: 'progress', message: 'Updating llama.cpp via Homebrew…' });
expect(progress).toHaveBeenCalledWith({ event: 'progress', message: 'Updated llama.cpp' });
expect(execPm2Calls.some((args) => args[0] === 'start' || args[0] === 'delete')).toBe(false);
});
it('restarts a managed server with its recovered launch configuration', async () => {
const binaryPath = homebrewBinaryPath;
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(binaryPath);
stubBrewInfo();
pm2State = {
name: LLAMA_APP,
status: 'online',
pid: 321,
args: ['-m', modelPath, '--port', String(PORTS.LLAMA_SERVER), '--alias', 'example-model'],
};
openAiModelsProbe.probeOpenAiModels.mockImplementation(async () => ({ reachable: pm2State?.status === 'online' }));
vi.spyOn(streamingSpawnModule, 'runStreamingCommand').mockResolvedValue({ success: true });
const result = await upgradeLlamaServer();
expect(result).toMatchObject({ success: true });
expect(execPm2Calls.some((args) => args[0] === 'delete')).toBe(true);
const startCall = execPm2Calls.find((args) => args[0] === 'start');
expect(startCall).toContain(modelPath);
expect(startCall).toContain('--alias');
expect(startCall).toContain('example-model');
});
it('restarts a managed server with the current saved idle window', async () => {
const binaryPath = homebrewBinaryPath;
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(binaryPath);
stubBrewInfo();
pm2State = {
name: LLAMA_APP,
status: 'online',
pid: 321,
args: ['-m', modelPath, '--port', String(PORTS.LLAMA_SERVER), '--sleep-idle-seconds', '0'],
};
openAiModelsProbe.probeOpenAiModels.mockImplementation(async () => ({ reachable: pm2State?.status === 'online' }));
vi.spyOn(childProcess, 'execFile').mockImplementation((_bin, _args, _opts, cb) => {
cb(null, '--sleep-idle-seconds SECONDS', '');
return fakeSpawnProcess();
});
vi.spyOn(streamingSpawnModule, 'runStreamingCommand').mockResolvedValue({ success: true });
resetForTest({ sleepIdleMinutes: 30 });
const result = await upgradeLlamaServer();
expect(result).toMatchObject({ success: true });
const startCall = execPm2Calls.find((args) => args[0] === 'start');
const idleFlagIndex = startCall.indexOf('--sleep-idle-seconds');
expect(startCall[idleFlagIndex + 1]).toBe('1800');
});
it('refuses to update when PM2 ownership cannot be determined', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(homebrewBinaryPath);
stubBrewInfo();
pm2Reads.failures = Infinity;
const stream = vi.spyOn(streamingSpawnModule, 'runStreamingCommand');
await expect(upgradeLlamaServer()).resolves.toMatchObject({
success: false,
error: expect.stringMatching(/could not read PM2/i),
});
expect(stream).not.toHaveBeenCalled();
});
it('refuses an automatic update when a source build owns the active binary', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
stubBrewInfo();
const stream = vi.spyOn(streamingSpawnModule, 'runStreamingCommand');
await expect(upgradeLlamaServer()).resolves.toMatchObject({
success: false,
manualUpdateRequired: true,
error: expect.stringMatching(/active llama-server is not the linked Homebrew/i),
});
expect(stream).not.toHaveBeenCalled();
});
it('does not report success when a managed restart never becomes ready', async () => {
const binaryPath = homebrewBinaryPath;
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(binaryPath);
stubBrewInfo();
pm2State = {
name: LLAMA_APP,
status: 'online',
pid: 321,
args: ['-m', modelPath, '--port', String(PORTS.LLAMA_SERVER), '--alias', 'example-model'],
};
openAiModelsProbe.probeOpenAiModels.mockResolvedValue({ reachable: false });
vi.spyOn(streamingSpawnModule, 'runStreamingCommand').mockResolvedValue({ success: true });
resetForTest({ startupWaitTimeout: 0, relaunchReadyTimeout: 0 });
const result = await upgradeLlamaServer();
expect(result).toMatchObject({
success: false,
updated: true,
error: expect.stringMatching(/could not be restarted/i),
});
expect(execPm2Calls.filter((args) => args[0] === 'delete').length).toBeGreaterThanOrEqual(2);
});
});
// ---- Windows (winget) ---------------------------------------------------
// llama.cpp ships an official winget package, so Windows gets the same
// install/update buttons as macOS rather than a Homebrew command it cannot
// run. Everything here pins `platform: 'win32'` so the branch is covered from
// any developer's OS.
describe('winget', () => {
const WINGET_BINARY = 'C:\\Users\\example\\AppData\\Local\\Microsoft\\WinGet\\Links\\llama-server.exe';
const wingetTable = (available) => [
'Name Id Version Available Source',
'-------------------------------------------------',
`llama.cpp ggml.llamacpp b10500 ${available ? 'b10730 ' : ''}winget`,
].join('\n');
// `--upgrade-available` lists the package ONLY when a newer build exists;
// the plain listing always does. That difference is the staleness signal.
const stubWinget = ({ installed = true, outdated = true } = {}) =>
vi.spyOn(bufferedSpawnModule, 'bufferedSpawn').mockImplementation(async (command, args) => {
if (command !== 'winget') return { success: true, code: 0, stdout: 'version: b10500', stderr: '', timedOut: false };
const checkingUpgrade = args.includes('--upgrade-available');
const listed = installed && (!checkingUpgrade || outdated);
return {
success: true,
code: 0,
stdout: listed ? wingetTable(checkingUpgrade) : 'No installed package found matching input criteria.',
stderr: '',
timedOut: false,
};
});
// Set by the PATH-adoption test below, which needs a real directory on disk
// for its existsSync probe; torn down here so a failed assertion can't leak it.
let wingetTmpDir = null;
beforeEach(() => {
resetForTest({ platform: 'win32' });
vi.stubEnv('LOCALAPPDATA', 'C:\\Users\\example\\AppData\\Local');
});
afterEach(async () => {
vi.unstubAllEnvs();
if (wingetTmpDir) await rm(wingetTmpDir, { recursive: true, force: true });
wingetTmpDir = null;
});
it('names the winget command on a host with no llama-server', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(null);
await expect(getLlamaServerStatus()).resolves.toMatchObject({
installed: false,
packageManager: 'winget',
packageManagerLabel: 'winget',
installCommand: 'winget install ggml.llamacpp',
});
});
it('reports an available winget build as an update PortOS can apply', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(WINGET_BINARY);
const buffered = stubWinget();
await expect(getLlamaServerUpdateStatus()).resolves.toMatchObject({
version: 'b10500',
latestVersion: 'b10730',
updateAvailable: true,
canUpgrade: true,
});
expect(buffered).toHaveBeenCalledWith(
'winget',
['list', '--id', 'ggml.llamacpp', '--exact', '--disable-interactivity', '--upgrade-available'],
{ timeoutMs: 60_000, shell: false },
);
});
it('keeps a hand-installed Windows build updateable by manual means only', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('C:\\tools\\llama.cpp\\llama-server.exe');
stubWinget();
await expect(getLlamaServerUpdateStatus()).resolves.toMatchObject({
latestVersion: 'b10730',
updateAvailable: true,
canUpgrade: false,
});
});
it('installs through winget without any Homebrew step', async () => {
vi.spyOn(commandExistsModule, 'commandExists').mockImplementation(async (cmd) => cmd === 'winget');
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(WINGET_BINARY);
const spawnCalls = [];
vi.spyOn(childProcess, 'spawn').mockImplementation((cmd, args) => {
spawnCalls.push({ cmd, args });
const child = fakeSpawnProcess();
setTimeout(() => endProcess(child, 0), 10);
return child;
});
await expect(installLlamaServer()).resolves.toMatchObject({ success: true });
expect(spawnCalls).toEqual([{
cmd: 'winget',
args: ['install', '--id', 'ggml.llamacpp', '--exact', '--disable-interactivity', '--accept-source-agreements', '--accept-package-agreements'],
}]);
});
it("adopts winget's Links directory when the new shim is not on PATH yet", async () => {
// winget adds that directory to the USER environment, a change an already
// running PortOS never inherits — so a perfectly successful install would
// otherwise report "llama-server was not found on PATH" until a restart.
const localAppData = await mkdtemp(join(tmpdir(), 'portos-winget-'));
wingetTmpDir = localAppData;
const links = join(localAppData, 'Microsoft', 'WinGet', 'Links');
await mkdir(links, { recursive: true });
vi.stubEnv('LOCALAPPDATA', localAppData);
vi.stubEnv('PATH', process.env.PATH || '');
vi.spyOn(commandExistsModule, 'commandExists').mockImplementation(async (cmd) => cmd === 'winget');
vi.spyOn(processEnv, 'findCommandOnPath').mockImplementation(() => (
(process.env.PATH || '').split(delimiter).includes(links) ? join(links, 'llama-server.exe') : null
));
vi.spyOn(childProcess, 'spawn').mockImplementation(() => {
const child = fakeSpawnProcess();
setTimeout(() => endProcess(child, 0), 10);
return child;
});
await expect(installLlamaServer()).resolves.toMatchObject({ success: true });
expect((process.env.PATH || '').split(delimiter)).toContain(links);
});
it('rejects install with a winget hint rather than a Homebrew one', async () => {
vi.spyOn(commandExistsModule, 'commandExists').mockResolvedValue(false);
await expect(installLlamaServer()).rejects.toThrow(/winget was not found/i);
});
it('updates a winget installation in place', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue(WINGET_BINARY);
stubWinget();
const stream = vi.spyOn(streamingSpawnModule, 'runStreamingCommand').mockResolvedValue({ success: true });
await expect(upgradeLlamaServer()).resolves.toMatchObject({ success: true });
expect(stream).toHaveBeenCalledWith(
'winget',
['upgrade', '--id', 'ggml.llamacpp', '--exact', '--disable-interactivity', '--accept-source-agreements', '--accept-package-agreements'],
expect.any(Function),
{ timeoutMs: 30 * 60 * 1000 },
);
});
it('refuses to update a llama-server winget did not install', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('C:\\tools\\llama.cpp\\llama-server.exe');
stubWinget();
const stream = vi.spyOn(streamingSpawnModule, 'runStreamingCommand');
await expect(upgradeLlamaServer()).resolves.toMatchObject({
success: false,
manualUpdateRequired: true,
error: expect.stringMatching(/not the winget-installed llama\.cpp/i),
});
expect(stream).not.toHaveBeenCalled();
});
});
// ---- tuning relaunch ----------------------------------------------------
// The sweep half of measured assessments: put new flags on the launch line
// between runs. Every failure mode here has to leave the daemon USABLE — it
// fronts the `llama` provider for the whole install.
describe('relaunchLlamaServerWithAlias', () => {
const startedAs = async (alias) => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
vi.spyOn(openAiModelsProbe, 'probeOpenAiModels')
.mockImplementation(async () => ({ reachable: pm2State?.status === 'online' }));
await startLlamaServer({ model: modelPath, port: PORTS.LLAMA_SERVER, alias });
};
const lastStartArgs = () => {
const start = [...execPm2Calls].reverse().find((c) => c[0] === 'start') || [];
const dash = start.indexOf('--');
return dash === -1 ? [] : start.slice(dash + 1);
};
// The mismatch this whole feature exists for: llama.cpp answers under the
// `--alias` its launch line set, so pointing the daemon at the id the
// provider sends is a rename of the weights already loaded.
it('relaunches the SAME weights under the requested id', async () => {
await startedAs('dflash');
execPm2Calls = [];
const result = await relaunchLlamaServerWithAlias('qwen3.8-27b-dflash2');
expect(result.applied).toBe(true);
expect(result.reason).toBeNull();
const args = lastStartArgs();
expect(args[args.indexOf('--alias') + 1]).toBe('qwen3.8-27b-dflash2');
// The weights never move — that is what makes this cheap.
expect(args[args.indexOf('-m') + 1]).toBe(modelPath);
});
// `null`, not `true`: nothing was restarted, and a toast claiming a restart
// would have the user waiting for a reload that never happened.
it('does not bounce a daemon that already answers under that id', async () => {
await startedAs('dflash');
execPm2Calls = [];
const result = await relaunchLlamaServerWithAlias('dflash');
expect(result.applied).toBeNull();
expect(execPm2Calls.some((c) => c[0] === 'start' || c[0] === 'delete')).toBe(false);
});
// An externally-launched llama-server belongs to whoever ran it. Refuse with
// the flag they can add themselves rather than killing their process.
it('refuses — and names the flag — when PortOS did not start the daemon', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
const result = await relaunchLlamaServerWithAlias('qwen3.8-27b-dflash2');
expect(result.applied).toBe(false);
expect(result.reason).toMatch(/--alias qwen3\.8-27b-dflash2/);
expect(execPm2Calls.some((c) => c[0] === 'start' || c[0] === 'delete')).toBe(false);
});
// The regression this pins: an untuned assessment RELAUNCHES the pre-tuning
// baseline. If that baseline keeps the old alias, it silently renames the
// daemon back and re-breaks the provider the rename just fixed.
it('carries the new id into the pre-tuning baseline a later untune restores', async () => {
await startedAs('dflash');
await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
await relaunchLlamaServerWithAlias('qwen3.8-27b-dflash2');
execPm2Calls = [];
// The untuned measurement: puts the pre-tuning launch line back.
await relaunchLlamaServerWithTuning({});
const args = lastStartArgs();
expect(args[args.indexOf('--alias') + 1]).toBe('qwen3.8-27b-dflash2');
// ...and it is still a genuine untune: the tuning flag is gone.
expect(args).not.toContain('-ub');
});
// A rejected rename leaves the ORIGINAL alias on the port, so the baseline
// must stay exactly as captured or the untune would rename to an id nothing
// ever served.
it('leaves the baseline alone when the rename never takes', async () => {
await startedAs('dflash');
await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
// Fail only the renamed launch; the restore that follows must succeed.
const fakeExec = pm2Module.execPm2.getMockImplementation();
let starts = 0;
vi.spyOn(pm2Module, 'execPm2').mockImplementation(async (args) => {
if (args[0] === 'start' && starts++ === 0) {
pm2State = { name: LLAMA_APP, status: 'errored', pid: null, args: [] };
execPm2Calls.push(args);
return { stdout: '', stderr: '' };
}
return fakeExec(args);
});
const result = await relaunchLlamaServerWithAlias('qwen3.8-27b-dflash2');
expect(result.applied).toBe(false);
execPm2Calls = [];
await relaunchLlamaServerWithTuning({});
const args = lastStartArgs();
expect(args[args.indexOf('--alias') + 1]).toBe('dflash');
});
it('refuses a blank model id without touching the daemon', async () => {
await startedAs('dflash');
execPm2Calls = [];
const result = await relaunchLlamaServerWithAlias(' ');
expect(result.applied).toBe(false);
expect(execPm2Calls).toEqual([]);
});
});
describe('relaunchLlamaServerWithTuning', () => {
// The default harness pins the endpoint unreachable so lifecycle tests don't
// collide with a developer's real llama-server. These tests need a FAITHFUL
// probe instead: "reachable" has to track the fake PM2 process, or the
// relaunch can never observe the server it just started answering — and the
// new `online` check would report every success as not-applied.
// The launch line of the most recent PM2 start — everything after the `--`
// separator, which is what llama-server actually received.
const launchArgs = () => {
const start = [...execPm2Calls].reverse().find((c) => c[0] === 'start') || [];
const dash = start.indexOf('--');
return dash === -1 ? [] : start.slice(dash + 1);
};
// Did the daemon actually bounce? Reading the status tails PM2's logs, which
// is not a restart — only a stop/start pair is.
const restarted = () => execPm2Calls.some((c) => c[0] === 'start' || c[0] === 'delete');
const probeTracksPm2 = () => vi.spyOn(openAiModelsProbe, 'probeOpenAiModels')
.mockImplementation(async () => ({ reachable: pm2State?.status === 'online' }));
const started = async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
probeTracksPm2();
await startLlamaServer({ model: modelPath, port: PORTS.LLAMA_SERVER });
};
it('refuses when nothing is running — there is no model path to reuse', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
const result = await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
expect(result.applied).toBe(false);
expect(result.reason).toMatch(/not running/);
});
// The user's launch flags live only in the running process, so an ordinary
// measurement must never reset them. Without `reset` an empty tuning asks
// for nothing and is answered before PM2 is touched at all.
// `null`, not `false`: nothing was refused. The daemon already serves the
// configuration the caller asked for, so an untuned assessment's "Backend
// defaults" label is accurate as it stands.
it('relaunches nothing for an empty tuning on a daemon it never tuned', async () => {
await started();
execPm2Calls = [];
const result = await relaunchLlamaServerWithTuning({});
expect(result.applied).toBeNull();
expect(result.reason).toBeNull();
expect(restarted()).toBe(false);
});
// The bug this pair exists for: an untuned run that followed a tuned one was
// sampling a daemon still carrying the tuned launch line, then filing the
// reading as "Backend defaults" — a record describing a configuration that
// never ran, which `compareTunings` ranks every real tuning against.
it('puts the pre-tuning launch line back for an empty tuning after a tuned one', async () => {
await started();
await relaunchLlamaServerWithTuning({ ubatchSize: 512, flashAttn: true });
execPm2Calls = [];
const result = await relaunchLlamaServerWithTuning({});
expect(result.applied).toBeNull();
const start = execPm2Calls.find((c) => c[0] === 'start');
expect(start).not.toContain('-ub');
expect(start).not.toContain('--flash-attn');
expect(start[start.indexOf('-m') + 1]).toBe(modelPath);
});
// The baseline is the line the FIRST tuning displaced. Crediting the second
// tuning's `previous` would make "untuned" mean "whatever the first sweep
// left running" — the same lie, one step removed.
it('keeps the original baseline across successive tunings', async () => {
await started();
await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
await relaunchLlamaServerWithTuning({ ubatchSize: 1024, flashAttn: true });
execPm2Calls = [];
await relaunchLlamaServerWithTuning({});
const start = execPm2Calls.find((c) => c[0] === 'start');
expect(start).not.toContain('-ub');
expect(start).not.toContain('--flash-attn');
});
it('relaunches nothing for a second empty tuning once the daemon is back at its baseline', async () => {
await started();
await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
await relaunchLlamaServerWithTuning({});
execPm2Calls = [];
expect(await relaunchLlamaServerWithTuning({})).toMatchObject({ applied: null });
expect(restarted()).toBe(false);
});
// A user-stopped daemon takes its tuning with it, so a later untuned run has
// nothing to undo — and must not resurrect the process to prove it.
it('drops the baseline when the daemon is stopped outside a relaunch', async () => {
await started();
await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
await stopLlamaServer();
execPm2Calls = [];
expect(await relaunchLlamaServerWithTuning({})).toMatchObject({ applied: null });
expect(restarted()).toBe(false);
});
// A start the user asked for supersedes the baseline. Restoring the old
// daemon's launch line over a configuration just chosen on the LLMs page
// would undo their change in the name of measuring defaults.
it('drops the baseline when a fresh server is started over a crashed one', async () => {
await started();
await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
// The daemon dies outside PortOS — nothing called stopLlamaServer, so
// nothing cleared the baseline — and the user starts a new one.
pm2State = null;
await startLlamaServer({ model: modelPath, port: PORTS.LLAMA_SERVER, ctxSize: 65536 });
execPm2Calls = [];
expect(await relaunchLlamaServerWithTuning({})).toMatchObject({ applied: null });
expect(restarted()).toBe(false);
});
// The regression this guards: a reset the caller did not ask for wipes the
// only copy of a configuration the user chose on the LLMs page.
it('keeps the flags a tuning does not name unless a reset was requested', async () => {
await started();
await relaunchLlamaServerWithTuning({ batchSize: 4096, flashAttn: true }, { reset: true });
const result = await relaunchLlamaServerWithTuning({ ubatchSize: 1024 });
expect(result.config.ubatchSize).toBe(1024);
expect(result.config.batchSize).toBe(4096);
expect(result.config.flashAttn).toBe(true);
});
// The bug `reset` exists for: variants applied in sequence accumulated, so
// the second launched with the first one's flags while its record's label
// claimed a single knob — and `compareTunings` credited the change to it.
it('clears the knobs a reset tuning does not name, so variants cannot accumulate', async () => {
await started();
await relaunchLlamaServerWithTuning({ batchSize: 4096 }, { reset: true });
execPm2Calls = [];
const result = await relaunchLlamaServerWithTuning({ ubatchSize: 1024 }, { reset: true });
expect(result.applied).toBe(true);
expect(result.config.ubatchSize).toBe(1024);
expect(result.config.batchSize).toBeNull();
expect(launchArgs()).not.toContain('4096');
});
// The baseline variant of a sweep: no knobs at all, which only means
// something when the caller asked for a complete tuning.
it('resets a tuned server back to backend defaults for an empty reset tuning', async () => {
await started();
await relaunchLlamaServerWithTuning({ ubatchSize: 1024, flashAttn: true }, { reset: true });
execPm2Calls = [];
const result = await relaunchLlamaServerWithTuning({}, { reset: true });
expect(result.applied).toBe(true);
expect(result.config.ubatchSize).toBeNull();
expect(result.config.flashAttn).toBe(false);
expect(launchArgs()).not.toContain('--flash-attn');
});
it('reports a reset already in effect without restarting', async () => {
await started();
execPm2Calls = [];
const result = await relaunchLlamaServerWithTuning({}, { reset: true });
expect(result.applied).toBe(true);
expect(restarted()).toBe(false);
});
// A sweep rewrites the launch line once per variant, and the running daemon
// is the only record of the flags the user chose.
it('captures and restores the launch configuration a sweep started from', async () => {
await started();
await relaunchLlamaServerWithTuning({ ubatchSize: 512, flashAttn: true }, { reset: true });
const captured = await captureLlamaServerConfig();
await relaunchLlamaServerWithTuning({ batchSize: 4096 }, { reset: true });
expect((await getLlamaServerStatus()).config.flashAttn).toBe(false);
const result = await restoreLlamaServerConfig(captured);
expect(result.restored).toBe(true);
const back = (await getLlamaServerStatus()).config;
expect(back.ubatchSize).toBe(512);
expect(back.flashAttn).toBe(true);
expect(back.batchSize).toBeNull();
});
// A sweep is EXPECTED to produce launch lines that do not work. When the
// failing variant's own fallback could not get the previous configuration up
// either, llama-server is stopped and the install's provider is dead —
// which is when a restore matters most, not a reason to decline it.
it('starts from the captured configuration when the daemon is down', async () => {
await started();
const captured = await captureLlamaServerConfig();
await stopLlamaServer();
expect((await getLlamaServerStatus()).running).toBe(false);
const result = await restoreLlamaServerConfig(captured);
expect(result.restored).toBe(true);
expect((await getLlamaServerStatus()).running).toBe(true);
});
// A server somebody else started belongs to them.
it('refuses a running server PortOS does not manage', async () => {
await started();
const captured = await captureLlamaServerConfig();
pm2State = { name: 'someone-elses-llama', status: 'online', pid: 999, args: [] };
execPm2Calls = [];
const result = await restoreLlamaServerConfig(captured);
expect(result.restored).toBe(false);
expect(result.reason).toMatch(/outside PortOS/);
expect(restarted()).toBe(false);
});
it('captures nothing when PortOS is not the one running llama-server', async () => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
expect(await captureLlamaServerConfig()).toBeNull();
});
// ── An unreadable PM2 is a THIRD answer ──────────────────────────────────
// `getAppStatusStrict` returning null means the read failed, not that the
// daemon is somebody else's. Telling a user who owns this server that they
// started it in a terminal points them at a fix for a problem they do not
// have — and, worse, files their baseline reading as un-applied.
it('refuses an unreadable PM2 as itself, not as an external llama-server', async () => {
await started();
execPm2Calls = [];
pm2Reads.failures = Infinity;
const readsBefore = pm2Reads.count;
const result = await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
expect(result.applied).toBe(false);
expect(result.retryable).toBe(true);
expect(result.reason).toMatch(/could not read PM2/i);
expect(result.reason).not.toMatch(/outside PortOS/i);
expect(restarted()).toBe(false);
// The bypass probe for the retry: it asked more than once before
// answering, so a single blip cannot decide this.
expect(pm2Reads.count - readsBefore).toBeGreaterThan(1);
});
it('applies the tuning anyway when only the first PM2 read fails', async () => {
await started();
execPm2Calls = [];
pm2Reads.failures = 1;
const result = await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
expect(result.applied).toBe(true);
expect(launchArgs()).toContain('-ub');
});
// The regression #4759 turned expensive: an UNTUNED run now relaunches too,
// and on an unreadable PM2 the old code took the `!running` exit, which
// cleared the pre-tuning launch line. The baseline that run existed to
// restore was gone by the time PM2 answered again, so the model's "Backend
// defaults" row — the one `compareTunings` ranks every tuned reading
// against — never appeared at all.
it('does not discard the pre-tuning baseline on an unreadable PM2', async () => {
await started();
await relaunchLlamaServerWithTuning({ ubatchSize: 512, flashAttn: true });
// PM2 unreadable AND the endpoint slow to answer — the state that used to
// read as "nothing is running, so there is nothing left to undo".
pm2Reads.failures = Infinity;
vi.spyOn(openAiModelsProbe, 'probeOpenAiModels').mockResolvedValue({ reachable: false });
const refused = await relaunchLlamaServerWithTuning({});
expect(refused.applied).toBe(false);
expect(refused.retryable).toBe(true);
// PM2 answers again, and the launch line PortOS displaced is still there
// to put back.
pm2Reads.failures = 0;
probeTracksPm2();
execPm2Calls = [];
const cleared = await relaunchLlamaServerWithTuning({});
expect(cleared.applied).toBeNull();
expect(restarted()).toBe(true);
expect(launchArgs()).not.toContain('--flash-attn');
expect(launchArgs()).not.toContain('-ub');
});
// Capturing is a READ. Capturing nothing because PM2 blipped is what costs
// the user their launch flags: the sweep clears them either way, and the
// restore then has no record of what they were.
it('captures the last known launch line when the PM2 read fails', async () => {
await started();
pm2Reads.failures = Infinity;
expect((await captureLlamaServerConfig())?.model).toBe(modelPath);
});
it('refuses a restore on an unreadable PM2 without calling it somebody else\'s server', async () => {
await started();
const captured = await captureLlamaServerConfig();
pm2Reads.failures = Infinity;
execPm2Calls = [];
const result = await restoreLlamaServerConfig(captured);
expect(result.restored).toBe(false);
expect(result.retryable).toBe(true);
expect(result.reason).toMatch(/could not read PM2/i);
expect(result.reason).not.toMatch(/outside PortOS/i);
expect(restarted()).toBe(false);
});
it('restoring nothing is a no-op rather than a restart', async () => {
await started();
execPm2Calls = [];
const result = await restoreLlamaServerConfig(null);
expect(result.restored).toBe(false);
expect(restarted()).toBe(false);
});
// A launcher-owned knob is not in the cleared set, so judging 'did anything
// change?' on that set alone would report a new context size as applied
// while the server kept serving the old window.
it('relaunches for a launcher-owned knob the cleared set does not cover', async () => {
await started();
execPm2Calls = [];
const result = await relaunchLlamaServerWithTuning({ ctxSize: 4096 });
expect(result.applied).toBe(true);
expect(result.config.ctxSize).toBe(4096);
expect(launchArgs()).toContain('4096');
});
it('skips the relaunch when a named knob already matches what is running', async () => {
await started();
const running = (await getLlamaServerStatus()).config;
execPm2Calls = [];
const result = await relaunchLlamaServerWithTuning({ ctxSize: running.ctxSize });
expect(result.applied).toBe(true);
expect(restarted()).toBe(false);
});
// The model path, the port, and the window the user picked on the LLMs page
// are not sweepable knobs — clearing them would resize the server out from
// under them.
it('leaves the model, port, and launcher-owned knobs alone', async () => {
await started();
const before = (await getLlamaServerStatus()).config;
const result = await relaunchLlamaServerWithTuning({ ubatchSize: 1024 });
expect(result.config.model).toBe(before.model);
expect(result.config.port).toBe(before.port);
expect(result.config.ctxSize).toBe(before.ctxSize);
expect(result.config.nGpuLayers).toBe(before.nGpuLayers);
});
it('puts the knobs on the new launch line, keeping the model it was serving', async () => {
await started();
execPm2Calls = [];
const result = await relaunchLlamaServerWithTuning({ ubatchSize: 512, flashAttn: true });
expect(result.applied).toBe(true);
const start = execPm2Calls.find((c) => c[0] === 'start');
expect(start).toContain('-ub');
expect(start[start.indexOf('-ub') + 1]).toBe('512');
expect(start).toContain('--flash-attn');
expect(start[start.indexOf('-m') + 1]).toBe(modelPath);
});
// A sweep is EXPECTED to produce launch lines llama.cpp rejects. Leaving the
// daemon down would break every later request, not just this measurement.
it('restores the previous configuration when the tuned launch line exits', async () => {
await started();
// The harness's fake, captured as a raw function — re-spying and calling
// `pm2Module.execPm2` would re-enter this wrapper and blow the stack.
const fakeExec = pm2Module.execPm2.getMockImplementation();
let starts = 0;
vi.spyOn(pm2Module, 'execPm2').mockImplementation(async (args) => {
// Fail only the FIRST start after the relaunch (the tuned one); the
// restore that follows must succeed.
if (args[0] === 'start' && starts++ === 0) {
pm2State = { name: LLAMA_APP, status: 'errored', pid: null, args: [] };
execPm2Calls.push(args);
return { stdout: '', stderr: '' };
}
return fakeExec(args);
});
execPm2Calls = [];
const result = await relaunchLlamaServerWithTuning({ ubatchSize: 999999 });
expect(result.applied).toBe(false);
expect(result.reason).toMatch(/rejected that tuning/);
// The restore ran, and it carried the ORIGINAL model with no `-ub`.
const restore = execPm2Calls.filter((c) => c[0] === 'start').at(-1);
expect(restore).not.toContain('-ub');
expect(restore[restore.indexOf('-m') + 1]).toBe(modelPath);
});
// PM2 reporting `online` is not the same as the server answering. A daemon
// that never opened its port has not had the tuning applied in any sense a
// measurement could rest on.
// `startLlamaServer` polls for only four seconds, and a large GGUF routinely
// takes longer than that to load. Treating "not ready yet" as "wedged" would
// tear down a launch that was about to succeed.
it('waits past the start probe for a slow load rather than calling it wedged', async () => {
await started();
let answerAfter = 3;
vi.spyOn(openAiModelsProbe, 'probeOpenAiModels').mockImplementation(async () => {
if (pm2State?.status !== 'online') return { reachable: false };
return { reachable: answerAfter-- <= 0 };
});
execPm2Calls = [];
const result = await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
expect(result.applied).toBe(true);
// One start only — the slow load was waited out, not restarted.
expect(execPm2Calls.filter((c) => c[0] === 'start')).toHaveLength(1);
});
it('reports not-applied when the relaunched server never answers', async () => {
await started();
// PM2 keeps reporting `online` while the endpoint stays silent — the exact
// split the check exists for.
vi.spyOn(openAiModelsProbe, 'probeOpenAiModels').mockResolvedValue({ reachable: false });
// Shrink the readiness budget: the give-up path is what's under test, and
// the production two minutes would just be two minutes of sleeping.
resetForTest({ startupWaitTimeout: 0, relaunchReadyTimeout: 0 });
execPm2Calls = [];
const result = await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
expect(result.applied).toBe(false);
expect(result.reason).toMatch(/never answered/);
// A silent process must not be LEFT running: this daemon fronts the llama
// provider for the whole install, so the previous configuration goes back
// exactly as it does for a launch line llama.cpp rejects outright.
const restore = execPm2Calls.filter((c) => c[0] === 'start').at(-1);
expect(restore).not.toContain('-ub');
expect(restore[restore.indexOf('-m') + 1]).toBe(modelPath);
// The timeout budgets are injected through the lifecycle seam: this
// verifies the give-up/restore behavior without sleeping through the
// production startup and readiness windows.
}, 30000);
});
// ===========================================================================
// IDLE UNLOAD (--sleep-idle-seconds)
// ===========================================================================
//
// llama.cpp releases a checkpoint IN PLACE and reloads it on the next request,
// so PortOS passes a flag rather than stopping the process the way it does for
// MTPLX. The flag is recent, and an older build rejects an unknown flag and
// exits before it binds — hence the capability probe these tests pin.
describe('idle unload', () => {
// Pin the binary like every other lifecycle test here: CI has no llama.cpp
// installed, so an unmocked `findCommandOnPath` makes `startLlamaServer`
// refuse before it ever builds a launch line — a failure that reproduces
// nowhere on a developer machine that has the real binary on PATH.
beforeEach(() => {
vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
});
/** Pin `llama-server --help` output for the capability probe. */
const mockHelp = (text) => vi.spyOn(childProcess, 'execFile')
.mockImplementation((_bin, _args, _opts, cb) => { cb(null, text, ''); return fakeSpawnProcess(); });
const startArgs = () => (execPm2Calls.find((c) => c[0] === 'start') || []).slice(8);
it('puts the window on the launch line in SECONDS when the build supports it', async () => {
mockHelp('--sleep-idle-seconds SECONDS number of seconds of idleness');
await startLlamaServer({ model: modelPath, draftModel: null, specType: '', port: 8080, sleepIdleMinutes: 30 });
expect(startArgs()).toContain('--sleep-idle-seconds');
// 30 minutes on the card, 1800 seconds on the flag — the unit is the
// flag's, and getting this backwards would idle-unload after 30 seconds.
expect(startArgs()[startArgs().indexOf('--sleep-idle-seconds') + 1]).toBe('1800');
});
it('leaves the flag off entirely when the window is 0', async () => {
mockHelp('--sleep-idle-seconds SECONDS number of seconds of idleness');
await startLlamaServer({ model: modelPath, draftModel: null, specType: '', port: 8080, sleepIdleMinutes: 0 });
expect(startArgs()).not.toContain('--sleep-idle-seconds');
});
it('leaves the flag off when keepLoaded is configured', async () => {
mockHelp('--sleep-idle-seconds SECONDS number of seconds of idleness');
_setLlamaKeepLoadedOverrideForTests(true);
await startLlamaServer({ model: modelPath, draftModel: null, specType: '', port: 8080, sleepIdleMinutes: 30 });
expect(startArgs()).not.toContain('--sleep-idle-seconds');
});
// The compatibility guarantee: an install on an older llama.cpp must keep
// starting. Emitting an unknown flag would make the daemon exit immediately.
it('omits the flag on a build that does not advertise it, rather than failing the start', async () => {
mockHelp('--port PORT port to listen on\n--ctx-size N context size');
const result = await startLlamaServer({ model: modelPath, draftModel: null, specType: '', port: 8080, sleepIdleMinutes: 30 });
expect(result.success).toBe(true);
expect(startArgs()).not.toContain('--sleep-idle-seconds');
// ...and the status reports what actually reached the process, not what was asked for.
expect(result.config.sleepIdleMinutes).toBe(0);
});
it('says so in the logs when the build cannot idle-unload', async () => {
mockHelp('--port PORT port to listen on');
await startLlamaServer({ model: modelPath, draftModel: null, specType: '', port: 8080, sleepIdleMinutes: 30 });
const status = await getLlamaServerStatus();
expect(status.recentLogs.join('\n')).toContain('--sleep-idle-seconds');
});
it('treats a --help probe that fails as "unsupported"', async () => {
vi.spyOn(childProcess, 'execFile')
.mockImplementation((_bin, _args, _opts, cb) => { cb(new Error('ENOENT'), '', ''); return fakeSpawnProcess(); });
const result = await startLlamaServer({ model: modelPath, draftModel: null, specType: '', port: 8080, sleepIdleMinutes: 30 });
expect(result.success).toBe(true);
expect(startArgs()).not.toContain('--sleep-idle-seconds');
});
it('probes --help once per binary, not once per start', async () => {
const spy = mockHelp('--sleep-idle-seconds SECONDS');
await startLlamaServer({ model: modelPath, draftModel: null, specType: '', port: 8080, sleepIdleMinutes: 30 });
await stopLlamaServer();
await startLlamaServer({ model: modelPath, draftModel: null, specType: '', port: 8080, sleepIdleMinutes: 30 });
expect(spy).toHaveBeenCalledTimes(1);
});
// A PortOS restart under a still-running daemon recovers the launch line
// from PM2's argv; the window has to come back with it or the status card
// would report idle unload as off on a server that has it on.
it('recovers the window from a running process argv, converting back to minutes', async () => {
pm2State = {
name: LLAMA_APP,
status: 'online',
pid: 4242,
args: ['-m', modelPath, '--port', '8080', '--sleep-idle-seconds', '1800'],
};
const status = await getLlamaServerStatus();
expect(status.config.sleepIdleMinutes).toBe(30);
});
it('reports 0 for a running process whose argv carries no window', async () => {
pm2State = { name: LLAMA_APP, status: 'online', pid: 4242, args: ['-m', modelPath, '--port', '8080'] };
const status = await getLlamaServerStatus();
expect(status.config.sleepIdleMinutes).toBe(0);
});
});
});