Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

74.4 KB
Raw Blame
/**
 * llama-server process manager
 *
 * Provides lifecycle management (status probe, start, stop, recent logs)
 * for a local `llama-server` instance running speculative decoding (e.g. DFlash 2)
 * managed as an optional PM2 process (`portos-llama-server`).
 */

import { realpath, stat } from 'fs/promises';
import { spawn } from '../lib/childProcess.js';
import { commandExists } from '../lib/commandExists.js';
import { adoptPathDirs, findCommandOnPath, safeChildProcessEnv, safeChildProcessOptions } from '../lib/processEnv.js';
import { expandHome, sleep } from '../lib/fileUtils.js';
import { createDaemonWatcher, pm2ArgValue, LLAMA_APP } from '../lib/managedDaemon.js';
import { execFile } from '../lib/childProcess.js';
import { resolveSpecModelPath } from './specDecodeModels.js';
import { parseSpecTypes, isDraftSpecType } from '../lib/specDecodePresets.js';
import { probeOpenAiModels } from '../lib/openAiModelsProbe.js';
import { isPortInUse } from '../lib/platform.js';
import {
  LLAMA_CPP_BREW_FORMULA,
  LLAMA_CPP_DOWNLOAD_URL,
  isWingetManagedPath,
  llamaCppInstallPlan,
  parseWingetPackageFields,
  wingetLinkDirs,
} from '../lib/llamaCppInstall.js';
import { PORTS } from '../lib/ports.js';
import { getNavPageForPath } from '../lib/navManifest.js';
import { LOCAL_RUNTIME_MANAGE_URLS } from '../lib/localProviderRuntime.js';
import { tuningSpecsFor } from '../lib/localModelTuning.js';
import { ServerError } from '../lib/errorHandler.js';
import { bufferedSpawn } from '../lib/bufferedSpawn.js';
import { runStreamingCommand } from '../lib/streamingSpawn.js';
import { execPm2, getAppStatusStrict, clearJlistCache, getSavedProcessNames } from './pm2.js';
// `settings.js` is lazy-imported at its call sites below, never statically: it
// eagerly resolves `fileUtils.PATHS` at module load, which drags PATHS into the
// module graph of every consumer of this manager and breaks the many suites that
// partial-mock fileUtils without it. Same reason `services/aiProvider.js` defers
// `localModelHealing`.


export { LLAMA_APP };

const PROBE_TIMEOUT_MS = 1500;
const STARTUP_WAIT_TIMEOUT_MS = 4000;
const STARTUP_POLL_DELAY_MS = 500;
const RELAUNCH_READY_TIMEOUT_MS = 120000;
const RELAUNCH_POLL_DELAY_MS = 1000;
const VERSION_PROBE_TIMEOUT_MS = 5000;
const BREW_INFO_TIMEOUT_MS = 15_000;
// winget rebuilds its source index on the first query of a session, which takes
// longer than any single `brew info`.
const WINGET_LIST_TIMEOUT_MS = 60_000;
const UPGRADE_TIMEOUT_MS = 30 * 60 * 1000;
// `llama-server --help` is a fast local exec; this only bounds a wedged binary.
const HELP_PROBE_TIMEOUT_MS = 5000;
// llama.cpp's own in-place idle unload. See `supportsSleepIdle`.
const SLEEP_IDLE_FLAG = '--sleep-idle-seconds';
// binary path -> does it understand SLEEP_IDLE_FLAG. Keyed by path so a user who
// swaps in a newer build at a different location is re-probed rather than
// inheriting the old answer.
const sleepIdleSupport = new Map();
// See `configuredSleepIdleMinutes`. Only `_resetLlamaServerStateForTests` writes it.
let sleepIdleMinutesOverride = null;
let sleepIdleKeepLoadedOverride = null;
// Which package manager PortOS drives for llama.cpp. Only the test seam writes
// it, so both the Homebrew and the winget path stay coverable from either OS.
let platformOverride = null;
const installPlan = () => llamaCppInstallPlan(platformOverride || process.platform);
// How long a relaunch waits for the kernel to release the old listener.
const PORT_RELEASE_TIMEOUT_MS = 5000;
// How long a relaunch waits for the new process to answer. `startLlamaServer`
// polls for only STARTUP_WAIT_TIMEOUT_MS, which a large GGUF routinely exceeds
// while loading — so a relaunch must not read "not ready yet" as "wedged".
// Mutable only through the test seam below: a suite asserting the give-up path
// cannot sit through two real minutes of polling.
let startupWaitTimeoutMs = STARTUP_WAIT_TIMEOUT_MS;
let startupPollDelayMs = STARTUP_POLL_DELAY_MS;
let relaunchReadyTimeoutMs = RELAUNCH_READY_TIMEOUT_MS;
let relaunchPollDelayMs = RELAUNCH_POLL_DELAY_MS;
// How many times a path that would REFUSE on an unreadable PM2 re-reads it
// first. See `readLlamaServerStatusRetrying`.
const PM2_READ_RETRIES = 2;
// Mutable only through the test seam below, for the same reason as the readiness
// budget above.
const PM2_READ_RETRY_DELAY_MS = 250;
let pm2ReadRetryDelayMs = PM2_READ_RETRY_DELAY_MS;

let currentConfig = null;
// The launch line the daemon was serving BEFORE PortOS put an assessment tuning
// on it. `null` means no PortOS-applied tuning is in effect — either none was
// ever applied, or it has since been cleared — and is what makes an UNTUNED
// assessment honest: without it a baseline run would sample whatever the
// previous measurement left running and file the numbers as "Backend defaults".
//
// Deliberately NOT the same thing as `CLEARED_TUNING`. That renders the launch
// line with every sweepable knob off, which also wipes flags the USER set on the
// LLMs page — correct for a sweep, which captured them and will put them back,
// and wrong for an ordinary measurement, which did not. Restoring the captured
// line undoes only what PortOS itself applied.
//
// Module state, so it does not survive a PortOS restart that re-adopts a still
// -tuned PM2 process. That case degrades to today's behaviour (the untuned run
// no-ops) rather than to a wrong relaunch, and the reconstructed launch line
// cannot tell a PortOS tuning apart from the same flags typed by the user.
let preTuningConfig = null;
let lastExitError = null;

/**
 * Run a small local command and retain its output without throwing. Version and
 * Homebrew metadata are optional decorations on status, so a missing/broken
 * command must not take the runtime status endpoint down with it.
 */
async function readCommandOutput(command, args, timeoutMs) {
  const result = await bufferedSpawn(command, args, { timeoutMs, shell: false });
  return {
    error: result.success
      ? null
      : result.error || new Error(result.timedOut ? `timed out after ${timeoutMs}ms` : `exited with code ${result.code}`),
    stdout: result.stdout || '',
    stderr: result.stderr || '',
  };
}

/**
 * Read the version printed by the installed llama-server binary. Different
 * llama.cpp builds have used both `version: X` and `version X`, so accept both
 * spellings while keeping the status field to the useful short version token.
 */
function parseLlamaServerVersion(output) {
  const match = String(output || '').match(/\bversion\s*:?\s+([^\s(]+)/i);
  return match?.[1]?.replace(/^v/, '') || null;
}

async function readLlamaServerVersion(binaryPath) {
  const result = await readCommandOutput(binaryPath, ['--version'], VERSION_PROBE_TIMEOUT_MS);
  return result.error ? null : parseLlamaServerVersion(`${result.stdout}\n${result.stderr}`);
}

/**
 * Read Homebrew's formula metadata for the installed llama.cpp package.
 * `outdated` is authoritative here: llama.cpp's installed Homebrew version is
 * often a build number while the current formula uses a release version, so a
 * generic semver comparison would report false positives.
 */
async function readBrewLlamaCppInfo() {
  const result = await readCommandOutput(
    'brew',
    ['info', '--json=v2', '--formula', LLAMA_CPP_BREW_FORMULA],
    BREW_INFO_TIMEOUT_MS,
  );
  if (result.error) return null;

  return Promise.resolve()
    .then(() => JSON.parse(result.stdout))
    .then(async (payload) => {
      const formula = payload?.formulae?.find((entry) => entry?.name === LLAMA_CPP_BREW_FORMULA);
      const installed = formula?.installed?.[0];
      if (!installed) return null;
      const prefixResult = await readCommandOutput(
        'brew',
        ['--prefix', LLAMA_CPP_BREW_FORMULA],
        BREW_INFO_TIMEOUT_MS,
      );
      return {
        installedVersion: String(installed.version || ''),
        latestVersion: formula?.versions?.stable ? String(formula.versions.stable) : null,
        outdated: formula.outdated === true,
        pinned: formula.pinned === true,
        linked: Boolean(formula.linked_keg),
        prefix: prefixResult.error ? null : prefixResult.stdout.trim().split(/\r?\n/)[0] || null,
      };
    })
    .catch(() => null);
}

/**
 * Confirm that the binary found on PATH resolves to Homebrew's linked keg.
 *
 * `brew info` reports whether the formula is linked, not which executable the
 * current PATH resolves. A source build earlier on PATH can therefore coexist
 * with a linked Homebrew formula; upgrading the formula in that state would
 * leave the serving process on the old source build. Canonicalize both paths so
 * the normal `/opt/homebrew/bin` and `/opt/homebrew/opt` symlinks compare as the
 * same Cellar executable.
 */
async function isHomebrewLlamaServer(binaryPath, brewInfo) {
  if (!binaryPath || !brewInfo?.prefix) return false;
  const expectedPath = `${brewInfo.prefix}/bin/llama-server`;
  const [activePath, homebrewPath] = await Promise.all([
    realpath(binaryPath).catch(() => null),
    realpath(expectedPath).catch(() => null),
  ]);
  return Boolean(activePath && homebrewPath && activePath === homebrewPath);
}

/**
 * Homebrew's answer to "what is installed, is it stale, and may PortOS update
 * it?", in the shape every caller below consumes.
 *
 * `blockedReason` is what makes this the single manager-specific step: a pinned
 * or unlinked formula, or a source build sitting earlier on PATH, all stay the
 * user's to update by hand, and each refusal has to say which. `null` means
 * PortOS may run the upgrade, so `canUpgrade` is simply its absence.
 */
async function readBrewPackageStatus(binaryPath) {
  const brewInfo = await readBrewLlamaCppInfo();
  if (!brewInfo) return null;
  const blockedReason = brewInfo.pinned
    ? 'llama.cpp is pinned in Homebrew. Unpin it before asking PortOS to update it.'
    : !brewInfo.linked
      ? 'Homebrew has llama.cpp installed, but its keg is not linked to the llama-server on PATH. Link the formula manually before asking PortOS to update it.'
      : !(await isHomebrewLlamaServer(binaryPath, brewInfo))
        ? `The active llama-server is not the linked Homebrew llama.cpp binary, so PortOS left it untouched. Update that installation manually: ${LLAMA_CPP_DOWNLOAD_URL}`
        : null;
  return {
    installedVersion: brewInfo.installedVersion || null,
    latestVersion: brewInfo.latestVersion || null,
    outdated: brewInfo.outdated === true,
    canUpgrade: blockedReason === null,
    blockedReason,
  };
}

/**
 * The same shape, read from winget.
 *
 * winget has no `brew info --json` equivalent, so this asks two questions in
 * parallel instead of parsing one payload: `winget list` says whether the
 * package is installed at all (and at what version), and the same query with
 * `--upgrade-available` lists it only when a newer build exists. Neither answer
 * is a version comparison PortOS performs — llama.cpp versions are build
 * numbers (`b10730`), which no semver comparison orders correctly.
 */
async function readWingetPackageStatus(binaryPath, plan) {
  const [listed, upgradable] = await Promise.all([
    readCommandOutput('winget', plan.listArgs, WINGET_LIST_TIMEOUT_MS),
    readCommandOutput('winget', plan.upgradeCheckArgs, WINGET_LIST_TIMEOUT_MS),
  ]);
  if (listed.error) return null;
  const installedFields = parseWingetPackageFields(listed.stdout, plan.packageId);
  if (!installedFields) return null;
  // Only the `--upgrade-available` table is guaranteed to carry the `Available`
  // column, so the newer version is read from that run and nowhere else.
  const upgradeFields = upgradable.error ? null : parseWingetPackageFields(upgradable.stdout, plan.packageId);
  const ours = isWingetManagedPath(binaryPath);
  return {
    installedVersion: installedFields[0] || null,
    latestVersion: upgradeFields?.[1] || null,
    outdated: Boolean(upgradeFields),
    canUpgrade: ours,
    blockedReason: ours
      ? null
      : `The active llama-server is not the winget-installed llama.cpp, so PortOS left it untouched. Update that installation manually: ${LLAMA_CPP_DOWNLOAD_URL}`,
  };
}

/** Package metadata for whichever manager this platform uses, or `null`. */
const readPackageStatus = (plan, binaryPath) => (
  plan.manager === 'winget'
    ? readWingetPackageStatus(binaryPath, plan)
    : readBrewPackageStatus(binaryPath)
);

/**
 * Probes whether an OpenAI-compatible endpoint responds at the given host/port.
 * Shares one implementation with the readiness checklist.
 */
const probeEndpoint = async (endpoint) =>
  (await probeOpenAiModels(endpoint, { timeoutMs: PROBE_TIMEOUT_MS })).reachable;

/**
 * Resolves the `llama-server` executable on the child-process PATH.
 *
 * Deliberately a filesystem probe only — `findCommandOnPath` already requires a
 * regular file carrying the execute bit, which is all "is it installed?" means.
 * Do NOT re-add a `commandExists`-style `llama-server --version` probe on top.
 *
 * @returns {string|null} absolute path, or null when it is not on PATH
 */
function resolveLlamaServerBinary() {
  return findCommandOnPath('llama-server');
}

/**
 * Does the installed `llama-server` understand `--sleep-idle-seconds`?
 *
 * That flag is how llama.cpp releases a checkpoint WITHOUT the process going
 * away: after N idle seconds the server unloads the model and reloads it on the
 * next request, keeping its PM2 entry, its port, and its launch line. Stopping
 * the process instead would reclaim the same memory at the cost of a full cold
 * start — which is why PortOS reaps MTPLX (it has no such flag) but not this.
 *
 * Probed rather than assumed because PortOS is distributed software: the flag
 * landed in a recent llama.cpp, and every install upgrades on its own schedule.
 * An older binary REJECTS an unknown flag and exits before it binds, so emitting
 * it blind would break the start outright for anyone who hasn't upgraded.
 *
 * Cached per binary path — `--help` is a subprocess, and a start already pays
 * for several. A probe that cannot run at all answers `false`: leaving the flag
 * off costs the idle release, while guessing `true` costs the daemon.
 *
 * @param {string} binaryPath
 * @returns {Promise<boolean>}
 */
async function supportsSleepIdle(binaryPath) {
  if (sleepIdleSupport.has(binaryPath)) return sleepIdleSupport.get(binaryPath);
  const help = await new Promise((resolve) => {
    execFile(binaryPath, ['--help'], { timeout: HELP_PROBE_TIMEOUT_MS, ...safeChildProcessOptions() },
      (err, stdout, stderr) => resolve(`${stdout || ''}${stderr || ''}`));
  }).catch(() => '');
  const supported = help.includes(SLEEP_IDLE_FLAG);
  sleepIdleSupport.set(binaryPath, supported);
  if (!supported) {
    console.log(`🦙 llama-server at ${binaryPath} has no ${SLEEP_IDLE_FLAG} — idle unload unavailable on this build`);
  }
  return supported;
}

/**
 * The configured idle window for llama-server, in minutes.
 *
 * `0` is a real choice ("keep the model resident", the behaviour every install
 * had before this setting existed) and is the default, so an upgrade changes
 * nothing until the user asks for it. An unreadable settings file degrades to
 * the same `0` rather than to a window nobody chose.
 *
 * @returns {Promise<number>}
 */
async function configuredSleepIdleMinutes() {
  // Test seam. Suites here assert EXACT launch lines, so reading the developer's
  // real `data/settings.json` would make them pass or fail on whether that
  // developer happens to use this feature. `null` (production) reads settings.
  if (sleepIdleMinutesOverride !== null) return sleepIdleMinutesOverride;
  const settings = await import('./settings.js').then((m) => m.getSettings()).catch(() => null);
  const raw = Number(settings?.localLlm?.llama?.idleMinutes);
  return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : 0;
}

async function configuredSleepIdleKeepLoaded() {
  if (sleepIdleKeepLoadedOverride !== null) return sleepIdleKeepLoadedOverride;
  const settings = await import('./settings.js').then((m) => m.getSettings()).catch(() => null);
  return Boolean(settings?.localLlm?.llama?.keepLoaded ?? settings?.localLlm?.llama?.pinned);
}

/**
 * Fails the start request when a GGUF the launch line names is not on disk.
 */
async function assertModelFileExists(label, modelPath) {
  const stats = await stat(resolveSpecModelPath(modelPath)).catch(() => null);
  if (stats?.isFile()) return;
  throw new ServerError(
    `${label} was not found at \`${modelPath}\`. Use the Download button next to this preset in the Speculative Decoding card to fetch it, or point this field at a file you already have.`,
    { status: 400, code: 'LLAMA_MODEL_FILE_MISSING' }
  );
}

/**
 * Reconstructs the launch config from PM2 process args if PortOS restarted
 * while the PM2 process remained online.
 */
function parseConfigFromArgs(args) {
  if (!args) return null;
  const list = Array.isArray(args) ? args : String(args).split(' ');
  const getArg = (flag) => pm2ArgValue(list, flag);

  const model = getArg('-m') || getArg('--model');
  if (!model) return null;

  const draftModel = getArg('--model-draft') || getArg('--spec-draft-model') || getArg('-md');
  // Absent means the recovered process is running text-only — a pre-#7611
  // PortOS could not emit this flag at all, so every launch line it left behind
  // reads back as no projector, which is the truth about that process.
  const projector = getArg('--mmproj');
  // Absent means the process is running WITHOUT speculative decoding — don't
  // invent a type the launch line never carried.
  const specType = getArg('--spec-type');
  const port = getArg('--port') ? Number(getArg('--port')) : PORTS.LLAMA_SERVER;
  const host = getArg('--host') || '127.0.0.1';
  const ctxSize = getArg('--ctx-size') ? Number(getArg('--ctx-size')) : 32768;
  const nGpuLayers = getArg('-ngl') !== null ? Number(getArg('-ngl')) : 99;
  const alias = getArg('--alias') || 'dflash';
  // Absent is not 1: an older launch line with no `--parallel` is running
  // llama.cpp's own default (often 4 slots), which is distinct from a value
  // PortOS chose. A relaunch still pins 1 — see startLlamaServer.
  const parallelRaw = getArg('--parallel') ?? getArg('-np');
  const parallel = parallelRaw !== null ? Number(parallelRaw) : null;

  return {
    model,
    draftModel,
    projector,
    specType,
    port,
    host,
    ctxSize,
    nGpuLayers,
    alias,
    parallel,
    // Tuning flags. `null` means the flag was NOT on the launch line, so
    // llama.cpp's own default applied — distinct from a value we chose. A
    // caller re-launching with this config must leave a null off the line
    // rather than substituting a number llama.cpp never saw.
    batchSize: getArg('-b') !== null ? Number(getArg('-b')) : null,
    ubatchSize: getArg('-ub') !== null ? Number(getArg('-ub')) : null,
    threads: getArg('-t') !== null ? Number(getArg('-t')) : null,
    flashAttn: list.includes('--flash-attn') || list.includes('-fa'),
    cacheTypeK: getArg('--cache-type-k'),
    cacheTypeV: getArg('--cache-type-v'),
    // Current llama.cpp calls this `--spec-draft-n-max`; retain the older
    // spelling on read so a pre-upgrade PortOS launch line can still be captured
    // and restored without inventing a new value.
    // Stored in MINUTES here, on the launch line in SECONDS — the flag's own
    // unit. Absent means the daemon is running without idle unload, which is 0
    // ("keep it resident"), not "unknown".
    sleepIdleMinutes: getArg(SLEEP_IDLE_FLAG) !== null
      ? Math.round(Number(getArg(SLEEP_IDLE_FLAG)) / 60)
      : 0,
    draftMax: getArg('--spec-draft-n-max') !== null
      ? Number(getArg('--spec-draft-n-max'))
      : (getArg('--draft-max') !== null ? Number(getArg('--draft-max')) : null),
  };
}

// The endpoint the current (or last-known) configuration serves on. Split out so
// the two callers below can't drift on how host/port are defaulted.
const endpointFor = (config) =>
  `http://${config?.host || '127.0.0.1'}:${config?.port ?? PORTS.LLAMA_SERVER}/v1`;

const daemon = createDaemonWatcher({
  appName: LLAMA_APP,
  defaultPort: PORTS.LLAMA_SERVER,
  endpointFor,
  parseConfigFromArgs,
  probe: probeEndpoint,
  isPortInUse: (...args) => isPortInUse(...args),
  sleep,
  getConfig: () => currentConfig,
  setConfig: (config) => { currentConfig = config; },
  getLastExitError: () => lastExitError,
  getAppStatus: (...args) => getAppStatusStrict(...args),
  getSavedProcessNames: (...args) => getSavedProcessNames(...args),
  execPm2: (...args) => execPm2(...args),
  getPortReleaseTimeoutMs: () => PORT_RELEASE_TIMEOUT_MS,
  preserveConfigOnReadFailure: true,
});
const appendLog = daemon.appendLog;

/**
 * Just the base URL llama-server is serving on — no endpoint probe, no PM2 log
 * fetch.
 *
 * `getLlamaServerStatus` answers a much bigger question and pays for it with a
 * network probe AND an `execPm2 logs` subprocess. A caller that only needs
 * "which port is it on?" (the assessments read path, which runs on every
 * Performance page load) must not spawn a process to find out and then discard
 * the logs it paid for.
 *
 * Reads the same recovered-config path as the status call, so a PortOS restart
 * that left the PM2 process online still resolves the real port rather than the
 * default.
 */
export async function getLlamaServerEndpoint() {
  // Reconstructs `currentConfig` from PM2's args as a side effect when PortOS
  // restarted under a still-running daemon — see `readLlamaServerLaunch`.
  if (!currentConfig) await readLlamaServerLaunch();
  return daemon.endpoint();
}

/**
 * Returns current status of llama-server (binary availability, running state, config, logs).
 */
export async function getLlamaServerStatus() {
  const binaryPath = resolveLlamaServerBinary();
  const installed = Boolean(binaryPath);
  const plan = installPlan();

  return {
    ...await daemon.getStatusBase({ installed }),
    // How llama.cpp is installed HERE, so the LLMs page can name the command
    // instead of guessing. Carried on the lifecycle status rather than on the
    // update probe below, because the not-installed card is exactly the one
    // that has to name it — and it never reaches that probe, which needs a
    // binary to ask about.
    packageManager: plan.manager,
    packageManagerLabel: plan.managerLabel,
    installCommand: plan.installCommand,
    // The SAVED window, which is what the settings field edits. What the running
    // process actually got is `config.sleepIdleMinutes` — they differ until the
    // next start, because this is a launch flag.
    idleMinutes: await configuredSleepIdleMinutes(),
    keepLoaded: await configuredSleepIdleKeepLoaded(),
  };
}

/**
 * Check whether this platform's package manager has a newer llama.cpp build.
 * This is kept separate from the lifecycle status probe because the latter is
 * also used by performance/ownership paths that must remain disk-and-process
 * local; the LLMs page calls this explicit update-information path.
 */
export async function getLlamaServerUpdateStatus() {
  const binaryPath = resolveLlamaServerBinary();
  if (!binaryPath) {
    return {
      version: null,
      latestVersion: null,
      updateAvailable: false,
      canUpgrade: false,
      downloadUrl: LLAMA_CPP_DOWNLOAD_URL,
    };
  }

  const [version, packageStatus] = await Promise.all([
    readLlamaServerVersion(binaryPath),
    readPackageStatus(installPlan(), binaryPath),
  ]);
  return {
    version: version || packageStatus?.installedVersion || null,
    latestVersion: packageStatus?.latestVersion || null,
    updateAvailable: Boolean(packageStatus?.outdated),
    canUpgrade: Boolean(packageStatus?.canUpgrade),
    downloadUrl: LLAMA_CPP_DOWNLOAD_URL,
  };
}

/**
 * `getLlamaServerStatus`, re-read while PM2 answers nothing at all.
 *
 * `managed: null` is "the PM2 read FAILED", and a failed read is a transient
 * subprocess/IPC hiccup far more often than a standing condition — `fetchJlist`
 * caches successes only, so each attempt is a genuine second read rather than
 * the same cached null handed back.
 *
 * Every caller below turns a `null` into a refusal, and a refusal costs a
 * measured assessment its whole reading: an untuned run that cannot be applied
 * is filed `applied: false`, which `getAssessmentReport` drops from `scorable`,
 * which takes the BASELINE row `compareTunings` ranks everything else against
 * off the table. One blip is not worth that, so ask again before answering.
 */
async function readLlamaServerStatusRetrying() {
  let status = await getLlamaServerStatus();
  for (let attempt = 0; status.managed === null && attempt < PM2_READ_RETRIES; attempt += 1) {
    await sleep(pm2ReadRetryDelayMs);
    status = await getLlamaServerStatus();
  }
  return status;
}

const trimmedOrNull = (value) => (typeof value === 'string' && value.trim() ? value.trim() : null);

/**
 * Fills defaults for the launch options. Every tuning knob defaults to `null` =
 * NOT SET: the flag is left off the launch line entirely so llama.cpp applies
 * its own default. Substituting a number would silently pin a value the user
 * never chose and make two "default" runs incomparable.
 */
function normalizeLaunchOptions({
  model,
  draftModel,
  // llama.cpp's multimodal projector (`--mmproj`). Independent of speculative
  // decoding: a vision-capable GGUF loads text-only without it.
  projector,
  specType = 'draft-dflash',
  port = PORTS.LLAMA_SERVER,
  host = '127.0.0.1',
  ctxSize = 32768,
  nGpuLayers = 99,
  alias = 'dflash',
  parallel = 1,
  batchSize = null,
  ubatchSize = null,
  threads = null,
  flashAttn = false,
  cacheTypeK = null,
  cacheTypeV = null,
  draftMax = null,
  // `undefined` = not supplied, so fall through to the saved setting; an
  // explicit number (including 0) is the caller's own choice and wins, which
  // is what lets a relaunch replay exactly the line it captured.
  sleepIdleMinutes = undefined,
}) {
  return {
    model, draftModel, projector, specType, port, host, ctxSize, nGpuLayers, alias, parallel,
    batchSize, ubatchSize, threads, flashAttn, cacheTypeK, cacheTypeV, draftMax, sleepIdleMinutes,
  };
}

/** Refuses a launch the host cannot honor: no binary, already online, port taken. */
async function assertLaunchable(binaryPath, { model, host, port }) {
  if (!binaryPath) {
    throw new ServerError(
      `llama-server binary was not found on PATH. Install it with \`${installPlan().installCommand}\` or build from source.`,
      { status: 400 }
    );
  }

  const pm2Status = await getAppStatusStrict(LLAMA_APP);
  if (pm2Status && pm2Status.status === 'online') {
    throw new ServerError(`llama-server is already running with PID ${pm2Status.pid}`, { status: 409 });
  }

  if (!model || typeof model !== 'string') {
    throw new ServerError('model path is required to start llama-server', { status: 400 });
  }

  const endpoint = `http://${host}:${port}/v1`;
  if (await probeEndpoint(endpoint)) {
    throw new ServerError(`Port ${port} is already in use by an active server at ${endpoint}`, { status: 409 });
  }
  if (await isPortInUse(port)) {
    throw new ServerError(
      `Port ${port} is already in use on ${host}. Choose a different port under Advanced options before starting llama-server.`,
      { status: 409, code: 'LLAMA_SERVER_PORT_IN_USE' }
    );
  }
  return endpoint;
}

/**
 * `--spec-type` is a comma-separated LIST, and only its `draft-*` entries want
 * a drafter GGUF — every `ngram-*` implementation speculates off the tokens
 * already in the context window. Emitting the flag only alongside a drafter
 * would throw away valid drafter-free launches (`--spec-type ngram-map-k`) and
 * silently ignore the ngram half of a mixed one (`draft-dflash,ngram-map-k`).
 *
 * The two halves are resolved against each other, because the launcher card
 * seeds BOTH fields from a preset: switching Spec Type to an `ngram-*` leaves
 * the preset's drafter path sitting in the form, and passing that as
 * `--model-draft` would load weights the run can't use — or fail the existence
 * check on a preset GGUF that was never downloaded. So a drafter is only
 * carried when some requested type actually drafts with one, and drafter-based
 * types are dropped when no drafter is set, which keeps the card's documented
 * "clear the field to run without it" working. An EMPTY spec type deliberately
 * still counts as wanting the drafter: llama.cpp speculates off a bare
 * `--model-draft`, so dropping it there would silently disable a working
 * configuration.
 */
function resolveSpecDrafter({ specType, draftModel }) {
  const configuredDraftPath = trimmedOrNull(draftModel);
  const requestedSpecTypes = parseSpecTypes(specType);
  const drafterInUse = requestedSpecTypes.length === 0 || requestedSpecTypes.some(isDraftSpecType);
  const draftPath = drafterInUse ? configuredDraftPath : null;
  const effectiveSpecTypes = draftPath
    ? requestedSpecTypes
    : requestedSpecTypes.filter((type) => !isDraftSpecType(type));
  const droppedSpecTypes = requestedSpecTypes.filter((type) => !effectiveSpecTypes.includes(type));
  return { configuredDraftPath, draftPath, effectiveSpecTypes, droppedSpecTypes };
}

/**
 * llama.cpp's in-place idle unload. A build without the flag gets the line it
 * would have got before this feature existed — never a rejected launch.
 * `effective` is what actually reached the process, so the status card reports
 * the truth rather than the request.
 */
async function resolveIdleMinutes({ sleepIdleMinutes, binaryPath }) {
  const requested = Number.isFinite(sleepIdleMinutes)
    ? Math.max(0, Math.floor(sleepIdleMinutes))
    : await configuredSleepIdleMinutes();
  const pinned = await configuredSleepIdleKeepLoaded();
  const effective = !pinned && requested > 0 && await supportsSleepIdle(binaryPath)
    ? requested
    : 0;
  return { requested, effective, pinned };
}

/** Everything the launch line and `currentConfig` are derived from. */
async function resolveLaunch(options, binaryPath) {
  const opts = normalizeLaunchOptions(options);
  // Validate the weights before building the launch line
  await assertModelFileExists('The base model', opts.model.trim());
  // Unlike the drafter, the projector is resolved against nothing — no spec type
  // turns vision on or off, so what the caller set is what goes on the line.
  const projectorPath = trimmedOrNull(opts.projector);
  if (projectorPath) await assertModelFileExists('The vision projector', projectorPath);
  const drafter = resolveSpecDrafter(opts);
  if (drafter.draftPath) await assertModelFileExists('The drafter model', drafter.draftPath);
  // Resolved AFTER the rest so a relaunch that carried an explicit value replays
  // it verbatim, and only then falls back to the saved setting.
  const idle = await resolveIdleMinutes({ sleepIdleMinutes: opts.sleepIdleMinutes, binaryPath });
  // Always on the line — leaving it off reverts to llama.cpp's multi-slot
  // default, which is the VRAM tax this pin exists to avoid. `null` from a
  // recovered pre-flag launch still means 1, not "daemon default".
  const parallelSlots = Number.isFinite(opts.parallel) && opts.parallel >= 1 ? opts.parallel : 1;
  return { ...opts, ...drafter, projectorPath, parallelSlots, idle };
}

/** The llama-server argv, in a stable order. Pure over the resolved launch. */
function buildLaunchArgs(r) {
  const args = ['-m', expandHome(r.model.trim())];
  if (r.draftPath) args.push('--model-draft', expandHome(r.draftPath));
  if (r.projectorPath) args.push('--mmproj', expandHome(r.projectorPath));
  if (r.effectiveSpecTypes.length > 0) args.push('--spec-type', r.effectiveSpecTypes.join(','));
  if (r.port) args.push('--port', String(r.port));
  if (r.host) args.push('--host', r.host);
  if (r.ctxSize) args.push('--ctx-size', String(r.ctxSize));
  if (r.nGpuLayers !== undefined && r.nGpuLayers !== null) args.push('-ngl', String(r.nGpuLayers));
  args.push('--parallel', String(r.parallelSlots));
  if (Number.isFinite(r.batchSize)) args.push('-b', String(r.batchSize));
  if (Number.isFinite(r.ubatchSize)) args.push('-ub', String(r.ubatchSize));
  if (Number.isFinite(r.threads)) args.push('-t', String(r.threads));
  if (r.flashAttn) args.push('--flash-attn');
  if (r.cacheTypeK) args.push('--cache-type-k', String(r.cacheTypeK));
  if (r.cacheTypeV) args.push('--cache-type-v', String(r.cacheTypeV));
  // Only meaningful alongside a drafter — passing it without one makes
  // llama-server reject the launch line outright.
  if (Number.isFinite(r.draftMax) && r.draftPath) args.push('--spec-draft-n-max', String(r.draftMax));
  if (r.alias) args.push('--alias', r.alias);
  if (r.idle.effective > 0) args.push(SLEEP_IDLE_FLAG, String(r.idle.effective * 60));
  return args;
}

/**
 * What the status card reports: the values actually on the launch line (the
 * drafter, projector and spec types that survived resolution), not what the
 * form happened to hold. `null` projector/drafter means the process runs without.
 */
function buildCurrentConfig(r) {
  return {
    model: r.model,
    draftModel: r.draftPath,
    projector: r.projectorPath,
    specType: r.effectiveSpecTypes.join(','),
    port: r.port,
    host: r.host,
    ctxSize: r.ctxSize,
    nGpuLayers: r.nGpuLayers,
    alias: r.alias,
    parallel: r.parallelSlots,
    batchSize: r.batchSize,
    ubatchSize: r.ubatchSize,
    threads: r.threads,
    flashAttn: r.flashAttn,
    cacheTypeK: r.cacheTypeK,
    cacheTypeV: r.cacheTypeV,
    draftMax: r.draftMax,
    sleepIdleMinutes: r.idle.effective,
  };
}

function logLaunchResolution(r, args) {
  if (!r.idle.pinned && r.idle.requested > 0 && r.idle.effective === 0) {
    appendLog(`This llama-server build has no ${SLEEP_IDLE_FLAG} — the model stays resident while idle`);
  }
  if (r.droppedSpecTypes.length > 0) {
    appendLog(`Ignoring spec-type ${r.droppedSpecTypes.join(',')} — no drafter model is set`);
    console.log(`🦙 llama-server dropping drafter-based spec types ${r.droppedSpecTypes.join(',')} (no --model-draft configured)`);
  }
  if (r.configuredDraftPath && !r.draftPath) {
    appendLog(`Ignoring drafter ${r.configuredDraftPath} — no requested spec type uses one`);
    console.log(`🦙 llama-server ignoring drafter ${r.configuredDraftPath} (spec types ${r.effectiveSpecTypes.join(',') || 'none'} need no drafter)`);
  }
  appendLog(`Starting: llama-server ${args.join(' ')}`);
}

const isDeadPm2Status = (proc) =>
  Boolean(proc) && (proc.status === 'errored' || proc.status === 'stopped' || proc.status === 'not_found');

/** Polls until the server answers, PM2 reports the process dead, or the wait times out. */
async function waitForServerOnline(endpoint) {
  const startTime = Date.now();
  let online = false;
  let proc = null;
  while (Date.now() - startTime < startupWaitTimeoutMs) {
    await sleep(startupPollDelayMs);
    clearJlistCache();
    proc = await getAppStatusStrict(LLAMA_APP);
    if (isDeadPm2Status(proc)) break;
    online = await probeEndpoint(endpoint);
    if (online) break;
  }
  return { online, proc };
}

/** Surfaces the dead process's log tail, cleans up its PM2 entry, and throws. */
async function failEarlyExit(proc) {
  const exitLogs = await execPm2(['logs', LLAMA_APP, '--nostream', '--lines', '15']).catch(() => null);
  const lines = (exitLogs?.stderr || exitLogs?.stdout || '').trim().split('\n').map((l) => l.trimEnd()).filter(Boolean);
  for (const line of lines) appendLog(line);
  const tail = (lines.length ? lines : daemon.snapshotLogs()).slice(-4).join(' | ');

  lastExitError = `PM2 status: ${proc.status}`;

  await execPm2(['delete', LLAMA_APP]).catch(() => {});
  clearJlistCache();
  throw new ServerError(
    `llama-server exited immediately (${lastExitError}).${tail ? ` Last output: ${tail}` : ''}`,
    { status: 500, code: 'LLAMA_SERVER_EXITED' }
  );
}

/**
 * Starts llama-server with the specified model and options under PM2.
 */
export async function startLlamaServer(options = {}) {
  const binaryPath = resolveLlamaServerBinary();
  const endpoint = await assertLaunchable(binaryPath, normalizeLaunchOptions(options));
  const resolved = await resolveLaunch(options, binaryPath);
  const args = buildLaunchArgs(resolved);
  const { host, port, model, draftPath, projectorPath } = resolved;

  lastExitError = null;
  daemon.resetLogs();
  logLaunchResolution(resolved, args);

  // A fresh launch line supersedes any baseline: whatever PortOS tuned is gone
  // with the process it tuned, and restoring the old daemon's configuration over
  // this one would undo a start the user just asked for. A relaunch re-arms this
  // AFTER the start returns, from the baseline it captured itself.
  preTuningConfig = null;
  currentConfig = buildCurrentConfig(resolved);

  // Delete stale PM2 entry so our own previous instance doesn't count as a collision
  await execPm2(['delete', LLAMA_APP]).catch(() => {});
  clearJlistCache();

  console.log(`🦙 llama-server starting on ${host}:${port} (model ${model}${draftPath ? `, drafter ${draftPath}` : ''}${projectorPath ? `, projector ${projectorPath}` : ''})`);
  await execPm2([
    'start', binaryPath,
    '--name', LLAMA_APP,
    '--interpreter', 'none',
    '--no-autorestart',
    '--',
    ...args,
  ]);
  clearJlistCache();

  const { online, proc } = await waitForServerOnline(endpoint);
  if (isDeadPm2Status(proc)) await failEarlyExit(proc);

  const finalProc = await getAppStatusStrict(LLAMA_APP);

  return {
    success: true,
    running: true,
    managed: true,
    pid: finalProc?.pid || null,
    endpoint,
    online,
    config: currentConfig,
  };
}

/**
 * Stops the managed llama-server process.
 */
export async function stopLlamaServer() {
  const pm2Status = await getAppStatusStrict(LLAMA_APP);
  const isManaged = Boolean(pm2Status && pm2Status.status === 'online');

  if (!isManaged) {
    const host = currentConfig?.host || '127.0.0.1';
    const port = currentConfig?.port ?? PORTS.LLAMA_SERVER;
    const endpoint = `http://${host}:${port}/v1`;
    const reachable = await probeEndpoint(endpoint);
    if (reachable) {
      return {
        success: false,
        message: `An external process is listening on ${endpoint}. It was not started by PortOS and cannot be stopped here.`,
      };
    }
    return { success: true, message: 'llama-server is not running' };
  }

  appendLog(`Stopping ${LLAMA_APP}`);
  try {
    await execPm2(['delete', LLAMA_APP]);
    clearJlistCache();
  } catch (err) {
    throw new ServerError(`Failed to stop llama-server: ${err.message}`, { status: 500 });
  }
  currentConfig = null;
  // The tuning died with the process, so there is no longer anything to clear.
  // A relaunch that stops the daemon itself re-establishes this from its own
  // captured baseline — see `relaunchLlamaServerWithTuning`.
  preTuningConfig = null;

  return { success: true, message: 'llama-server stopped' };
}

/**
 * Block until nothing is listening on `port`, or the timeout elapses.
 *
 * `startLlamaServer` refuses when the port is still bound, and PM2's delete
 * returns before the kernel has released the listener — without this a relaunch
 * loses a race with itself and reports "port already in use" for the server it
 * just stopped.
 */
const waitForPortRelease = daemon.waitForPortRelease;

/**
 * Put a managed server back after an upgrade attempt. The model may take longer
 * than the normal port-release race, so wait before asking `startLlamaServer`
 * to validate the preserved launch configuration.
 */
async function restartManagedLlamaServer(config, baseline) {
  await waitForPortRelease(config.port ?? PORTS.LLAMA_SERVER);
  // The recovered PM2 config carries the effective idle flag from the old
  // process. Leave this field undefined so startLlamaServer reads the current
  // saved setting when an upgrade is also the next launch.
  const started = await startLlamaServer({ ...config, sleepIdleMinutes: undefined }).catch((err) => {
    console.error(`❌ llama-server: could not restore the previous configuration after update: ${err.message}`);
    return null;
  });
  if (!started) return null;
  if (!started.online && !(await waitForEndpoint(started.endpoint))) {
    console.error('❌ llama-server: restored process never answered after update');
    await stopLlamaServer().catch(() => {});
    return null;
  }
  preTuningConfig = baseline;
  return started;
}

async function upgradeFailureResult(message, managedConfig, baseline, updated = false) {
  const restored = managedConfig ? await restartManagedLlamaServer(managedConfig, baseline) : null;
  const restoreNote = managedConfig && !restored
    ? ' PortOS could not restore the llama-server process.'
    : '';
  return {
    success: false,
    ...(updated ? { updated: true } : {}),
    error: `${message}${restoreNote}`,
  };
}

/**
 * Refuse an upgrade PortOS has no business running, with the reason.
 *
 * A package PortOS did not install — a source build, a hand-extracted release
 * zip, a pinned or unlinked keg — must be left alone: swapping the package would
 * not touch the binary actually on PATH, so the user would be told their runtime
 * was updated while the serving process stayed exactly as stale as before.
 *
 * Manager-agnostic on purpose: `readPackageStatus` is the ONE place that knows
 * how each package manager answers "may PortOS update this?", so a new manager
 * is registered there and in its plan rather than adding a branch here.
 *
 * @returns {Promise<object|null>} a failure result, or `null` to proceed
 */
async function upgradePreflight(plan, binaryPath) {
  const status = await readPackageStatus(plan, binaryPath);
  if (!status) return { success: false, error: plan.notInstalledError };
  if (status.canUpgrade) return null;
  return { success: false, manualUpdateRequired: true, error: status.blockedReason };
}

/**
 * Upgrade a package-manager-installed llama.cpp binary in place.
 *
 * A managed llama-server is stopped before the package manager swaps the binary
 * and then restarted with the exact launch configuration it was using. An
 * external process is never stopped; changing the package only takes effect for
 * it when its owner starts it again.
 */
export async function upgradeLlamaServer({ onProgress = () => {} } = {}) {
  const plan = installPlan();
  const binaryPath = resolveLlamaServerBinary();
  const refusal = await upgradePreflight(plan, binaryPath);
  if (refusal) return refusal;

  const launch = await readLlamaServerLaunch();
  if (launch.readFailed) {
    return {
      success: false,
      error: 'PortOS could not read PM2 to determine whether it owns llama-server. Try the update again in a moment.',
    };
  }
  if (launch.managed && !launch.config?.model) {
    return {
      success: false,
      error: 'PortOS could not recover llama-server\'s launch configuration, so it left the running process untouched.',
    };
  }
  const managedConfig = launch.managed ? launch.config : null;
  const baseline = preTuningConfig;

  if (managedConfig) {
    onProgress({ event: 'progress', message: 'Stopping llama-server before updating…' });
    const stopped = await stopLlamaServer();
    if (!stopped.success) {
      return {
        success: false,
        error: stopped.message || 'PortOS could not stop the managed llama-server before updating.',
      };
    }
  }

  onProgress({ event: 'progress', message: `Updating llama.cpp via ${plan.managerLabel}…` });
  const upgraded = await runStreamingCommand(
    plan.manager,
    plan.upgradeArgs,
    (message) => onProgress({ event: 'progress', message }),
    { timeoutMs: UPGRADE_TIMEOUT_MS },
  );

  if (!upgraded.success) {
    return upgradeFailureResult(
      `llama.cpp upgrade failed: ${upgraded.error || `${plan.managerLabel} reported an unknown error`}.`,
      managedConfig,
      baseline,
    );
  }

  if (!resolveLlamaServerBinary()) {
    const { repaired, detail } = await repairBinaryOnPath(plan, safeChildProcessEnv(), onProgress);
    if (!repaired) {
      return upgradeFailureResult(
        `llama.cpp was upgraded, but its llama-server binary is not on PATH${detail ? `: ${detail}` : '.'}`,
        managedConfig,
        baseline,
        true,
      );
    }
  }

  // A new binary may gain support for a cached compatibility flag such as
  // --sleep-idle-seconds. Re-probe it before the preserved launch line returns.
  sleepIdleSupport.clear();

  if (managedConfig) {
    onProgress({ event: 'progress', message: 'Restarting llama-server with its previous settings…' });
    const restarted = await restartManagedLlamaServer(managedConfig, baseline);
    if (!restarted) {
      return {
        success: false,
        updated: true,
        error: 'llama.cpp was upgraded, but llama-server could not be restarted with its previous settings.',
      };
    }
    return {
      success: true,
      note: 'Updated llama.cpp and restarted llama-server with its previous settings.',
    };
  }

  return {
    success: true,
    note: 'Updated llama.cpp. The new binary will be used the next time llama-server starts.',
  };
}

/**
 * Block until the endpoint answers, or the readiness budget elapses.
 * `false` means it never answered — which is a wedged process, not a slow one.
 */
async function waitForEndpoint(endpoint) {
  const deadline = Date.now() + relaunchReadyTimeoutMs;
  while (Date.now() < deadline) {
    if (await probeEndpoint(endpoint)) return true;
    await sleep(relaunchPollDelayMs);
  }
  return false;
}

/**
 * Stop whatever is running and bring `next` up in its place, putting `previous`
 * back when the new launch line is rejected or never answers.
 *
 * Shared by the two relaunch paths below — both are "same weights, one field
 * changed", and this restore ceremony is the entire reason either of them is
 * safe to offer from a button: this daemon fronts the `llama` provider for the
 * whole install, so a relaunch that leaves it down breaks every later request,
 * not just the one that was being tuned or renamed.
 *
 * @returns {Promise<{ok: boolean, rejected: string|null, config: object|null}>}
 *   `ok: false` with `rejected: null` means the process came up but never
 *   answered on its port.
 */
async function relaunchWithConfig(next, previous) {
  // Both failure paths below land here: the port is free (or about to be), and
  // the known-good launch line goes back on it. Resolves to `null` rather than
  // throwing — a restore that fails is already the worst case, and replacing the
  // caller's own reason with a second error would hide why the relaunch failed.
  const restorePrevious = async () => {
    await waitForPortRelease(previous.port ?? PORTS.LLAMA_SERVER);
    const restored = await startLlamaServer(previous).catch((err) => {
      console.error(`❌ llama-server: could not restore the previous configuration: ${err.message}`);
      return null;
    });
    return restored?.config || null;
  };

  await stopLlamaServer();
  await waitForPortRelease(next.port ?? PORTS.LLAMA_SERVER);

  // A tuning sweep EXPECTS launch lines that don't work — `--flash-attn` on a
  // build without it, a `--cache-type-k` this build lacks, a `-ub` past what the
  // GPU can hold. llama-server exits immediately and `startLlamaServer` throws.
  const started = await startLlamaServer(next).catch(async (err) => {
    console.error(`❌ llama-server: relaunch failed (${err.message}) — restoring the previous configuration`);
    return { failure: err.message, config: await restorePrevious() };
  });
  if (started.failure) return { ok: false, rejected: started.failure, config: started.config };

  // PM2 reporting `online` is not the same as the server answering. But
  // `startLlamaServer` only polls for four seconds, and a large GGUF routinely
  // takes longer than that to load — so `online: false` is "not ready YET",
  // not "wedged". Give it a real readiness budget before judging.
  if (started.online || await waitForEndpoint(started.endpoint)) {
    return { ok: true, rejected: null, config: started.config };
  }

  // Still silent. Treat it exactly like a rejected launch line: put the previous
  // configuration back, so the install's llama provider is not left pointing at
  // a process that never serves.
  console.error('❌ llama-server: relaunched process never answered — restoring the previous configuration');
  await stopLlamaServer().catch(() => {});
  return { ok: false, rejected: null, config: await restorePrevious() };
}

/**
 * The launch line PM2 is currently running llama-server on, without the log
 * fetch and endpoint probe `getLlamaServerStatus` spends on a full status.
 *
 * `managed: false` covers both "nothing is running" and "something is listening
 * that PortOS did not start" — neither is a launch line PortOS may change.
 *
 * `readFailed: true` is a THIRD answer and must not collapse into either: it
 * means PM2 itself could not be read (`getAppStatusStrict`'s null sentinel), so
 * PortOS does not know what is running. Reporting that as "not PortOS's" would
 * tell a user who owns the daemon to go add `--alias` to a launch line PortOS
 * wrote — the fix is to retry. `getLlamaServerStatus` draws the same
 * distinction (`isReadFailed`).
 *
 * @returns {Promise<{managed: boolean, config: object|null, readFailed: boolean}>}
 */
export async function readLlamaServerLaunch() {
  return daemon.readLaunch();
}

/**
 * Relaunch llama-server serving the SAME weights under a different model id.
 *
 * llama.cpp answers `GET /v1/models` with exactly one entry, and that entry is
 * whatever `--alias` was on its launch line. So a provider pinned to
 * `qwen3.8-27b-dflash2` against a server started as `dflash` fails every request
 * even when the GGUF loaded IS Qwen3.8-27B + DFlash 2 — the ids simply never
 * matched. Nothing is missing and nothing needs downloading; the fix is a label.
 *
 * Refuses (never throws) when PortOS did not start the process: an
 * externally-launched llama-server belongs to whoever ran it, and stopping it to
 * change a name PortOS invented would kill a server the user owns.
 *
 * @param {string} alias the model id the provider will send
 * @returns {Promise<{applied: boolean|null, reason: string|null, config: object|null}>}
 *   `applied: null` means the daemon already answers under that id.
 */
export async function relaunchLlamaServerWithAlias(alias) {
  const wanted = String(alias ?? '').trim();
  if (!wanted) return { applied: false, reason: 'No model id was given to serve under.', config: null };

  const { managed, config, readFailed } = await readLlamaServerLaunch();
  if (readFailed) {
    return {
      applied: false,
      retryable: true,
      reason: 'PortOS could not read PM2 to find out what llama-server is running. Try again in a moment.',
      config: null,
    };
  }
  if (!managed || !config?.model) {
    const page = getNavPageForPath(LOCAL_RUNTIME_MANAGE_URLS.llama)?.breadcrumb;
    return {
      applied: false,
      reason: `llama-server is not running under PortOS, so its launch line is not PortOS's to change. ${page ? `Start it from ${page}` : 'Start it'}, or add \`--alias ${wanted}\` to your own launch line.`,
      config: null,
    };
  }
  if (config.alias === wanted) return { applied: null, reason: null, config };

  // A rename carries the whole launch line forward, tuning flags included, so
  // the baseline PortOS displaced when it tuned this daemon is still the line to
  // put back later. `startLlamaServer` clears it on every fresh launch, so it is
  // captured here and re-armed on success.
  const baseline = preTuningConfig;
  console.log(`🦙 llama-server: relaunching to answer as ${wanted} (was ${config.alias || 'unset'})`);
  const outcome = await relaunchWithConfig({ ...config, alias: wanted }, config);
  // The baseline is the line a later untuned assessment RELAUNCHES, so it has to
  // carry the new id or that assessment silently renames the daemon back and
  // re-breaks the provider this button just fixed. Only the alias moves: every
  // other field of the baseline is the user's own launch line, which a rename
  // does not touch. On either failure path the original alias is what is back on
  // the port, so the baseline stays exactly as captured.
  preTuningConfig = outcome.ok && baseline ? { ...baseline, alias: wanted } : baseline;
  if (outcome.rejected) {
    return { applied: false, reason: `llama-server rejected the relaunch: ${outcome.rejected}`, config: outcome.config };
  }
  if (!outcome.ok) {
    return {
      applied: false,
      reason: 'llama-server relaunched but never answered on its port — the previous model id is back.',
      config: outcome.config,
    };
  }
  return { applied: true, reason: null, config: outcome.config };
}

/**
 * Relaunch llama-server with a different tuning, keeping the model/drafter it is
 * already serving.
 *
 * This is the "evaluate tuning parameters for launching these" half of the
 * measured-assessment feature: a sweep across micro-batch sizes or KV-cache
 * types is only possible if something can put those flags on the launch line
 * between runs.
 *
 * It refuses rather than guesses in the three cases where it cannot know what to
 * relaunch, or must not:
 *   - PM2 could not be read (even after `readLlamaServerStatusRetrying` asked
 *     again), so PortOS cannot prove it owns the process. Reported as its own
 *     `retryable` refusal, never as the external-ownership one below;
 *   - nothing is running, so there is no model path to reuse;
 *   - something IS listening but PortOS did not start it (an externally-launched
 *     llama-server), so stopping it would kill a process the user owns.
 *
 * Every one of those returns `{ applied: false, reason }` instead of throwing:
 * the caller (an assessment run) can still measure whatever is actually serving
 * and record that the requested tuning was NOT applied, which is far more useful
 * than failing the whole run. A launch line llama-server rejects, and a relaunch
 * that never answers on its port, land on the same shape — and the rejected case
 * puts the PREVIOUS configuration back, because a tuning sweep is expected to
 * produce launch lines that don't work and must not leave the daemon down.
 *
 * `reset` makes the request the COMPLETE tuning rather than a patch: every
 * sweepable knob it omits goes back to off. A SWEEP needs that — merging onto
 * the previous launch line is what made its second variant inherit the first
 * variant's flags (`-b 4096` then `-ub 1024` launching with both, while the
 * record's label claimed one knob and `deltaPercent` credited an accumulated
 * line to a single change) — and with `reset` an EMPTY tuning becomes a real
 * instruction, "run backend defaults", which is the baseline every variant is
 * compared against.
 *
 * It is OPT-IN, and must stay that way. The user's launch flags live only in the
 * running process (the LLMs page seeds its form from hardcoded defaults, not
 * from what is running), so a reset the caller did not ask for destroys the only
 * copy of a configuration they chose. Only a sweep — which captures the
 * configuration first and puts it back afterwards — may request one.
 *
 * The reset covers the knobs llama-server leaves OFF the launch line when they
 * are absent. `ctxSize`, `nGpuLayers` and `parallel` are deliberately excluded:
 * every one of them is emitted unconditionally with a real launcher default
 * rather than having an unset state, the user picks them on the LLMs page, and
 * no sweep varies them — "clearing" them would just substitute the default and
 * silently resize the window (or the slot count) out from under a running
 * server.
 *
 * An EMPTY tuning WITHOUT `reset` is the third request: an untuned measurement
 * asking to be taken at backend defaults. It restores the launch line PortOS
 * displaced when it first tuned this daemon — which undoes what PortOS applied
 * and leaves the user's own flags exactly where they were, so it needs none of
 * the capture/restore ceremony a `reset` does. When PortOS has tuned nothing,
 * nothing is relaunched.
 *
 * @param {object} tuning launch knobs from `lib/localModelTuning.js`
 * @param {{reset?: boolean}} [options] `reset` clears every sweepable knob the
 *   tuning does not name. Never pass it from an ordinary measurement.
 * @returns {Promise<{applied: boolean|null, reason: string|null, config: object|null}>}
 *   `applied: null` means nothing needed to change — the daemon already serves
 *   the requested configuration, which is not a refusal and must not be recorded
 *   as one.
 */
const LAUNCHER_OWNED_KNOBS = new Set(['ctxSize', 'nGpuLayers', 'parallel']);

// Derived from the catalog rather than listed again here, so a knob added to
// llama.cpp's spec list is reset by default instead of quietly persisting into
// the next variant of a sweep. The value each knob resets TO is the one
// `startLlamaServer` reads as "leave the flag off the launch line".
const CLEARED_TUNING = Object.freeze(Object.fromEntries(
  tuningSpecsFor('llama')
    .filter((spec) => spec.config && !LAUNCHER_OWNED_KNOBS.has(spec.id))
    .map((spec) => [spec.id, spec.type === 'boolean' ? false : null])
));

// `undefined`, `null`, and a false toggle all render the same launch line — the
// flag is simply absent — so they must compare equal, or every untuned run on an
// untuned server would look like a change and pay for a restart.
const asLaunched = (value) => (value === undefined || value === null || value === false ? null : value);

/**
 * The launch configuration currently in effect, or `null` when llama-server is
 * not running or is somebody else's (nothing to put back, and nothing PortOS may
 * touch). An unreadable PM2 is NOT one of those: see below.
 *
 * Paired with `restoreLlamaServerConfig`: a tuning sweep relaunches the daemon
 * once per variant and would otherwise leave the last variant's flags in place
 * with the user's own flags cleared — and the running process is the only record
 * of what they chose.
 */
export async function captureLlamaServerConfig() {
  const status = await readLlamaServerStatusRetrying();
  // `managed !== false`, not `managed`: capturing is a READ, and the only thing
  // a capture can cost is `restoreLlamaServerConfig` later refusing to act on
  // it — which does its own ownership check against a fresh status. Capturing
  // nothing because PM2 blipped is the expensive answer: the sweep then clears
  // the user's own launch flags with no record of what they were.
  return status.running && status.managed !== false && status.config?.model ? status.config : null;
}

/**
 * Put a captured launch configuration back, exactly as it was.
 *
 * A no-op for `null` (nothing was captured) and for a configuration already in
 * effect, so a sweep that never actually changed anything costs no restart.
 *
 * A daemon that is DOWN is the case a restore matters most, not one to decline:
 * a sweep is expected to produce launch lines that do not work, and when the
 * failing one's own fallback could not get the previous configuration up either,
 * llama-server is stopped and the install's `llama` provider is dead until
 * somebody notices. The captured configuration is a known-good launch line and
 * nothing holds the port (`startLlamaServer` refuses if anything does), so it is
 * started rather than refused. Only a RUNNING server PortOS does not own is off
 * limits — either because it belongs to somebody else (`managed: false`) or
 * because PM2 could not be read to find out (`managed: null`, reported as its
 * own retryable refusal rather than as somebody else's process).
 */
export async function restoreLlamaServerConfig(config) {
  if (!config?.model) return { restored: false, reason: 'nothing was captured' };
  const status = await readLlamaServerStatusRetrying();
  // Split from the refusal below, not folded into it: a server PortOS cannot
  // READ is not a server somebody else started. Both still refuse — PortOS must
  // not stop a process it cannot prove it owns — but only one of them is worth
  // trying again, and only one of them is true.
  if (status.running && status.managed === null) {
    return {
      restored: false,
      retryable: true,
      reason: 'PortOS could not read PM2, so it cannot tell whether this llama-server is its own to restart',
    };
  }
  if (status.running && status.managed === false) {
    return { restored: false, reason: 'llama-server is now running outside PortOS, so its launch line is not PortOS\'s to change' };
  }

  if (status.running) {
    const differs = Object.keys(CLEARED_TUNING).some((id) => asLaunched(status.config?.[id]) !== asLaunched(config[id]));
    if (!differs) return { restored: true, reason: null };
    await stopLlamaServer();
  }
  await waitForPortRelease(config.port ?? PORTS.LLAMA_SERVER);

  console.log('🦙 llama-server: restoring the launch configuration the sweep started from');
  const started = await startLlamaServer(config).catch((err) => {
    console.error(`❌ llama-server: could not restore the launch configuration: ${err.message}`);
    return null;
  });
  return {
    restored: Boolean(started),
    reason: started ? null : 'llama-server would not start with the captured configuration, and is now stopped',
  };
}

export async function relaunchLlamaServerWithTuning(tuning = {}, { reset = false } = {}) {
  const empty = Object.values(tuning).every((v) => v === null || v === undefined);
  // An empty tuning with no `reset` is an UNTUNED measurement asking to be taken
  // at backend defaults. If PortOS put a tuning on this daemon, it has to come
  // back off — the reading is stored as "Backend defaults" either way, and
  // `compareTunings` ranks every real tuning against it. Restoring the captured
  // pre-tuning line, NOT `CLEARED_TUNING`: the reset clears sweepable knobs the
  // user may have set themselves, which only a sweep is entitled to do.
  const clearing = empty && !reset && Boolean(preTuningConfig);
  // Nothing PortOS applied is outstanding, so an empty tuning asks for nothing —
  // and answering before touching PM2 keeps an ordinary untuned measurement free
  // of a status round trip.
  if (!reset && empty && !clearing) {
    return { applied: null, reason: null, config: currentConfig };
  }

  const status = await readLlamaServerStatusRetrying();
  // Checked BEFORE `running`, and before anything is discarded. `managed: null`
  // means the PM2 read failed even on retry, so PortOS knows neither what is
  // running nor whether it owns it — and the `!running` branch below would clear
  // `preTuningConfig`, throwing away the only record of the launch line PortOS
  // displaced, on the strength of a read that never happened.
  //
  // It still refuses: PortOS cannot confirm it may restart this daemon, and
  // filing the reading as a trustworthy baseline would be worse. But it refuses
  // as itself — "could not read PM2", retryable — rather than borrowing the
  // wording of an external-ownership refusal, which sends a user who owns the
  // daemon looking for a process that does not exist.
  if (status.managed === null) {
    return {
      applied: false,
      retryable: true,
      reason: 'PortOS could not read PM2, so it cannot tell whether it owns this llama-server. Try again in a moment.',
      config: status.config || null,
    };
  }
  if (!status.running) {
    // Whatever carried the tuning is gone; the next start is untuned by
    // construction, so there is nothing left to undo.
    preTuningConfig = null;
    if (clearing) return { applied: null, reason: null, config: null };
    return { applied: false, reason: 'llama-server is not running, so PortOS has no model path to relaunch with', config: null };
  }
  if (!status.managed || !status.config?.model) {
    return {
      applied: false,
      // Worded for the direction it was asked in. A clearing run requested NO
      // tuning, and its row already reads "backend defaults" — telling that user
      // their tuning could not be applied contradicts what they are looking at.
      reason: clearing
        ? 'llama-server was started outside PortOS, so PortOS cannot relaunch it without the tuning it is carrying'
        : 'llama-server was started outside PortOS — start it from the LLMs page to let PortOS apply tuning',
      config: status.config || null,
    };
  }

  const previous = status.config;
  // Captured before `stopLlamaServer` clears it, and put back explicitly on
  // every exit below — the stop is part of this relaunch, not the user ending
  // the tuning session.
  const baseline = preTuningConfig;
  const next = clearing ? baseline : { ...previous, ...(reset ? CLEARED_TUNING : {}), ...tuning };
  // Nothing to do when the daemon is ALREADY running this exact tuning — the
  // common case for an untuned measurement on an untuned server, and the reason
  // making an empty tuning meaningful costs a plain assessment nothing.
  //
  // Compared across the cleared set UNION whatever the request names, not the
  // cleared set alone: a request for a launcher-owned knob (`ctxSize`,
  // `nGpuLayers`) changes nothing in the cleared set, and judging on that would
  // report a context-size change as applied while the server kept serving the
  // old window.
  //
  // A CLEARING run compares the whole captured line, not just the knob set: the
  // baseline is a complete configuration, and any field of it that drifted is a
  // difference this relaunch exists to undo.
  const compared = clearing
    ? new Set([...Object.keys(baseline), ...Object.keys(previous)])
    : new Set([...Object.keys(CLEARED_TUNING), ...Object.keys(tuning || {})]);
  const changed = [...compared].some((id) => asLaunched(next[id]) !== asLaunched(previous[id]));
  if (!changed) {
    // The daemon already serves the pre-tuning line, so the tuning is gone.
    if (clearing) preTuningConfig = null;
    return { applied: clearing ? null : true, reason: null, config: previous };
  }

  const knobs = Object.entries(tuning).filter(([, v]) => v !== null && v !== undefined);
  console.log(clearing
    ? '🦙 llama-server: relaunching without the tuning PortOS applied'
    : `🦙 llama-server: relaunching to apply tuning (${knobs.length ? knobs.map(([k, v]) => `${k}=${v}`).join(', ') : 'backend defaults'})`);
  const outcome = await relaunchWithConfig(next, previous);
  if (outcome.rejected) {
    // `previous` is back on the port, so whatever tuning it carried is still in
    // effect and still needs undoing later.
    preTuningConfig = baseline;
    return {
      applied: false,
      reason: clearing
        ? `llama-server rejected its pre-tuning launch line: ${outcome.rejected}`
        : `llama-server rejected that tuning: ${outcome.rejected}`,
      config: outcome.config,
    };
  }
  if (!outcome.ok) {
    // The relaunched process never answered. `previous` is back, so the caller
    // measures a live endpoint rather than recording timeouts as evidence.
    preTuningConfig = baseline;
    return {
      applied: false,
      reason: 'llama-server relaunched but never answered on its port',
      config: outcome.config,
    };
  }
  // The daemon now serves `next`. Clearing leaves it untuned, so there is
  // nothing left to undo; a first tuning records the line it displaced, and a
  // second must NOT overwrite that with the line the first one left.
  preTuningConfig = clearing ? null : (baseline || previous);
  // `null`, not `true`, for a clear: there was no tuning to apply, and the
  // reading that follows describes backend defaults truthfully.
  return { applied: clearing ? null : true, reason: null, config: outcome.config };
}

/**
 * Runs `brew link --overwrite llama.cpp`, resolving `{ linked, output }` on exit
 * rather than rejecting — a failed link attempt should fall through to the
 * caller's own error message, not replace it with a `brew link` failure. The
 * captured output is what makes a failure diagnosable ("Could not symlink…",
 * a permissions error on the prefix); discarding it left the caller guessing.
 */
function linkLlamaCpp(env) {
  return new Promise((resolve) => {
    const spawnOpts = safeChildProcessOptions({
      env,
      shell: false,
      stdio: ['ignore', 'pipe', 'pipe'],
    });
    const child = spawn('brew', ['link', '--overwrite', LLAMA_CPP_BREW_FORMULA], spawnOpts);
    let output = '';
    const collect = (chunk) => { output += chunk.toString(); };
    child.stdout?.on('data', collect);
    child.stderr?.on('data', collect);
    child.on('error', (err) => resolve({ linked: false, output: err.message }));
    // 'close' rather than 'exit': 'exit' fires the moment the process ends, with
    // stdio possibly still buffered, which would hand back a truncated (often
    // empty) `output` exactly when a failure needs explaining. `brew link` is a
    // short, single-process command, so waiting for the pipes to close is safe.
    child.on('close', (code) => resolve({ linked: code === 0, output: output.trim() }));
  });
}

/**
 * Adopt winget's portable-shim directory onto this process's PATH.
 *
 * winget links a portable package's executables into a Links directory and adds
 * that directory to the USER environment — a change an already-running PortOS
 * does not inherit, so a perfectly successful install would otherwise report
 * "llama-server was not found on PATH" until the server was restarted. See
 * `adoptPathDirs` for why extending `process.env.PATH` is the fix.
 *
 * @returns {string|null} the newly-visible binary path, or null if it is not there
 */
function adoptWingetLinkDirs() {
  adoptPathDirs(wingetLinkDirs());
  return resolveLlamaServerBinary();
}

/**
 * Put llama.cpp's binary on PATH when the package manager left it off.
 *
 * The two managers land in this state for different reasons and get out of it
 * in different ways, but from here they look identical — the manager reports a
 * package it installed, and PATH still finds nothing. `brew install`/`brew
 * upgrade` can leave a keg unlinked ("Warning: llama.cpp X is already
 * installed, it's just not linked."); winget puts its portable shims in a Links
 * directory it adds to the USER environment, which an already-running PortOS has
 * not inherited. Shared by the install and upgrade paths so a change to either
 * story is made once.
 *
 * Resolves rather than rejecting: a failed repair falls through to the caller's
 * own error message, and `detail` is what makes that message diagnosable.
 *
 * @returns {Promise<{repaired: boolean, detail: string|null}>}
 */
async function repairBinaryOnPath(plan, env, onProgress) {
  if (plan.manager === 'winget') {
    return { repaired: Boolean(adoptWingetLinkDirs()), detail: null };
  }
  onProgress({ event: 'progress', message: 'llama.cpp is installed but not linked — linking…' });
  const { linked, output } = await linkLlamaCpp(env);
  return {
    repaired: Boolean(linked && resolveLlamaServerBinary()),
    // A `brew link` conflict can list every clashing file, so cap what rides
    // along into the caller's error message.
    detail: output ? output.slice(0, 500) : null,
  };
}

/**
 * Installs llama.cpp through this platform's package manager — Homebrew on
 * macOS/Linux, winget on Windows.
 */
export async function installLlamaServer({ onProgress = () => {} } = {}) {
  const plan = installPlan();
  const managerExists = await commandExists(plan.manager, ['--version']);
  if (!managerExists) {
    throw new ServerError(plan.missingManagerError, { status: 400 });
  }

  onProgress({ event: 'start', message: `Installing llama.cpp via ${plan.managerLabel}…` });
  const env = safeChildProcessEnv();
  const spawnOpts = safeChildProcessOptions({
    env,
    shell: false,
    stdio: ['ignore', 'pipe', 'pipe'],
  });

  const child = spawn(plan.manager, plan.installArgs, spawnOpts);

  return new Promise((resolve, reject) => {
    child.stdout?.on('data', (d) => {
      onProgress({ event: 'progress', message: d.toString().trim() });
    });
    child.stderr?.on('data', (d) => {
      onProgress({ event: 'progress', message: d.toString().trim() });
    });
    // `error` and `exit` can both fire for one child (an `error` raised after a
    // successful spawn is followed by the process's own exit). Without this
    // guard the exit path would still emit a `complete` progress event to the
    // UI after the request had already been rejected — the client would render
    // "installed successfully" alongside a 500.
    let settled = false;
    child.on('error', (err) => {
      if (settled) return;
      settled = true;
      reject(new ServerError(`Failed to run ${plan.manager}: ${err.message}`, { status: 500 }));
    });
    // Async listener on a child-process event: it runs outside the Express
    // lifecycle AND outside this Promise executor's own throw path, so an
    // unguarded throw here would surface as an unhandled rejection while the
    // install request hung forever. Route every failure to `reject`.
    child.on('exit', async (code) => {
      if (settled) return;
      settled = true;
      try {
        let binaryPath = resolveLlamaServerBinary();
        // The manager reported success but PATH finds nothing — an unlinked keg
        // or a Links directory this process has not picked up. Repair it rather
        // than leaving that as a dead end for the caller.
        let repairDetail = null;
        if (!binaryPath && code === 0) {
          const repair = await repairBinaryOnPath(plan, env, onProgress);
          repairDetail = repair.detail;
          if (repair.repaired) binaryPath = resolveLlamaServerBinary();
        }

        if (binaryPath) {
          onProgress({ event: 'complete', message: `llama.cpp installed successfully (${binaryPath})` });
          resolve({ success: true, message: 'llama.cpp installed successfully' });
          return;
        }

        const hint = repairDetail ? `${plan.manager} said: ${repairDetail}` : plan.pathRepairHint;
        const msg = code === 0
          ? `${plan.manager} completed but llama-server was not found on PATH — ${hint}`
          : `${plan.installCommand} failed (exit code ${code})`;
        reject(new ServerError(msg, { status: 500 }));
      } catch (err) {
        reject(new ServerError(`Failed to verify the llama.cpp install: ${err.message}`, { status: 500 }));
      }
    });
  });
}

/**
 * Clears in-memory test state (used by test suites).
 */
export function _resetLlamaServerStateForTests({
  startupWaitTimeout,
  startupPollDelay,
  relaunchReadyTimeout,
  relaunchPollDelay,
  pm2ReadRetryDelay,
  sleepIdleMinutes = 0,
  sleepIdleKeepLoaded = null,
  platform = null,
} = {}) {
  sleepIdleSupport.clear();
  // Pinned rather than read from disk — see `configuredSleepIdleMinutes`.
  sleepIdleMinutesOverride = sleepIdleMinutes;
  sleepIdleKeepLoadedOverride = sleepIdleKeepLoaded;
  // `null` = use the real host platform, which is what every suite that is not
  // specifically exercising the other package manager's path wants.
  platformOverride = platform;
  currentConfig = null;
  preTuningConfig = null;
  daemon.resetLogs();
  lastExitError = null;
  // Restored to the production budget unless a suite asks for a shorter one.
  startupWaitTimeoutMs = Number.isFinite(startupWaitTimeout) ? startupWaitTimeout : STARTUP_WAIT_TIMEOUT_MS;
  startupPollDelayMs = Number.isFinite(startupPollDelay) ? startupPollDelay : STARTUP_POLL_DELAY_MS;
  relaunchReadyTimeoutMs = Number.isFinite(relaunchReadyTimeout) ? relaunchReadyTimeout : RELAUNCH_READY_TIMEOUT_MS;
  relaunchPollDelayMs = Number.isFinite(relaunchPollDelay) ? relaunchPollDelay : RELAUNCH_POLL_DELAY_MS;
  pm2ReadRetryDelayMs = Number.isFinite(pm2ReadRetryDelay) ? pm2ReadRetryDelay : PM2_READ_RETRY_DELAY_MS;
}

export function _setLlamaKeepLoadedOverrideForTests(val) {
  sleepIdleKeepLoadedOverride = val;
}