Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

38.2 KB
Raw Blame
import { getMemoryStats } from './memoryStats.js';
import { localEndpointPort } from './localEndpoint.js';
import { isLocalEndpointBusy, localEndpointsSharePort, onLocalEndpointSettled } from './localLlmConcurrency.js';
import { createSingleFlight } from './singleFlight.js';

/**
 * Shared plumbing for a local daemon PortOS runs as an optional PM2 process
 * (`llamaServerManager.js` → `portos-llama-server`, `mtplxServerManager.js` →
 * `portos-mtplx`, `slotstreamServerManager.js` → `portos-slotstream`).
 *
 * Both managers answer the same two questions the same way, and the answers are
 * fiddly enough that two copies drift:
 *
 *   - **What did it print?** A launcher card is useless without the daemon's
 *     recent output, and the output lives in two places — the lines PortOS
 *     itself logged around the launch, and what `pm2 logs` has. They have to be
 *     merged without duplicating the overlap and without growing unbounded.
 *   - **What was it launched with?** After a PortOS restart the only record of a
 *     still-online daemon's configuration is its PM2 argv, so both managers
 *     recover the launch flags by reading values back out of that array.
 *
 * Deliberately NOT a "daemon manager" abstraction: what each daemon's launch
 * line means, when it may be started, and what a refusal should say are exactly
 * the parts that differ, and folding them together would produce a base class
 * with two special cases. This is the shared *mechanism* only.
 */

/**
 * The PM2 process names of the local model servers PortOS manages.
 *
 * Declared here rather than in each manager so process names remain consistent;
 * `llamaServerManager.js`, `mtplxServerManager.js`, and
 * `slotstreamServerManager.js` re-export these as `LLAMA_APP` / `MTPLX_APP` /
 * `SLOTSTREAM_APP`.
 */
export const LLAMA_APP = 'portos-llama-server';
export const MTPLX_APP = 'portos-mtplx';
export const SLOTSTREAM_APP = 'portos-slotstream';

/** Same cap both managers used, and what the launcher cards render. */
const DEFAULT_MAX_LINES = 100;

/**
 * A bounded, timestamped ring buffer of a daemon's recent output.
 *
 * `withPm2Logs` does NOT fold the PM2 output into the buffer: PM2 owns those
 * lines and re-reads them on every status call, so remembering them here would
 * grow a second copy that outlives the process they came from. It returns the
 * merged VIEW a status response renders.
 *
 * @param {{maxLines?: number}} [options]
 */
export function createDaemonLogBuffer({ maxLines = DEFAULT_MAX_LINES } = {}) {
  let lines = [];

  const append = (line) => {
    if (!line) return;
    const text = String(line).trimEnd();
    if (!text) return;
    lines.push(`[${new Date().toISOString()}] ${text}`);
    if (lines.length > maxLines) lines = lines.slice(-maxLines);
  };

  return {
    append,
    maxLines,
    reset: () => { lines = []; },
    snapshot: () => [...lines],
    /**
     * This buffer's lines followed by anything in `pm2Output` it does not
     * already hold, capped to the same budget.
     * @param {string} pm2Output combined stdout + stderr from `pm2 logs`
     * @returns {string[]}
     */
    withPm2Logs(pm2Output) {
      const merged = [...lines];
      const seen = new Set(merged);
      for (const line of String(pm2Output || '').split('\n').map((l) => l.trimEnd()).filter(Boolean)) {
        if (seen.has(line)) continue;
        merged.push(line);
        seen.add(line);
      }
      return merged.length > maxLines ? merged.slice(-maxLines) : merged;
    },
  };
}

/**
 * The value following `flag` in a PM2 process's recorded argv, or `null`.
 *
 * `null` means the flag was NOT on the launch line, which is distinct from a
 * flag whose value happens to be falsy — a caller reconstructing a config must
 * leave an absent flag off a relaunch rather than substituting a default the
 * daemon never saw.
 *
 * @param {string[]|string} args PM2's `args` (an array, or the space-joined string it sometimes reports)
 * @param {string} flag
 * @returns {string|null}
 */
export function pm2ArgValue(args, flag) {
  const list = Array.isArray(args) ? args : String(args || '').split(' ');
  const idx = list.indexOf(flag);
  return idx !== -1 && idx + 1 < list.length ? list[idx + 1] : null;
}

/**
 * Shared watcher for a local daemon PortOS owns through PM2.
 *
 * Managers supply their daemon-specific launch-line parser and endpoint probe;
 * the watcher owns the common PM2 adoption, status skeleton, bounded logs, and
 * stop-then-relaunch port-release wait. State remains in the manager through
 * `getConfig` / `setConfig`, so install and tuning paths can keep their domain
 * rules without reaching into this mechanism.
 *
 * Dependency callbacks are explicit both to keep this module side-effect free
 * and to preserve the managers' existing test seams around PM2 and networking.
 */
export function createDaemonWatcher({
  appName,
  defaultHost = '127.0.0.1',
  defaultPort,
  endpointFor,
  parseConfigFromArgs,
  probe,
  isPortInUse,
  sleep,
  getConfig,
  setConfig,
  getLastExitError,
  getAppStatus,
  getSavedProcessNames,
  execPm2,
  getPortReleaseTimeoutMs,
  preserveConfigOnReadFailure = false,
  maxLogLines,
}) {
  const logs = createDaemonLogBuffer({ maxLines: maxLogLines });

  const recoverConfig = (pm2Status) => {
    const current = getConfig();
    if (current || pm2Status?.status !== 'online' || !pm2Status.args) return current;
    const recovered = parseConfigFromArgs(pm2Status.args);
    setConfig(recovered);
    return recovered;
  };

  const readLaunch = async () => {
    const pm2Status = await getAppStatus(appName);
    if (pm2Status === null) return { managed: false, config: null, readFailed: true };
    if (pm2Status.status !== 'online') return { managed: false, config: null, readFailed: false };
    return { managed: true, config: recoverConfig(pm2Status), readFailed: false };
  };

  const endpoint = () => endpointFor(getConfig());

  const getStatusBase = async ({ installed }) => {
    const [pm2Status, savedApps] = await Promise.all([getAppStatus(appName), getSavedProcessNames()]);
    const isReadFailed = pm2Status === null;
    const isManagedActive = pm2Status?.status === 'online';
    const config = recoverConfig(pm2Status);
    const resolvedEndpoint = endpointFor(config);
    const reachable = await probe(resolvedEndpoint);
    const pm2Logs = pm2Status && pm2Status.status !== 'not_found'
      ? await execPm2(['logs', appName, '--nostream', '--lines', String(logs.maxLines)]).catch(() => null)
      : null;

    return {
      installed,
      running: isManagedActive || reachable,
      managed: isReadFailed ? null : isManagedActive,
      pid: isManagedActive ? (pm2Status.pid || null) : null,
      host: config?.host || defaultHost,
      port: config?.port ?? defaultPort,
      endpoint: resolvedEndpoint,
      config: isManagedActive || (isReadFailed && preserveConfigOnReadFailure) ? config : null,
      runAtStartup: savedApps === null ? null : savedApps.includes(appName),
      recentLogs: logs.withPm2Logs(`${pm2Logs?.stdout || ''}\n${pm2Logs?.stderr || ''}`),
      lastExitError: isReadFailed ? 'Failed to read PM2 status' : getLastExitError(),
      releaseReason: isManagedActive ? null : (idleDaemons.get(appName)?.releaseReason ?? null),
    };
  };

  const waitForPortRelease = async (port) => {
    const deadline = Date.now() + getPortReleaseTimeoutMs();
    while (Date.now() < deadline && await isPortInUse(port)) await sleep(200);
  };

  return {
    appendLog: logs.append,
    endpoint,
    getStatusBase,
    readLaunch,
    resetLogs: logs.reset,
    snapshotLogs: logs.snapshot,
    waitForPortRelease,
  };
}

/**
 * The last few lines PM2 has for a dead daemon, folded into the manager's own
 * log buffer and summarized alongside the PM2 status word.
 *
 * Shared because `startMtplxServer`'s relaunch-wait loop and
 * `ensureSlotstreamRunning`'s used to diagnose the identical failure two
 * different ways: MTPLX's wait loop tailed the PM2 log (6a9348344, 7480e0476),
 * Slotstream's reported only `PM2 status: errored`. A launch line that dies
 * mid-load deserves the same diagnosis regardless of which daemon it was.
 *
 * @param {{appName: string, execPm2: (args: string[]) => Promise<{stdout?: string, stderr?: string}>, appendLog: (line: string) => void, lines?: number, tailLines?: number}} options
 * @returns {(status: string) => Promise<string>}
 */
export function createPm2ExitTail({ appName, execPm2, appendLog, lines = 15, tailLines = 4 }) {
  return async (status) => {
    const pm2Logs = await execPm2(['logs', appName, '--nostream', '--lines', String(lines)]).catch(() => null);
    const outputLines = `${pm2Logs?.stderr || pm2Logs?.stdout || ''}`.split('\n').map((l) => l.trimEnd()).filter(Boolean);
    for (const line of outputLines) appendLog(line);
    const tail = outputLines.slice(-tailLines).join(' | ');
    return tail ? `PM2 status: ${status} — ${tail}` : `PM2 status: ${status}`;
  };
}

/**
 * Shared on-demand wake mechanism for a local daemon that stops on idle
 * (`registerIdleDaemon` above) and must come back up lazily for the next
 * request. Extracted from `mtplxServerManager.js` and `slotstreamServerManager.js`,
 * which had drifted into two copies with different bugs (#8105): MTPLX's port
 * arm never matched (a string compared with `===` to a number), and only
 * MTPLX's death-detecting readiness loop carried a PM2 log tail.
 *
 * Mechanism only, same contract as `createDaemonWatcher` above: what a launch
 * line MEANS (which knobs it carries, how a saved one merges with a live one)
 * stays in `resolveLaunch`, which each manager supplies.
 *
 * @param {{
 *   appName: string,
 *   label: string,
 *   emoji: string,
 *   readSection: () => Promise<object|null>,
 *   endpointFor: (config: object|null) => string,
 *   probe: (endpoint: string) => Promise<boolean>,
 *   start: (config: object) => Promise<object>,
 *   stop: () => Promise<unknown>,
 *   getStatusStrict: () => Promise<object|null>,
 *   clearStatusCache?: () => void,
 *   exitTail?: (status: string) => Promise<string>,
 *   resolveLaunch: (current: object|null, saved: {model: string|null, port: number|null, raw: object}) => object,
 *   getConfig?: () => object|null,
 *   parseConfigFromArgs?: (args: string[]|string) => object|null,
 *   sleep: (ms: number) => Promise<void>,
 *   getRelaunchReadyTimeoutMs: () => number,
 *   getRelaunchPollMs: () => number,
 * }} options
 */
export function createOnDemandDaemon({
  appName,
  label,
  emoji,
  readSection,
  endpointFor,
  probe,
  start,
  stop,
  getStatusStrict,
  clearStatusCache = () => {},
  exitTail,
  resolveLaunch,
  getConfig = () => null,
  parseConfigFromArgs = () => null,
  sleep,
  getRelaunchReadyTimeoutMs,
  getRelaunchPollMs,
}) {
  // Test seam, same shape as each manager's own `_set*OverrideForTests`: a
  // suite must not depend on whether this developer has an idle window
  // configured. `null` means "read settings"; both live here now rather than
  // in the manager, since `registerIdle` below is what reads them.
  let idleMinutesOverride = null;
  let keepLoadedOverride = null;

  const configuredIdleMinutes = async () => {
    if (idleMinutesOverride !== null) return idleMinutesOverride;
    const section = await readSection();
    const raw = Number(section?.idleMinutes);
    return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : 0;
  };

  const configuredKeepLoaded = async () => {
    if (keepLoadedOverride !== null) return keepLoadedOverride;
    const section = await readSection();
    // Legacy fallback: `pinned` was the setting's name before `keepLoaded`.
    return Boolean(section?.keepLoaded ?? section?.pinned);
  };

  let inFlightStarts = 0;
  // Explicit starts return after a short HTTP budget while weights are still
  // loading. This stays raised until the endpoint answers or PM2 leaves online.
  let inFlightLoads = 0;
  // In-memory launch config is null after a PortOS restart until something
  // recovers argv. The provider slot is the port the process is actually on.
  let boundEndpoint = null;

  const portEndpoint = (config) => (
    Number.isFinite(Number(config?.port)) ? endpointFor({ port: Number(config.port) }) : null
  );
  const busyEndpoint = () => portEndpoint(getConfig()) || boundEndpoint || endpointFor(getConfig());
  const isBusy = () => inFlightStarts > 0 || inFlightLoads > 0 || isLocalEndpointBusy(busyEndpoint());

  let unsubscribeSettled = () => {};

  /** The launch line last saved on the settings card, sanitized. */
  const savedLaunchConfig = async () => {
    const section = await readSection();
    const launch = section?.launch || {};
    return {
      model: typeof launch.model === 'string' && launch.model.trim() ? launch.model.trim() : null,
      port: Number.isFinite(Number(launch.port)) ? Number(launch.port) : null,
      // Whatever else the daemon's own knobs need (MTPLX's `tuning`,
      // Slotstream's `memoryGb`) — shaped and normalized by `resolveLaunch`,
      // never generically here, since only the manager knows that shape.
      raw: launch,
    };
  };

  const refreshBoundEndpoint = async (knownStatus) => {
    const live = portEndpoint(getConfig());
    if (live) {
      boundEndpoint = live;
      return;
    }
    const status = knownStatus ?? await Promise.resolve(getStatusStrict()).catch(() => null);
    if (status?.status === 'online' && status.args) {
      try {
        const fromArgs = portEndpoint(parseConfigFromArgs(status.args));
        if (fromArgs) {
          boundEndpoint = fromArgs;
          return;
        }
      } catch {
        // Unparseable argv is not a port. The saved launch is the next source.
      }
    }
    const saved = await savedLaunchConfig().catch(() => null);
    if (saved?.port != null) boundEndpoint = endpointFor({ port: saved.port });
  };

  // Resolves once, then stays sync: the reaper's last busy check cannot await.
  const ensureBoundEndpoint = () => (
    portEndpoint(getConfig()) || boundEndpoint ? Promise.resolve() : refreshBoundEndpoint()
  );

  const registerIdle = () => {
    unsubscribeSettled();
    settledUnsubscribers.delete(unsubscribeSettled);
    unsubscribeSettled = onLocalEndpointSettled((endpoint) => {
      if (localEndpointsSharePort(endpoint, busyEndpoint())) markDaemonUsed(appName);
    });
    settledUnsubscribers.add(unsubscribeSettled);
    registerIdleDaemon({
      name: appName,
      getIdleMs: async () => idleWindowMs(await configuredIdleMinutes()),
      isPinned: async () => configuredKeepLoaded(),
      isRunning: async () => Boolean((await getStatusStrict())?.status === 'online'),
      isBusy,
      warm: ensureBoundEndpoint,
      stop: () => stop(),
    });
  };

  /**
   * Block until the relaunched daemon answers, or until it is proven dead.
   * MTPLX's `waitForRelaunchedEndpoint`, generalized: poll PM2 alongside the
   * endpoint, because "still loading" and "already died" look identical from
   * the endpoint alone and cost wildly different amounts of time to wait out.
   */
  const waitForReady = async (endpoint) => {
    const deadline = Date.now() + getRelaunchReadyTimeoutMs();
    while (Date.now() < deadline) {
      if (await probe(endpoint)) return { ready: true, reason: null };
      clearStatusCache();
      const proc = await getStatusStrict();
      if (proc && ['errored', 'stopped', 'not_found'].includes(proc.status)) {
        const tail = exitTail ? await exitTail(proc.status) : `PM2 status: ${proc.status}`;
        return { ready: false, reason: `${label} exited while loading (${tail})` };
      }
      await sleep(getRelaunchPollMs());
    }
    return { ready: false, reason: `${label} relaunched but never answered on its port` };
  };

  const markUsed = () => markDaemonUsed(appName);

  /**
   * Bring the daemon up if the idle reaper (or the user) stopped it, and mark
   * it used either way. A no-op when already online — the overwhelmingly
   * common case, and it has to stay cheap enough to sit in front of every
   * request: the one PM2 status read it costs is the same read a status poll
   * already does.
   *
   * Resolves `{ ready, reason }` rather than throwing: a caller in front of an
   * inference request wants to report "could not be started" alongside its own
   * error, not have a lazy start unwind its stack.
   *
   * Single-flight: concurrent callers (two CoS agents, an agent plus a voice
   * turn) share one wake. Without it each saw the daemon offline and called
   * `start`, whose `pm2 delete` killed the sibling launch, so both failed with
   * "exited while loading". Joining is safe because the launch line comes from
   * live/saved state, not from the caller. The slot clears on settle, so a later
   * wake after another stop starts the daemon again.
   *
   * The slot is exclusive and shared with `runStart` (#12723): an explicit start
   * or a tuning relaunch would otherwise `pm2 delete` an in-flight wake's
   * process, the same race between a different pair of callers. A wake that
   * arrives during one waits for it, then finds the daemon online.
   */
  const starts = createSingleFlight({ exclusive: true });

  const trackStart = async (key, fn) => {
    inFlightStarts += 1;
    markUsed();
    try {
      return await starts.run(key, fn);
    } finally {
      inFlightStarts = Math.max(0, inFlightStarts - 1);
      markUsed();
    }
  };

  const ensureRunning = () => trackStart('wake', wake);

  const armLoadWatch = (endpoint) => {
    inFlightLoads += 1;
    const settle = () => {
      inFlightLoads = Math.max(0, inFlightLoads - 1);
      markUsed();
    };
    // Two-arm then, not finally: a floating finally re-raises on a promise
    // nobody awaits. The watch must not hold the caller's start open — launch*
    // returns after its short budget while a cold checkpoint is still loading.
    waitForReady(endpoint).then(settle, settle);
  };

  /**
   * Run an explicit start/stop sequence (a user-clicked Start, a tuning
   * relaunch) in the daemon's exclusive start slot. Each call gets its own key,
   * so it queues behind an in-flight wake or start rather than joining it: its
   * launch config can differ from the one being loaded. `fn` must call the
   * manager's raw start, never `ensureRunning` or `runStart` — either would
   * wait on the slot `fn` itself holds.
   *
   * @template T
   * @param {() => Promise<T>} fn
   * @returns {Promise<T>}
   */
  const runStart = (fn) => trackStart(Symbol('start'), async () => {
    const result = await fn();
    if (result?.online === false && typeof result.endpoint === 'string') armLoadWatch(result.endpoint);
    return result;
  });

  const wake = async () => {
    const pm2Status = await getStatusStrict();
    if (pm2Status?.status === 'online') {
      // Reuse the status just read. A second PM2 read here would sit on every
      // request, and getConfig() is still null until argv is recovered.
      if (!portEndpoint(getConfig())) await refreshBoundEndpoint(pm2Status);
      return { ready: true, reason: null };
    }

    // Resolve the launch line BEFORE probing, so the probe below asks about
    // the port this start would actually bind. `resolveLaunch` sets the
    // precedence: the config recovered from the last live process, then the
    // launch the user saved, then the daemon's own default.
    const saved = await savedLaunchConfig();
    const config = resolveLaunch(getConfig(), saved);
    if (Number.isFinite(Number(config?.port))) boundEndpoint = endpointFor({ port: Number(config.port) });

    // Something else is already serving that port — a daemon the user started
    // outside PortOS, or another process entirely. Either way this is not
    // ours to start, and probing beats racing `start` into a port conflict.
    const endpoint = endpointFor(config);
    if (await probe(endpoint)) return { ready: true, reason: null };

    console.log(`${emoji} ${label} is stopped — starting it for an incoming request`);
    const started = await start(config).catch((err) => ({ error: err }));

    if (started.error) return { ready: false, reason: started.error.message };
    // `start` returns as soon as it knows the process did not die on the
    // spot; a multi-gigabyte checkpoint routinely outlasts that window, so the
    // caller's request has to wait for the real readiness signal.
    if (started.online) return { ready: true, reason: null };
    return waitForReady(started.endpoint ?? endpoint);
  };

  /** Is `provider` served by the port THIS daemon's live launch is bound to? */
  const servesPort = (provider, managedPort) =>
    Boolean(managedPort) && Number(localEndpointPort(provider?.endpoint)) === managedPort;

  return {
    ensureRunning,
    runStart,
    markUsed,
    registerIdle,
    waitForReady,
    servesPort,
    // Card-status reads. Kept on the returned object rather than re-declared
    // in the manager — that is exactly the "no longer define
    // configuredIdleMinutes/configuredKeepLoaded" half of #8105.
    idleMinutes: configuredIdleMinutes,
    keepLoaded: configuredKeepLoaded,
    savedLaunch: savedLaunchConfig,
    setIdleMinutesOverrideForTests: (value) => { idleMinutesOverride = value; },
    setKeepLoadedOverrideForTests: (value) => { keepLoadedOverride = value; },
  };
}

// =============================================================================
// IDLE REAPER
// =============================================================================

/**
 * Shared "stop this daemon when nothing has used it for a while" mechanism.
 *
 * ONLY for a daemon that cannot release its weights any other way. `llama-server`
 * deliberately does NOT register here: it carries its own `--sleep-idle-seconds`,
 * which unloads the model in place and reloads it on the next request without the
 * process ever going away (see `llamaServerManager.js`). Stopping that process to
 * reclaim the same memory would trade a cheap internal reload for a full PM2
 * cold start, and lose the launch line with it. MTPLX has no such flag — its
 * `--retrieval-idle-timeout` unloads retrieval models only, never the main
 * checkpoint — so stopping the process is the only way to get its 20GB back, and
 * it is the one registrant.
 *
 * One `setInterval` for every registrant, not one per daemon: the beat is a
 * coarse poll against a timestamp, so N timers would buy nothing but N chances
 * to leak one.
 */

/** How often the reaper checks. Coarse on purpose — the windows are minutes. */
const IDLE_REAP_INTERVAL_MS = 60_000;

/** Default free-memory threshold below which host memory is considered under pressure (4 GB). */
export const DEFAULT_PRESSURE_THRESHOLD_BYTES = 4 * 1024 * 1024 * 1024;
/** Dead-band: memory must rise above threshold + dead-band (4 GB + 2 GB = 6 GB) to exit pressure. */
export const DEFAULT_PRESSURE_DEAD_BAND_BYTES = 2 * 1024 * 1024 * 1024;
/** How long pressure must be sustained before a daemon is released early (30s). */
export const DEFAULT_SUSTAINED_PRESSURE_MS = 30_000;
/** Calm-down window after releasing a daemon before another daemon can be released (60s). */
export const DEFAULT_PRESSURE_CALM_DOWN_MS = 60_000;

/** Format a timestamp into HH:MM (e.g. 09:14) for human-legible release notes. */
export function formatReleaseTime(timestamp = Date.now()) {
  const d = new Date(timestamp);
  const hh = String(d.getHours()).padStart(2, '0');
  const mm = String(d.getMinutes()).padStart(2, '0');
  return `${hh}:${mm}`;
}

/** name → `{ getIdleMs, isPinned, isRunning, isBusy, warm, stop, lastUsedAt, releaseReason, releasedAt }`. */
const idleDaemons = new Map();
/** Unsubscribes for per-daemon slot listeners. Cleared with the idle map in tests. */
const settledUnsubscribers = new Set();
let reaperTimer = null;

let pressureHistory = [];
let lastPressureReleaseAt = null;
const MAX_PRESSURE_HISTORY = 100;

export function recordPressureSample({ at = Date.now(), free, used, total }) {
  pressureHistory.push({ at, free, used, total });
  if (pressureHistory.length > MAX_PRESSURE_HISTORY) {
    pressureHistory = pressureHistory.slice(-MAX_PRESSURE_HISTORY);
  }
}

export function getPressureHistory() {
  return [...pressureHistory];
}

export function getLastPressureReleaseTime() {
  return lastPressureReleaseAt;
}

export function setLastPressureReleaseTime(time) {
  lastPressureReleaseAt = time;
}

/**
 * A user-supplied idle window in minutes, as milliseconds.
 *
 * `0` means "never stop" and is returned as `0`, NOT as null — it is a real
 * choice (today's always-on behaviour) and must survive a round-trip through
 * settings distinguishably from "no value stored". Anything unparseable or
 * negative is `null` = not configured, which the reaper also treats as never.
 *
 * @param {unknown} minutes
 * @returns {number|null}
 */
export function idleWindowMs(minutes) {
  // `Number(null)` and `Number('')` are both 0, which would make "nothing
  // stored" indistinguishable from the user explicitly choosing "never stop".
  // They mean the same thing to the reaper, but not to a caller reporting what
  // is configured — so absent stays null.
  if (minutes === null || minutes === undefined || minutes === '') return null;
  const n = Number(minutes);
  if (!Number.isFinite(n) || n < 0) return null;
  return Math.floor(n) * 60_000;
}

/**
 * Register a daemon the reaper may stop.
 *
 * `lastUsedAt` is seeded to NOW rather than to null, so a daemon that was just
 * started by hand — or one PortOS re-adopted after its own restart — gets a
 * full idle window before it is eligible. Seeding null and treating it as
 * "infinitely idle" would reap a server the user started seconds ago.
 *
 * Re-registering the same name refreshes the hooks and leaves `lastUsedAt`
 * alone, so a manager reloaded under test doesn't reset a live clock.
 *
 * @param {{
 *   name: string,
 *   getIdleMs: () => Promise<number|null>|number|null,
 *   isPinned?: () => Promise<boolean>|boolean,
 *   isRunning?: () => Promise<boolean>|boolean,
 *   isBusy?: () => Promise<boolean>|boolean,
 *   warm?: () => Promise<void>|void,
 *   stop: () => Promise<unknown>
 * }} daemon
 *   `getIdleMs` resolves the CURRENT configured window on every sweep; `null`/`0` = never stop.
 *   `isPinned` returns true if user pinned the server ("keep loaded") — pinned servers are never stopped.
 *   `isRunning` checks if the daemon process is online.
 *   `isBusy` returns true while a wake/start, a still-loading checkpoint, or an inference request is in flight — busy servers are never stopped.
 *   `warm` may resolve the port `isBusy` reads before the synchronous re-check. It must not itself decide to stop.
 */
export function registerIdleDaemon({ name, getIdleMs, isPinned, isRunning, isBusy, warm, stop }) {
  const existing = idleDaemons.get(name);
  idleDaemons.set(name, {
    getIdleMs,
    isPinned: typeof isPinned === 'function' ? isPinned : () => Boolean(isPinned),
    isRunning: typeof isRunning === 'function' ? isRunning : null,
    isBusy: typeof isBusy === 'function' ? isBusy : () => Boolean(isBusy),
    warm: typeof warm === 'function' ? warm : null,
    stop,
    lastUsedAt: existing?.lastUsedAt ?? Date.now(),
    releaseReason: existing?.releaseReason ?? null,
    releasedAt: existing?.releasedAt ?? null,
  });
}

/**
 * Record that something just used `name` — the signal the whole mechanism runs
 * on. Call it on real traffic (an inference request, a lazy start), never on a
 * status poll: a status card that refreshes every few seconds would otherwise
 * hold a 24GB checkpoint resident forever while nobody used it.
 *
 * Clears any prior release reason now that the server is active again.
 *
 * @param {string} name
 */
export function markDaemonUsed(name) {
  const entry = idleDaemons.get(name);
  if (entry) {
    entry.lastUsedAt = Date.now();
    entry.releaseReason = null;
    entry.releasedAt = null;
  }
}

/** The recorded last-use timestamp for `name`, or `null`. Exposed for status cards. */
export function daemonLastUsedAt(name) {
  return idleDaemons.get(name)?.lastUsedAt ?? null;
}

/** The recorded release reason for `name`, or `null`. */
export function daemonReleaseReason(name) {
  return idleDaemons.get(name)?.releaseReason ?? null;
}

/** Explicitly clear release reason for `name`. */
export function clearDaemonReleaseReason(name) {
  const entry = idleDaemons.get(name);
  if (entry) {
    entry.releaseReason = null;
    entry.releasedAt = null;
  }
}

/**
 * Pure policy function for memory pressure daemon eviction.
 * Evaluates current state, pressure reading, and recent history.
 *
 * Returns `{ shouldRelease: boolean, target?: object, reason?: string, ... }`.
 */
export function evaluateMemoryPressurePolicy({
  daemons = [],
  memoryStats = null,
  history = [],
  now = Date.now(),
  lastReleasedAt = null,
  options = {},
} = {}) {
  const thresholdBytes = options.pressureThresholdBytes ?? DEFAULT_PRESSURE_THRESHOLD_BYTES;
  const deadBandBytes = options.deadBandBytes ?? DEFAULT_PRESSURE_DEAD_BAND_BYTES;
  const sustainedDurationMs = options.sustainedDurationMs ?? DEFAULT_SUSTAINED_PRESSURE_MS;
  const calmDownMs = options.calmDownMs ?? DEFAULT_PRESSURE_CALM_DOWN_MS;

  if (!memoryStats || typeof memoryStats.free !== 'number') {
    return { shouldRelease: false, target: null, reason: 'memory stats unavailable' };
  }

  const free = memoryStats.free;
  const wasUnderPressure = Boolean(options.wasUnderPressure ?? (lastReleasedAt && (now - lastReleasedAt < calmDownMs * 2)));
  const exitThresholdBytes = thresholdBytes + deadBandBytes;
  const isRelieved = wasUnderPressure
    ? free >= exitThresholdBytes
    : free >= thresholdBytes;

  if (isRelieved) {
    return {
      shouldRelease: false,
      target: null,
      reason: 'host memory not under pressure',
      free,
      thresholdBytes,
      exitThresholdBytes,
      wasUnderPressure,
    };
  }

  if (lastReleasedAt && (now - lastReleasedAt < calmDownMs)) {
    return {
      shouldRelease: false,
      target: null,
      reason: 'in calm-down window',
      remainingCalmDownMs: calmDownMs - (now - lastReleasedAt),
    };
  }

  if (sustainedDurationMs > 0) {
    // Track "under pressure" against the same bar isRelieved just used above —
    // when wasUnderPressure, that's the higher exit threshold, not the base
    // threshold — otherwise samples sitting in the dead-band (below exitThresholdBytes
    // but above thresholdBytes) never count as sustained and eviction starves.
    const activeThresholdBytes = wasUnderPressure ? exitThresholdBytes : thresholdBytes;
    const validSamples = (history || [])
      .map((s) => ({
        at: s.at ?? s.timestamp ?? s.time ?? now,
        free: s.free ?? (s.total != null && s.used != null ? s.total - s.used : null),
      }))
      .filter((s) => s.at <= now && typeof s.free === 'number')
      .sort((a, b) => a.at - b.at);

    let earliestUnderPressureAt = null;
    for (let i = validSamples.length - 1; i >= 0; i--) {
      if (validSamples[i].free < activeThresholdBytes) {
        earliestUnderPressureAt = validSamples[i].at;
      } else {
        break;
      }
    }

    const sustainedMs = earliestUnderPressureAt != null ? (now - earliestUnderPressureAt) : 0;
    if (sustainedMs < sustainedDurationMs) {
      return {
        shouldRelease: false,
        target: null,
        reason: 'pressure not sustained',
        sustainedMs,
        requiredMs: sustainedDurationMs,
      };
    }
  }

  const list = Array.isArray(daemons)
    ? daemons
    : Array.from(daemons?.values?.() || []);
  const eligible = list.filter((d) => (
    d
    && d.running !== false
    && !d.busy
    && !d.pinned
    && !d.keepLoaded
  ));

  if (eligible.length === 0) {
    return {
      shouldRelease: false,
      target: null,
      reason: 'no eligible daemons to release',
    };
  }

  // Least recently used ordering: smallest lastUsedAt first
  eligible.sort((a, b) => (a.lastUsedAt ?? 0) - (b.lastUsedAt ?? 0));
  const target = eligible[0];

  return {
    shouldRelease: true,
    target,
    reason: 'host memory pressure',
    free,
    thresholdBytes,
  };
}

// Synchronous on purpose. Awaiting here is the yield in which a request can
// start after the snapshot and still be stopped. A thenable fails safe to busy;
// its rejection is swallowed so the second call cannot become unhandled.
function readBusySync(entry) {
  try {
    const value = entry?.isBusy?.();
    if (value && typeof value.then === 'function') {
      value.catch(() => {});
      return true;
    }
    return Boolean(value);
  } catch {
    return true;
  }
}

/**
 * Sweep: stop daemons whose idle window has elapsed, and run pressure-aware pass
 * to release the least recently used unpinned daemon under sustained host memory pressure.
 *
 * @param {number} [now]
 * @param {object} [options]
 * @returns {Promise<string[]>} the names actually stopped
 */
export async function reapIdleDaemons(now = Date.now(), options = {}) {
  const stopped = [];

  // 1. Normal idle timeout pass
  for (const [name, entry] of idleDaemons) {
    // Fail-safe to pinned/busy on a transient read error, matching the pressure-aware
    // pass below — assuming "not pinned" or "not busy" here would risk stopping a
    // keepLoaded or actively serving daemon on a flaky read.
    await Promise.resolve(entry.warm?.()).catch(() => {});
    const isPinned = await Promise.resolve(entry.isPinned?.()).catch(() => true);
    if (isPinned) continue; // Pinned servers are exempt

    const isBusy = await Promise.resolve(entry.isBusy?.()).catch(() => true);
    if (isBusy) continue; // Active wake/start or in-flight request is exempt

    // Already stopped (by the pressure-aware pass, or externally) — skip so this
    // pass doesn't overwrite its release reason and doesn't preempt the
    // pressure-aware pass below via the early return once its idle window re-elapses.
    const isRunning = entry.isRunning
      ? await Promise.resolve(entry.isRunning()).catch(() => true)
      : true;
    if (!isRunning) continue;

    // Resolved per sweep, so lowering the window in Settings applies to the very
    // next beat rather than to the next server restart.
    const windowMs = await Promise.resolve(entry.getIdleMs()).catch(() => null);
    if (!windowMs || windowMs <= 0) continue;
    if (now - entry.lastUsedAt < windowMs) continue;
    if (readBusySync(entry)) continue;

    const idleMin = Math.round((now - entry.lastUsedAt) / 60_000);
    console.log(`💤 Stopping ${name} — idle ${idleMin}m (window ${Math.round(windowMs / 60_000)}m)`);
    // `stop` reaches PM2 over a subprocess. A failure here must not kill the
    // interval that every other daemon's reaping depends on.
    const failed = await Promise.resolve(entry.stop()).then(() => null, (err) => err);
    if (failed) {
      console.error(`❌ Idle stop of ${name} failed: ${failed.message}`);
      continue;
    }
    // Only on success: a failed stop that left the daemon up would otherwise
    // retry every beat forever with the clock reset each time.
    entry.lastUsedAt = now;
    entry.releasedAt = now;
    entry.releaseReason = `released at ${formatReleaseTime(now)} — idle timeout`;
    stopped.push(name);
  }

  // Release at most one daemon per tick and re-read before the next
  if (stopped.length > 0) {
    return stopped;
  }

  // 2. Pressure-aware pass
  const memoryStats = options.memoryStats ?? await getMemoryStats().catch(() => null);
  if (memoryStats) {
    recordPressureSample({
      at: now,
      free: memoryStats.free,
      used: memoryStats.used,
      total: memoryStats.total,
    });

    const daemonList = [];
    for (const [name, entry] of idleDaemons) {
      await Promise.resolve(entry.warm?.()).catch(() => {});
      const isPinned = await Promise.resolve(entry.isPinned?.()).catch(() => true);
      const isBusy = await Promise.resolve(entry.isBusy?.()).catch(() => true);
      const isRunning = entry.isRunning
        ? await Promise.resolve(entry.isRunning()).catch(() => false)
        : true;
      daemonList.push({
        name,
        entry,
        lastUsedAt: entry.lastUsedAt,
        pinned: isPinned,
        busy: isBusy,
        running: isRunning,
      });
    }

    const policyOptions = { ...options.policyOptions, ...options };
    const decision = evaluateMemoryPressurePolicy({
      daemons: daemonList,
      memoryStats,
      history: options.history ?? getPressureHistory(),
      now,
      lastReleasedAt: getLastPressureReleaseTime(),
      options: policyOptions,
    });

    if (decision.shouldRelease && decision.target && !readBusySync(decision.target.entry)) {
      const { target } = decision;
      const freeMb = Math.round((memoryStats.free || 0) / (1024 * 1024));
      console.log(`⚠️ Stopping ${target.name} — host memory pressure (free ${freeMb}MB)`);
      const failed = await Promise.resolve(target.entry.stop()).then(() => null, (err) => err);
      if (failed) {
        console.error(`❌ Pressure stop of ${target.name} failed: ${failed.message}`);
      } else {
        target.entry.lastUsedAt = now;
        target.entry.releasedAt = now;
        target.entry.releaseReason = `released at ${formatReleaseTime(now)} — host memory pressure`;
        setLastPressureReleaseTime(now);
        stopped.push(target.name);
      }
    }
  }

  return stopped;
}

/**
 * Arm the single reaper timer. Idempotent — a second call is a no-op rather than
 * a second interval.
 *
 * Boot-safe by construction: it arms a timer and reads timestamps. It makes no
 * AI provider call, which is what lets `server/index.js` start it unconditionally
 * under AGENTS.md's "No cold-bootstrap LLM calls" rule.
 *
 * @param {{intervalMs?: number}} [options]
 */
export function startIdleReaper({ intervalMs = IDLE_REAP_INTERVAL_MS } = {}) {
  if (reaperTimer) return;
  reaperTimer = setInterval(() => {
    // Outside the Express request lifecycle: an unhandled rejection here would
    // take the process down, so the sweep's own failures are swallowed after
    // logging (each daemon's stop failure is already reported individually).
    reapIdleDaemons().catch((err) => console.error(`❌ Idle reaper sweep failed: ${err.message}`));
  }, intervalMs);
  // Never hold the event loop open for this — a shutdown must not wait a minute
  // for a poll that has nothing to do.
  reaperTimer.unref?.();
  console.log(`💤 Idle reaper armed (checking every ${Math.round(intervalMs / 1000)}s)`);
}

/** Disarm the reaper. For shutdown and for test isolation. */
export function stopIdleReaper() {
  if (!reaperTimer) return;
  clearInterval(reaperTimer);
  reaperTimer = null;
}

/** Test seam: drop every registration, reset pressure state, and disarm the timer. */
export function _resetIdleDaemonsForTests() {
  stopIdleReaper();
  for (const unsubscribe of settledUnsubscribers) unsubscribe();
  settledUnsubscribers.clear();
  idleDaemons.clear();
  pressureHistory = [];
  lastPressureReleaseAt = null;
}