Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

36.6 KB
Raw Blame
import { spawn } from '../lib/childProcess.js';
import { existsSync } from 'fs';
import { writeFile, unlink } from 'fs/promises';
import { join, dirname } from 'path';
import { createRequire } from 'module';
import { randomUUID } from 'crypto';
import { homedir } from 'os';
import { atomicWrite, safeJSONParse, tryReadFile } from '../lib/fileUtils.js';
import { parseCommandArgs } from '../lib/commandSecurity.js';
import { parsePm2JlistStdout } from '../lib/pm2Jlist.js';
import { beginLoadSample, formatLoadAttribution, roundMs } from '../lib/eventLoopLagProbe.js';
import { PATHS } from '../lib/paths.js';
import { serverSecretEnvKeys, withoutServerSecretsEnv } from '../lib/processEnv.js';
// The leaf, not readinessNotify.js: its observer's lazy imports would join
// pm2.js's ~280 suites to the CoS cluster's CI selection (#12285).
import { noteReadinessChanged } from './readinessSignal.js';

const IS_WIN = process.platform === 'win32';

// TTL cache for jlist results to reduce CLI churn during rapid UI refreshes
const JLIST_TTL_MS = 400;
const JLIST_TIMEOUT_MS = 15_000;
const COMMAND_TIMEOUT_MS = 60_000;
const KILL_GRACE_MS = 1_000;
const jlistCache = new Map();
const jlistInflight = new Map();
const cacheKey = (pm2Home) => pm2Home || '_default';

// Track PM2 read failures per home so we log only on state change or reason change
const jlistFailureState = new Map(); // home -> { lastError: string }

/**
 * Invalidate the jlist TTL cache (e.g. after mutations like start/stop/delete).
 * @param {string|null} [pm2Home=null]
 */
export function clearJlistCache(pm2Home = null) {
  noteReadinessChanged('health');
  noteReadinessChanged('capabilities');
  if (pm2Home !== undefined && pm2Home !== null) {
    jlistCache.delete(cacheKey(pm2Home));
  } else {
    jlistCache.clear();
  }
}

/**
 * Record a PM2 read failure and log it only on state change or reason change.
 * @param {string|null} pm2Home The PM2 home being read
 * @param {string} errorReason Formatted error description
 * @param {string} [attribution] Numbers appended to the line, never part of the dedupe key
 * @private
 */
function logJlistFailure(pm2Home, errorReason, attribution = '') {
  const key = cacheKey(pm2Home);
  const state = jlistFailureState.get(key);
  const homeDesc = pm2Home ? 'custom' : 'default';

  // Log only if no prior state, or if the reason changed
  if (!state || state.lastError !== errorReason) {
    console.error(`❌ PM2 read failed (home=${homeDesc}): ${errorReason}${attribution}`);
    jlistFailureState.set(key, { lastError: errorReason });
  }
}

/**
 * Record recovery from a PM2 read failure.
 * @param {string|null} pm2Home The PM2 home that recovered
 * @private
 */
function logJlistRecovery(pm2Home) {
  const key = cacheKey(pm2Home);
  const state = jlistFailureState.get(key);

  if (state) {
    const homeDesc = pm2Home ? 'custom' : 'default';
    console.log(`✅ PM2 read recovered (home=${homeDesc})`);
    jlistFailureState.delete(key);
  }
}

// Resolve PM2 CLI binary path from our local dependency using require.resolve
// to handle hoisted node_modules correctly.
const require = createRequire(import.meta.url);
const ECOSYSTEM_FILE = 'ecosystem.config.cjs';
const LEGACY_ECOSYSTEM_FILE = 'ecosystem.config.js';
const PM2_BIN = join(dirname(require.resolve('pm2/package.json')), 'bin', 'pm2');

/**
 * Check if a script path is a JS file that PM2 can fork through Node.
 * Other executable commands (including shell scripts) must run directly so PM2
 * does not try to parse them as JavaScript.
 */
function isJsScript(script) {
  return /\.(?:js|mjs|cjs|ts)$/i.test(script);
}

/**
 * PM2 exports a managed process's own `pm2_env` into that process's environment
 * as lowercase config keys, and the PM2 CLI reads those keys back as config —
 * where they outrank an explicit CLI flag. Since PortOS itself runs under PM2,
 * every `pm2 start` it spawns would otherwise inherit *portos-server's* config.
 *
 * That is how a loaded, serving llama-server kept dying: it was handed
 * portos-server's 4GB `max_memory_restart`, so PM2's memory watchdog (which
 * fires regardless of `--no-autorestart`) killed the 25GB model process seconds
 * after every successful load, forever. `exec_mode` and `watch` leak the same
 * way — verified by starting a process with each key set in the environment.
 *
 * `node_args` joins them for the same reason: portos-server carries its own V8
 * heap cap (ecosystem.config.cjs), and leaking that into a `pm2 start` would cap
 * every app PortOS launches at portos-server's ceiling — including an app whose
 * whole job is to hold more than that.
 */
const INHERITED_PM2_CONFIG_KEYS = ['max_memory_restart', 'exec_mode', 'watch', 'node_args'];

/** Drop the PM2 config keys PM2 injected into our own environment. */
function withoutInheritedPm2Config(env) {
  const clean = { ...env };
  for (const key of INHERITED_PM2_CONFIG_KEYS) delete clean[key];
  return clean;
}

/**
 * Spawn PM2 CLI via local binary (node pm2/bin/pm2).
 * Always uses the local PM2 binary to avoid depending on a global pm2 install.
 * On Windows this also skips the pm2.cmd shim, dropping the cmd.exe -> pm2.cmd
 * -> node hops and the PATH-resolution ambiguity that comes with them.
 *
 * A default 'error' listener is attached so that an ENOENT or EACCES error
 * (e.g. missing Node binary) doesn't crash Node via the uncaught-EventEmitter
 * exception path when no caller attaches its own 'error' listener.
 * Callers that attach their own 'error' handler are unaffected — multiple
 * listeners are fine and all receive the event.
 *
 * @param {string[]} pm2Args PM2 CLI arguments (e.g. ['jlist'], ['start', 'ecosystem.config.cjs'])
 * @param {object} opts Spawn options (cwd, env, etc.)
 * @returns {ChildProcess}
 */
export function spawnPm2(pm2Args, opts = {}) {
  // windowsHide comes from lib/childProcess.js. Re-asserting it here would also
  // outrank a caller's explicit opt-out, which the wrapper deliberately honors.
  // The env sanitize applies to a caller-supplied env too — it inherits
  // `process.env`, so it carries the same injected keys.
  const child = spawn(process.execPath, [PM2_BIN, ...pm2Args], {
    ...opts,
    env: withoutInheritedPm2Config(opts.env || process.env),
  });
  // Default error handler — prevents an uncaught EventEmitter 'error' crash
  // when no caller attaches its own handler. Callers that do attach their own
  // 'error' listener receive the event too (multiple listeners are additive).
  child.on('error', (err) => {
    console.error(`❌ spawnPm2 error [${pm2Args[0]}]: ${err.message}`);
  });
  return child;
}

/**
 * Execute a PM2 CLI command and return stdout/stderr as a promise.
 * Drop-in replacement for execAsync('pm2 ...') that bypasses pm2.cmd on Windows.
 *
 * TESTING — `vi.spyOn(pm2Module, 'execPm2')` is NOT enough on its own. It
 * intercepts callers in OTHER modules (mtplxServerManager, llamaServerManager),
 * but every export in THIS file that calls `execPm2`/`spawnPm2` reads the local
 * binding and runs the real implementation regardless of the spy — silently, with
 * the test still passing. `pm2.savedProcesses.test.js` relied on that and really
 * ran `pm2 save`, and PM2 forked a God Daemon per test that outlived the suite:
 * 641 orphans holding 38 GB accumulated on one dev machine before anyone noticed.
 * A test that calls into this module must mock the spawn seam instead —
 * `vi.mock('../lib/childProcess.js', … spawn: mockSpawn)`, as pm2.launch.test.js
 * and pm2.savedProcesses.test.js do — and assert the mock was reached.
 *
 * @param {string[]} pm2Args PM2 CLI arguments (e.g. ['jlist'])
 * @param {object} opts Spawn options (env, cwd, etc.) plus timeoutMs (default 60s)
 * @returns {Promise<{stdout: string, stderr: string}>}
 */
export function execPm2(pm2Args, { timeoutMs = COMMAND_TIMEOUT_MS, ...opts } = {}) {
  return runPm2Command(pm2Args, opts, timeoutMs);
}

function runPm2Command(pm2Args, opts, timeoutMs, allowNonzeroWithoutStderr = false) {
  return new Promise((resolve, reject) => {
    const child = spawnPm2(pm2Args, opts);
    let stdout = '';
    let stderr = '';
    let settled = false;
    let timedOut = false;
    let cleanupTimer;
    const onStdout = data => { stdout += data.toString(); };
    const onStderr = data => { stderr += data.toString(); };
    const finish = (error, cleanupConfirmed = true) => {
      if (settled) return;
      settled = true;
      clearTimeout(timer);
      clearTimeout(cleanupTimer);
      child.stdout?.removeListener('data', onStdout);
      child.stderr?.removeListener('data', onStderr);
      child.removeListener('close', onClose);
      child.removeListener('error', onError);
      if (timedOut) {
        // Killing the CLI cannot undo a mutation already accepted by the daemon.
        error = Object.assign(new Error(`pm2 ${pm2Args.join(' ')} timed out after ${timeoutMs / 1000}s; CLI cleanup ${cleanupConfirmed ? 'confirmed' : 'unconfirmed'}, daemon outcome unknown`), {
          code: 'PM2_TIMEOUT', timedOut: true, cleanupConfirmed, outcome: 'unknown',
        });
        if (!cleanupConfirmed) {
          child.stdin?.destroy();
          child.stdout?.destroy();
          child.stderr?.destroy();
          child.unref?.();
        }
      }
      if (error) reject(error);
      else resolve({ stdout, stderr });
    };
    const onClose = code => finish(code === 0 || (allowNonzeroWithoutStderr && !stderr)
      ? null : new Error(stderr || `pm2 exited with code ${code}`));
    // An error after spawn (e.g. a failed kill) does not prove the child exited.
    const onError = error => { if (!timedOut) finish(error); };
    const signal = value => {
      try { child.kill(value); }
      catch (error) { console.error(`❌ PM2 CLI ${value} failed: ${error.message}`); }
    };
    const timer = setTimeout(() => {
      timedOut = true;
      cleanupTimer = setTimeout(() => {
        cleanupTimer = setTimeout(() => finish(null, false), KILL_GRACE_MS);
        signal('SIGKILL');
      }, KILL_GRACE_MS);
      // The direct child is Node running PM2, never a shell or managed app.
      signal('SIGTERM');
    }, timeoutMs);
    child.stdout?.on('data', onStdout);
    child.stderr?.on('data', onStderr);
    child.on('close', onClose);
    child.on('error', onError);
  });
}

/**
 * Build environment object with optional custom PM2_HOME.
 *
 * Exported because the log-stream socket handler spawns `pm2 logs` directly
 * (not through spawnPm2Cli) and must target the same home the app's processes
 * live in — otherwise an app with its own PM2 instance streams an empty log.
 *
 * @param {string} pm2Home Optional custom PM2_HOME path
 * @returns {object} Environment variables
 */
export function buildEnv(pm2Home) {
  const env = { ...process.env };
  if (pm2Home) {
    env.PM2_HOME = pm2Home;
  }
  // Strip PortOS env vars to avoid conflicts
  delete env.PORT;
  delete env.HOST;
  return env;
}

// Env for a CLI `pm2 start` of a managed app: buildEnv() minus server secrets.
function buildStartEnv(pm2Home) {
  return withoutServerSecretsEnv(buildEnv(pm2Home));
}

// The Node API merges process.env into the child; its `filter_env` option (an
// array of names) is the only way to drop inherited keys. Explicit `env`
// entries are applied after the filter, so configured variables still arrive.
const serverSecretFilterEnv = () => serverSecretEnvKeys(process.env);

/**
 * Spawn a PM2 CLI command with optional custom PM2_HOME
 * @param {string} action PM2 action (stop, restart, delete)
 * @param {string} name PM2 process name
 * @param {string} pm2Home Optional custom PM2_HOME path
 * @returns {Promise<{success: boolean}>}
 */
async function spawnPm2Cli(action, name, pm2Home) {
  await execPm2([action, name], { env: buildEnv(pm2Home) });
  clearJlistCache(pm2Home);
  return { success: true };
}

/** Each operation owns its native client, including deadline teardown. */
async function connectAndRun(action, timeoutMs = COMMAND_TIMEOUT_MS) {
  // Importing PM2 constructs its singleton and probes the interactor socket.
  // Merely loading a reconciliation/peer-sync dependency must not contact it.
  const { default: pm2 } = await import('pm2');
  return new Promise((resolve, reject) => {
    const client = new pm2.custom();
    let settled = false;
    const disconnect = () => new Promise(resolveCleanup => {
      let cleaned = false;
      const done = error => {
        if (cleaned) return;
        cleaned = true;
        clearTimeout(cleanupTimer);
        if (!error) { resolveCleanup(null); return; }
        console.error(`❌ PM2 client disconnect failed: ${error.message}`);
        resolveCleanup(Object.assign(new Error('PM2 client cleanup unconfirmed'), {
          cause: error, code: 'PM2_CLEANUP_UNKNOWN', cleanupConfirmed: false, outcome: 'unknown',
        }));
      };
      const cleanupTimer = setTimeout(() => done(new Error('disconnect timed out')), KILL_GRACE_MS);
      try { client.disconnect(done); }
      catch (error) { done(error); }
    });
    const finish = (error, result) => {
      if (settled) return;
      settled = true;
      clearTimeout(timer);
      void disconnect().then(cleanupError => {
        if (cleanupError) error = Object.assign(error || cleanupError, { cleanupConfirmed: false });
        if (error) reject(error);
        else resolve(result);
      });
    };
    const timer = setTimeout(() => finish(Object.assign(new Error('pm2 RPC timed out; client disconnect requested, daemon outcome unknown'), {
      code: 'PM2_TIMEOUT', timedOut: true, cleanupConfirmed: false, outcome: 'unknown',
    })), timeoutMs);
    const onConnect = error => {
      // Connect can finish after its deadline and open new socket handles.
      if (settled) { void disconnect(); return; }
      if (error) { finish(error); return; }
      Promise.resolve().then(() => {
        if (settled) return;
        return action(client);
      }).then(result => finish(null, result), finish);
    };
    try { client.connect(onConnect); }
    catch (error) { finish(error); }
  });
}

/**
 * Start an app with PM2
 * @param {string} name PM2 process name
 * @param {object} options Start options
 */
export async function startApp(name, options = {}) {
  return connectAndRun((pm2) => {
    return new Promise((resolve, reject) => {
      const script = options.script || 'npm';
      const startOptions = {
        name,
        script,
        args: options.args || 'run dev',
        cwd: options.cwd,
        env: options.env || {},
        filter_env: serverSecretFilterEnv(),
        watch: false,
        autorestart: true,
        max_restarts: 10,
        min_uptime: '10s',
        restart_delay: 5000,
        windowsHide: IS_WIN
      };

      // Non-JS commands must run directly: PM2 otherwise forks them through
      // Node, which parses shell scripts as JavaScript on every platform.
      if (!isJsScript(script)) {
        startOptions.interpreter = 'none';
      }

      pm2.start(startOptions, (err, proc) => {
        if (err) return reject(err);
        clearJlistCache();
        resolve({ success: true, process: proc });
      });
    });
  });
}

/**
 * Stop an app
 * @param {string} name PM2 process name
 * @param {string} pm2Home Optional custom PM2_HOME path
 */
export async function stopApp(name, pm2Home = null) {
  // Use CLI for custom PM2_HOME
  if (pm2Home) {
    return spawnPm2Cli('stop', name, pm2Home);
  }

  return connectAndRun((pm2) => {
    return new Promise((resolve, reject) => {
      pm2.stop(name, (err) => {
        if (err) return reject(err);
        clearJlistCache();
        resolve({ success: true });
      });
    });
  });
}

/**
 * Restart an app
 * @param {string} name PM2 process name
 * @param {string} pm2Home Optional custom PM2_HOME path
 */
export async function restartApp(name, pm2Home = null) {
  return spawnPm2Cli('restart', name, pm2Home);
}

/**
 * Delete an app from PM2
 * @param {string} name PM2 process name
 * @param {string} pm2Home Optional custom PM2_HOME path
 */
export async function deleteApp(name, pm2Home = null) {
  // Use CLI for custom PM2_HOME
  if (pm2Home) {
    return spawnPm2Cli('delete', name, pm2Home);
  }

  return connectAndRun((pm2) => {
    return new Promise((resolve, reject) => {
      pm2.delete(name, (err) => {
        if (err) return reject(err);
        clearJlistCache();
        resolve({ success: true });
      });
    });
  });
}

/**
 * Get status of a specific process using CLI (avoids connection deadlocks)
 * @param {string} name PM2 process name
 * @param {string} pm2Home Optional custom PM2_HOME path
 */
export async function getAppStatus(name, pm2Home = null) {
  const processes = await fetchJlist(pm2Home);
  // `error` here means "PM2 read failed", but it's the same forgiving shape the
  // generic callers expect. App-status callers that must distinguish a failed
  // read from a genuine state use getAppStatusStrict (null sentinel) instead.
  if (!processes) return { name, status: 'error', pm2_env: null };
  const proc = processes.find(p => p.name === name);

  if (!proc) {
    return { name, status: 'not_found', pm2_env: null };
  }

  return shapeProcStatus(proc);
}

/**
 * Strict variant of {@link getAppStatus}: returns `null` when the PM2 read
 * FAILED, distinct from a `{ status: 'not_found' }` for a read that succeeded
 * but found no matching process. The detail endpoint uses this so a daemon
 * blip surfaces as `degraded` rather than a confident `not_started`.
 *
 * @returns {Promise<object|null>} Process status, `{ status: 'not_found' }`, or
 *   `null` on read failure.
 */
export async function getAppStatusStrict(name, pm2Home = null) {
  const processes = await fetchJlist(pm2Home);
  if (!processes) return null;
  const proc = processes.find(p => p.name === name);
  if (!proc) return { name, status: 'not_found', pm2_env: null };
  return shapeProcStatus(proc);
}

// Shared shaping for getAppStatus / getAppStatusStrict so the two never drift.
function shapeProcStatus(proc) {
  return {
    name: proc.name,
    status: proc.pm2_env?.status || 'unknown',
    pid: proc.pid,
    pm_id: proc.pm_id,
    cpu: proc.monit?.cpu || 0,
    memory: proc.monit?.memory || 0,
    uptime: proc.pm2_env?.pm_uptime ? Date.now() - proc.pm2_env.pm_uptime : null,
    restarts: proc.pm2_env?.restart_time || 0,
    unstableRestarts: proc.pm2_env?.unstable_restarts || 0,
    createdAt: proc.pm2_env?.created_at || null,
    args: proc.pm2_env?.args || null
  };
}

/**
 * Parse the stdout of a `pm2 jlist` (custom PM2_HOME CLI path) into a raw
 * process array, or `null` when the read effectively failed.
 *
 * Compatibility re-export: the strict parser now lives in the pure
 * `server/lib/pm2Jlist.js` (issue #8164) so package-local consumers like the
 * autofixer can import it without the rest of this service. Kept exported here
 * under its original name for existing first-party callers/tests.
 *
 * @param {string} stdout Raw `pm2 jlist` stdout (may carry ANSI noise).
 * @returns {Array|null} The parsed process array (incl. `[]`), or `null` on failure.
 */
export const parseJlistStdout = parsePm2JlistStdout;

/**
 * Fetch PM2 process list with TTL caching.
 * Uses the PM2 Node.js API for the default PM2_HOME (no subprocess spawning — avoids
 * visible cmd windows on Windows). Falls back to CLI only for custom PM2_HOME paths.
 * @param {string} pm2Home Optional custom PM2_HOME path
 * @returns {Promise<Array|null>} Raw process list from PM2, or null on error
 */
function fetchJlist(pm2Home = null) {
  const key = cacheKey(pm2Home);
  const cached = jlistCache.get(key);
  if (cached && Date.now() - cached.ts < JLIST_TTL_MS) return Promise.resolve(cached.data);

  const inflight = jlistInflight.get(key);
  if (inflight) return inflight;

  // A timed-out read is the usual first symptom of a stalled loop, so it carries
  // its own busy-versus-starved attribution (#12852).
  const startedAt = performance.now();
  const sample = beginLoadSample();
  // Defer the read until the single-flight entry exists, even for a sync callback.
  const promise = Promise.resolve().then(() => pm2Home
    ? execPm2(['jlist'], { env: buildEnv(pm2Home), timeoutMs: JLIST_TIMEOUT_MS })
      .then(({ stdout }) => parseJlistStdout(stdout))
    : connectAndRun(client => new Promise((resolve, reject) => {
      client.list((error, list) => error ? reject(error) : resolve(list));
    }), JLIST_TIMEOUT_MS)
  ).then(list => {
    if (!Array.isArray(list)) throw new Error('Invalid PM2 process list response');
    logJlistRecovery(pm2Home);
    jlistCache.set(key, { data: list, ts: Date.now() });
    return list;
  }).catch(error => {
    const attribution = error?.timedOut
      ? ` wallMs=${roundMs(performance.now() - startedAt)}${formatLoadAttribution(sample.read())}`
      : '';
    logJlistFailure(pm2Home, error.message || String(error), attribution);
    return null;
  }).finally(() => {
    sample.end();
    if (jlistInflight.get(key) === promise) jlistInflight.delete(key);
  });

  jlistInflight.set(key, promise);
  return promise;
}

/**
 * List all PM2 processes
 * @param {string} pm2Home Optional custom PM2_HOME path
 */
export async function listProcesses(pm2Home = null) {
  const list = await fetchJlist(pm2Home);
  if (!list) return [];
  return list.map(mapProcess);
}

/**
 * Strict variant of {@link listProcesses}: returns `null` when the PM2 read
 * FAILED (daemon unreachable / non-zero exit) and the mapped array otherwise —
 * including `[]` for a successful read with no processes.
 *
 * `listProcesses()` flattens a failed read into `[]`, which is the right
 * forgiving behavior for log/command/voice callers that just want "whatever is
 * running." But app-status getters MUST distinguish "PM2 unreachable" from
 * "nothing running" (AGENTS.md absent-vs-empty rule) — collapsing them records
 * a transient blip as every app `not_started`. Those callers use this instead.
 *
 * @param {string} pm2Home Optional custom PM2_HOME path
 * @returns {Promise<Array|null>} Mapped process list, or `null` on read failure.
 */
export async function listProcessesStrict(pm2Home = null) {
  const list = await fetchJlist(pm2Home);
  if (!list) return null;
  return list.map(mapProcess);
}

/**
 * The running daemon's own pm2 and Node versions — what the Node runtime panel
 * compares against the pm2 PortOS ships. Only those two fields leave this
 * function: the daemon's report also carries its whole environment.
 * @returns {Promise<{pm2Version: string|null, nodeVersion: string|null}|null>} null when unreachable
 */
export async function getDaemonIdentity() {
  const report = await connectAndRun(client => new Promise((resolve, reject) => {
    client.Client.executeRemote('getReport', {}, (error, value) => error ? reject(error) : resolve(value));
  }), JLIST_TIMEOUT_MS).catch(() => null);
  if (!report) return null;
  const text = (value) => (typeof value === 'string' && value && value !== 'N/A' ? value : null);
  return { pm2Version: text(report.pm2_version), nodeVersion: text(report.node_version) };
}

/** Internal maintenance inventory: fresh daemon read, no UI cache or env dump. */
export async function listMaintenanceProcesses() {
  return connectAndRun(pm2 => new Promise((resolve, reject) => {
    pm2.list((err, list) => {
      if (err || !Array.isArray(list)) {
        reject(new Error('PM2 maintenance inventory unavailable.'));
        return;
      }
      resolve(list.map(proc => ({
        name: proc?.name, pmId: proc?.pm_id, pid: proc?.pid,
        status: proc?.pm2_env?.status,
        cwd: proc?.pm2_env?.pm_cwd, script: proc?.pm2_env?.pm_exec_path,
      })));
    });
  }));
}

/**
 * Internal maintenance restart of one PortOS producer. Re-evaluates the
 * install's ecosystem file with --update-env so the app receives the saved
 * database configuration a cutover just committed, never PM2's cached env.
 * `env` is the environment the ecosystem file is evaluated in.
 * PM2 success is not proof of the backend: the restarted process must still
 * prove its own pool (services/databaseCutoverHandshake.js).
 */
export async function restartMaintenanceProducer(name, env = process.env) {
  if (!['portos-server', 'portos-cos'].includes(name)) throw new Error('Unknown maintenance producer.');
  await execPm2(['restart', join(PATHS.installRoot, 'ecosystem.config.cjs'), '--only', name, '--update-env'],
    { cwd: PATHS.installRoot, env });
  return { success: true };
}

// Shared shaping for listProcesses / listProcessesStrict so the two never drift.
function mapProcess(proc) {
  return {
    name: proc.name,
    status: proc.pm2_env?.status || 'unknown',
    pid: proc.pid,
    pm_id: proc.pm_id,
    cpu: proc.monit?.cpu || 0,
    memory: proc.monit?.memory || 0,
    uptime: proc.pm2_env?.pm_uptime ? Date.now() - proc.pm2_env.pm_uptime : null,
    restarts: proc.pm2_env?.restart_time || 0,
    unstableRestarts: proc.pm2_env?.unstable_restarts || 0
  };
}

/**
 * Get logs for a process using pm2 CLI (more reliable for log retrieval)
 * @param {string} name PM2 process name
 * @param {number} lines Number of lines to retrieve
 * @param {string} pm2Home Optional custom PM2_HOME path
 */
export async function getLogs(name, lines = 100, pm2Home = null) {
  const args = ['logs', name, '--lines', String(lines), '--nostream', '--raw'];
  // Preserve the logs reader's historical partial-output contract on nonzero exit.
  const { stdout } = await runPm2Command(args, { env: buildEnv(pm2Home) }, JLIST_TIMEOUT_MS, true);
  return stdout;
}

/**
 * Start an app using a specific command in cwd
 * @param {string} name PM2 process name
 * @param {string} cwd Working directory
 * @param {string} command Command to run (e.g., "npm run dev")
 * @param {object} [options]
 * @param {boolean} [options.autorestart=true] When false, PM2 will NOT relaunch
 *   the process after it exits. Portless/desktop apps (a game window) pass false:
 *   the user closing the window is a normal exit, not a crash to loop on. The
 *   restart-tuning fields (max_restarts/min_uptime/restart_delay) are omitted in
 *   that mode since they only apply when autorestart is on.
 * @param {number} [options.maxRestarts=10] Restart cap when autorestart is on.
 * @param {string} [options.pm2Home=null] Custom PM2_HOME. When set, starts via
 *   CLI (like deleteApp/getAppStatus) instead of the default-daemon Node API —
 *   otherwise the process would launch under PortOS's own PM2 daemon and be
 *   invisible to every subsequent status/log/delete call that targets pm2Home.
 * @param {number} [options.port=null] The app's own port, pinned as PORT in the
 *   child's environment so PortOS's inherited PORT cannot reach it. Pass null
 *   when no single port applies (a portless app, or several processes sharing
 *   one start call) — see the EADDRINUSE note at the env assignment below.
 */
export async function startWithCommand(name, cwd, command, options = {}) {
  const { autorestart = true, maxRestarts = 10, maxMemoryRestart = '500M', pm2Home = null, port = null } = options;
  // Parse with quote-awareness so `node --opt "arg with spaces"` survives;
  // a bare split(' ') would shred quoted segments. PM2 accepts `args` as an
  // array, which avoids re-joining and re-splitting on the way through.
  const [script, ...args] = parseCommandArgs(command);

  if (pm2Home) {
    return spawnPm2StartCommand(name, cwd, script, args, { autorestart, maxRestarts, maxMemoryRestart, pm2Home, port });
  }

  return connectAndRun((pm2) => {
    return new Promise((resolve, reject) => {
      const opts = {
        name,
        script,
        args,
        cwd,
        watch: false,
        autorestart,
        // Restart tuning only matters when autorestart is on; a desktop process
        // exiting cleanly should stay stopped, not be nursed back up. This
        // includes max_memory_restart: PM2's memory monitor relaunches the
        // process independently of autorestart, so leaving a 500M cap on a
        // desktop app (a game window routinely exceeds it) would kill and
        // respawn the live window — exactly the relaunch loop this avoids.
        ...(autorestart
          ? { max_restarts: maxRestarts, min_uptime: '10s', restart_delay: 5000, max_memory_restart: maxMemoryRestart }
          : {}),
        // PM2 snapshots the launching process's environment into the child, so
        // PortOS's own PORT would otherwise reach a managed app and win: both
        // Bun's `--env-file` and dotenv yield to an already-set variable, so the
        // app's env file cannot rescue it. The app then binds PortOS's port and
        // whichever process loses the boot race crashloops on EADDRINUSE.
        // buildEnv() strips PORT/HOST on the CLI path below; the Node API has no
        // way to *unset* an inherited key (it merges `env` over process.env), so
        // pin the app's own recorded port instead. Last writer wins, so this
        // also overrides the app's env file — they agree by construction, since
        // PortOS generates that file from the same port it records.
        ...(port ? { env: { PORT: String(port) } } : {}),
        filter_env: serverSecretFilterEnv(),
        windowsHide: IS_WIN
      };

      // PM2 defaults to Node for command scripts. Run executables directly so
      // native launchers such as `./scripts/game` honor their shebang.
      if (!isJsScript(script)) {
        opts.interpreter = 'none';
      }

      pm2.start(opts, (err, proc) => {
        if (err) return reject(err);
        clearJlistCache();
        resolve({ success: true, process: proc });
      });
    });
  });
}

/**
 * CLI-based counterpart to startWithCommand's Node-API path, used whenever a
 * custom pm2Home is given. `min_uptime` has no CLI flag equivalent — it only
 * tunes the autorestart=true nursing behavior, which no current caller uses
 * together with a custom pm2Home.
 */
async function spawnPm2StartCommand(name, cwd, script, args, { autorestart, maxRestarts, maxMemoryRestart, pm2Home, port = null }) {
  const cliArgs = ['start', script, '--name', name, '--cwd', cwd];
  if (!isJsScript(script)) cliArgs.push('--interpreter', 'none');
  if (autorestart) {
    cliArgs.push('--max-restarts', String(maxRestarts), '--restart-delay', '5000', '--max-memory-restart', String(maxMemoryRestart));
  } else {
    cliArgs.push('--no-autorestart');
  }
  if (args.length > 0) cliArgs.push('--', ...args);
  const env = buildStartEnv(pm2Home);
  if (port) env.PORT = String(port);
  await execPm2(cliArgs, { env });
  clearJlistCache(pm2Home);
  return { success: true };
}

/**
 * Spawn pm2 start with an ecosystem config file
 * @param {string} cwd Working directory
 * @param {string} ecosystemFile Config filename
 * @param {string[]} processNames Processes to start (--only flag)
 * @param {string} pm2Home Optional custom PM2_HOME
 */
async function spawnPm2StartEcosystem(cwd, ecosystemFile, processNames, pm2Home) {
  const args = ['start', ecosystemFile];
  if (processNames.length > 0) args.push('--only', processNames.join(','));
  const { stdout } = await execPm2(args, { cwd, env: buildStartEnv(pm2Home) });
  clearJlistCache(pm2Home);
  return { success: true, output: stdout };
}

/**
 * Persist the CURRENT PM2 process list to PM2's dump file (`pm2 save`).
 *
 * This is the half of "start at boot" PortOS can own. `pm2 startup` writes a
 * system init/launchd unit and is deliberately blocked (see
 * `PM2_BLOCKED_SUBCOMMANDS` in `lib/commandSecurity.js`) — it is a one-time,
 * privileged, machine-level decision the operator makes in a terminal. `pm2
 * save` only rewrites `$PM2_HOME/dump.pm2`, which that unit resurrects, so once
 * the operator has run `pm2 startup` once, saving here is what makes a
 * newly-started daemon (llama-server, MTPLX) come back after a reboot.
 *
 * The dump is process-list wide by design: it snapshots everything currently
 * running, which is exactly what a resurrect needs.
 *
 * @param {string|null} [pm2Home=null]
 */
export async function saveProcessList(pm2Home = null, { exclude = [] } = {}) {
  await execPm2(['save'], { env: buildEnv(pm2Home) });
  const dropped = await dropFromSavedList(exclude, pm2Home);
  return { success: true, excluded: dropped };
}

/**
 * Remove app entries from PM2's dump AFTER `pm2 save` wrote it.
 *
 * `pm2 save` is process-list wide with no exclusion flag — it snapshots whatever
 * is running. That is the wrong answer for a daemon PortOS starts on demand:
 * MTPLX is lazily started by the first request that needs it and stopped again
 * when idle, so resurrecting it at boot would put 20GB back on a machine nobody
 * has asked anything of yet — precisely the waste the idle stop exists to end.
 *
 * Rewriting the dump rather than stopping the daemon first keeps the running
 * process untouched: the user's MTPLX stays up for the session, it just isn't in
 * the list a reboot replays.
 *
 * Best-effort by design. An unreadable or non-JSON dump means PM2 owns a format
 * this doesn't understand, and a save that persisted one extra process is a far
 * better outcome than a corrupted dump that resurrects none.
 *
 * @param {string[]} names app names to drop
 * @param {string|null} [pm2Home]
 * @returns {Promise<string[]>} the names actually removed
 */
async function dropFromSavedList(names, pm2Home = null) {
  if (!names?.length) return [];
  const home = pm2Home || process.env.PM2_HOME || join(homedir(), '.pm2');
  const dumpPath = join(home, 'dump.pm2');
  const raw = await tryReadFile(dumpPath);
  if (raw === null) return [];
  const parsed = safeJSONParse(raw, null, { allowArray: true });
  if (!Array.isArray(parsed)) return [];

  const drop = new Set(names);
  const kept = parsed.filter((app) => !drop.has(app?.name));
  const removed = parsed.length - kept.length;
  if (removed === 0) return [];

  await atomicWrite(dumpPath, kept);
  const dropped = parsed.filter((app) => drop.has(app?.name)).map((app) => app.name);
  console.log(`💾 Excluded ${dropped.join(', ')} from the PM2 boot list (started on demand instead)`);
  return dropped;
}

/**
 * The app names in PM2's saved dump — i.e. what `pm2 resurrect` (and therefore
 * a boot-time `pm2 startup` unit) would bring back.
 *
 * Returns `null` when the dump could not be READ (absent, unreadable, not JSON)
 * — deliberately NOT the same value as `[]` ("read, and it saves nothing"), so a
 * caller can say "unknown" instead of reporting a daemon as not-persisted when
 * PortOS simply could not tell.
 *
 * @param {string|null} [pm2Home=null]
 * @returns {Promise<string[]|null>}
 */
export async function getSavedProcessNames(pm2Home = null) {
  const home = pm2Home || process.env.PM2_HOME || join(homedir(), '.pm2');
  const raw = await tryReadFile(join(home, 'dump.pm2'));
  if (raw === null) return null;
  const parsed = safeJSONParse(raw, null, { allowArray: true });
  if (!Array.isArray(parsed)) return null;
  return parsed.map((app) => app?.name).filter((name) => typeof name === 'string');
}

/**
 * Start app(s) using ecosystem.config.cjs/js file
 * This properly uses all env vars, scripts, args defined in the config
 * @param {string} cwd Working directory containing ecosystem config
 * @param {string[]} processNames Optional: specific processes to start (--only flag)
 * @param {string} pm2Home Optional custom PM2_HOME path
 */
export async function startFromEcosystem(cwd, processNames = [], pm2Home = null) {
  const ecosystemFile = [ECOSYSTEM_FILE, LEGACY_ECOSYSTEM_FILE]
    .find(f => existsSync(`${cwd}/${f}`));

  if (!ecosystemFile) {
    throw new Error('No ecosystem.config.cjs or ecosystem.config.js found');
  }

  // On Windows, PM2 fork mode can't execute .cmd batch files (npm, npx, etc.)
  // Load the config, patch non-JS scripts with interpreter:'none', write temp config
  if (IS_WIN) {
    return startFromEcosystemWindows(cwd, ecosystemFile, processNames, pm2Home);
  }

  return spawnPm2StartEcosystem(cwd, ecosystemFile, processNames, pm2Home);
}

/**
 * Evaluate the app's config inside the sanitized PM2 child, never in the
 * server: a config can copy process.env into an app-specific key before the
 * CLI's inheritance filter runs. Requiring the original file preserves its
 * relative imports and __dirname while this wrapper patches Windows launchers.
 */
async function startFromEcosystemWindows(cwd, ecosystemFile, processNames, pm2Home) {
  const tempFile = `_portos_pm2_${randomUUID()}.config.cjs`;
  const tempPath = join(cwd, tempFile);
  const content = `const config = require(${JSON.stringify(`./${ecosystemFile}`)});
const selected = ${JSON.stringify(processNames)};
for (const app of config.apps || []) {
  if (selected.length && !selected.includes(app.name)) continue;
  if (app.script && !/\\.(?:js|mjs|cjs|ts)$/i.test(app.script) && app.interpreter !== 'none') app.interpreter = 'none';
  if (!app.windowsHide) app.windowsHide = true;
  if (app.autorestart !== false && !app.max_restarts) app.max_restarts = 10;
}
module.exports = config;
`;
  await writeFile(tempPath, content, { flag: 'wx', mode: 0o600 });
  try {
    return await spawnPm2StartEcosystem(cwd, tempFile, processNames, pm2Home);
  } finally {
    // The CLI has closed, so it has finished evaluating this invocation's file.
    await unlink(tempPath).catch(() => {});
  }
}