Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
34.6 KB
Raw Blame
/**
 * GBRA-175: one upgrade restarts every gbrain process the brain runs on this
 * host, or refuses with the exact step for each process it cannot restart, and
 * then proves that every process reports one build and that the warm CLI
 * socket accepts it. A long-running process keeps the code it loaded at start
 * (#7245), so an upgrade that restarts nothing leaves serve, the jobs
 * supervisor and its worker, autopilot and every MCP server on the old build
 * while one-shot commands and job children run the new one.
 *
 * Three steps, each with its side effects behind an interface so tests drive
 * them with fakes:
 *
 * - `collectHostProcesses`: the processes this brain runs on this host. The
 *   process table is the authority on what runs (role by command line, every
 *   ancestor pid, start time, the version the process's install holds; on
 *   Linux only processes whose GBRAIN_HOME is this brain's); the consumer
 *   heartbeat rows (Postgres), the supervisor PID file and the worker registry
 *   add the build each pid loaded, and a record whose pid the table does not
 *   list as a gbrain process is stale and dropped. Read-only.
 * - `planRestart`: which processes are not on the installed build and how each
 *   restarts: serve and autopilot through their installed service, only when
 *   the service's own process runs them; the supervisor with `jobs supervisor
 *   stop` and `start --detach` with the options it was launched with (its
 *   worker restarts with it); an MCP stdio server by stopping it when
 *   `--stop-mcp` says so (its harness starts the new build on reconnect).
 *   Anything else is a manual step, and any manual step refuses the whole
 *   restart before anything changes.
 * - `runRestartSteps` then `verifyRestart`: run the steps in that order, then
 *   poll until no restarted pid is alive, every process reports the installed
 *   build (or, reporting none, started after the restart from an install that
 *   holds it), each restarted role and supervised worker came back, and the
 *   warm CLI socket, when serve had one, accepts this build.
 */
import { basename, dirname, isAbsolute, resolve } from 'node:path';
import { parseGlobalFlags } from './cli-options.ts';

export type ProcessRole = 'serve' | 'autopilot' | 'supervisor' | 'worker' | 'mcp' | 'sync';

export interface HostProcess {
  pid: number;
  /** The nearest gbrain process above it in the process table; null when none is. */
  ppid: number | null;
  role: ProcessRole;
  /** The build the process runs (in memory); null when it records none. */
  version: string | null;
  /** Epoch ms the process started; null when unknown. */
  started_ms: number | null;
  /** Every pid above it in the process table, nearest first (which service runs it). */
  ancestors?: number[];
  /** Its arguments after the gbrain entry point, from the process table; absent when the table did not list it. */
  args?: string[];
  /** The version the install it was started from holds now; null when unknown (a compiled binary). */
  install_version?: string | null;
  /** The package directory of the gbrain install it was started from (realpath); null when unknown (a compiled binary). */
  install_root?: string | null;
  /** A worker with no gbrain process above it: the process that started it, which was not recognised as gbrain. */
  unclassified_parent?: { pid: number; executable: string };
}

export interface ProcessTableRow { pid: number; ppid: number; command: string; started_ms: number | null; gbrain_home: string | null | undefined; cwd?: string | null }

export interface ProcessSources {
  /** Live consumer heartbeat rows on this host (Postgres only; PGLite keeps none). */
  heartbeats: () => Promise<Array<{ pid: number; kind: string; version: string | null }>>;
  supervisor: () => { pid: number; version: string | null } | null;
  workers: () => Array<{ pid: number; version: string | null }>;
  /** This user's processes, null when ps cannot run; `gbrain_home` is the GBRAIN_HOME (else HOME) from the process's environment, undefined when unreadable. */
  processTable: () => ProcessTableRow[] | null;
  /** The gbrain install a command line started in `cwd` runs from: its package directory and the version it holds now (null when unknown). */
  install?: (command: string, cwd?: string | null) => { root: string | null; version: string | null };
  /** The package name at a directory, undefined when unreadable: which `src/cli.ts` entry points are gbrain. */
  packageName?: (root: string) => string | null | undefined;
  /** This brain's GBRAIN_HOME (else HOME), resolved. */
  home: string;
  /** Pids never listed: this process and its parents (the upgrade that spawned it). */
  exclude: readonly number[];
}

const SERVE_SUBCOMMANDS_TRANSIENT = new Set(['status', 'install', 'uninstall', 'stop', 'logs', 'help']);
/** `gbrain autopilot` forms that report or change the service and exit; anything else is the daemon (`--repo <path>`, `start`). */
const AUTOPILOT_TRANSIENT = new Set(['status', 'install', 'uninstall', 'pause', 'resume', 'help']);
const INTERPRETERS = new Set(['bun', 'node']);
/** Interpreter flags whose value is the next token (gbrain's own sanitized re-run passes them this way: cli-preflight.ts PATH_RUNTIME_FLAGS). */
const VALUE_FLAGS = new Set(['--preload', '-r', '--require', '--import', '--config', '-c', '--env-file', '--tsconfig-override', '--cwd']);

/**
 * What decides whether a script entry point is gbrain: the process's working directory, which a relative entry
 * resolves against (null when unreadable), and the package name at a directory (undefined when unreadable).
 */
export interface EntryLookup { cwd?: string | null; packageName?: (root: string) => string | null | undefined }

/** A `src/cli.ts` entry is gbrain when the package two levels up is named gbrain; an unreadable package falls back to a gbrain path. */
function isGbrainCli(path: string, packageName: EntryLookup['packageName']): boolean {
  if (!/(^|\/)src\/cli\.ts$/.test(path)) return false;
  const name = packageName?.(dirname(dirname(path)));
  return name === undefined ? /gbrain[^\s]*\/src\/cli\.ts$/.test(path) : name === 'gbrain';
}

/**
 * The gbrain entry point of a command line and the arguments after it, or null when the command is not gbrain
 * itself: a wrapper that starts gbrain as its child (tini, a shell, `timeout`, `bunx`, `bun x`, `bun run <bin>`)
 * is not the gbrain process, while `bun run <file>` runs the file in its own process. The value of an interpreter
 * flag the walk does not list is skipped: after it, the entry is the next path. A relative entry is
 * returned resolved against the process's working directory; when that directory is unknown it stays relative and
 * only a gbrain path matches.
 */
export function gbrainEntry(command: string, lookup: EntryLookup = {}): { entry: string; args: string[] } | null {
  const tokens = command.trim().split(/\s+/);
  const isPath = (token: string) => token.includes('/') || /\.[cm]?[jt]s$/.test(token);
  let at = 0;
  if (INTERPRETERS.has(basename(tokens[0]!))) {
    for (at = 1; tokens[at]?.startsWith('-'); at++) if (VALUE_FLAGS.has(tokens[at]!)) at++;
    if (basename(tokens[0]!) === 'bun') {
      if (tokens[at] === 'x') return null;
      if (tokens[at] === 'run') for (at++; tokens[at]?.startsWith('-'); at++) if (VALUE_FLAGS.has(tokens[at]!)) at++;
    }
    if (tokens[at] !== undefined && !isPath(tokens[at]!)) {
      if (tokens[at - 1]!.startsWith('-')) at = tokens.findIndex((token, i) => i > at && isPath(token));
      else if (basename(tokens[0]!) === 'bun') return null;
    }
  }
  const token = tokens[at];
  if (token === undefined) return null;
  const unresolved = isPath(token) && !isAbsolute(token) && !lookup.cwd;
  const entry = isPath(token) && !isAbsolute(token) && lookup.cwd ? resolve(lookup.cwd, token) : token;
  if (basename(entry) !== 'gbrain' && !isGbrainCli(entry, unresolved ? undefined : lookup.packageName)) return null;
  return { entry, args: tokens.slice(at + 1) };
}

/** The role of a gbrain command line, or null when it is not a long-running gbrain process. */
export function classifyGbrainCommand(command: string, lookup: EntryLookup = {}): ProcessRole | null {
  const args = gbrainEntry(command, lookup)?.args;
  if (!args) return null;
  const [sub, second] = args.filter(arg => !arg.startsWith('-'));
  if (sub === 'serve') {
    if (second && SERVE_SUBCOMMANDS_TRANSIENT.has(second)) return null;
    return args.includes('--http') ? 'serve' : 'mcp';
  }
  if (sub === 'jobs' && second === 'work') return 'worker';
  if (sub === 'jobs' && second === 'supervisor') return args.some(arg => arg === 'stop' || arg === 'status') ? null : 'supervisor';
  if (sub === 'autopilot') return args.some(arg => AUTOPILOT_TRANSIENT.has(arg.replace(/^--/, ''))) ? null : 'autopilot';
  if (sub === 'sync') return 'sync';
  return null;
}

const installFields = (install: { root: string | null; version: string | null } | undefined) => ({ install_version: install?.version ?? null, install_root: install?.root ?? null });

const HEARTBEAT_ROLES: Record<string, ProcessRole> = { serve: 'serve', jobs: 'worker', autopilot: 'autopilot', mcp: 'mcp', sync: 'sync' };

/** Every gbrain process this brain runs on this host, one per pid. */
export async function collectHostProcesses(sources: ProcessSources): Promise<HostProcess[]> {
  const byPid = new Map<number, HostProcess>();
  const excluded = new Set(sources.exclude);
  const table = sources.processTable();
  const rows = new Map((table ?? []).map(row => [row.pid, row]));
  const ancestry = (ppid: number) => {
    const chain: number[] = [];
    for (let pid = ppid; pid > 1 && chain.length < 64 && !chain.includes(pid); pid = rows.get(pid)?.ppid ?? 0) chain.push(pid);
    return chain;
  };
  for (const row of table ?? []) {
    if (excluded.has(row.pid)) continue;
    if (row.gbrain_home !== undefined && row.gbrain_home !== null && resolve(row.gbrain_home) !== resolve(sources.home)) continue;
    const lookup = { cwd: row.cwd, packageName: sources.packageName };
    const role = classifyGbrainCommand(row.command, lookup);
    if (!role) continue;
    byPid.set(row.pid, { pid: row.pid, ppid: null, role, version: null, started_ms: row.started_ms, ancestors: ancestry(row.ppid),
      args: gbrainEntry(row.command, lookup)!.args, ...installFields(sources.install?.(row.command, row.cwd)) });
  }
  const merge = (pid: number, role: ProcessRole | null, version: string | null) => {
    if (excluded.has(pid)) return;
    const known = byPid.get(pid);
    if (known) {
      if (role === 'supervisor' || role === 'worker') known.role = role;
      known.version = version ?? known.version;
    } else if (role && table === null) {
      byPid.set(pid, { pid, ppid: null, role, version, started_ms: null });
    }
  };
  for (const row of await sources.heartbeats()) merge(row.pid, HEARTBEAT_ROLES[row.kind] ?? null, row.version || null);
  for (const worker of sources.workers()) merge(worker.pid, 'worker', worker.version);
  const supervisor = sources.supervisor();
  if (supervisor) merge(supervisor.pid, 'supervisor', supervisor.version);
  for (const p of byPid.values()) {
    p.ppid = p.ancestors?.find(pid => byPid.has(pid)) ?? null;
    const launcher = p.role === 'worker' && p.ppid === null && p.ancestors?.[0] !== undefined ? rows.get(p.ancestors[0]) : undefined;
    if (launcher) p.unclassified_parent = { pid: launcher.pid, executable: basename(launcher.command.trim().split(/\s+/)[0]!) };
  }
  return [...byPid.values()].sort((a, b) => a.pid - b.pid);
}

/** An installed service: its restart command and the pid its manager runs now (null when it is not running). */
export interface ServiceHandle { argv: string[]; pid: number | null }

export interface RestartContext {
  /** The installed build every process must run. */
  expected: string;
  /** The installed serve service; null when none is installed. */
  serveService: ServiceHandle | null;
  /** The installed autopilot service; null when none is installed. */
  autopilotService: ServiceHandle | null;
  /** Stop MCP stdio servers (their harness starts the new build on reconnect) instead of refusing. */
  stopMcp: boolean;
  /** The package directory of the install this upgrade replaces (realpath); null when unknown (a compiled binary). */
  ownInstallRoot?: string | null;
  /** What starts autopilot again when it exits, for an autopilot no installed service runs; null when nothing is known to. */
  autopilotLauncher?: AutopilotLauncher | null;
}

/**
 * #8776: a launcher outside launchd and systemd that starts autopilot again when it exits: this brain's own
 * gbrain cron line (the linux-cron install target), or a launcher the operator declared with
 * `self_upgrade.autopilot_launcher external` (a cron entry or wrapper of their own). The restart hands autopilot to
 * it: stop the daemon, then wait `wait_ms` for the launcher to start the installed build. Only the daemon this
 * brain's autopilot lock names (`pid`) is handed over, so another brain's autopilot is never signalled on a host
 * whose process environments cannot be read (macOS).
 */
export interface AutopilotLauncher { kind: 'cron' | 'declared'; wait_ms: number; pid: number | null }

export const launcherName = (launcher: AutopilotLauncher) => launcher.kind === 'cron'
  ? 'this brain\'s gbrain cron line' : 'the launcher self_upgrade.autopilot_launcher declares';

/** Before the swap the installed build is not known yet: every process is stale. */
export const PENDING_BUILD = 'pending';

/** A gbrain package directory inside a Bun global install (`<BUN_INSTALL>/install/global/node_modules/gbrain`): the BUN_INSTALL it belongs to. */
export function bunGlobalCopyOf(pkgDir: string): { bun_install: string } | null {
  const parts = pkgDir.replace(/\/+$/, '').split('/');
  if (parts.length < 6 || parts.slice(-4).join('/') !== 'install/global/node_modules/gbrain') return null;
  return { bun_install: parts.slice(0, -4).join('/') || '/' };
}

/** The steps that move another gbrain install onto the target build, as argv, plus the sentence that names them. */
export function copyFix(root: string, holds: string | null, expected: string): { argv: string[][]; env: Record<string, string>; instruction: string } {
  const release = expected === PENDING_BUILD ? 'the release being installed' : `v${expected}`;
  const copy = bunGlobalCopyOf(root);
  if (copy) {
    const argv = [['bun', 'remove', '-g', 'gbrain'], ['bun', 'add', '-g', `github:garrytan/gbrain#v${expected}`]];
    const shown = argv.map(a => `BUN_INSTALL=${copy.bun_install} ${a.join(' ')}`).join(' && ');
    return { argv, env: { BUN_INSTALL: copy.bun_install }, instruction: `replace the Bun global copy at ${root} (${holds ? `v${holds}` : 'version unknown'}; a pinned copy keeps its pin through bun add alone, so remove it first): \`${shown}\`` };
  }
  const argv = [['git', '-C', root, 'fetch', '--tags', 'origin'], ['git', '-C', root, 'checkout', `v${expected}`], ['bun', 'install', '--cwd', root]];
  return { argv, env: {}, instruction: `move the install at ${root} (${holds ? `v${holds}` : 'version unknown'}) to ${release}: for a git clone, \`${argv.map(a => a.join(' ')).join(' && ')}\`; otherwise start this process from the upgraded install instead` };
}

export type RestartStep =
  | { kind: 'service'; role: 'serve' | 'autopilot'; argv: string[]; pids: number[] }
  | { kind: 'supervisor'; pid: number; workers: number[]; stop: string[]; start: string[]; pid_file: string | null }
  | { kind: 'stop'; role: 'mcp'; pid: number; started_ms: number | null }
  | { kind: 'handoff'; role: 'autopilot'; pid: number; started_ms: number | null; launcher: AutopilotLauncher }
  | { kind: 'replace_copy'; root: string; bun_install: string; from: string | null; to: string; pids: number[] };

export interface ManualStep { role: ProcessRole; pid: number; version: string | null; argv: string[]; instruction: string }

export interface RestartPlan {
  expected: string;
  /** Processes already on the installed build; left running. */
  current: HostProcess[];
  /** Processes on another build, or reporting none. */
  stale: HostProcess[];
  steps: RestartStep[];
  manual: ManualStep[];
}

const label = (p: Pick<HostProcess, 'role' | 'pid' | 'version'>) => `${p.role} pid ${p.pid} (${p.version ? `v${p.version}` : 'build not recorded'})`;

/**
 * The supervisor's stop and start commands from the arguments it runs with, so the restart keeps its queue, concurrency and PID file.
 * A `--flag=value` token that is neither a global flag nor --job-isolation was ignored by the supervisor that holds it (an older release) and is refused by this one, so it is dropped: the restart runs what the supervisor applied.
 */
export function supervisorRestartArgv(args: readonly string[]): { stop: string[]; start: string[]; pid_file: string | null } {
  let local: ReadonlySet<string> = new Set();
  try { local = new Set(parseGlobalFlags([...args]).rest); } catch { /* an unparseable global flag: keep every token, as before */ }
  const rest = args.slice(args.indexOf('supervisor') + 1).filter(arg => arg !== 'start' && arg !== '--detach' && arg !== '--json' && !(/^--[^=]+=/.test(arg) && local.has(arg) && !arg.startsWith('--job-isolation=')));
  const at = rest.indexOf('--pid-file');
  const pidFile = at === -1 ? null : rest[at + 1] ?? null;
  return {
    stop: ['gbrain', 'jobs', 'supervisor', 'stop', ...(pidFile ? ['--pid-file', pidFile] : []), '--json'],
    start: ['gbrain', 'jobs', 'supervisor', 'start', '--detach', ...rest, '--json'],
    pid_file: pidFile,
  };
}

export function planRestart(processes: readonly HostProcess[], ctx: RestartContext): RestartPlan {
  const stale = processes.filter(p => p.version !== ctx.expected);
  const current = processes.filter(p => p.version === ctx.expected);
  const manual: ManualStep[] = [];
  const services = new Map<'serve' | 'autopilot', number[]>();
  const supervisors = new Map<number, number[]>();
  const stops: RestartStep[] = [];
  const byPid = new Map(processes.map(p => [p.pid, p]));
  const addService = (role: 'serve' | 'autopilot', pid: number) => services.set(role, [...(services.get(role) ?? []), pid]);
  const handoffs = new Map<number, Extract<RestartStep, { kind: 'handoff' }>>();
  const launcherOf = (p: HostProcess) => (ctx.autopilotLauncher?.pid === p.pid && p.started_ms !== null && p.args ? ctx.autopilotLauncher : null);
  const addHandoff = (p: HostProcess, launcher: AutopilotLauncher) => handoffs.set(p.pid, { kind: 'handoff', role: 'autopilot', pid: p.pid, started_ms: p.started_ms, launcher });
  const runsUnder = (p: HostProcess, service: ServiceHandle | null) => !!service && service.pid !== null && (p.pid === service.pid || !!p.ancestors?.includes(service.pid));
  const kill = (p: HostProcess, instruction: string) => manual.push({ role: p.role, pid: p.pid, version: p.version, argv: ['kill', String(p.pid)], instruction });

  const copies = new Map<string, Extract<RestartStep, { kind: 'replace_copy' }>>();
  for (const p of stale) {
    const parent = p.ppid === null ? undefined : byPid.get(p.ppid);
    if (ctx.ownInstallRoot && p.install_root && p.install_root !== ctx.ownInstallRoot && p.install_version !== ctx.expected) {
      const copy = bunGlobalCopyOf(p.install_root);
      if (!copy) {
        const fix = copyFix(p.install_root, p.install_version ?? null, ctx.expected);
        manual.push({ role: p.role, pid: p.pid, version: p.version, argv: fix.argv[0]!,
          instruction: `${label(p)} runs from another gbrain install, which this upgrade does not change, so a restart brings it back on the old build: ${fix.instruction}; then run the upgrade again.` });
        continue;
      }
      const known = copies.get(p.install_root);
      if (known) known.pids.push(p.pid);
      else copies.set(p.install_root, { kind: 'replace_copy', root: p.install_root, bun_install: copy.bun_install, from: p.install_version ?? null, to: ctx.expected, pids: [p.pid] });
    }
    switch (p.role) {
      case 'serve':
        if (runsUnder(p, ctx.serveService)) addService('serve', p.pid);
        else kill(p, `${label(p)} was not started by the installed serve service${ctx.serveService ? (ctx.serveService.pid === null ? ' (the service is not running)' : ` (the service runs pid ${ctx.serveService.pid})`) : ''}: stop it (\`kill ${p.pid}\`, SIGTERM drains it) and start it again the way it was launched.`);
        break;
      case 'autopilot':
        if (runsUnder(p, ctx.autopilotService)) addService('autopilot', p.pid);
        else if (launcherOf(p)) addHandoff(p, launcherOf(p)!);
        else if (ctx.autopilotLauncher?.pid === p.pid) kill(p, `${label(p)} is known only from its heartbeat row (the process table could not be read), so the upgrade cannot confirm it is the process to stop before handing autopilot to its launcher: stop it (\`kill ${p.pid}\`) and let the launcher start it again, then run the upgrade.`);
        else if (ctx.autopilotLauncher) kill(p, `${label(p)} is not the daemon this brain's autopilot lock names (it names ${ctx.autopilotLauncher.pid === null ? 'no running autopilot' : `pid ${ctx.autopilotLauncher.pid}`}), so the upgrade does not hand it to ${launcherName(ctx.autopilotLauncher)}: if it is this brain's autopilot, stop it (\`kill ${p.pid}\`) and let the launcher start it again, then run the upgrade; if it serves another brain, upgrade from that brain.`);
        else manual.push({ role: p.role, pid: p.pid, version: p.version, argv: ['gbrain', 'autopilot', '--install'],
          instruction: `${label(p)} was started outside gbrain's autopilot service (by a cron entry, a wrapper or by hand), so the upgrade cannot restart it. `
            + 'Either run `gbrain autopilot --install`, which runs autopilot under launchd, systemd or a gbrain cron line so every upgrade restarts it (retire the launcher that starts it now first, or both race for the autopilot lock), '
            + `or, when that launcher starts autopilot again within 5 minutes of it exiting, \`gbrain config set self_upgrade.autopilot_launcher external\` lets the upgrade stop it and wait for the launcher. By hand: \`kill ${p.pid}\` and start it again the way it was launched.` });
        break;
      case 'supervisor':
        if (p.args) supervisors.set(p.pid, supervisors.get(p.pid) ?? []);
        else kill(p, `${label(p)}: its launch options could not be read from the process table, so it is not restarted with defaults: stop it (\`gbrain jobs supervisor stop\`) and start it again the way it was launched.`);
        break;
      case 'worker':
        if (parent?.role === 'supervisor' && parent.args) supervisors.set(parent.pid, [...(supervisors.get(parent.pid) ?? []), p.pid]);
        else if (parent?.role === 'autopilot' && runsUnder(parent, ctx.autopilotService)) addService('autopilot', parent.pid);
        else if (parent?.role === 'autopilot' && launcherOf(parent)) addHandoff(parent, launcherOf(parent)!);
        else kill(p, `${label(p)} is not run by a jobs supervisor this restart can relaunch${parent?.role === 'autopilot' ? ' (its autopilot runs under no installed service)' : ''}${p.unclassified_parent ? ` (its parent, pid ${p.unclassified_parent.pid} (\`${p.unclassified_parent.executable}\`), is not a process this restart recognises as gbrain)` : ''}: stop it (\`kill ${p.pid}\`, SIGTERM drains its jobs) and start \`gbrain jobs work\` again the way it was launched, or run workers under \`gbrain jobs supervisor start --detach\`.`);
        break;
      case 'mcp':
        if (ctx.stopMcp && p.args) stops.push({ kind: 'stop', role: 'mcp', pid: p.pid, started_ms: p.started_ms });
        else if (ctx.stopMcp) kill(p, `${label(p)} is known only from its heartbeat row (the process table could not be read), so it is not signalled: stop it yourself if it is a gbrain MCP server, and reconnect gbrain in its harness afterwards.`);
        else kill(p, `${label(p)} is an MCP server an agent harness started: rerun with --stop-mcp to stop it during the restart and reconnect gbrain in that harness afterwards (Claude Code: /mcp, then reconnect; other harnesses: start a new session).`);
        break;
      case 'sync':
        manual.push({ role: p.role, pid: p.pid, version: p.version, argv: ['gbrain', 'sync', 'status', '--json'],
          instruction: `${label(p)} is a running gbrain sync: let it finish (\`gbrain sync status --json\` shows its progress), then run the restart again.` });
        break;
    }
  }
  const steps: RestartStep[] = [...copies.values()];
  const serve = services.get('serve');
  if (serve && ctx.serveService) steps.push({ kind: 'service', role: 'serve', argv: ctx.serveService.argv, pids: [...new Set(serve)] });
  const autopilot = services.get('autopilot');
  if (autopilot && ctx.autopilotService) steps.push({ kind: 'service', role: 'autopilot', argv: ctx.autopilotService.argv, pids: [...new Set(autopilot)] });
  for (const [pid, workers] of supervisors) steps.push({ kind: 'supervisor', pid, workers, ...supervisorRestartArgv(byPid.get(pid)!.args!) });
  steps.push(...handoffs.values(), ...stops);
  return { expected: ctx.expected, current, stale, steps, manual };
}

export interface RunResult { status: number | null; output: string }

export interface RestartExecutor {
  run: (argv: string[], env?: Record<string, string>) => RunResult | Promise<RunResult>;
  /** SIGTERM the MCP server or autopilot at pid only if it is still the process the plan saw: 'changed' when the pid now names another process. */
  terminate: (pid: number, startedMs: number | null, role: 'mcp' | 'autopilot') => 'stopped' | 'gone' | 'changed';
  /** The live supervisor's pid now (read from its PID file, the default one when null), or null. */
  supervisorPid: (pidFile: string | null) => number | null;
  sleep: (ms: number) => Promise<void>;
}

export interface StepResult { step: RestartStep; ok: boolean; detail: string }

const SUPERVISOR_RESPAWN_GRACE_MS = 3_000;
const failed = (argv: string[], r: { status: number | null; output: string }) => `\`${argv.join(' ')}\` exited ${r.status ?? 'without a status'}: ${r.output.trim().split('\n')[0] ?? ''}`;

export async function runRestartSteps(plan: RestartPlan, exec: RestartExecutor): Promise<StepResult[]> {
  const results: StepResult[] = [];
  for (const step of plan.steps) {
    if (step.kind === 'replace_copy') {
      const env = { BUN_INSTALL: step.bun_install };
      const probe = ['git', 'ls-remote', '--exit-code', '--tags', 'https://github.com/garrytan/gbrain', `refs/tags/v${step.to}`];
      const add = ['bun', 'add', '-g', `github:garrytan/gbrain#v${step.to}`];
      const probed = await exec.run(probe);
      if (probed.status !== 0) { results.push({ step, ok: false, detail: `v${step.to} has no release tag, so the copy at ${step.root} was left as it is (${failed(probe, probed)})` }); continue; }
      const removed = await exec.run(['bun', 'remove', '-g', 'gbrain'], env);
      const added = removed.status === 0 ? await exec.run(add, env) : removed;
      if (added.status === 0) { results.push({ step, ok: true, detail: `replaced the Bun global copy at ${step.root} with v${step.to}` }); continue; }
      const restored = removed.status === 0 && step.from ? (await exec.run(['bun', 'add', '-g', `github:garrytan/gbrain#v${step.from}`], env)).status === 0 : false;
      results.push({ step, ok: false, detail: `could not replace the copy at ${step.root} (${failed(removed.status === 0 ? add : ['bun', 'remove', '-g', 'gbrain'], added)})${restored ? `; restored v${step.from}` : ''}` });
    } else if (step.kind === 'service') {
      const r = await exec.run(step.argv);
      results.push({ step, ok: r.status === 0, detail: r.status === 0 ? `ran \`${step.argv.join(' ')}\`` : failed(step.argv, r) });
    } else if (step.kind === 'supervisor') {
      const stop = await exec.run(step.stop);
      if (stop.status !== 0) { results.push({ step, ok: false, detail: failed(step.stop, stop) }); continue; }
      await exec.sleep(SUPERVISOR_RESPAWN_GRACE_MS);
      const respawned = exec.supervisorPid(step.pid_file);
      if (respawned !== null && respawned !== step.pid) {
        results.push({ step, ok: true, detail: `stopped supervisor pid ${step.pid}; its service manager started pid ${respawned}` });
        continue;
      }
      const start = await exec.run(step.start);
      results.push({ step, ok: start.status === 0, detail: start.status === 0 ? `stopped supervisor pid ${step.pid} and started it again with \`${step.start.join(' ')}\`` : failed(step.start, start) });
    } else if (step.kind === 'handoff') {
      const outcome = exec.terminate(step.pid, step.started_ms, 'autopilot');
      results.push({ step, ok: outcome !== 'changed', detail: outcome === 'changed'
        ? `pid ${step.pid} is no longer the autopilot the plan saw, so it was not signalled`
        : `stopped autopilot pid ${step.pid}; ${launcherName(step.launcher)} starts it again` });
    } else {
      const outcome = exec.terminate(step.pid, step.started_ms, 'mcp');
      results.push({ step, ok: outcome !== 'changed', detail: outcome === 'changed'
        ? `pid ${step.pid} is no longer the MCP server the plan saw, so it was not signalled`
        : `stopped mcp pid ${step.pid}; reconnect gbrain in its harness` });
    }
  }
  return results;
}

/** `mismatch` names the identity part an `identity_mismatch` refusal differed on (`version`, `env GBRAIN_MODEL`, ...). */
export type WarmSocketState = { state: 'absent' } | { state: 'accepted' } | { state: 'refused'; reason: string; mismatch?: string };

/** A refusal for anything but a non-version identity part means serve does not accept this build; an env or config difference is this shell's, not the upgrade's. */
function refusesThisBuild(warm: WarmSocketState): boolean {
  return warm.state === 'refused' && !(warm.reason === 'identity_mismatch' && warm.mismatch && warm.mismatch !== 'version');
}

const restartedPids = (plan: RestartPlan) => new Set(plan.steps.flatMap(step => step.kind === 'service' ? step.pids : step.kind === 'supervisor' ? [step.pid, ...step.workers] : step.kind === 'stop' || step.kind === 'handoff' ? [step.pid] : []));

/** Processes that came back on a build other than the target, with the install each one runs from. */
export function mismatchedBuilds(plan: RestartPlan, processes: readonly HostProcess[]): HostProcess[] {
  const restarted = restartedPids(plan);
  return processes.filter(p => !restarted.has(p.pid) && (p.version ? p.version !== plan.expected : !!p.install_version && p.install_version !== plan.expected));
}

/** What still stands between the host and one build after a restart; empty when it is done. */
export function restartProblems(plan: RestartPlan, processes: readonly HostProcess[], restartStartedMs: number,
  warm: { before: WarmSocketState; after: WarmSocketState }, lockPid: number | null = null): string[] {
  const problems: string[] = [];
  const live = new Map(processes.map(p => [p.pid, p]));
  const restarted = restartedPids(plan);
  for (const pid of restarted) {
    const p = live.get(pid);
    if (p) problems.push(`${label(p)} is still running after the restart.`);
  }
  const fresh = processes.filter(p => !restarted.has(p.pid));
  const mismatched = new Set(mismatchedBuilds(plan, processes));
  for (const p of fresh) {
    const from = p.install_root ? `; it runs from ${p.install_root}, which holds ${p.install_version ? `v${p.install_version}` : 'an unknown version'}: ${copyFix(p.install_root, p.install_version ?? null, plan.expected).instruction}` : '';
    if (mismatched.has(p) && p.version) problems.push(`${label(p)} runs a build other than v${plan.expected}${from}.`);
    else if (mismatched.has(p)) problems.push(`${label(p)} reports no build and was started from an install that holds v${p.install_version}${from}.`);
    else if (!p.version && (p.started_ms === null || p.started_ms < restartStartedMs)) problems.push(`${label(p)} reports no build and started before the restart.`);
  }
  for (const step of plan.steps) {
    if (step.kind === 'stop' || step.kind === 'replace_copy') continue;
    if (step.kind === 'handoff') {
      if (!fresh.some(p => p.role === 'autopilot' && p.pid === lockPid && p.started_ms !== null && p.started_ms >= restartStartedMs)) {
        problems.push(`autopilot did not come back after it was stopped for ${launcherName(step.launcher)} to start again (this brain's autopilot lock names ${lockPid === null ? 'no autopilot' : `pid ${lockPid}, which is not an autopilot started after the restart`}).`);
      }
      continue;
    }
    const role = step.kind === 'supervisor' ? 'supervisor' : step.role;
    if (!fresh.some(p => p.role === role)) {
      problems.push(`${role} did not come back after ${step.kind === 'service' ? `\`${step.argv.join(' ')}\`` : 'the supervisor restart'}.`);
    } else if (step.kind === 'supervisor' && step.workers.length && !fresh.some(p => p.role === 'worker' && live.get(p.ppid ?? -1)?.role === 'supervisor' && !restarted.has(p.ppid!))) {
      problems.push(`the jobs worker did not come back under the restarted supervisor (\`${step.start.join(' ')}\`).`);
    }
  }
  if (warm.after.state === 'refused' && refusesThisBuild(warm.after)) problems.push(`The warm CLI socket refused this build (${warm.after.reason}${warm.after.mismatch ? `: ${warm.after.mismatch}` : ''}).`);
  else if (warm.before.state !== 'absent' && warm.after.state === 'absent') problems.push('The warm CLI socket serve had before the restart is gone.');
  return problems;
}

/** Processes that pass verification only by their start time: they record no build and their install's version is unknown (a compiled binary). */
export function unverifiedBuilds(plan: RestartPlan, processes: readonly HostProcess[]): string[] {
  const restarted = restartedPids(plan);
  return processes.filter(p => !restarted.has(p.pid) && !p.version && !p.install_version)
    .map(p => `${label(p)} started after the restart but records no build, so its build is inferred from its start time.`);
}

export interface VerifyDeps {
  collect: () => Promise<HostProcess[]>;
  probeWarmSocket: () => Promise<WarmSocketState>;
  sleep: (ms: number) => Promise<void>;
  now: () => number;
  /** This brain's autopilot lock pid, re-read on every poll: a hand-off is done only when an autopilot started after the restart holds it. */
  autopilotLockPid?: () => number | null;
}

export interface VerifyResult { ok: boolean; processes: HostProcess[]; warm_socket: WarmSocketState; problems: string[]; unverified: string[] }

export async function verifyRestart(plan: RestartPlan, restartStartedMs: number, warmBefore: WarmSocketState, deps: VerifyDeps,
  opts: { timeoutMs: number; pollMs: number }): Promise<VerifyResult> {
  const deadline = deps.now() + opts.timeoutMs;
  for (;;) {
    const processes = await deps.collect();
    const warm = await deps.probeWarmSocket();
    let lockPid: number | null = null;
    try { lockPid = deps.autopilotLockPid?.() ?? null; } catch { lockPid = null; }
    const problems = restartProblems(plan, processes, restartStartedMs, { before: warmBefore, after: warm }, lockPid);
    if (!problems.length || deps.now() >= deadline) return { ok: !problems.length, processes, warm_socket: warm, problems, unverified: unverifiedBuilds(plan, processes) };
    await deps.sleep(opts.pollMs);
  }
}