Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

44.0 KB
Raw Blame
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { persistRunnerCompletion } from './completion.js';
import { createSecurityScanManager } from './securityScan.js';
import { securityScanRequestSchema } from '../lib/securityScanEvidence.js';
import { maintenance } from '../lib/maintenanceAdmission.js';
import '../services/databaseBootFence.js';
/**
 * CoS Agent Runner - Standalone PM2 Process
 *
 * This service runs as a separate PM2 app (portos-cos) that doesn't restart
 * when portos-server restarts. It manages Claude CLI agent spawning and
 * prevents orphaned processes when the main server cycles.
 *
 * Communication with portos-server happens via HTTP on port 5558.
 */

import express from 'express';
import { runnerRequestGuard, runnerRequestRefusal } from './requestGuard.js';
import { assertDatabaseAdmission } from '../lib/databaseMaintenanceJournal.js';
import { spawn } from '../lib/childProcess.js';
import * as pty from 'node-pty';
import { join, basename } from 'path';
import { writeFile, readFile } from 'fs/promises';
import { existsSync } from 'fs';
import http from 'http';
import { Server as SocketServer } from 'socket.io';
import { ensureDir, PATHS } from '../lib/fileUtils.js';
import { prepareCliSpawn, killProcessTree, guardChildStdin, deliverChildStdin } from '../lib/bufferedSpawn.js';
import { buildCliChildEnv } from '../lib/cliChildEnv.js';
import { prepareCliPrompt } from '../lib/cliProviderArgs.js';
import { commandExists } from '../lib/commandExists.js';
import { adoptNpmGlobalBinDir } from '../lib/npmGlobalBin.js';
import { findCommandOnPath } from '../lib/processEnv.js';
import { diagnosePtySpawnFailure, probePtyRuntime, PTY_WORKSPACE_MISSING_PREFIX } from '../lib/ptySpawnDiagnostics.js';
import { createCodexStderrFormatter } from '../lib/codexCliOutput.js';
import { isKnownCliStderrNoise } from '../lib/cliStderrNoise.js';
import { createStreamingAnsiStripper } from '../lib/ansiStrip.js';
import { createStreamJsonParser } from '../lib/streamJsonParser.js';
import { withState, drainState } from './runnerState.js';
import { getProcessStats, checkProcessRunning } from './processStats.js';
import { usableAgentPid, runnerAgentLivenessFields } from '../lib/runnerAgentLiveness.js';
import { ALLOWED_COMMANDS, isAllowedCommand } from './allowedCommands.js';
import { armForceKill as armForceKillShared } from './forceKill.js';
import { createTuiExitHandler } from './tuiExit.js';
import { bodyErrorMessage, btwBodySchema, pauseBodySchema, spawnBodySchema, spawnTuiBodySchema, terminateBodySchema } from './requestSchemas.js';
import { createRunnerShutdown, registerRunnerShutdownSignals } from './shutdown.js';
import { createHttpDrain } from '../lib/httpDrain.js';
import { PORTS } from '../lib/ports.js';
import { setupProcessErrorHandlers } from '../lib/errorHandler.js';
import { parseSentinelPayload } from '../lib/agentSentinel.js';
import { createAgentSentinelAccess } from '../services/agentSentinelWatcher.js';
import { SENTINEL_COMPLETION_MARKER } from '../lib/agentOutputMarkers.js';

// Process-level safety net (defense-in-depth, see issue #1878). The main server
// (server/index.js) already wires this same shared helper via
// setupProcessErrorHandlers(io); the CoS runner — a separate long-lived PM2
// process that spawns and supervises child agents — had no such net, so a stray
// async handler that rejected/threw outside the request lifecycle would crash it
// with Node's default unhandled output. Reusing the shared helper gives the runner
// the identical, already-tested convention: log via the emoji-prefixed console.error
// style, and on an uncaughtException exit cleanly after flushing so PM2 restarts a
// process that would otherwise be left in an undefined state (boot then reaps any
// orphaned agents). Called with no `io` — the runner's socket server fans agent
// output to portos-server, not the error-alert UI, so the UI emit is skipped.
setupProcessErrorHandlers();

// Timing constants for process termination/cleanup (ms). Named so the grace
// windows are obvious at each call site instead of bare literals.
const SIGKILL_GRACE_MS = 5000;       // wait after SIGTERM before forcing SIGKILL
const ORPHAN_CLEANUP_DELAY_MS = 3000; // delay on boot before reaping orphaned agents
// Agentic CLIs can spend several seconds loading config/plugins before their
// lightweight version command returns. Keep this aligned with commandExists'
// documented heavy-CLI probe budget so a cold but runnable provider is not
// rejected before its PTY opens.
const TUI_CAPABILITY_PROBE_TIMEOUT_MS = 15 * 1000;

// Path + listen constants. These lived adjacent to the command allowlist before
// it was extracted to ./allowedCommands.js; they must stay here — they're
// referenced throughout (AGENTS_DIR/ROOT_DIR for spawn cwd + dirs, and
// server.listen(PORT, HOST)) and dropping them crashes the runner on boot.
const ROOT_DIR = PATHS.root;
const AGENTS_DIR = PATHS.cosAgents;

const PORT = process.env.PORT || PORTS.COS;
const HOST = process.env.HOST || '127.0.0.1';

// Active agent processes (in memory)
const activeAgents = new Map();

const TUI_SIGNALS = new Set(['SIGTERM', 'SIGKILL', 'SIGINT']);

// Bind the shared escalation to this process's map, grace window, and durable
// state. See forceKill.js for the contract (notably: a fresh termination always
// re-arms, and `dropState` is only for a kill relayed after the PortOS server
// has already finalized the agent).
const armForceKill = (agentId, agent, opts = {}) =>
  armForceKillShared(activeAgents, agentId, agent, {
    graceMs: SIGKILL_GRACE_MS,
    ...opts,
    onDropState: (id) => {
      withState((state) => { delete state.agents[id]; })
        .catch(err => console.error(`❌ Reap state write failed for ${id}: ${err.message}`));
    },
  });

// Express app setup
const app = express();
app.use(runnerRequestGuard);
app.use(express.json({ limit: '10mb' }));

const server = http.createServer(app);
const io = new SocketServer(server, {
  allowRequest: (req, callback) => callback(null, runnerRequestRefusal(req) === null)
});

// Check subsequent polling requests and upgrades too, even with an existing sid.
io.engine.use(runnerRequestGuard);

const httpDrain = createHttpDrain([server]);
let transportClose;
const lifecycle = createRunnerShutdown({
  admission: maintenance,
  stopIntake: (deadlineMs) => httpDrain.begin(deadlineMs),
  closeTransports: () => {
    transportClose ??= new Promise(resolve => {
      io.close(resolve);
      server.closeAllConnections?.();
    });
    return transportClose;
  },
  drainState,
  exit: (code) => process.exit(code),
});
registerRunnerShutdownSignals(process, lifecycle);

// Keep established sockets for outgoing completion events, but stop incoming
// commands and new namespace connections as soon as shutdown starts.
io.use((socket, next) => next(lifecycle.isStopping() ? new Error('Runner is shutting down') : undefined));

const persistCompletion = (agentId, output, metadata) => persistRunnerCompletion(AGENTS_DIR, agentId, output, metadata);

const securityScans = createSecurityScanManager({
  agentsDir: AGENTS_DIR,
  getAgent: id => activeAgents.get(id),
  trackWork: work => lifecycle.trackWork(work),
  emitEvidence: async (agentId, evidence) => {
    const agent = activeAgents.get(agentId);
    if (agent) agent.securityScan = evidence;
    emitToServer('agent:security-scan', { agentId, evidence });
  },
});

app.post('/agents/:agentId/security-scan', async (req, res) => {
  const parsed = securityScanRequestSchema.safeParse(req.body ?? {});
  if (!parsed.success) return res.status(400).json({ error: bodyErrorMessage(parsed.error) });
  if (lifecycle.rejectSpawn(res)) return;
  try {
    assertDatabaseAdmission();
    res.json(await securityScans.start(req.params.agentId, parsed.data));
  } catch (err) { res.status(err.status || 500).json({ error: err.message }); }
});
app.post('/agents/:agentId/security-scan/stop', async (req, res) => {
  try { res.json({ evidence: await securityScans.stop(req.params.agentId) }); }
  catch (err) { res.status(err.status || 500).json({ error: err.message }); }
});
app.get('/agents/:agentId/security-scan', async (req, res) => {
  try { res.json({ evidence: await securityScans.read(req.params.agentId) }); }
  catch (err) { res.status(err.status || 500).json({ error: err.message }); }
});

/**
 * Emit event to connected portos-server instances
 */
function emitToServer(event, data) {
  io.emit(event, data);
  console.log(`📡 Emitted ${event}`);
}

/**
 * Health check endpoint
 */
app.get('/health', (req, res) => {
  res.json({
    status: 'ok',
    activeAgents: activeAgents.size,
    uptime: process.uptime()
  });
});

async function inspectAgentProcess(agent) {
  const pid = usableAgentPid(agent.pid);
  const stats = pid ? await getProcessStats(pid) : null;
  return { stats, ...runnerAgentLivenessFields(agent, stats) };
}

/**
 * Get list of active agents with process stats
 */
app.get('/agents', async (req, res) => {
  const agents = [];
  for (const [agentId, agent] of activeAgents) {
    // onExit stamps exited then awaits disk I/O; skip so a dying handle is
    // not advertised as a stale listing that sweeps would reap before
    // agent:completed lands.
    if (agent.exited === true) continue;
    const inspected = await inspectAgentProcess(agent);
    agents.push({
      id: agentId,
      taskId: agent.taskId,
      pid: agent.pid,
      startedAt: agent.startedAt,
      runningTime: Date.now() - agent.startedAt,
      processActive: inspected.processActive,
      cpu: inspected.cpu,
      memoryMb: inspected.memoryMb,
      state: inspected.state,
      liveness: inspected.liveness,
      kind: agent.kind || 'cli',
      sessionId: agent.sessionId || null,
      command: agent.command || null,
      workspacePath: agent.workspacePath || null,
      securityScan: agent.securityScan || null,
      // null means an older caller did not opt into submission tracking.
      promptSubmitted: agent.promptSubmitted ?? null,
    });
  }
  res.json(agents);
});

/**
 * Spawn an interactive TUI in a runner-owned PTY. PortOS remains responsible
 * for prompt timing and rich completion analysis while it is connected; the
 * runner owns process survival and emits normal agent completion if the server
 * restarts before the TUI exits.
 */
app.post('/spawn-tui', lifecycle.spawnRoute(async (req, res) => {
  assertDatabaseAdmission();
  const body = spawnTuiBodySchema.safeParse(req.body ?? {});
  if (!body.success) return res.status(400).json({ error: bodyErrorMessage(body.error) });
  const {
    agentId,
    taskId,
    sessionId = agentId,
    command,
    args = [],
    workspacePath,
    envVars = {},
    // Non-secret provider identity used to retain that provider's ambient auth
    // allowlist when the runner builds the child environment.
    providerAuth = null,
    cols = 80,
    rows = 24,
    doneSentinelPath = null,
    trackPromptSubmission = false,
  } = body.data;

  // Name the offending field. A single opaque "missing or invalid fields" 400
  // sent a grok-tui agent to the zombie reaper with no shell and no clue why —
  // the real cause (`grok` absent from ALLOWED_COMMANDS) was invisible.
  const missing = Object.entries({ agentId, taskId, sessionId })
    .filter(([, value]) => !value)
    .map(([key]) => key);
  if (missing.length) {
    return res.status(400).json({ error: `Missing TUI spawn fields: ${missing.join(', ')}` });
  }
  if (!isAllowedCommand(command)) {
    return res.status(400).json({
      error: `Command not allowed: ${command}. Permitted commands: ${[...ALLOWED_COMMANDS].join(', ')}`
    });
  }
  if (activeAgents.has(agentId)) {
    return res.status(409).json({ error: `Agent ${agentId} is already running` });
  }

  const cwd = workspacePath && typeof workspacePath === 'string' ? workspacePath : ROOT_DIR;
  // Ahead of the executable checks below, because both of them run the child IN
  // this directory: a worktree reaped before the spawn makes the `--version`
  // probe fail too, and the runner would then blame the provider CLI and tell a
  // human to reinstall a binary that is sitting right there on PATH.
  if (!existsSync(cwd)) {
    return res.status(422).json({
      error: `${PTY_WORKSPACE_MISSING_PREFIX} the workspace for this spawn does not exist. Its worktree was probably removed before the agent launched.`
    });
  }
  const childEnv = buildCliChildEnv({ before: envVars, provider: providerAuth, cwd });
  // node-pty reports a missing executable as an immediate exit with no data.
  // Check the exact child PATH first so the caller gets a usable configuration
  // error instead of a generic startup-failure after a blank PTY transcript.
  // Keep the resolved path private: it may include the local account name.
  const executable = findCommandOnPath(command, { env: childEnv, cwd });
  if (!executable) {
    return res.status(422).json({
      error: `Command executable unavailable: ${basename(command)} is not on the CoS Runner PATH. Install it or update the provider command.`
    });
  }
  // A PATH hit can still be a broken npm shim or an incomplete CLI install.
  // Probe the same launch shape that the PTY will use: on Windows a .cmd/.bat
  // shim must run through cmd.exe, while on POSIX it remains the direct binary.
  // That prevents node-pty from reducing an immediate CLI error to a blank exit.
  const versionProbe = prepareCliSpawn(executable, ['--version'], childEnv);
  const runnable = await commandExists(versionProbe.command, versionProbe.args, {
    timeoutMs: TUI_CAPABILITY_PROBE_TIMEOUT_MS,
    env: childEnv,
    cwd,
  });
  if (lifecycle.rejectSpawn(res)) return;
  if (!runnable) {
    return res.status(422).json({
      error: `Command executable unavailable: ${basename(command)} did not pass the CoS Runner capability check. Reinstall it or update the provider command.`
    });
  }
  // Use the same safe wrapper for the actual PTY launch. In particular, this
  // preserves the shared escaping contract for paths/args passed to cmd.exe.
  const { command: ptyCommand, args: ptyArgs } = prepareCliSpawn(executable, args, childEnv);
  await securityScans.register(agentId, body.data.securityScanGrant);
  // Read BEFORE pty.spawn so onData/onExit subscribe with no await after the
  // spawn: a PTY that exits during the read would otherwise lose its exit (#11761).
  const securityScan = await securityScans.read(agentId);
  // Both checks above cleared the executable, so a throw here is about the PTY
  // layer or the workspace — not the command. node-pty collapses those into one
  // opaque `posix_spawn failed: No such file or directory`, which read as a
  // transient runner refusal and sent the whole fleet into a retry storm. Name
  // the actual fault instead, and let the caller block on the unrecoverable ones.
  let tuiProcess;
  assertDatabaseAdmission();
  try {
    tuiProcess = pty.spawn(ptyCommand, ptyArgs, {
      name: 'xterm-256color',
      cols,
      rows,
      cwd,
      env: childEnv,
    });
  } catch (err) {
    const { diagnosed, message } = diagnosePtySpawnFailure(err, {
      cwd,
      probeCwd: ROOT_DIR,
      runtimeProbe: (probeCwd) => probePtyRuntime(pty, probeCwd),
    });
    // Neither known fault: let the original bubble to the error middleware exactly
    // as it did before, rather than dressing an unknown cause in a confident 422.
    if (!diagnosed) throw err;
    console.error(`❌ PTY spawn failed for ${agentId}: ${message}`);
    return res.status(422).json({ error: message });
  }
  const startedAt = Date.now();
  const agent = {
    kind: 'tui',
    process: tuiProcess,
    taskId,
    pid: tuiProcess.pid,
    startedAt,
    outputBuffer: '',
    workspacePath: cwd,
    sessionId,
    command,
    doneSentinelPath,
    completedBySentinel: false,
    // Kept beside the live PTY, so a portos-server restart cannot erase the
    // distinction between a running task and an empty startup composer.
    promptSubmitted: trackPromptSubmission ? false : null,
    doneWatcher: null,
    // node-pty reports pid 0 for ConPTY on Windows, so a pid probe is not a
    // liveness signal. onExit is the runner's authority; GET /agents reads this.
    exited: false,
    securityScan,
    stopSecurityScan: () => securityScans.stop(agentId).catch(err => lifecycle.reportFailure(err)),
  };
  activeAgents.set(agentId, agent);
  const sentinelAccess = createAgentSentinelAccess({
    workspacePath: cwd,
    agentId,
    sentinelPath: doneSentinelPath,
    startedAt,
    getActiveAgentIds: () => activeAgents.keys(),
  });

  tuiProcess.onData((data) => {
    const current = activeAgents.get(agentId);
    if (!current) return;
    current.outputBuffer += data;
    if (current.outputBuffer.length > 512 * 1024) {
      current.outputBuffer = current.outputBuffer.slice(-512 * 1024);
    }
    io.emit('tui:output', { sessionId, agentId, data });
  });

  tuiProcess.onExit(lifecycle.agentExit(agentId, agent, createTuiExitHandler({
    agentId, taskId, sessionId, agent, activeAgents, io, emitToServer, withState,
    persistCompletion, onError: lifecycle.reportFailure,
  })));

  if (doneSentinelPath) {
    const handleSentinel = () => {
      agent.sentinelWork = lifecycle.trackWork(async () => {
        const current = activeAgents.get(agentId);
        if (!current) return;
        const sentinelSourcePath = sentinelAccess.resolvedPath?.();
        let sentinelReadSucceeded = true;
        const contents = await sentinelAccess.read().catch(err => {
          sentinelReadSucceeded = false;
          console.error(`❌ TUI agent ${agentId} sentinel read failed: ${err.message}`);
          return '';
        });
        let sentinelValidated = sentinelReadSucceeded;
        let validatedContents = contents;
        if (sentinelReadSucceeded) {
          sentinelValidated = await sentinelAccess.promote(contents).catch(err => {
            console.error(`❌ TUI agent ${agentId} canonical sentinel promotion failed: ${err.message}`);
            return false;
          });
          if (sentinelValidated && sentinelSourcePath && sentinelSourcePath !== sentinelAccess.path) {
            let canonicalReadSucceeded = true;
            validatedContents = await sentinelAccess.read().catch(err => {
              canonicalReadSucceeded = false;
              console.error(`❌ TUI agent ${agentId} canonical sentinel read failed: ${err.message}`);
              return '';
            });
            sentinelValidated = canonicalReadSucceeded;
          }
        }
        if (!sentinelValidated) {
          if (activeAgents.get(agentId) === current) {
            current.doneWatcher?.();
            current.doneWatcher = sentinelAccess.watch(handleSentinel);
          }
          return;
        }
        current.completedBySentinel = true;
        const { summary } = parseSentinelPayload(validatedContents);
        if (summary) {
          current.completionOutput = `${SENTINEL_COMPLETION_MARKER}\n${summary}\n`;
          emitToServer('agent:output', {
            agentId,
            text: `${SENTINEL_COMPLETION_MARKER}\n${summary.slice(0, 4096)}\n`,
          });
        }
        if (!current.exited) current.process.kill();
        await sentinelAccess.cleanup();
      });
      return agent.sentinelWork;
    };
    agent.doneWatcher = sentinelAccess.watch(handleSentinel);
  }

  await withState((state) => {
    state.agents[agentId] = {
      pid: tuiProcess.pid,
      taskId,
      startedAt,
      kind: 'tui',
      sessionId,
      workspacePath: cwd,
      doneSentinelPath,
      securityScanGrant: body.data.securityScanGrant,
    };
    state.stats.spawned++;
  });

  console.log(`📟 Runner-owned TUI ${agentId} started (PID: ${tuiProcess.pid})`);
  res.json({ success: true, agentId, sessionId, pid: tuiProcess.pid });
}));

/**
 * Get process stats for a specific agent
 */
app.get('/agents/:agentId/stats', async (req, res) => {
  const { agentId } = req.params;
  const agent = activeAgents.get(agentId);

  if (!agent) {
    return res.status(404).json({ error: 'Agent not found or not running' });
  }

  const inspected = await inspectAgentProcess(agent);
  res.json({
    agentId,
    pid: agent.pid,
    active: inspected.processActive,
    cpu: inspected.cpu,
    memoryKb: inspected.stats?.memoryKb ?? 0,
    memoryMb: inspected.memoryMb,
    state: inspected.state,
    liveness: inspected.liveness,
  });
});

/**
 * Spawn a new agent
 */
app.post('/spawn', lifecycle.spawnRoute(async (req, res) => {
  assertDatabaseAdmission();
  const body = spawnBodySchema.safeParse(req.body ?? {});
  if (!body.success) return res.status(400).json({ error: bodyErrorMessage(body.error) });
  const {
    agentId,
    taskId,
    prompt,
    workspacePath,
    model,
    envVars = {},
    // Non-secret provider identity used to retain that provider's ambient auth
    // allowlist when the runner builds the child environment.
    providerAuth = null,
    // New: CLI-agnostic parameters
    cliCommand,
    cliArgs,
    // Legacy: Claude-specific (deprecated)
    claudePath = process.env.CLAUDE_PATH || 'claude'
  } = body.data;

  if (!agentId || !taskId || !prompt) {
    return res.status(400).json({ error: 'Missing required fields: agentId, taskId, prompt' });
  }

  // Use new CLI params if provided, otherwise fallback to legacy Claude defaults
  let command, spawnArgs;
  if (cliCommand) {
    // Validate command against allowlist to prevent arbitrary code execution
    if (!isAllowedCommand(cliCommand)) {
      return res.status(400).json({
        error: `Command not allowed: ${cliCommand}. Permitted commands: ${[...ALLOWED_COMMANDS].join(', ')}`
      });
    }
    command = cliCommand;
    // cliArgs is an array or a single string (the schema rejects anything else);
    // default to empty args when not provided.
    const args = cliArgs ?? [];
    spawnArgs = Array.isArray(args) ? args : [args];
  } else {
    // Legacy: Claude-specific args
    command = claudePath;
    spawnArgs = [
      '--dangerously-skip-permissions',
      '--print',
      '--output-format', 'stream-json',
      '--verbose',
      '--include-partial-messages'
    ];
    if (model) {
      spawnArgs.push('--model', model);
    }
  }

  console.log(`🤖 Spawning agent ${agentId} for task ${taskId} (CLI: ${command})`);

  // Ensure workspacePath is valid
  const cwd = workspacePath && typeof workspacePath === 'string' ? workspacePath : ROOT_DIR;

  // The env arrives already composed — agentLifecycle built it with
  // composeProviderEnv (provider.envVars + the OpenCode models map) and POSTed it
  // — so this side only needs the base, the PWD pin, and the CLAUDECODE strip.
  // The pin is the path #3193 was reported through: the log line above named the
  // app's workspace correctly while every OpenCode agent still ran in the PortOS
  // folder. No `guard` — this separate process has never carried the pm2 shim.
  const childEnv = buildCliChildEnv({ before: envVars, provider: providerAuth, cwd });

  // Resolve a bare npm-installed CLI (opencode/codex/claude/… — a .cmd/.bat
  // shim on Windows) to its real path and wrap a shim as `cmd.exe /c <path>` so
  // spawn() under shell:false can launch it. Without this, Windows can't find
  // `opencode.cmd` from the bare name → spawn ENOENT (errno -4058) → empty
  // output → startup-failure. Mirrors the working "Run Prompt" path
  // (server/services/runner.js); resolved against childEnv so a provider PATH
  // override is honored. See issue #2243.
  // Deliver the prompt per provider convention BEFORE resolving the spawn shim:
  //   - Antigravity (`agy`): spliced in as the --print VALUE (agy does not read
  //     stdin) → useStdin=false. Without this the prompt never reaches the model
  //     and the trailing --print marker swallows the next flag as its "prompt".
  //   - Grok on Windows: `/dev/stdin` rewritten to a temp file → useStdin=false.
  //   - Every other provider: unchanged, prompt over stdin → useStdin=true.
  assertDatabaseAdmission();
  const { args: deliveredArgs, useStdin, cleanup: cleanupPromptFile } = prepareCliPrompt(command, spawnArgs, prompt, { cwd });
  const { command: spawnCommand, args: finalSpawnArgs } = prepareCliSpawn(command, deliveredArgs, childEnv);

  if (lifecycle.rejectSpawn(res)) {
    cleanupPromptFile();
    return;
  }

  // Spawn the CLI process
  await securityScans.register(agentId, body.data.securityScanGrant);
  // Read BEFORE spawn: Node reports a failed spawn (ENOENT/EACCES/EMFILE) via
  // process.nextTick, so no await may sit between spawn() and the 'error'
  // listener or the unhandled event kills the runner (#11761).
  const securityScan = await securityScans.read(agentId);
  const claudeProcess = spawn(spawnCommand, finalSpawnArgs, {
    cwd,
    shell: false,
    stdio: ['pipe', 'pipe', 'pipe'],
    env: childEnv
  });

  // Detect if stream-json format is active (Claude CLI with streaming)
  const isStreamJson = spawnArgs.includes('stream-json');
  const streamParser = isStreamJson ? createStreamJsonParser() : null;
  const isCodexCli = basename(command).replace(/\.exe$/i, '') === 'codex';
  const codexStderrFormatter = isCodexCli ? createCodexStderrFormatter(prompt) : null;

  // Store in memory
  const agent = {
    process: claudeProcess,
    taskId,
    pid: claudeProcess.pid,
    startedAt: Date.now(),
    outputBuffer: '',
    rawStreamBuffer: '',
    streamParser,
    codexStderrFormatter,
    // Per-stream ANSI strippers, mirroring spawnDirectly's headless path — the
    // same providers land here when the run is dispatched through the runner, so
    // without these the `[stderr] [0m` noise reproduces on that half of the
    // dispatch matrix. Stateful: each buffers an escape split across two chunks.
    // A stream-json stdout is NDJSON with no raw ESC byte, so it skips the scan.
    stripStdoutAnsi: isStreamJson ? (text) => text : createStreamingAnsiStripper(),
    stripStderrAnsi: createStreamingAnsiStripper(),
    workspacePath: cwd,
    securityScan,
    stopSecurityScan: () => securityScans.stop(agentId).catch(err => lifecycle.reportFailure(err)),
  };
  activeAgents.set(agentId, agent);

  // Guard the stdin pipe BEFORE writing: a child that exits before reading it
  // (bad flag, missing CLI) emits EPIPE, and an unlistened stream 'error' out
  // here crashes the runner process. The 'error'/'close' handlers settle the run.
  guardChildStdin(claudeProcess);

  // Send prompt via stdin (skipped when the prompt was delivered via argv —
  // antigravity's --print value, or grok's Windows temp file).
  deliverChildStdin(claudeProcess, useStdin ? prompt : null, `agent ${agentId}`);

  // Handle stdout
  claudeProcess.stdout.on('data', (data) => {
    const text = agent?.stripStdoutAnsi ? agent.stripStdoutAnsi(data.toString()) : data.toString();

    if (agent?.streamParser) {
      // Parse stream-json and emit extracted text lines (cap buffer at 512KB for error analysis)
      agent.rawStreamBuffer += text;
      if (agent.rawStreamBuffer.length > 512 * 1024) {
        agent.rawStreamBuffer = agent.rawStreamBuffer.slice(-512 * 1024);
      }
      const lines = agent.streamParser.processChunk(text);
      for (const line of lines) {
        agent.outputBuffer += line + '\n';
        emitToServer('agent:output', { agentId, text: line + '\n' });
      }
    } else {
      // Non-stream providers: emit stdout as-is once decolored. A chunk that was
      // purely terminal control has nothing left to show. Unlike stderr below,
      // whitespace is NOT dropped here — a blank line is legitimate formatting
      // when it isn't wearing an `[stderr]` tag.
      if (!text) return;
      if (agent) {
        agent.outputBuffer += text;
      }
      emitToServer('agent:output', { agentId, text });
    }
  });

  // Handle stderr
  claudeProcess.stderr.on('data', (data) => {
    const decolored = agent?.stripStderrAnsi ? agent.stripStderrAnsi(data.toString()) : data.toString();
    if (agent?.codexStderrFormatter) {
      const lines = agent.codexStderrFormatter.processChunk(decolored);
      for (const line of lines) {
        agent.outputBuffer += line + '\n';
        emitToServer('agent:output', { agentId, text: line + '\n' });
      }
      return;
    }

    // A chunk that decolors down to whitespace was pure terminal control
    // (`opencode run` emits a bare reset per progress redraw). Tagging it
    // `[stderr]` would add one blank noise line to the tail per redraw.
    const trimmedDecolored = decolored.trim();
    if (!trimmedDecolored || isKnownCliStderrNoise(trimmedDecolored)) return;
    const text = `[stderr] ${decolored}`;
    if (agent) agent.outputBuffer += text;
    emitToServer('agent:output', { agentId, text });
  });

  // Handle errors
  claudeProcess.on('error', (err) => {
    cleanupPromptFile();
    console.error(`❌ Agent ${agentId} spawn error: ${err.message}`);
    emitToServer('agent:error', { agentId, error: err.message });
    activeAgents.delete(agentId);
  });

  // Handle process exit
  claudeProcess.on('close', lifecycle.agentExit(agentId, agent, async (code) => {
    cleanupPromptFile();
    try {
      agent.exited = true;
      await agent.stopSecurityScan?.();
      // Cancel any pending SIGKILL timer — process already exited.
      if (agent?.killTimer) {
        clearTimeout(agent.killTimer);
        agent.killTimer = null;
      }
      const duration = Date.now() - (agent?.startedAt || Date.now());

      // Flush remaining stream parser data
      if (agent?.streamParser) {
        const remaining = agent.streamParser.flush();
        for (const line of remaining) {
          agent.outputBuffer += line + '\n';
          emitToServer('agent:output', { agentId, text: line + '\n' });
        }
        // Use the parsed final result for the output file if available
        const finalResult = agent.streamParser.getFinalResult();
        if (finalResult) {
          agent.outputBuffer = finalResult;
        }
      }
      if (agent?.codexStderrFormatter) {
        const remaining = agent.codexStderrFormatter.flush();
        for (const line of remaining) {
          agent.outputBuffer += line + '\n';
          emitToServer('agent:output', { agentId, text: line + '\n' });
        }
      }

      const output = agent?.outputBuffer || '';
      const paused = agent?.paused === true;

      // A non-zero exit is a failure line, so it belongs on stderr where a
      // stderr-based monitor can see it (#7945). A pause or a clean exit is
      // ordinary progress and stays on stdout.
      const logExit = !paused && code !== 0 ? console.error : console.log;
      logExit(`${paused ? '⏸️' : code === 0 ? '✅' : '❌'} Agent ${agentId} exited with code ${code}${paused ? ' after pause' : ''}`);

      // Persist output and terminal evidence before publishing completion or
      // removing durable ownership. Failed writes leave recovery evidence intact.
      await withBackupAssetPublication(async () => {
        await persistCompletion(agentId, output, paused ? null : {
          taskId,
          completedAt: new Date().toISOString(),
          exitCode: code,
          success: code === 0,
          duration,
          securityScan: agent.securityScan,
        });
        if (paused) {
          activeAgents.delete(agentId);
          return;
        }

        // Emit completion event
        emitToServer('agent:completed', {
          agentId,
          taskId,
          exitCode: code,
          success: code === 0,
          duration,
          outputLength: output.length,
          securityScan: agent.securityScan,
        });

        // Update state — serialize with the spawn write path via withState.
        await withState((state) => {
          state.stats.completed++;
          if (code !== 0) state.stats.failed++;
          delete state.agents[agentId];
        });
      });
      activeAgents.delete(agentId);
    } catch (err) {
      lifecycle.reportFailure(err);
      activeAgents.delete(agentId);
      return false;
    }
  }));

  // Update state — use withState to serialize the read-modify-write with the
  // close handler's own state update, preventing concurrent mutation of the
  // same state file.
  await withState((state) => {
    state.agents[agentId] = {
      pid: claudeProcess.pid,
      securityScanGrant: body.data.securityScanGrant,
      taskId,
      startedAt: Date.now()
    };
    state.stats.spawned++;
  });

  res.json({
    success: true,
    agentId,
    pid: claudeProcess.pid
  });
}));

/**
 * Terminate an agent (graceful with SIGTERM, then SIGKILL after timeout)
 */
app.post('/terminate/:agentId', async (req, res) => {
  const body = terminateBodySchema.safeParse(req.body ?? {});
  if (!body.success) return res.status(400).json({ error: bodyErrorMessage(body.error) });
  const { agentId } = req.params;
  const agent = activeAgents.get(agentId);

  if (!agent) {
    return res.status(404).json({ error: 'Agent not found or not running' });
  }

  if (body.data.onlyIfPromptUnsubmitted === true) {
    if (agent.kind !== 'tui' || agent.promptSubmitted !== false) {
      return res.json({ success: true, agentId, terminated: false });
    }
    // Claim cancellation synchronously with the receipt check, before scanner
    // settlement yields. Late server writes must not start this canceled task.
    agent.startupInterrupted = true;
  }

  console.log(`🔪 Terminating agent ${agentId}`);
  await agent.stopSecurityScan?.();
  killProcessTree(agent.process, 'SIGTERM');
  armForceKill(agentId, agent);

  res.json({ success: true, agentId, terminated: true });
});

/**
 * Force kill an agent immediately with SIGKILL (no graceful shutdown)
 */
app.post('/kill/:agentId', async (req, res) => {
  const { agentId } = req.params;
  const agent = activeAgents.get(agentId);

  if (!agent) {
    return res.status(404).json({ error: 'Agent not found or not running' });
  }

  console.log(`💀 Force killing agent ${agentId} (PID: ${agent.pid})`);
  await agent.stopSecurityScan?.();
  killProcessTree(agent.process, 'SIGKILL');

  // Clean up immediately
  activeAgents.delete(agentId);

  // Update state
  await withState((state) => {
    delete state.agents[agentId];
  });

  res.json({ success: true, agentId, pid: agent.pid, signal: 'SIGKILL' });
});

/**
 * Pause an agent: stop the child process without reporting normal completion.
 * PortOS server persists the paused agent/task state and preserves the
 * worktree; the runner just ensures the process stops spending tokens.
 */
app.post('/pause/:agentId', async (req, res) => {
  const { agentId } = req.params;
  const body = pauseBodySchema.safeParse(req.body ?? {});
  if (!body.success) return res.status(400).json({ error: bodyErrorMessage(body.error) });
  const { reason = null } = body.data;
  const agent = activeAgents.get(agentId);

  if (!agent) {
    return res.status(404).json({ error: 'Agent not found or not running' });
  }

  const pausedAt = new Date().toISOString();
  console.log(`⏸️ Pausing agent ${agentId} (PID: ${agent.pid})${reason ? `: ${reason}` : ''}`);

  agent.paused = true;
  agent.pausedAt = pausedAt;
  agent.pauseReason = reason;
  await agent.stopSecurityScan?.();
  killProcessTree(agent.process, 'SIGTERM');
  // NOT armForceKill: a pause keeps its map entry (the agent is meant to be
  // resumable) and only escalates while the pause is still in force.
  agent.killTimer = setTimeout(() => {
    agent.killTimer = null;
    const current = activeAgents.get(agentId);
    if (current?.paused) killProcessTree(current.process, 'SIGKILL');
  }, SIGKILL_GRACE_MS);

  res.json({ success: true, agentId, pid: agent.pid, pausedAt });
});

/**
 * Kill all agents
 */
app.post('/terminate-all', async (req, res) => {
  const agentIds = Array.from(activeAgents.keys());

  // Per-agent SIGKILL fallback timers. Each agent gets its OWN timer (stored
  // as agent.killTimer so its close handler can cancel it on clean exit). A
  // single shared timer would be cleared by the FIRST agent to exit, leaving
  // any agent that ignores SIGTERM running with no force-kill escalation.
  for (const agentId of agentIds) {
    const agent = activeAgents.get(agentId);
    if (agent) {
      await agent.stopSecurityScan?.();
      killProcessTree(agent.process, 'SIGTERM');
      armForceKill(agentId, agent);
    }
  }

  res.json({ success: true, killed: agentIds.length });
});

/**
 * Send a BTW (additional context) message to a running agent.
 * Writes the message to a BTW.md file in the agent's workspace so the
 * agent can discover it during file operations.
 */
app.post('/btw/:agentId', async (req, res) => {
  const { agentId } = req.params;
  const agent = activeAgents.get(agentId);

  if (!agent) {
    return res.status(404).json({ error: 'Agent not found or not running' });
  }

  const body = btwBodySchema.safeParse(req.body ?? {});
  if (!body.success) return res.status(400).json({ error: bodyErrorMessage(body.error) });
  const { message } = body.data;

  // Derive workspace from the agent's known record, not from request body
  const agentWorkspace = agent.workspacePath;
  if (!agentWorkspace || typeof agentWorkspace !== 'string') {
    return res.status(400).json({ error: 'Agent has no known workspacePath' });
  }

  const timestamp = new Date().toISOString();
  const entry = `\n---\n**[${timestamp}]** ${message}\n`;
  const btwPath = join(agentWorkspace, 'BTW.md');

  // Append to BTW.md (create if first message)
  const existing = await readFile(btwPath, 'utf-8').catch(() => '');
  const header = existing ? '' : '# Additional Context from User\n\nThe user has sent you additional context while you are working. Read and incorporate this information.\n';
  await writeFile(btwPath, header + existing + entry);

  console.log(`💬 BTW message delivered to agent ${agentId}: ${message.substring(0, 80)}`);

  // Emit the btw event so the main server can track it
  emitToServer('agent:btw', { agentId, message, timestamp });

  res.json({ success: true, agentId, btwPath });
});

/**
 * Get agent output
 */
app.get('/agents/:agentId/output', (req, res) => {
  const { agentId } = req.params;
  const agent = activeAgents.get(agentId);

  if (!agent) {
    return res.status(404).json({ error: 'Agent not found' });
  }

  res.json({ agentId, output: agent.outputBuffer });
});

/**
 * Socket.IO connection handling
 */
io.on('connection', (socket) => {
  if (lifecycle.isStopping()) { socket.disconnect(true); return; }
  socket.use((packet, next) => {
    if (!lifecycle.isStopping()) next();
  });
  console.log(`🔌 Client connected: ${socket.id}`);

  socket.on('tui:input', ({ sessionId, data, promptSubmission }) => {
    try {
      const agent = [...activeAgents.values()].find(candidate => candidate.sessionId === sessionId);
      if (agent?.kind === 'tui' && !agent.startupInterrupted && typeof data === 'string') {
        agent.process.write(data);
        // The write and receipt are synchronous in the runner. A queued socket
        // write in the server is not evidence that the Enter reached the PTY.
        if (promptSubmission === true && data === '\r' && agent.promptSubmitted !== null) {
          agent.promptSubmitted = true;
        }
      }
    } catch (err) {
      console.error(`❌ TUI input relay failed: ${err.message}`);
    }
  });

  socket.on('tui:resize', ({ sessionId, cols, rows }) => {
    try {
      const agent = [...activeAgents.values()].find(candidate => candidate.sessionId === sessionId);
      if (agent?.kind !== 'tui') return;
      if (!Number.isInteger(cols) || !Number.isInteger(rows) || cols < 1 || rows < 1) return;
      agent.process.resize(cols, rows);
    } catch (err) {
      console.error(`❌ TUI resize relay failed: ${err.message}`);
    }
  });

  socket.on('tui:kill', async ({ sessionId, signal = 'SIGTERM' }) => {
    try {
      if (!TUI_SIGNALS.has(signal)) return;
      const entry = [...activeAgents.entries()].find(([, candidate]) => candidate.sessionId === sessionId);
      if (!entry) return;
      const [agentId, agent] = entry;
      if (agent.kind !== 'tui') return;
      await agent.stopSecurityScan?.();
      killProcessTree(agent.process, signal);
      // `dropState` only for an agent the server has already FINALIZED. A pause
      // also relays a kill (the server stops a TUI session through this socket),
      // and its durable record is exactly what a later resume reads — reaping it
      // would strand the paused run.
      armForceKill(agentId, agent, { dropState: agent.paused !== true });
    } catch (err) {
      console.error(`❌ TUI termination relay failed: ${err.message}`);
    }
  });

  socket.on('disconnect', () => {
    console.log(`🔌 Client disconnected: ${socket.id}`);
  });
});

/**
 * Cleanup orphaned agents on startup
 * Checks if PIDs from state are still running
 * Emits a batch completion event for dead agents so main server can retry tasks
 */
async function cleanupOrphanedAgents() {
  const orphaned = [];
  // Recovery can overlap a signal and finalizers. Keep its entire read/check/
  // write in the same queue so an old snapshot cannot resurrect completed runs.
  await withState(async (state) => {
    for (const [agentId, agentInfo] of Object.entries(state.agents)) {
      if (agentInfo.securityScanGrant) await securityScans.recover(agentId);
      const isRunning = await checkProcessRunning(agentInfo.pid);
      if (!isRunning) {
        orphaned.push({ agentId, taskId: agentInfo.taskId });
        delete state.agents[agentId];
      }
    }
  });

  if (orphaned.length > 0) {
    console.log(`🧹 Cleaned up ${orphaned.length} orphaned agents from state`);

    // Emit a single batch event with all orphaned agents
    // This avoids log spam when many agents were orphaned
    io.emit('agents:orphaned', {
      agents: orphaned.map(o => ({
        agentId: o.agentId,
        taskId: o.taskId,
        exitCode: -1,
        success: false,
        orphaned: true,
        error: 'Agent process died (runner restart detected dead PID)'
      })),
      count: orphaned.length
    });
    console.log(`📡 Emitted agents:orphaned (${orphaned.length} agents)`);
  }

  return orphaned;
}

// Terminal JSON error handler: without it Express answers a thrown route error
// with an HTML page and drops err.code, which cosRunnerClient cannot classify.
app.use((err, _req, res, next) => {
  if (res.headersSent) return next(err);
  const isValidation = err?.name === 'ZodError' || Array.isArray(err?.issues);
  const status = isValidation ? 400 : Number.isInteger(err?.status) ? err.status : 500;
  const message = Array.isArray(err?.issues) ? bodyErrorMessage(err) : err?.message || 'Internal runner error';
  res.status(status).json({
    error: message,
    code: isValidation ? 'VALIDATION_ERROR' : err?.code || 'RUNNER_ERROR',
  });
});

/**
 * Start the server
 */
server.listen(PORT, HOST, async () => {
  console.log(`🤖 CoS Agent Runner started on http://${HOST}:${PORT}`);

  // The runner is its own PM2 app with its own environment, and it spawns
  // provider CLIs by bare name — so it must adopt npm's global bin directory
  // itself. Without this a CLI the AI Providers page reports as installed
  // (the main server adopted it there) still 422s here as "not on the CoS
  // Runner PATH". Fire-and-forget: never blocks accepting work.
  adoptNpmGlobalBinDir().catch((err) => console.error(`❌ npm global bin adoption failed: ${err.message}`));

  // Ensure agents directory exists. try/catch is mandatory: this listener runs
  // outside the request lifecycle, so a rejected await here escapes as an
  // unhandled rejection and takes the runner down at boot (fatal on Node >= 15).
  try {
    if (!existsSync(AGENTS_DIR)) {
      await ensureDir(AGENTS_DIR);
    }
  } catch (err) {
    console.error(`❌ Agents dir setup failed: ${err.message}`);
  }

  // Delay orphan cleanup to allow socket connections to establish
  // This ensures completion events reach the main server for task retry
  setTimeout(async () => {
    // Same rule, and this body is the one that reads and rewrites the agent
    // state file: a truncated/corrupt `agents.json` or an EACCES rejects out of
    // an async timer callback with no owner, killing the runner seconds after
    // boot and leaving PM2 to restart-loop it against the same bad file.
    // Log and continue — a corrupt state file must not stop the runner from
    // accepting new work, and the sweep re-runs on the next restart.
    try {
      if (lifecycle.isStopping()) return;
      const orphaned = await lifecycle.trackWork(cleanupOrphanedAgents);
      if (orphaned.length > 0) {
        console.log(`🧹 Cleaned ${orphaned.length} orphaned agent(s)`);
      }
    } catch (err) {
      console.error(`❌ Orphan cleanup failed: ${err.message}`);
    }
  }, ORPHAN_CLEANUP_DELAY_MS);
});