Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

19.2 KB
Raw Blame
/**
 * paneProbe — run the smoke checks against a LIVE agent pane, the way a person would: type the
 * prompt into the tmux pane, wait for the marker to show up in what the pane prints.
 *
 * tmux only. The harness runs every agent in a tmux pane (`tmuxBackend.ts`), and `respawn-pane`
 * keeps the pane id across a grid switch, so one `%N` follows the agent through all three legs.
 *
 * Pure-ish: the tmux calls are injected (`Tmux`) so the sequencing — settle, type, poll, tail — is
 * unit-testable offline; `realTmux` is the one that shells out.
 */
import { execFile } from 'node:child_process'
import { promisify } from 'node:util'
import { scenarioFor, type CheckOutcome, type Leg, type LegOutcome, type LogKind, type SmokeCheck } from './smokeChecks.js'
import type { ToolResult } from './sessionTools.js'

const execFileP = promisify(execFile)
let typeSequence = 0

export interface Tmux {
  /** The pane's visible text plus scrollback tail (`capture-pane -p -S -<lines>`). */
  capture(pane: string, lines: number): Promise<string>
  /** Type `text` literally, then Enter. */
  type(pane: string, text: string): Promise<void>
  /** Press one key by tmux name (`1`, `Enter`, `Escape`) — for a dialog, not for a prompt. */
  key(pane: string, key: string): Promise<void>
  sleep(ms: number): Promise<void>
  now(): number
}

export const realTmux: Tmux = {
  async capture(pane, lines) {
    const { stdout } = await execFileP('tmux', ['capture-pane', '-p', '-t', pane, '-S', `-${lines}`], { timeout: 5000 })
    return stdout
  },
  async type(pane, text) {
    // Not `send-keys -l`: a whole line arriving as one keystroke burst trips the TUI's paste
    // detector (codex 0.155.1 kept the text in the composer and swallowed the Enter). The harness's
    // own injection (`lib/tmux.ts sendToTmux`) pastes from a named buffer, lets it land, then sends
    // Enter on its own — same sequence here.
    const buffer = `e2e-probe-${process.pid}-${++typeSequence}`
    await execFileP('tmux', ['set-buffer', '-b', buffer, '--', text], { timeout: 5000 })
    await execFileP('tmux', ['paste-buffer', '-t', pane, '-b', buffer, '-d'], { timeout: 5000 })
    await new Promise((r) => setTimeout(r, 400))
    await execFileP('tmux', ['send-keys', '-t', pane, 'Enter'], { timeout: 5000 })
  },
  async key(pane, key) {
    await execFileP('tmux', ['send-keys', '-t', pane, key], { timeout: 5000 })
  },
  sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
  now: () => Date.now(),
}

/**
 * Startup dialogs a fresh coding tool puts in front of its prompt. A check typed into one of these
 * is not a check: the text is swallowed and Enter picks whatever is highlighted (measured: codex
 * 0.155's "Update available" took the Enter as "1. Update now" and ran `npm install -g`). Each is
 * answered the way a person running a test would — and each answer is recorded in the leg's
 * evidence, since a tool that greets every resume with a dialog is itself a finding.
 */
export const STARTUP_DIALOGS: ReadonlyArray<{ id: string; match: RegExp; keys: string[] }> = [
  // codex: "Do you trust the contents of this directory?" 1. Yes, continue
  { id: 'codex-trust', match: /Do you trust the contents of this directory/, keys: ['1', 'Enter'] },
  // codex: "Update available! x -> y" 1. Update now / 2. Skip / 3. Skip until next version
  { id: 'codex-update', match: /Update available!/, keys: ['2', 'Enter'] },
  // codex 0.156: "Hooks need review — 2 hooks are new or changed" with the cursor on "Review hooks".
  // The hooks are the harness's own (how it tracks the agent), so "Trust all and continue" is what a
  // person using the harness answers too — Down once from "Review hooks", then Enter.
  { id: 'codex-hooks-trust', match: /Hooks need review/, keys: ['Down', 'Enter'] },
  // codex: "GPT-5.5 retires on <date>. Switch to <next> to continue" 1. Try new model / 2. Use existing
  // model. Keep the model the run was configured with: a test that quietly changes its own model is
  // no longer the test that was asked for.
  { id: 'codex-model-retire', match: /retires on [^\n]*\n?[^\n]*Switch to|1\. Try new model/, keys: ['2', 'Enter'] },
  // claude: "Do you trust the files in this folder?" — Enter accepts the highlighted "Yes, proceed"
  { id: 'claude-trust', match: /Do you trust the files in this folder/, keys: ['Enter'] },
  // claude 2.1.281's rewording — "Quick safety check: Is this a project you created or one you
  // trust?" — and its default moved to **"No, exit"**: the old Enter would now quit the engine.
  // Down to "Yes, I trust this folder", then Enter. (The run pre-trusts its own folder, so this is
  // the fallback for when that does not take.)
  { id: 'claude-trust-v2', match: /Yes, I trust this folder/, keys: ['Down', 'Enter'] },
  // claude: a project .mcp.json must be approved — Enter accepts the highlighted "Use this and all future MCP servers"
  { id: 'claude-mcp-approve', match: /MCP server[s]? (found|configured) in \.mcp\.json|Use this and all future MCP servers|Use this MCP server/, keys: ['Enter'] },
  // claude, on `--dangerously-skip-permissions` (permission mode `full`): a full-screen Bypass
  // Permissions warning whose DEFAULT is "No, exit" — Enter alone exits 1 and leaves the pane a
  // shell. Down first, then Enter, picks "Yes, I accept". This is not a fallback in practice: with
  // `bypassPermissionsModeAccepted` already written to ~/.claude.json, claude 2.1.281 still drew the
  // warning, and answering it here is what let the run's first leg happen at all.
  { id: 'claude-bypass-accept', match: /running in Bypass Permissions mode/, keys: ['Down', 'Enter'] },
  // claude, MID-TURN, on any gateway (measured on grid.autonomous.ai AND a local relay): an "auto mode
  // classifier … your requests go through <host>, which isn't compatible" notice that blocks the turn
  // until a key is pressed. A finding about the relay's Anthropic contract, and a modal to get past.
  { id: 'claude-automode-notice', match: /isn't compatible with this update|ask your gateway to/, keys: ['Enter'] },
]

/**
 * The tool saying its account is out of usage — the same words in both, read out of the binaries
 * rather than guessed:
 *
 *   codex   "You've hit your usage limit. … try again at Sep 24th, 2026 6:02 PM."
 *   claude  "You've hit your session limit · resets 6pm"   (also weekly / monthly / team budget)
 *           "You're out of usage credits. /model to switch models."
 *
 * Without this, a check on an exhausted account waited out its 90s, came back `stuck`, and the run
 * went to the reviewer as if it were a bug to diagnose — for a condition the pane had already stated
 * in plain words. "Approaching usage limit" (claude's early warning) is deliberately not matched.
 */
export const QUOTA_EXHAUSTED = /You['\u2019]ve hit your [^\n]{0,40}?(limit|budget)|You['\u2019]re out of usage credits|usage limit reached/i

/** When the tool says the quota comes back, in its own words — or null if it did not say. */
export function quotaResets(text: string): string | null {
  const m = text.match(/(resets? (?:at |in |on )?[^\n\u00b7|.]{1,40}|try again (?:at|in) [^\n|]{1,50}?)(?:\.(?:\s|$)|\n|$)/im)
  return m ? m[1].trim() : null
}

/**
 * The dialog that is actually waiting — the LOWEST one on screen, not the first in the list. A fresh
 * codex stacks them (the hooks review, then the model notice under it), and the earlier text stays
 * in the scrollback after it is answered; picking by list order answered a screen that was already
 * gone and sent its keys into the one that was not. One answer per dialog per round of dismissals.
 */
function activeDialog(screen: string, answered: readonly string[]): (typeof STARTUP_DIALOGS)[number] | undefined {
  let best: (typeof STARTUP_DIALOGS)[number] | undefined
  let at = -1
  for (const d of STARTUP_DIALOGS) {
    if (answered.includes(d.id)) continue
    const all = [...screen.matchAll(new RegExp(d.match.source, d.match.flags.includes('g') ? d.match.flags : d.match.flags + 'g'))]
    const last = all.length ? all[all.length - 1].index! : -1
    if (last > at) { at = last; best = d }
  }
  return best
}

/**
 * Answer any startup dialog on screen, then wait for the pane to settle again. Returns the ids of
 * the dialogs answered (evidence). Bounded: a dialog that comes back after being answered is left
 * alone after `max` rounds so the check that follows records it as stuck, with the dialog on the tail.
 */
export async function dismissStartupDialogs(tmux: Tmux, pane: string, opts: ProbeOptions = {}, max = 5): Promise<string[]> {
  const o = { ...DEFAULTS, ...opts }
  const answered: string[] = []
  for (let round = 0; round < max; round++) {
    const screen = await tmux.capture(pane, o.tailLines)
    const dialog = activeDialog(screen, answered)
    if (!dialog) break
    for (const k of dialog.keys) {
      await tmux.key(pane, k)
      await tmux.sleep(300)
    }
    answered.push(dialog.id)
    await waitForPaneSettle(tmux, pane, opts)
  }
  return answered
}

export interface ProbeOptions {
  /** How long one check may take before it is `stuck`. Grid models can be slow; default 90s. */
  checkTimeoutMs?: number
  /** The tool / MCP log lines right now (`workspace.ts readLog`); a step with a `log` needs one new matching line. */
  readLog?: (kind: LogKind) => string[]
  /** A workspace file's content right now, or null when it does not exist (`workspace.ts readWorkspaceFile`). */
  readFile?: (relPath: string) => string | null
  /** What the engine's session file says `tool` did for the step tagged `ref` (`sessionTools.ts toolResult`). */
  toolResult?: (ref: string, tool: string) => ToolResult
  /** Whose wording the requests use (smokeChecks.ts scenarioFor): each names that engine's tool. */
  engine?: string
  /** The pane is "ready" once its screen stops changing for this long. A resumed TUI needs it. */
  settleMs?: number
  /** Give up waiting for the pane to settle after this long (the TUI itself may be stuck). */
  settleTimeoutMs?: number
  pollMs?: number
  tailLines?: number
}

const DEFAULTS: Required<ProbeOptions> = {
  checkTimeoutMs: 90_000,
  readLog: () => [],
  readFile: () => null,
  toolResult: () => ({ called: false, answered: false }),
  engine: 'claude',
  settleMs: 2_000,
  settleTimeoutMs: 60_000,
  pollMs: 1_000,
  tailLines: 40,
}

/** Wait until two consecutive captures match — the TUI has drawn and is waiting on input. */
export async function waitForPaneSettle(tmux: Tmux, pane: string, opts: ProbeOptions = {}): Promise<boolean> {
  const o = { ...DEFAULTS, ...opts }
  const start = tmux.now()
  let last = await tmux.capture(pane, o.tailLines)
  while (tmux.now() - start < o.settleTimeoutMs) {
    await tmux.sleep(o.settleMs)
    const next = await tmux.capture(pane, o.tailLines)
    if (next === last) return true
    last = next
  }
  return false
}

/**
 * Where [ref] last is on [screen], as `[start, end)`, or null. A tool's TUI wraps a long prompt itself,
 * indenting what follows, so the tag can reach the screen cut in two (`(ref-2u` / `  j4gc)`): it is
 * matched with a line break allowed between any two of its characters, with a box's side border and
 * padding on either side of the break and a prompt lead after it. Searched as one string, a wrapped tag
 * looked dropped, and every write step was typed three times (grid-dev, 2026-10-20).
 */
export function findRef(screen: string, ref: string): [number, number] | null {
  // The tag is [a-z0-9-]: only `-` needs escaping, and only outside a character class.
  const wrap = '(?:[ \\t│┃|]*\\r?\\n[ \\t│┃|>›]*)?'
  const pattern = new RegExp(ref.split('').map((c) => (c === '-' ? '\\-' : c)).join(wrap), 'g')
  let last: [number, number] | null = null
  for (const m of screen.matchAll(pattern)) last = [m.index, m.index + m[0].length]
  return last
}

/** A short tag typed after the prompt so THIS check's echo can be told from earlier ones. */
export function checkRef(now: number): string {
  return `ref-${now.toString(36).slice(-4).padStart(4, '0')}${Math.floor(Math.random() * 1296).toString(36).padStart(2, '0')}`
}

/**
 * Type one check into the pane and wait for its marker. Evidence (`tail`) comes back either way.
 *
 * The marker only counts AFTER the echo of this check's own `ref` tag. Counting occurrences was
 * wrong: a resume redraws the whole transcript, so the previous leg's answer is on screen again,
 * and as the new turn scrolls the old one out the count stays flat — a real `ANSWER_42` on the
 * back-home leg was recorded as stuck (measured, codex 0.155.1). The tag pins the search to the
 * new turn instead, found even when the tool wrapped it across lines (findRef).
 */
export async function runCheck(tmux: Tmux, pane: string, check: SmokeCheck, opts: ProbeOptions = {}): Promise<CheckOutcome> {
  const o = { ...DEFAULTS, ...opts }
  const before = await tmux.capture(pane, o.tailLines)
  const logBefore = check.log ? o.readLog(check.log).length : 0
  // read: the file's token, made fresh for this run and written nowhere else (workspace.ts).
  const token = check.answerFromFile ? (o.readFile(check.answerFromFile) ?? '').trim() : ''
  if (check.answerFromFile && !token) {
    return { id: check.id, status: 'stuck', elapsedMs: 0, tail: before.trimEnd(), note: `the workspace has no ${check.answerFromFile} — the run's setup, not the tool` }
  }
  const start = tmux.now()
  const ref = checkRef(start)
  await tmux.type(pane, `${check.prompt} (${ref})`)
  // Make sure it landed. A tool that has only just started can drop input while it is still
  // loading — measured on grid-dev: claude's first question of a run never reached it (its own
  // transcript has every later question and not that one). The tag is on screen within a second
  // when the text arrived, in the composer or in the transcript; if it is nowhere, type it again.
  for (let retry = 0; retry < 2; retry++) {
    let landed = false
    for (let waited = 0; waited < 5_000 && !landed; waited += o.pollMs) {
      await tmux.sleep(o.pollMs)
      landed = findRef(await tmux.capture(pane, o.tailLines), ref) !== null
    }
    if (landed) break
    await tmux.type(pane, `${check.prompt} (${ref})`)
  }

  // The proof, never the reply: a log line for exactly these numbers, the file on disk as asked,
  // the read file's token on the pane after this prompt (nobody guesses `kiwi-4821-tulip`),
  // or the engine's own record of the tool coming back with results.
  const proven = (afterEcho: string | null, tail: string): { logLines?: string[] } | null => {
    if (check.log) {
      const logLines = o.readLog(check.log).slice(logBefore)
      return logLines.some((l) => l.includes(check.logPattern!)) ? { logLines } : null
    }
    if (check.sessionTool) return o.toolResult(ref, check.sessionTool).answered ? {} : null
    if (check.file) {
      const content = o.readFile(check.file.path)
      if (content === null) return null
      const f = check.file
      if (f.equals !== undefined && content.trim() !== f.equals) return null
      if (f.contains !== undefined && !content.includes(f.contains)) return null
      if (f.lacks !== undefined && content.includes(f.lacks)) return null
      return {}
    }
    if (token) return (afterEcho ?? (before.includes(token) ? '' : tail)).includes(token) ? {} : null
    return null
  }

  let tail = before
  const dialogs: string[] = []
  while (tmux.now() - start < o.checkTimeoutMs) {
    await tmux.sleep(o.pollMs)
    tail = await tmux.capture(pane, o.tailLines)
    const echo = findRef(tail, ref)
    const afterEcho = echo ? tail.slice(echo[1]) : null
    // A modal can land in the middle of a turn (claude's gateway notice); answer it and keep waiting.
    // Only one that appeared after THIS prompt: startup screens answered before it stay in the
    // scrollback, and answering them again would send their keys into the running turn.
    const modal = activeDialog(afterEcho ?? tail, dialogs)
    if (modal) {
      for (const k of modal.keys) { await tmux.key(pane, k); await tmux.sleep(300) }
      dialogs.push(modal.id)
      continue
    }
    // Out of usage: said once, right after our prompt. Only text after THIS prompt's echo counts
    // (or, when the echo scrolled away, only if the screen did not already say it before we typed).
    const answer = afterEcho ?? (QUOTA_EXHAUSTED.test(before) ? '' : tail)
    if (QUOTA_EXHAUSTED.test(answer)) {
      const resets = quotaResets(answer)
      return { id: check.id, ref, status: 'no-quota', elapsedMs: tmux.now() - start, tail: tail.trimEnd(), ...(resets ? { resets } : {}), ...(dialogs.length ? { dialogs } : {}) }
    }
    const proof = proven(afterEcho, tail)
    if (proof) {
      const elapsedMs = tmux.now() - start
      // Proven — but the turn may still be running (the model reporting what it did). Let the pane
      // go quiet before the next request, the way a person waits for an answer before typing.
      await waitForPaneSettle(tmux, pane, { ...opts, settleTimeoutMs: 30_000 })
      return { id: check.id, ref, status: 'ok', elapsedMs, tail: (await tmux.capture(pane, o.tailLines)).trimEnd(), ...proof, ...(dialogs.length ? { dialogs } : {}) }
    }
  }
  // Not proven in time: say what was missing, with what was actually there.
  const logLines = check.log ? o.readLog(check.log).slice(logBefore) : undefined
  let note: string
  if (check.log) note = `no ${check.log} log line for "${check.logPattern}"${logLines && logLines.length ? ` (got: ${logLines.join(' | ')})` : ''}`
  else if (check.sessionTool) {
    const used = o.toolResult(ref, check.sessionTool)
    note = !used.called ? `${check.sessionTool} was never called` : `${check.sessionTool} came back without results${used.output ? `: ${JSON.stringify(used.output)}` : ''}`
  } else if (check.file) {
    const content = o.readFile(check.file.path)
    note = content === null ? `${check.file.path} does not exist` : `${check.file.path} is ${JSON.stringify(content.trim().slice(0, 120))}`
  } else note = `${check.answerFromFile}'s token never appeared after the question`
  return { id: check.id, ref, status: 'stuck', elapsedMs: tmux.now() - start, tail: tail.trimEnd(), note, ...(logLines ? { logLines } : {}), ...(dialogs.length ? { dialogs } : {}) }
}

/**
 * Run this leg's steps of the scenario, in order. Stops at the first stuck step: a pane that is
 * not answering will not answer the next prompt either, and the queued text would only muddy the evidence.
 */
export async function probeLeg(tmux: Tmux, pane: string, leg: Leg, opts: ProbeOptions = {}): Promise<LegOutcome> {
  const steps = scenarioFor(opts.engine ?? 'claude')[leg]
  const checks: CheckOutcome[] = []
  const settled = await waitForPaneSettle(tmux, pane, opts)
  if (!settled) {
    const tail = (await tmux.capture(pane, opts.tailLines ?? DEFAULTS.tailLines)).trimEnd()
    // Nothing typed: the TUI never stopped redrawing, so the first check is stuck before it began.
    checks.push({ id: steps[0].id, status: 'stuck', elapsedMs: 0, tail })
    for (const c of steps.slice(1)) checks.push({ id: c.id, status: 'not-run' })
    return { leg, checks }
  }
  const dialogs = await dismissStartupDialogs(tmux, pane, opts)
  for (const [i, check] of steps.entries()) {
    const outcome = await runCheck(tmux, pane, check, opts)
    checks.push(outcome)
    if (outcome.status === 'stuck' || outcome.status === 'no-quota') {
      for (const rest of steps.slice(i + 1)) checks.push({ id: rest.id, status: 'not-run' })
      break
    }
  }
  return dialogs.length ? { leg, checks, dialogs } : { leg, checks }
}