Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

39.4 KB
Raw Blame
import { describe, expect, it } from 'vitest'
import { gridSpawn, backHomeSpawn } from './gridArgv.js'
import { MockVersionSource, NpmVersionSource } from './versionSource.js'
import { compareVersions, planMatrixRuns } from './trigger.js'
import { sessionName, workspaceDirName } from './sessionName.js'
import { buildMatrixEntry } from './matrix.js'
import type { AgentEngine } from '../engines/types.js'
import { SMOKE_CHECKS, SCENARIO, LEGS, firstStuck, plannedLegs, quotaHit, scenarioFor, type CheckId } from './smokeChecks.js'
import { pickGridModel } from './gridSwitchDriver.js'
import { prepareWorkspace, readLog, logPath, readWorkspaceFile, preAcceptClaudeBypassMode, CALC_MCP_MJS } from './workspace.js'
import { toolResult, useByRef } from './sessionTools.js'
import { execFileSync } from 'node:child_process'
import { findRef, probeLeg, runCheck, dismissStartupDialogs, STARTUP_DIALOGS, type Tmux } from './paneProbe.js'
import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, chmodSync, existsSync, rmSync } from 'node:fs'
import { join } from 'node:path'
import { tmpdir } from 'node:os'

const GRID = {
  networkId: 'grid-1',
  networkName: 'someone-grid',
  baseUrl: 'https://grid.autonomous.ai/grid-1/relay/v1',
  apiKey: 'sk-probe',
  model: 'grid:gpt-5-mini',
}

describe('argv-package (spawn exactly like the harness)', () => {
  it('codex grid: provider argv + wire_api + clearEnv', () => {
    const spec = gridSpawn('codex', GRID, 'S1')
    expect('ok' in spec ? false : true).toBe(true) // not an error spec
    const s = spec as { command: string[]; env: Record<string, string>; clearEnv: string[] }
    expect(s.command[0]).toBe('codex')
    expect(s.command).toContain('resume') // codex resume <id>
    expect(s.command).toContain('S1')
    expect(s.command.join(' ')).toContain('model_provider="grid"')
    expect(s.command.join(' ')).toContain('model_providers.grid.wire_api="responses"')
    expect(s.command.join(' ')).toContain('https://grid.autonomous.ai/grid-1/relay/v1')
    expect(Object.keys(s.env).length).toBeGreaterThan(0) // the grid key goes in env, not argv
    expect(s.clearEnv.length).toBeGreaterThan(0)
  })

  it('claude grid: --resume + ANTHROPIC_BASE_URL env', () => {
    const spec = gridSpawn('claude', GRID, 'S1') as { command: string[]; env: Record<string, string> }
    expect(spec.command[0]).toBe('claude')
    expect(spec.command).toContain('--resume')
    expect(spec.env.ANTHROPIC_BASE_URL).toContain('grid.autonomous.ai/grid-1')
  })

  it('back home clears the grid env vars', () => {
    const spec = backHomeSpawn('claude', 'S1') as { clearEnv: string[] }
    expect(spec.clearEnv).toContain('ANTHROPIC_BASE_URL')
  })
})

describe('version source (mock first, npm later)', () => {
  it('mock returns fixtures offline', async () => {
    const src = new MockVersionSource()
    expect(await src.latestVersion('codex')).toBe('0.156.0')
    expect(await src.latestVersion('claude')).toBe('2.1.279')
  })

  it('npm impl reads the registry via fetch (stubbed)', async () => {
    let called = ''
    let sentInit: RequestInit | undefined
    const fetchImpl = (async (url: string, init?: RequestInit) => {
      called = url
      sentInit = init
      return { ok: true, json: async () => ({ version: '9.9.9' }) } as unknown as Response
    }) as typeof fetch
    const src = new NpmVersionSource(fetchImpl)
    expect(await src.latestVersion('codex')).toBe('9.9.9')
    expect(called).toContain('@openai/codex')
    // No Accept header: `/latest` answers 406 to the abbreviated-metadata type, and the failure is
    // silent (latest=null reads as "nothing new"), so the request is pinned bare. See versionSource.
    expect(sentInit).toBeUndefined()
  })

  it('reads each engine\'s STABLE channel, and never a pre-release', async () => {
    const asked: string[] = []
    const answer = (version: string) => (async (url: string) => {
      asked.push(url)
      return { ok: true, json: async () => ({ version }) } as unknown as Response
    }) as typeof fetch
    expect(await new NpmVersionSource(answer('2.1.273')).latestVersion('claude')).toBe('2.1.273')
    expect(await new NpmVersionSource(answer('0.156.1')).latestVersion('codex')).toBe('0.156.1')
    // claude's stable is its own `stable` tag, not `latest` (its fast channel); codex's is `latest`.
    expect(asked).toEqual(['https://registry.npmjs.org/@anthropic-ai/claude-code/stable', 'https://registry.npmjs.org/@openai/codex/latest'])
    // Whatever tag it came from, a pre-release is never "the new version".
    for (const v of ['0.158.0-alpha.8', '2.1.300-beta.1', '1.0.0-rc.2', '0.1.2505172116-nightly']) {
      expect(await new NpmVersionSource(answer(v)).latestVersion('codex'), v).toBeNull()
    }
  })

  // A stub cannot catch a registry that refuses our headers, and that exact bug once kept the whole
  // pipeline quiet. This talks to npm for real, so it is opt-in: `RUN_REAL_NPM=1 vitest run src/e2e`.
  it.runIf(process.env.RUN_REAL_NPM === '1')('npm impl really answers for both engines', async () => {
    const src = new NpmVersionSource()
    for (const engine of ['codex', 'claude'] as const) {
      expect(await src.latestVersion(engine)).toMatch(/^\d+\.\d+\.\d+/)
    }
  }, 30_000)
})

describe('claude bypass-permissions warning (what killed a run once)', () => {
  it('accepts it in a config copy, is idempotent, and leaves an unreadable one alone', () => {
    const home = mkdtempSync(join(tmpdir(), 'wd-home-'))
    try {
      expect(preAcceptClaudeBypassMode(home)).toBe('skipped') // no ~/.claude.json at all
      writeFileSync(join(home, '.claude.json'), JSON.stringify({ projects: { '/x': { allowedTools: [] } } }))
      expect(preAcceptClaudeBypassMode(home)).toBe('accepted')
      const after = JSON.parse(readFileSync(join(home, '.claude.json'), 'utf8'))
      expect(after.bypassPermissionsModeAccepted).toBe(true)
      expect(after.projects['/x']).toEqual({ allowedTools: [] }) // nothing else touched
      expect(preAcceptClaudeBypassMode(home)).toBe('already')
      writeFileSync(join(home, '.claude.json'), 'not json')
      expect(preAcceptClaudeBypassMode(home)).toBe('skipped')
    } finally {
      rmSync(home, { recursive: true, force: true })
    }
  })

  it('the probe answers the warning by moving off the default "No, exit"', () => {
    const dialog = STARTUP_DIALOGS.find((d) => d.id === 'claude-bypass-accept')!
    expect(dialog.match.test('  WARNING: Claude Code running in Bypass Permissions mode')).toBe(true)
    expect(dialog.keys).toEqual(['Down', 'Enter'])
  })
})

describe('trigger (current vs latest)', () => {
  it('runs on upgrade, not on unchanged', async () => {
    const src = new MockVersionSource({ codex: '0.156.0', claude: '2.1.279' })
    const cur = {
      currentVersion: async (e: AgentEngine) => (e === 'codex' ? '0.155.1' : '2.1.279'),
    }
    const plans = await planMatrixRuns(src, cur, ['codex', 'claude'])
    const codex = plans.find((p) => p.engine === 'codex')!
    const claude = plans.find((p) => p.engine === 'claude')!
    expect(codex.plan).toBe(true)
    expect(codex.reason).toBe('upgrade')
    expect(claude.plan).toBe(false)
    expect(claude.reason).toBe('unchanged')
  })

  it('compares dotted numeric versions', () => {
    expect(compareVersions('0.155.0', '0.155.1')).toBe(-1)
    expect(compareVersions('0.156.0', '0.155.1')).toBe(1)
    expect(compareVersions('2.1.279', '2.1.279')).toBe(0)
  })
})

describe('workspace folder name (what claude 2.1.274 tripped on)', () => {
  it('keeps the session readable but drops every character a tool might rewrite', () => {
    const session = sessionName({ engine: 'claude', version: '2.1.274', testcase: 'grid-switch', at: new Date('2026-09-24T20:22:39Z') })
    expect(session).toContain('->') // the bundle keeps its readable name
    const dir = workspaceDirName(session)
    expect(dir).toBe('claude-2.1.274_to_grid-switch-none--20260924T202239Z')
    expect(dir).toMatch(/^[A-Za-z0-9._-]+$/)
  })
})

describe('session naming', () => {
  it('is namespaced and grep-able', () => {
    const name = sessionName({ engine: 'codex', version: '0.156.0', testcase: 'grid-switch', gridModel: 'grid:gpt-5-mini', at: new Date('2026-09-21T18:05:00Z') })
    expect(name).toMatch(/^codex@0\.156\.0->grid-switch@grid-gpt-5-mini--/)
  })
})

describe('matrix entry (dry run)', () => {
  it('emits argv + notify when not logged in', async () => {
    const plan = { engine: 'claude' as AgentEngine, current: '2.1.278', latest: '2.1.279', reason: 'upgrade' as const, plan: true }
    const entry = await buildMatrixEntry(plan, {
      status: async () => ({ binary: true, login: false, grid: true }),
      grid: async () => null,
      at: new Date('2026-09-21T18:05:00Z'),
    })
    expect(entry.session).toContain('[email protected]')
    expect(entry.notify.join(' ')).toMatch(/NOT_LOGGED_IN/)
    expect(entry.status.login).toBe(false)
    expect(entry.subscriptionModel).toBe('claude-sonnet-5')
    // Dry run: the journey is planned, nothing typed yet.
    expect(entry.legs.map((l) => l.leg)).toEqual(['subscription', 'grid'])
    expect(entry.legs.flatMap((l) => l.checks).every((c) => c.status === 'not-run')).toBe(true)
  })

  it('uses the fixed models for the subscription leg', async () => {
    const src = new MockVersionSource({ codex: '0.156.0' })
    const plans = await planMatrixRuns(src, { currentVersion: async () => '0.155.0' }, ['codex'])
    const entry = await buildMatrixEntry(plans[0], {
      status: async () => ({ binary: true, login: true, grid: true }),
      grid: async () => null,
    })
    expect(entry.subscriptionModel).toBe('gpt-5.5')
  })
})

describe('scenario (what a person does, on each side of the switch)', () => {
  it('four requests on each side — read, write, edit, then MCP — with nothing shared', () => {
    expect(LEGS).toEqual(['subscription', 'grid'])
    // The own account calls the workspace's MCP server; the grid, the harness's web search.
    expect(SCENARIO.subscription.map((c) => c.id)).toEqual(['read', 'write', 'edit', 'mcp'])
    expect(SCENARIO.grid.map((c) => c.id)).toEqual(['read', 'write', 'edit', 'web'])
    expect(SCENARIO.grid[3].sessionTool).toBe('mcp__harness__web_search')
    // Nothing the first side leaves behind can pass the second: its own numbers and its own files.
    const patterns = SMOKE_CHECKS.filter((c) => c.logPattern).map((c) => c.logPattern)
    expect(new Set(patterns).size).toBe(patterns.length)
    const files = SMOKE_CHECKS.map((c) => c.answerFromFile ?? c.file?.path).filter(Boolean)
    expect(new Set(files).size).toBe(files.length)
    expect(plannedLegs().flatMap((l) => l.checks).every((c) => c.status === 'not-run')).toBe(true)
  })

  it('names each engine its own tool and the exact path, and never its own proof', () => {
    for (const engine of ['codex', 'claude'] as const) {
      const scenario = scenarioFor(engine)
      for (const c of [...scenario.subscription, ...scenario.grid]) {
        expect(c.prompt).not.toMatch(/reply with exactly|<result>|\bRECALL_|\bTOOL_|\bMCP_(?!_)/i)
        if (c.logPattern) expect(c.prompt).not.toContain(c.logPattern) // "a=30 b=12" is asked; "= 42" is only the log's
        expect(!!c.log || !!c.answerFromFile || !!c.file || !!c.sessionTool).toBe(true) // proven by the machine, never by the reply
        const path = c.answerFromFile ?? c.file?.path
        if (path) expect(c.prompt).toContain(path)
        if (engine === 'codex' && ['read', 'write', 'edit'].includes(c.id)) expect(c.prompt).toContain('exec_command')
      }
    }
    const claude = scenarioFor('claude')
    for (const c of [...claude.subscription, ...claude.grid]) {
      if (c.id === 'read') expect(c.prompt).toContain('Read tool')
      if (c.id === 'write') expect(c.prompt).toContain('Write tool')
      if (c.id === 'edit') expect(c.prompt).toContain('Edit tool')
    }
    // The engine's own tool name is what differs; a step is still proven only by its result.
    expect(scenarioFor('codex')).not.toEqual(scenarioFor('claude'))
  })

  it('firstStuck names the first leg/step that did not pass', () => {
    const legs = plannedLegs()
    legs[0].checks.forEach((c) => (c.status = 'ok'))
    legs[1].checks[0].status = 'ok'
    legs[1].checks[1].status = 'stuck'
    expect(firstStuck(legs)).toEqual({ leg: 'grid', check: 'write', status: 'stuck' })
    legs.forEach((l) => l.checks.forEach((c) => (c.status = 'ok')))
    expect(firstStuck(legs)).toBeNull()
  })
})

/** A pane that answers each typed prompt with `reply(prompt)` on the next capture. */
function fakePane(reply: (prompt: string) => string | null): Tmux & { typed: string[] } {
  let screen = '> '
  let clock = 0
  const typed: string[] = []
  return {
    typed,
    async capture() {
      return screen
    },
    async type(_pane, text) {
      typed.push(text)
      screen += `\n> ${text}`
      const r = reply(text)
      if (r !== null) screen += `\n${r}`
    },
    async key(_pane, key) {
      typed.push(`<${key}>`)
      // Any key answers a dialog: the dialog text leaves the screen.
      screen = screen.replace(/Update available![^\n]*\n?/, '').replace(/Do you trust[^\n]*\n?/, '')
    },
    async sleep(ms) {
      clock += ms
    },
    now: () => clock,
  }
}

/** An MCP log that gains a line whenever the fake tool "uses" the server. */
function fakeLogs() {
  const lines: Record<'mcp', string[]> = { mcp: [] }
  return { lines, readLog: (k: 'mcp') => [...lines[k]] }
}

/**
 * A fake coding tool on a fake machine: it does what each request asks — reads the file, writes
 * it, edits it, calls the MCP server (a log line), searches the web (a result in its
 * session, as `toolResult` would read it). With `lie`, it only SAYS it did those steps, which is the
 * case the proofs exist for; with `empty`, its web search comes back with nothing.
 */
function fakeAgent(opts: { lie?: CheckId[]; empty?: boolean } = {}) {
  const files: Record<string, string> = {
    'notes/info-1.txt': 'kiwi-4821-tulip\n', 'notes/info-2.txt': 'otter-1234-ember\n',
    'notes/todo-1.txt': '# todo 1\nstatus: pending\n', 'notes/todo-2.txt': '# todo 2\nstatus: pending\n',
  }
  const logs = fakeLogs()
  const searched: Record<string, string> = {}
  const lie = new Set(opts.lie ?? [])
  const reply = (p: string): string | null => {
    let m: RegExpMatchArray | null
    if ((m = p.match(/Read tool on (notes\/info-\d\.txt)/))) return lie.has('read') ? 'It says hello.' : `It says: ${files[m[1]].trim()}`
    if ((m = p.match(/mcp__harness__web_search tool .*\((ref-[0-9a-z]{6})\)/))) {
      if (!lie.has('web')) searched[m[1]] = opts.empty ? 'No results.' : 'Node.js 24.11.0 — https://nodejs.org/en/download'
      return 'The latest stable Node.js is 24.11.0.'
    }
    if ((m = p.match(/Write tool to create (out\/hello-\d\.txt) .*text: (.*) \(ref-/))) {
      if (!lie.has('write')) files[m[1]] = `${m[2]}\n`
      return 'Done.'
    }
    if ((m = p.match(/Edit tool on (notes\/todo-\d\.txt) .*change pending to done/))) {
      if (!lie.has('edit')) files[m[1]] = files[m[1]].replace('pending', 'done')
      return 'Updated.'
    }
    if ((m = p.match(/e2e_calc .*add tool with a=(\d+) and b=(\d+)/))) {
      if (!lie.has('mcp')) logs.lines.mcp.push(`t add ${m[1]} ${m[2]} = ${+m[1] + +m[2]}`)
      return `${+m[1] + +m[2]}`
    }
    return null
  }
  const pane = fakePane(reply)
  const toolResult = (ref: string, tool: string) => {
    const out = tool === 'mcp__harness__web_search' ? searched[ref] : undefined
    return { called: out !== undefined, answered: !!out && out.includes('https://'), ...(out ? { output: out } : {}) }
  }
  const probe = { settleMs: 1, pollMs: 1, checkTimeoutMs: 5_000, readLog: logs.readLog, readFile: (f: string) => files[f] ?? null, toolResult }
  return { pane, files, logs, probe }
}

const READ_STEP = SCENARIO.subscription[0]
const MCP_STEP = SCENARIO.subscription[3]
const WEB_STEP = SCENARIO.grid[3]

describe('pane probe (live steps on a tmux pane)', () => {
  it('a tool that really does the four requests passes both sides, and each proof is recorded', async () => {
    const { pane, probe, files } = fakeAgent()
    const own = await probeLeg(pane, '%1', 'subscription', probe)
    const grid = await probeLeg(pane, '%1', 'grid', probe)
    expect(own.checks.map((c) => `${c.id}=${c.status}`)).toEqual(['read=ok', 'write=ok', 'edit=ok', 'mcp=ok'])
    expect(grid.checks.map((c) => `${c.id}=${c.status}`)).toEqual(['read=ok', 'write=ok', 'edit=ok', 'web=ok'])
    expect(own.checks[3].logLines).toEqual(['t add 30 12 = 42'])
    expect(files['out/hello-2.txt']).toBe('hello from step 2\n')
    expect(files['notes/todo-1.txt']).toContain('status: done')
    // Every step carries the tag its prompt was typed with — how its tools are found afterwards.
    expect([...own.checks, ...grid.checks].every((c) => /^ref-[0-9a-z]{6}$/.test(c.ref ?? ''))).toBe(true)
  })

  // grid-dev, 2026-10-20: codex's TUI wraps a long prompt, so its `(ref-…)` tag reached the screen in two
  // pieces; the probe took it for dropped and typed every write step three times, and the third ran into
  // the next step's prompt on the grid leg (edit=stuck).
  it('types a step once when the tool wraps its prompt across lines', async () => {
    const { pane, probe, files } = fakeAgent()
    // At 72 columns the tag of this step's prompt (from column 69) is cut in two, as on grid-dev's screen.
    const wrap = (text: string) => text.split('\n').map((line) => line.match(/.{1,72}/g)?.join('\n  ') ?? line).join('\n')
    const wrapped: Tmux = { ...pane, capture: async (p, lines) => wrap(await pane.capture(p, lines)) }
    const write = SCENARIO.subscription[1]
    const out = await runCheck(wrapped, '%1', write, probe)
    expect(out.status).toBe('ok')
    expect(pane.typed.filter((t) => t.startsWith(write.prompt))).toHaveLength(1)
    expect(files['out/hello-1.txt']).toBe('hello from step 1\n')
  })

  it('finds a tag cut by a wrap inside a bordered box, and only its own', () => {
    const ref = 'ref-2uj4gc'
    // A plain indent (codex), a side border before and after the break, and a quote/prompt lead.
    for (const screen of [`… (ref-2u\n  j4gc)`, `… (ref-2u   │\n│   j4gc)`, `… (ref-2u│\n│j4gc)`, `… (ref-2u\n› j4gc)`]) {
      expect(findRef(screen, ref), JSON.stringify(screen)).not.toBeNull()
    }
    // Text between the pieces is not a wrap; nor is another tag.
    expect(findRef('… (ref-2u x\nj4gc)', ref)).toBeNull()
    expect(findRef('(ref-2uj4gd)', ref)).toBeNull()
    // The last echo is the one an answer is read after.
    const screen = `(ref-2uj4gc) first\n(ref-2u\n  j4gc) second`
    expect(screen.slice(findRef(screen, ref)![1])).toBe(') second')
  })

  it('saying is not doing: each step fails when the tool only claims it, with what was missing', async () => {
    const notes: Record<CheckId, RegExp> = {
      read: /notes\/info-1\.txt's token never appeared/,
      write: /out\/hello-1\.txt does not exist/,
      edit: /notes\/todo-1\.txt is "# todo 1\\nstatus: pending"/,
      mcp: /no mcp log line for "add 30 12 = 42"/,
      web: /mcp__harness__web_search was never called/,
    }
    for (const step of [...SCENARIO.subscription, WEB_STEP]) {
      const { pane, probe } = fakeAgent({ lie: [step.id] })
      const out = await runCheck(pane, '%1', step, probe)
      expect(out.status, step.id).toBe('stuck')
      expect(out.note, step.id).toMatch(notes[step.id])
    }
  })

  it('web: a search that came back with no results is not a search that worked', async () => {
    const { pane, probe } = fakeAgent({ empty: true })
    const out = await runCheck(pane, '%1', WEB_STEP, probe)
    expect(out.status).toBe('stuck')
    expect(out.note).toBe('mcp__harness__web_search came back without results: "No results."')
  })

  it('read: the token must come after THIS question — one already on screen is no proof', async () => {
    const { pane, probe } = fakeAgent({ lie: ['read'] })
    await pane.type('%1', 'kiwi-4821-tulip') // on screen before the question was asked
    expect((await runCheck(pane, '%1', READ_STEP, probe)).status).toBe('stuck')
  })

  it('write: the exact text, not merely a file', async () => {
    const { pane, probe, files } = fakeAgent({ lie: ['write'] })
    files['out/hello-1.txt'] = 'hello from step one\n'
    const out = await runCheck(pane, '%1', SCENARIO.subscription[1], probe)
    expect(out.status).toBe('stuck')
    expect(out.note).toBe('out/hello-1.txt is "hello from step one"')
  })

  it('read with no info file in the workspace is the run\'s setup failing, said as such', async () => {
    const { pane, probe } = fakeAgent()
    const out = await runCheck(pane, '%1', READ_STEP, { ...probe, readFile: () => null })
    expect(out.status).toBe('stuck')
    expect(out.note).toMatch(/the run's setup, not the tool/)
    expect(pane.typed).toHaveLength(0) // nothing typed for a step that cannot be proven
  })

  it('out of usage is recognised at once, with its reset time, instead of waiting out the step', async () => {
    // What each tool really prints — codex and claude, from their own binaries.
    const cases: Array<[string, string | null]> = [
      ["■ You've hit your usage limit. Upgrade to Pro, or try again at Sep 24th, 2026 6:02 PM.", 'try again at Sep 24th, 2026 6:02 PM'],
      ["You've hit your session limit · resets 6pm", 'resets 6pm'],
      ["You’ve hit your weekly limit · resets Mon 9am", 'resets Mon 9am'],
      ["You're out of usage credits. /model to switch models.", null],
    ]
    for (const [said, resets] of cases) {
      const pane = fakePane(() => said)
      const out = await runCheck(pane, '%1', MCP_STEP, { settleMs: 1, pollMs: 1, checkTimeoutMs: 120_000 })
      expect(out.status, said).toBe('no-quota')
      expect(out.resets ?? null, said).toBe(resets)
      expect(out.elapsedMs!, said).toBeLessThan(10) // one poll, not the whole budget
    }
    // Claude's early warning is not the limit: the step carries on and passes.
    const logs = fakeLogs()
    const warned = fakePane(() => { logs.lines.mcp.push('t add 30 12 = 42'); return 'Approaching usage limit · resets 6pm\n42' })
    expect((await runCheck(warned, '%1', MCP_STEP, { settleMs: 1, pollMs: 1, readLog: logs.readLog })).status).toBe('ok')
  })

  it('an old limit message already on screen does not count against a new request', async () => {
    const { pane, probe } = fakeAgent()
    await pane.type('%1', "You've hit your session limit · resets 6pm") // yesterday's, still in the scrollback
    pane.typed.length = 0
    expect((await runCheck(pane, '%1', MCP_STEP, probe)).status).toBe('ok')
  })

  it('out of usage ends the leg: nothing more is typed into an account that cannot answer', async () => {
    const pane = fakePane(() => "You've hit your usage limit. try again at 6:02 PM.")
    const leg = await probeLeg(pane, '%1', 'subscription', { settleMs: 1, pollMs: 1, readFile: () => 'kiwi-4821-tulip\n' })
    expect(leg.checks.map((c) => `${c.id}=${c.status}`)).toEqual(['read=no-quota', 'write=not-run', 'edit=not-run', 'mcp=not-run'])
    expect(pane.typed).toHaveLength(1)
    expect(quotaHit([leg])).toEqual({ leg: 'subscription', check: 'read', resets: 'try again at 6:02 PM' })
  })

  it('answers codex 0.156\'s stacked startup screens — the one on top first, each once', async () => {
    const { pane, probe } = fakeAgent()
    // What the pane held on grid-dev: the hooks review, and the model notice drawn under it.
    await pane.type('%1', 'Hooks need review\n  2 hooks are new or changed.\n›    Review hooks\n     Trust all and continue')
    await pane.type('%1', 'GPT-5.5 retires on October 14, 2026. Switch to GPT-5.6 Sol to continue working in Codex.\n› 1. Try new model\n  2. Use existing model')
    pane.typed.length = 0
    const answered = await dismissStartupDialogs(pane, '%1', { settleMs: 1, pollMs: 1 })
    expect(answered).toEqual(['codex-model-retire', 'codex-hooks-trust'])
    expect(pane.typed).toEqual(['<2>', '<Enter>', '<Down>', '<Enter>'])
    // And the step typed after them is not disturbed by their text still being in the scrollback.
    pane.typed.length = 0
    expect((await runCheck(pane, '%1', MCP_STEP, probe)).status).toBe('ok')
    expect(pane.typed).toHaveLength(1)
  })

  it('types the request again when a tool that is still loading dropped it', async () => {
    const { pane, probe } = fakeAgent()
    // The first paste vanishes, the way claude's first question did on grid-dev; the second lands.
    const realType = pane.type.bind(pane)
    let dropped = false
    pane.type = async (id, text) => { if (!dropped) { dropped = true; pane.typed.push(text); return } await realType(id, text) }
    // The real per-step budget: the 5s spent seeing whether the first paste landed comes out of it.
    const out = await runCheck(pane, '%1', MCP_STEP, { ...probe, pollMs: 100, checkTimeoutMs: 120_000 })
    expect(out.status).toBe('ok')
    expect(pane.typed).toHaveLength(2) // typed, not seen, typed again
    expect(pane.typed[0]).toBe(pane.typed[1]) // the same request with the same tag
  })

  it('answers claude 2.1.281\'s reworded folder trust by moving off its "No, exit" default', async () => {
    const pane = fakePane(() => null)
    await pane.type('%1', 'Quick safety check: Is this a project you created or one you trust?\n ❯ No, exit\n   Yes, I trust this folder')
    pane.typed.length = 0
    expect(await dismissStartupDialogs(pane, '%1', { settleMs: 1, pollMs: 1 })).toEqual(['claude-trust-v2'])
    expect(pane.typed).toEqual(['<Down>', '<Enter>'])
  })

  it('a step that is not proven stops the leg there: nothing more is typed after it', async () => {
    const { pane, probe } = fakeAgent({ lie: ['write'] })
    const leg = await probeLeg(pane, '%1', 'subscription', { ...probe, checkTimeoutMs: 5 })
    expect(leg.checks.map((c) => `${c.id}=${c.status}`)).toEqual(['read=ok', 'write=stuck', 'edit=not-run', 'mcp=not-run'])
    expect(pane.typed).toHaveLength(2)
  })

  it('answers a startup dialog (codex update → Skip) before typing the first request, and records it', async () => {
    const { pane, probe } = fakeAgent()
    // What a fresh codex shows before its prompt; a request typed here would pick "1. Update now".
    await pane.type('%1', 'Update available! 0.155.0 -> 0.155.1  › 1. Update now  2. Skip  Press enter to continue')
    pane.typed.length = 0
    const leg = await probeLeg(pane, '%1', 'subscription', probe)
    expect(pane.typed.slice(0, 2)).toEqual(['<2>', '<Enter>'])
    expect(pane.typed[2]).toContain(READ_STEP.prompt)
    expect(leg.dialogs).toEqual(['codex-update'])
    expect(leg.checks.every((c) => c.status === 'ok')).toBe(true)
    expect(await dismissStartupDialogs(pane, '%1', { settleMs: 1 })).toEqual([]) // nothing left on screen
  })
})

describe('session tools and models (read from the engine\'s own session file)', () => {
  // Records shaped exactly like the real ones — a codex 0.156.1 rollout and a claude 2.1.273
  // session from grid-dev, 2026-09-24 — trimmed to the fields that matter.
  it('codex: code-mode exec calls, classic calls, a namespaced MCP call, and each turn\'s model', () => {
    const home = mkdtempSync(join(tmpdir(), 'wd-sess-'))
    try {
      const dir = join(home, '.codex', 'sessions', '2026', '09', '24')
      mkdirSync(dir, { recursive: true })
      const rec = (o: object) => JSON.stringify(o)
      const user = (t: string) => rec({ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: t }] } })
      writeFileSync(join(dir, 'rollout-x.jsonl'), [
        rec({ type: 'turn_context', payload: { model: 'gpt-6-luna' } }), // written BEFORE the turn's message
        user('Run tools/calc.sh add 40 2 and tell me the result. (ref-aaaaaa)'),
        rec({ type: 'response_item', payload: { type: 'custom_tool_call', name: 'exec', input: 'const r = await tools.exec_command({cmd:"tools/calc.sh add 40 2"})' } }),
        rec({ type: 'turn_context', payload: { model: 'DeepSeek-V4-Flash-0731' } }),
        user('Use the MCP server e2e_calc to add 60 and 18. (ref-bbbbbb)'),
        rec({ type: 'response_item', payload: { type: 'function_call', name: 'sub', namespace: 'mcp__e2e_calc', arguments: '{}' } }),
        rec({ type: 'response_item', payload: { type: 'function_call', name: 'apply_patch', arguments: '{}' } }),
      ].join('\n'))
      expect(useByRef('codex', '/unused', ['ref-aaaaaa', 'ref-bbbbbb'], home)).toEqual({
        'ref-aaaaaa': { tools: ['exec→exec_command'], models: ['gpt-6-luna'] },
        'ref-bbbbbb': { tools: ['mcp__e2e_calc__sub', 'apply_patch'], models: ['DeepSeek-V4-Flash-0731'] },
      })
    } finally {
      rmSync(home, { recursive: true, force: true })
    }
  })

  it('claude: tool_use names per step, ToolSearch included, and the model of every reply', () => {
    const home = mkdtempSync(join(tmpdir(), 'wd-sess-'))
    try {
      const cwd = '/tmp/grid-matrix-out/agents/[email protected]>grid-switch@none--20260924T081321Z'
      const dir = join(home, '.claude', 'projects', '-tmp-grid-matrix-out-agents-claude-2-1-273--grid-switch-none--20260924T081321Z')
      mkdirSync(dir, { recursive: true })
      const rec = (o: object) => JSON.stringify(o)
      const reply = (model: string, ...tools: string[]) => rec({ type: 'assistant', message: { model, content: tools.map((name) => ({ type: 'tool_use', name, input: {} })) } })
      writeFileSync(join(dir, 's.jsonl'), [
        rec({ type: 'user', message: { role: 'user', content: 'Read notes/info-1.txt and tell me what it says. (ref-cccccc)' } }),
        reply('claude-haiku-4-5-20251001', 'Read'),
        rec({ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', content: 'kiwi-4821-tulip' }] } }),
        reply('claude-haiku-4-5-20251001'),
        rec({ type: 'user', message: { role: 'user', content: 'Use the MCP server e2e_calc to add 30 and 12. (ref-dddddd)' } }),
        reply('DeepSeek-V4-Flash-0731', 'ToolSearch'),
        reply('DeepSeek-V4-Flash-0731', 'mcp__e2e_calc__add'),
        reply('<synthetic>'),
      ].join('\n'))
      expect(useByRef('claude', cwd, ['ref-cccccc', 'ref-dddddd'], home)).toEqual({
        'ref-cccccc': { tools: ['Read'], models: ['claude-haiku-4-5-20251001'] },
        'ref-dddddd': { tools: ['ToolSearch', 'mcp__e2e_calc__add'], models: ['DeepSeek-V4-Flash-0731'] },
      })
    } finally {
      rmSync(home, { recursive: true, force: true })
    }
  })

  it('codex: the web search of THIS step, and whether it came back with results', () => {
    const home = mkdtempSync(join(tmpdir(), 'wd-sess-'))
    try {
      const dir = join(home, '.codex', 'sessions', '2026', '10', '06')
      mkdirSync(dir, { recursive: true })
      const rec = (o: object) => JSON.stringify(o)
      const item = (payload: object) => rec({ type: 'response_item', payload })
      const user = (t: string) => item({ type: 'message', role: 'user', content: [{ type: 'input_text', text: t }] })
      // The output shape codex 0.160.0 recorded for an MCP call on the grid (grid-dev, 2026-10-01).
      const out = (callId: string, text: string) => item({ type: 'function_call_output', call_id: callId, output: [{ type: 'input_text', text: 'Wall time: 0.4 seconds\nOutput:' }, { type: 'input_text', text }] })
      writeFileSync(join(dir, 'rollout-x.jsonl'), [
        user('Delete the file notes/old-2.txt. (ref-aaaaaa)'),
        user('<environment_context>a context line in the user role, mid-step</environment_context>'),
        user('Search the web for the latest stable version of Node.js and tell me the version number. (ref-bbbbbb)'),
        user('<environment_context>a context line in the user role, mid-step</environment_context>'),
        item({ type: 'function_call', name: 'web_search', namespace: 'mcp__harness', arguments: '{"query":"node"}', call_id: 'c1' }),
        out('c1', 'Node.js 24.11.0 (LTS) https://nodejs.org/en/download'),
        user('Search again. (ref-cccccc)'),
        item({ type: 'function_call', name: 'web_search', namespace: 'mcp__harness', arguments: '{}', call_id: 'c2' }),
        out('c2', 'MCP tool call requires approval, which the policy denies'),
      ].join('\n'))
      expect(toolResult('codex', '/unused', 'ref-aaaaaa', 'mcp__harness__web_search', home)).toEqual({ called: false, answered: false })
      expect(toolResult('codex', '/unused', 'ref-bbbbbb', 'mcp__harness__web_search', home)).toMatchObject({ called: true, answered: true })
      // Called, refused: no results, and the refusal is what the probe quotes.
      expect(toolResult('codex', '/unused', 'ref-cccccc', 'mcp__harness__web_search', home)).toEqual({
        called: true, answered: false, output: 'Wall time: 0.4 seconds Output: MCP tool call requires approval, which the policy denies',
      })
    } finally {
      rmSync(home, { recursive: true, force: true })
    }
  })

  it('claude: the web search tool_use, its tool_result, and an error result', () => {
    const home = mkdtempSync(join(tmpdir(), 'wd-sess-'))
    try {
      const cwd = '/tmp/ws'
      const dir = join(home, '.claude', 'projects', '-tmp-ws')
      mkdirSync(dir, { recursive: true })
      const rec = (o: object) => JSON.stringify(o)
      const prompt = (t: string) => rec({ type: 'user', message: { role: 'user', content: t } })
      const call = (id: string) => rec({ type: 'assistant', message: { model: 'DeepSeek-V4-Flash-0731', content: [{ type: 'tool_use', id, name: 'mcp__harness__web_search', input: { query: 'node' } }] } })
      const result = (id: string, text: string, isError = false) => rec({ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: id, is_error: isError, content: [{ type: 'text', text }] }] } })
      writeFileSync(join(dir, 's.jsonl'), [
        prompt('Search the web for the latest stable version of Node.js and tell me the version number. (ref-dddddd)'),
        call('t1'), result('t1', 'Node.js 24.11.0 https://nodejs.org'),
        prompt('Search again. (ref-eeeeee)'),
        call('t2'), result('t2', 'see https://example.com — upstream timed out', true),
      ].join('\n'))
      expect(toolResult('claude', cwd, 'ref-dddddd', 'mcp__harness__web_search', home)).toMatchObject({ called: true, answered: true })
      expect(toolResult('claude', cwd, 'ref-eeeeee', 'mcp__harness__web_search', home)).toMatchObject({ called: true, answered: false })
    } finally {
      rmSync(home, { recursive: true, force: true })
    }
  })
})

describe('grid model pick (live)', () => {
  it('takes the smallest model on any listed grid, own grid first, even when the top-level list is empty', () => {
    const pick = pickGridModel({
      gridName: 'mine',
      models: [],
      localModelEngines: null,
      grids: [
        { name: 'mine', own: true, models: [] },
        { name: 'autonomous.ai', own: false, models: [
          { id: 'DeepSeek-V4-Flash-0731', node: 'a' },
          { id: 'Qwen/Qwen3.8-27B', node: 'b' },
          { id: 'Qwen3.6-35B-A3B', node: 'c' },
          { id: 'gemma-4-31B-it', node: 'd' },
        ] },
      ],
    })
    expect(pick).toEqual({ model: 'Qwen/Qwen3.8-27B', gridName: 'autonomous.ai' })
    expect(pickGridModel({ gridName: null, models: [], grids: [], localModelEngines: null })).toBeNull()
  })
})

describe('workspace (the person\'s project the agent is created in)', () => {
  it('lays out the notes, the MCP config for the engine, and the log the MCP step is proven by', () => {
    const cwd = mkdtempSync(join(tmpdir(), 'wd-ws-'))
    try {
      // A stand-in for the codex binary: it records the argv it was called with and answers
      // `mcp list` the way a codex that accepted the registration would.
      const codexBin = join(cwd, 'fake-codex')
      const argvLog = join(cwd, 'fake-codex.log')
      writeFileSync(codexBin, `#!/bin/sh\necho "$@" >> ${JSON.stringify(argvLog)}\n[ "$2" = list ] && echo "e2e_calc node /x/calc-mcp.mjs"\nexit 0\n`)
      chmodSync(codexBin, 0o755)
      // The tool approvals are appended to codex's config: a copy here, never the developer's own.
      const codexConfig = join(cwd, 'codex-config.toml')
      writeFileSync(codexConfig, 'model = "gpt-6-luna"\n')
      const laid = prepareWorkspace(cwd, 'codex', { codexBin, codexConfig })
      // The file steps' files: a fresh unguessable secret per side, a todo to edit.
      const secrets = [1, 2].map((n) => readWorkspaceFile(cwd, `notes/info-${n}.txt`)?.trim() ?? '')
      for (const t of secrets) expect(t).toMatch(/^[a-z]+-\d{4}-[a-z]+$/)
      expect(secrets[0]).not.toBe(secrets[1])
      expect(readWorkspaceFile(cwd, 'notes/todo-1.txt')).toContain('status: pending')
      expect(existsSync(join(cwd, 'out'))).toBe(true)
      expect(readWorkspaceFile(cwd, 'out/hello-1.txt')).toBeNull() // written by the step, never by us
      // Fresh per run: two workspaces never share a secret.
      const other = mkdtempSync(join(tmpdir(), 'wd-ws-'))
      prepareWorkspace(other, 'claude')
      expect(readWorkspaceFile(other, 'notes/info-1.txt')).not.toBe(readWorkspaceFile(cwd, 'notes/info-1.txt'))
      rmSync(other, { recursive: true, force: true })
      // The registration is codex's own command, with the server and its log as argv — and a stale
      // entry is removed before the add, so an interrupted run cannot leave one behind.
      const calls = readFileSync(argvLog, 'utf8').trim().split('\n')
      expect(calls[0]).toBe('mcp remove e2e_calc')
      expect(calls[1]).toBe(`mcp add e2e_calc -- node ${laid.mcpServer} ${logPath(cwd, 'mcp')}`)
      expect(calls[2]).toBe('mcp list')
      // The string shipped in the bundle is the file in workspace/, byte for byte.
      expect(CALC_MCP_MJS).toBe(readFileSync(join(__dirname, 'workspace', 'calc-mcp.mjs'), 'utf8'))
      expect(existsSync(join(cwd, 'tools', 'calc-mcp.mjs'))).toBe(true)
      // codex has no project-level MCP config — a `<cwd>/.codex/config.toml` is never read (measured:
      // `codex mcp list` inside such a workspace does not list the server), so the workspace must not
      // pretend otherwise. Registration goes through `codex mcp add`, which this test does not run:
      // it would write the developer's own ~/.codex/config.toml. What is pinned is that the run says
      // out loud which of the two happened.
      expect(existsSync(join(cwd, '.codex', 'config.toml'))).toBe(false)
      expect(laid.mcpConfig).toBe(codexConfig)
      expect(laid.mcpNote).toMatch(/codex mcp add/)
      // Its tools always allowed, as on a developer's machine: codex then never sends them to its
      // reviewer, whose `codex-auto-review` model the grid does not serve (openai/codex#24879).
      const toml = readFileSync(codexConfig, 'utf8')
      expect(toml.startsWith('model = "gpt-6-luna"\n')).toBe(true) // appended, nothing rewritten
      for (const tool of ['add', 'sub']) expect(toml).toContain(`[mcp_servers.e2e_calc.tools.${tool}]\napproval_mode = "approve"\n`)
      // The MCP server answers tools/call and leaves its line in the MCP log.
      const rpc = ['{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}', '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"sub","arguments":{"a":60,"b":18}}}'].join('\n') + '\n'
      const out = execFileSync('node', [laid.mcpServer, logPath(cwd, 'mcp')], { input: rpc }).toString()
      expect(out).toContain('"text":"42"')
      expect(readLog(cwd, 'mcp').at(-1)).toContain('sub 60 18 = 42')
      // claude gets .mcp.json instead.
      const claude = prepareWorkspace(cwd, 'claude')
      expect(JSON.parse(readFileSync(join(cwd, '.mcp.json'), 'utf8')).mcpServers.e2e_calc.command).toBe('node')
      expect(claude.mcpConfig).toBe(join(cwd, '.mcp.json'))
      // …and the project settings a person who said "don't ask again" has: the steps' tools allowed.
      const settings = JSON.parse(readFileSync(join(cwd, '.claude', 'settings.json'), 'utf8'))
      expect(settings.permissions.allow).toEqual(expect.arrayContaining(['Write', 'Edit', 'mcp__e2e_calc', 'mcp__harness']))
      expect(settings.enabledMcpjsonServers).toEqual(['e2e_calc'])
    } finally {
      rmSync(cwd, { recursive: true, force: true })
    }
  })
})