Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

10.8 KB
Raw Blame
/**
 * Questions as a workload: an agent asks, a window answers through `question_response`, and the answer
 * the engine took is read back from its own transcript. Shared by the opt-in performance run
 * (perf.e2e.ts, `PERF_QUESTIONS`) and the soak (endurance.e2e.ts).
 *
 * The fake engines draw what the real CLIs draw (fakeEngine.mjs): a single choice (`!ask`), Claude Code's
 * "Type something." row (a typed answer to `!ask`), Claude Code's multiSelect with its review (`!askmany`)
 * and a permission prompt (`!permit`). Codex's recorded request_user_input has no free-text row and no
 * checkboxes, so Codex answers choices and permissions only.
 */
import { existsSync, readdirSync, readFileSync } from 'node:fs'
import { join } from 'node:path'
import type { Frame, LocalClient } from './client.js'

export type QuestionEngine = 'claude' | 'codex'
export type QuestionKind = 'select' | 'multi' | 'text' | 'permit'
export const QUESTION_KINDS: Record<QuestionEngine, QuestionKind[]> = {
  claude: ['select', 'multi', 'text', 'permit'],
  codex: ['select', 'permit'],
}

export interface QuestionOutcome {
  engine: QuestionEngine
  kind: QuestionKind
  token: string
  requestId: string | null
  answer: string | null
  /** What the engine says once it has the answer: the transcript's last word on that turn. */
  expected: string | null
  /** performance.now() when the first answer was sent. */
  sentAt: number | null
  /** Answer sent → its `question_response_result` (core answers once the dialog is gone or submitted). */
  resultMs: number | null
  /** Answer sent → `commander_question_close` for it, when one came by the time the turn ended. */
  closeMs: number | null
  /** Answer sent → the turn's `turn_ended` (the engine took the answer and finished). */
  endedMs: number | null
  attempts: number
  errors: string[]
  ok: boolean
}

const sleep = (ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms))
const isTurnEnd = (agentId: string) => (frame: Frame) => frame.type === 'turn_ended' && frame.agentId === agentId

function prompt(kind: QuestionKind, token: string): { content: string; command: string } {
  const command = `printf ${token || 'hi'}`
  if (kind === 'multi') return { content: `!askmany ${token}`.trim(), command }
  if (kind === 'permit') return { content: `!permit ${command}`, command }
  return { content: `!ask ${token}`.trim(), command }
}

function answerFor(engine: QuestionEngine, kind: QuestionKind, n: number, options: string[], command: string): { answer: string; expected: string } {
  if (kind === 'permit') {
    const yes = n % 2 === 0
    return { answer: options[yes ? 0 : 2] ?? options[0], expected: yes ? `ran ${command}` : `did not run ${command}` }
  }
  if (kind === 'multi') {
    const answer = n % 2 === 0 ? 'Cheese, Basil' : 'Ham'
    return { answer, expected: `you chose ${answer}` }
  }
  if (kind === 'text') {
    const answer = `hot chocolate ${n}`
    return { answer, expected: `you chose ${answer}` }
  }
  const choices = engine === 'codex' ? ['Coffee', 'Tea', 'None of the above'] : ['Coffee', 'Tea']
  const answer = choices[n % choices.length]
  return { answer, expected: `you chose ${answer}` }
}

/**
 * One question: the prompt that asks it, the question as the window hears it, and the answer sent back
 * under its request id. A refused answer is retried (`retries`) as a new intent, a second later: what a
 * person does when the app says the answer did not go in. Never throws; what went wrong is in `errors`.
 */
export async function askAndAnswer(client: LocalClient, agent: { id: string; engine: QuestionEngine }, kind: QuestionKind, n: number,
  options: { token?: string; askMs?: number; resultMs?: number; endMs?: number; retries?: number } = {}): Promise<QuestionOutcome> {
  const token = options.token ?? ''
  const { content, command } = prompt(kind, token)
  const outcome: QuestionOutcome = { engine: agent.engine, kind, token, requestId: null, answer: null, expected: null,
    sentAt: null, resultMs: null, closeMs: null, endedMs: null, attempts: 0, errors: [], ok: false }
  const asked = client.next(frame => frame.type === 'commander_question' && frame.agentId === agent.id, options.askMs ?? 60_000, `question (${kind} ${token})`)
  const ended = client.next(isTurnEnd(agent.id), options.endMs ?? 120_000, `turn_ended (${kind} ${token})`).then(() => performance.now())
  ended.catch(() => {})
  client.send('message', { agentId: agent.id, content })
  let question: Record<string, any>
  try { question = (await asked).payload ?? {} } catch (error) {
    outcome.errors.push(`asked: ${error instanceof Error ? error.message : String(error)}`)
    return outcome
  }
  const shaped = (question.questions as Array<{ q: string; options: string[] }> | undefined)?.[0]
  if (!shaped || typeof question.requestId !== 'string') {
    outcome.errors.push(`asked: unreadable question ${JSON.stringify(question).slice(0, 200)}`)
    ended.catch(() => {})
    return outcome
  }
  const { answer, expected } = answerFor(agent.engine, kind, n, shaped.options ?? [], command)
  Object.assign(outcome, { requestId: question.requestId, answer, expected })
  const closed = client.next(frame => frame.type === 'commander_question_close' && frame.agentId === agent.id
    && frame.payload?.requestId === question.requestId, (options.endMs ?? 120_000) + 5_000, 'commander_question_close')
    .then(() => performance.now(), () => null)
  let closeAt: number | null | undefined
  void closed.then(at => { closeAt = at })
  for (let attempt = 0; attempt <= (options.retries ?? 0); attempt++) {
    if (attempt) await sleep(1_000)
    outcome.attempts++
    const result = client.next(frame => frame.type === 'question_response_result' && frame.payload?.requestId === question.requestId,
      options.resultMs ?? 45_000, `question_response_result (${kind} ${token})`)
    const at = performance.now()
    outcome.sentAt ??= at
    client.send('question_response', { requestId: question.requestId, agentId: agent.id, answers: { [shaped.q]: answer } })
    try {
      const reply = (await result).payload ?? {}
      if (reply.error === undefined) { outcome.resultMs = performance.now() - outcome.sentAt; outcome.ok = true; break }
      outcome.errors.push(`answer ${attempt + 1}: ${String(reply.error)} ${String(reply.detail ?? reply.message ?? '').slice(0, 160)}`.trim())
    } catch (error) {
      outcome.errors.push(`answer ${attempt + 1}: ${error instanceof Error ? error.message : String(error)}`)
    }
  }
  // Never answered: the dialog is still up and the turn cannot end. The caller stops asking this agent.
  if (!outcome.ok) return outcome
  try { outcome.endedMs = (await ended) - outcome.sentAt! } catch (error) {
    outcome.ok = false
    outcome.errors.push(`ended: ${error instanceof Error ? error.message : String(error)}`)
  }
  await sleep(0)
  outcome.closeMs = typeof closeAt === 'number' ? closeAt - outcome.sentAt! : null
  return outcome
}

/** The answers a question turn can end on, as the fake engines write them (fakeEngine.mjs `finish`). */
const QUESTION_OUTCOME = /^(?:you chose .+|ran printf .+|did not run printf .+|\(question cancelled\))$/s

function transcripts(directory: string, sessionId: string): string[] {
  if (!existsSync(directory)) return []
  return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
    const path = join(directory, entry.name)
    if (entry.isDirectory()) return transcripts(path, sessionId)
    return entry.isFile() && entry.name.endsWith(`${sessionId}.jsonl`) ? [path] : []
  })
}

/** What an agent's engine recorded as the end of each question turn, in order, from its transcript. */
export function engineAnswers(root: string, engine: QuestionEngine, sessionId: string): string[] {
  const files = transcripts(engine === 'claude' ? join(root, 'claude', 'projects') : join(root, 'codex', 'sessions'), sessionId)
  const found: string[] = []
  for (const file of files) {
    for (const line of readFileSync(file, 'utf8').split('\n')) {
      if (!line.trim()) continue
      let record: Record<string, any>
      try { record = JSON.parse(line) } catch { continue }
      const text = engine === 'claude'
        ? record.type === 'assistant' && record.message?.stop_reason === 'end_turn' ? record.message?.content?.[0]?.text : undefined
        : record.type === 'event_msg' && record.payload?.type === 'task_complete' ? record.payload?.last_agent_message : undefined
      if (typeof text === 'string' && QUESTION_OUTCOME.test(text)) found.push(text)
    }
  }
  return found
}

/** The answers that went in against what the engine took: none lost, none taken twice, none out of order. */
export function compareAnswers(expected: string[], seen: string[]): { expected: number; seen: number; equal: boolean; lost: number; duplicated: number; firstDifference: number | null } {
  const count = (list: string[]) => list.reduce((map, value) => map.set(value, (map.get(value) ?? 0) + 1), new Map<string, number>())
  const want = count(expected), got = count(seen)
  let lost = 0, duplicated = 0
  for (const [value, n] of want) lost += Math.max(0, n - (got.get(value) ?? 0))
  for (const [value, n] of got) duplicated += Math.max(0, n - (want.get(value) ?? 0))
  const length = Math.max(expected.length, seen.length)
  let firstDifference: number | null = null
  for (let i = 0; i < length; i++) if (expected[i] !== seen[i]) { firstDifference = i; break }
  return { expected: expected.length, seen: seen.length, equal: firstDifference === null, lost, duplicated, firstDifference }
}

export function percentile(values: number[], p: number): number | null {
  if (!values.length) return null
  const sorted = [...values].sort((a, b) => a - b)
  return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))]
}

/** Count, p50 and p95 of each latency, overall and by engine and kind. */
export function summarizeAnswers(outcomes: QuestionOutcome[]) {
  const stats = (list: QuestionOutcome[]) => {
    const pick = (key: 'resultMs' | 'endedMs' | 'closeMs') => list.map(o => o[key]).filter((v): v is number => typeof v === 'number')
    const result = pick('resultMs'), ended = pick('endedMs'), close = pick('closeMs')
    return {
      count: list.length, ok: list.filter(o => o.ok).length, errors: list.reduce((sum, o) => sum + o.errors.length, 0),
      retried: list.filter(o => o.attempts > 1).length,
      resultMs: { n: result.length, p50: percentile(result, 0.5), p95: percentile(result, 0.95), max: result.length ? Math.max(...result) : null },
      endedMs: { n: ended.length, p50: percentile(ended, 0.5), p95: percentile(ended, 0.95) },
      closeMs: { n: close.length, p50: percentile(close, 0.5), p95: percentile(close, 0.95) },
    }
  }
  const groups = (key: (o: QuestionOutcome) => string) => Object.fromEntries([...new Set(outcomes.map(key))].sort()
    .map(name => [name, stats(outcomes.filter(o => key(o) === name))]))
  return { all: stats(outcomes), byEngine: groups(o => o.engine), byKind: groups(o => `${o.engine}:${o.kind}`) }
}