Skip to content

Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World

16.4 KB
Raw Blame
import type { BenchClient } from './recorder.ts';
import { SCENARIOS, type SampleResult, type ScenarioName } from './scenarios.ts';
import { bootstrapMedian, type Summary, summarizeSamples } from './stats.ts';

/**
 * Where a session runs and how durable it is.
 *
 * - `mode`: `server` (a Function drives it), `tab` (this tab drives it; with
 *   durability on its World is IndexedDB), `hybrid` (this tab drives it on
 *   the server's World, and it moves to a Function or a Sandbox for tools
 *   that need one).
 * - `durability`: `off` is the same harness with no World at all (the raw
 *   baseline, in the same place); `on` is today's awaited writes;
 *   `run-ahead` lets the harness run ahead of durability (Server mode only).
 */
export type Durability = 'off' | 'on' | 'run-ahead' | 'native';
export type Mode = 'server' | 'tab' | 'hybrid' | 'sandbox';
/**
 * `pi`, `fx`, `eve` (eve's own harness on Vercel Workflow, `apps/demo/agent`,
 * durable, in a Function), or a process harness the server registered
 * (`claude-code`, `codex`, `opencode`, `ai-sdk`): Server cells only.
 */
export type HarnessName = string;
/** The harnesses with a raw (no World) and a native lane. */
export const RAW_HARNESSES = ['pi', 'fx'];

export interface Cell {
  mode: Mode;
  durability: Durability;
  harness: HarnessName;
}

export const cellKey = (c: Cell) => `${c.mode}/${c.durability}/${c.harness}`;

/** The cell key that selects eve: one cell (`server/on/eve`), whatever harnesses the rest use. */
export const EVE_CELL_KEY = 'server/eve';

/**
 * `mode/durability` keys × harnesses, plus eve's own cell for `server/eve`.
 * A process harness has no Wrapped (no World) or Native lane and runs on the
 * server only: such a cell throws, or with `onSkip` is left out and named.
 */
export function parseCells(keys: string[], harnesses: string[], onSkip?: (message: string) => void): Cell[] {
  return keys.flatMap((k): Cell[] => {
    if (k === EVE_CELL_KEY) return [{ mode: 'server', durability: 'on', harness: 'eve' }];
    const [mode, durability] = k.split('/') as [Mode, Durability];
    return harnesses
      .filter((h) => h !== 'eve')
      .flatMap((harness): Cell[] => {
        const process = !RAW_HARNESSES.includes(harness);
        const why =
          process && durability === 'off'
            ? `${k}: Wrapped (no World) runs Pi and fx only; ${harness} has no raw lane.`
            : process && durability === 'native'
              ? `${k}: no native baseline for ${harness} yet (Native runs Pi and fx).`
              : process && mode !== 'server'
                ? `${k}: ${harness} runs on the server only.`
                : undefined;
        if (!why) return [{ mode, durability, harness }];
        if (!onSkip) throw new Error(why);
        onSkip(why);
        return [];
      });
  });
}
/** Where a cell's harness physically runs. */
export const placeOf = (mode: Mode): 'function' | 'tab' | 'vm' => (mode === 'server' ? 'function' : mode === 'sandbox' ? 'vm' : 'tab');

/** The raw baseline measured in the same place as `cell`. */
export function baselineOf(cell: Cell): Cell {
  // The Sandbox mode's baseline is the Function raw cell (the VM raw lane is a separate run, `--vm-raw`).
  const mode = cell.mode === 'hybrid' ? 'tab' : cell.mode === 'sandbox' ? 'server' : cell.mode;
  return { mode, durability: 'off', harness: cell.harness };
}

/** Which scenarios mean something in a cell. */
export function applies(scenario: ScenarioName, cell: Cell): boolean {
  // Native: plain Pi or fx in a Function, one turn per request: no steer of ours, no lanes, no crash drill.
  if (cell.durability === 'native') return cell.mode === 'server' && (scenario === 'chat' || scenario === 'chat-long' || scenario === 'tool-write' || scenario === 'tool-bash');
  // eve has no lanes to hand off between and no crash drill of ours.
  if (cell.harness === 'eve') return scenario === 'chat' || scenario === 'chat-long' || scenario === 'tool-write' || scenario === 'tool-bash' || scenario === 'steer';
  if (scenario.startsWith('handoff-')) return cell.mode === 'hybrid' && cell.durability !== 'off';
  // The Sandbox mode (home: the session's VM) only exists durable.
  if (cell.mode === 'sandbox') return cell.durability !== 'off' && scenario !== 'crash' && scenario !== 'crash-silent';
  if (scenario === 'crash' || scenario === 'crash-silent') return cell.mode === 'server' && cell.durability !== 'off';
  return true;
}

export interface CellResult {
  cell: Cell;
  scenario: ScenarioName;
  samples: (SampleResult & { error?: string; at: string })[];
  stats: Record<string, Summary>;
}

/** Budgets: over them is a warning in P0–P2. */
export interface Budget {
  scenario: ScenarioName;
  metric: string;
  /** Allowed excess over the same-place raw baseline's p50, ms. */
  overRaw?: number;
  /** Or an absolute ceiling for the p50, ms. */
  absolute?: number;
  label: string;
}

export const BUDGETS: Budget[] = [
  { scenario: 'chat', metric: 'ttft', overRaw: 150, label: 'TTFT ≤ raw + 150 ms' },
  { scenario: 'chat', metric: 'followupTtft', overRaw: 150, label: 'Follow-up TTFT ≤ raw + 150 ms' },
  { scenario: 'tool-write', metric: 'turn', overRaw: 200, label: 'Safe-tool turn ≤ raw + 200 ms' },
  { scenario: 'tool-bash', metric: 'turn', overRaw: 200, label: 'just-bash turn ≤ raw + 200 ms' },
  { scenario: 'tool-write', metric: 'toolGap', overRaw: 150, label: 'Model → tool → model gap ≈ raw' },
  { scenario: 'steer', metric: 'steerToEffect', overRaw: 150, label: 'Steer to effect ≤ raw + 150 ms' },
  { scenario: 'handoff-function', metric: 'tabToServer', absolute: 300, label: 'Tab → Function ≤ 300 ms' },
  { scenario: 'handoff-function', metric: 'serverToTab', absolute: 250, label: 'Function → tab ≤ 250 ms' },
  { scenario: 'handoff-sandbox', metric: 'tabToServer', absolute: 300, label: 'Tab → Sandbox ≤ 300 ms' },
  { scenario: 'handoff-sandbox', metric: 'serverToTab', absolute: 250, label: 'Sandbox → tab ≤ 250 ms' },
  { scenario: 'crash', metric: 'crashToDrive', absolute: 12_000, label: 'Crash drill (proof + self-wake) → recovered drive ≤ 12 s' },
  { scenario: 'crash-silent', metric: 'crashToDrive', absolute: 30_000, label: 'Silent crash (no record) → recovered drive ≤ 30 s' },
];

export interface Comparison {
  scenario: ScenarioName;
  metric: string;
  cell: Cell;
  p50: number;
  p90: number;
  raw?: { p50: number; p90: number };
  /** Which cell the delta is against: the native lane where it ran, else the raw one. */
  against?: string;
  /** p50 − raw p50. */
  delta?: number;
  /** 90% bootstrap interval of the median of per-round differences (rounds interleave the cells), or of the p50 for an absolute budget. */
  interval?: { lo: number; hi: number };
  budget?: string;
  pass?: boolean;
  /** `inconclusive`: the interval spans the budget, so the samples cannot say. */
  verdict?: 'ok' | 'over' | 'inconclusive';
}

export interface BenchMeta {
  startedAt: string;
  finishedAt?: string;
  origin?: string;
  deploymentId?: string | null;
  sha?: string | null;
  region?: string | null;
  world?: string;
  model?: string;
  /** Round trip to a warm Function from where the bench ran. */
  rttMs?: { p50: number; p90: number };
  runner: string;
  n: number;
  warmup: number;
  userAgent?: string;
  /** How the tab's model calls reached AI Gateway: `direct` (a minted token) or `proxy` (`/api/gateway`). */
  gateway?: 'direct' | 'proxy';
  /** Page flags the tab cells ran with (apps/browser flags.ts). */
  drive?: string;
  hints?: 'on' | 'off';
  relay?: string;
}

export interface BenchReport {
  meta: BenchMeta;
  results: CellResult[];
  comparisons: Comparison[];
}

export interface RunOptions {
  cells: Cell[];
  scenarios: ScenarioName[];
  n: number;
  /** Discarded samples per cell and scenario before measuring. Default 1. */
  warmup?: number;
  client(cell: Cell): Promise<BenchClient>;
  meta?: Partial<BenchMeta>;
  rttMs?: number;
  /** `chat`: idle time before the follow-up. Default 3000. */
  idleMs?: number;
  /** Pause between samples. Default 0. */
  pauseMs?: number;
  /** A throttled sample (HTTP 429) is taken again this many times, after `throttleWaitMs` (default 2, 60s). */
  throttleRetries?: number;
  throttleWaitMs?: number;
  onProgress?(p: { scenario: ScenarioName; cell: Cell; round: number; sample?: SampleResult & { error?: string }; warmup: boolean }): void;
  signal?: AbortSignal;
  log?(message: string): void;
}

const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
const harnessFor = (cell: Cell) => (cell.mode === 'hybrid' || cell.mode === 'sandbox' ? (cell.harness === 'fx' ? 'hybrid-fx' : 'hybrid') : cell.harness);

/**
 * Run every applicable (scenario, cell) `n` times. Rounds interleave the
 * cells (round r runs sample r of every cell before sample r+1 of any), so a
 * slow minute at the model provider lands on all of them alike.
 */
export async function runBench(options: RunOptions): Promise<BenchReport> {
  const warmup = options.warmup ?? 1;
  const meta: BenchMeta = { startedAt: new Date().toISOString(), runner: 'unknown', n: options.n, warmup, ...options.meta };
  const clients = new Map<string, Promise<BenchClient>>();
  const clientFor = (cell: Cell) => {
    const k = cellKey(cell);
    if (!clients.has(k)) clients.set(k, options.client(cell));
    return clients.get(k)!;
  };
  const results: CellResult[] = [];
  for (const scenario of options.scenarios) {
    const cells = options.cells.filter((c) => applies(scenario, c) && (c.durability !== 'run-ahead' || c.mode === 'server'));
    const byCell = new Map(cells.map((c) => [cellKey(c), { cell: c, scenario, samples: [], stats: {} } as CellResult]));
    for (let round = -warmup; round < options.n; round++) {
      for (const cell of cells) {
        if (options.signal?.aborted) throw new Error('aborted');
        let sample: SampleResult & { error?: string } = { metrics: {} };
        for (let attempt = 0; ; attempt++) {
          try {
            const client = await clientFor(cell);
            sample = await SCENARIOS[scenario]({ client, harness: harnessFor(cell), ...(options.rttMs !== undefined ? { rttMs: options.rttMs } : {}), ...(options.idleMs !== undefined ? { idleMs: options.idleMs } : {}), log: options.log });
            break;
          } catch (err) {
            const sessionId = (err as { sessionId?: string } | undefined)?.sessionId;
            sample = { metrics: {}, error: err instanceof Error ? err.message : String(err), ...(sessionId ? { sessionId } : {}) };
            // Rate limited (the World's backend throttles bursts of new
            // sessions): wait it out and take the sample again.
            if (attempt < (options.throttleRetries ?? 2) && /too many requests|\b429\b|throttl/i.test(sample.error!)) {
              options.log?.(`throttled; retrying in ${(options.throttleWaitMs ?? 60_000) / 1000}s`);
              await sleep(options.throttleWaitMs ?? 60_000);
              continue;
            }
            break;
          }
        }
        if (options.pauseMs) await sleep(options.pauseMs);
        options.onProgress?.({ scenario, cell, round, sample, warmup: round < 0 });
        if (round >= 0) byCell.get(cellKey(cell))!.samples.push({ ...sample, at: new Date().toISOString() });
      }
    }
    for (const r of byCell.values()) {
      r.stats = summarizeSamples(r.samples.filter((s) => !s.error));
      results.push(r);
    }
  }
  meta.finishedAt = new Date().toISOString();
  return { meta, results, comparisons: compare(results) };
}

/**
 * Every durable cell's metrics against the baseline in the same place (the
 * native lane where it ran, else the raw one), with a 90% bootstrap interval
 * of the per-round difference, and the budgets: ok when the interval is
 * within the budget, over when it is past it, inconclusive when it spans it.
 */
export function compare(results: CellResult[]): Comparison[] {
  const out: Comparison[] = [];
  const find = (scenario: ScenarioName, cell: Cell) =>
    results.find((r) => r.scenario === scenario && cellKey(r.cell) === cellKey(cell));
  for (const r of results) {
    if (r.cell.durability === 'off' || r.cell.durability === 'native') continue;
    const native = r.cell.mode === 'server' ? find(r.scenario, { mode: 'server', durability: 'native', harness: r.cell.harness }) : undefined;
    const base = native ?? find(r.scenario, baselineOf(r.cell));
    for (const [metric, s] of Object.entries(r.stats)) {
      const raw = base?.stats[metric];
      const budget = BUDGETS.find((b) => b.scenario === r.scenario && b.metric === metric);
      const delta = raw ? s.p50 - raw.p50 : undefined;
      const mine = r.samples.map((x) => (x.error ? undefined : x.metrics[metric]));
      const theirs = base?.samples.map((x) => (x.error ? undefined : x.metrics[metric])) ?? [];
      const diffs = mine.flatMap((v, i) => (v !== undefined && theirs[i] !== undefined && Number.isFinite(v) && Number.isFinite(theirs[i]!) ? [v - theirs[i]!] : []));
      const interval = budget?.absolute !== undefined ? bootstrapMedian(mine.filter((v): v is number => v !== undefined)) : diffs.length ? bootstrapMedian(diffs) : undefined;
      const limit = budget ? (budget.absolute ?? budget.overRaw) : undefined;
      const verdict = budget && interval && limit !== undefined ? (interval.hi <= limit ? 'ok' : interval.lo > limit ? 'over' : 'inconclusive') : undefined;
      const pass = budget
        ? budget.absolute !== undefined
          ? s.p50 <= budget.absolute
          : delta !== undefined && budget.overRaw !== undefined
            ? delta <= budget.overRaw
            : undefined
        : undefined;
      out.push({
        scenario: r.scenario,
        metric,
        cell: r.cell,
        p50: s.p50,
        p90: s.p90,
        ...(raw ? { raw: { p50: raw.p50, p90: raw.p90 }, delta, against: cellKey(base!.cell) } : {}),
        ...(interval ? { interval: { lo: Math.round(interval.lo), hi: Math.round(interval.hi) } } : {}),
        ...(budget ? { budget: budget.label } : {}),
        ...(pass !== undefined ? { pass } : {}),
        ...(verdict ? { verdict } : {}),
      });
    }
  }
  return out;
}

/** A plain-text report: one table per scenario, p50 / p90 per metric and cell, and the budgets. */
export function formatReport(report: BenchReport): string {
  const lines: string[] = [];
  const m = report.meta;
  lines.push(`durable-harness bench · ${m.startedAt}${m.deploymentId ? ` · ${m.deploymentId}` : ''}${m.sha ? ` · ${m.sha.slice(0, 7)}` : ''}`);
  lines.push(`runner ${m.runner} · model ${m.model ?? '?'} · world ${m.world ?? '?'} · n=${m.n} (+${m.warmup} warm-up)${m.rttMs ? ` · Function RTT p50 ${m.rttMs.p50} ms` : ''}${m.gateway ? ` · tab gateway ${m.gateway}` : ''}`);
  const scenarios = [...new Set(report.results.map((r) => r.scenario))];
  for (const scenario of scenarios) {
    const rows = report.results.filter((r) => r.scenario === scenario);
    const metrics = [...new Set(rows.flatMap((r) => Object.keys(r.stats)))];
    lines.push('', `## ${scenario}`);
    const header = ['cell', ...metrics];
    const table = rows.map((r) => {
      const errors = r.samples.filter((s) => s.error).length;
      return [
        `${cellKey(r.cell)}${errors ? ` (${errors} failed)` : ''}`,
        ...metrics.map((k) => {
          const st = r.stats[k];
          return st ? `${st.p50}/${st.p90}${st.missing ? ` (${st.missing} unmeasured)` : ''}` : '–';
        }),
      ];
    });
    lines.push(...renderTable([header, ...table]));
  }
  const budgets = report.comparisons.filter((c) => c.budget);
  if (budgets.length) {
    lines.push('', '## budgets (p50; warnings until P3)');
    lines.push(
      ...renderTable([
        ['scenario', 'metric', 'cell', 'p50', 'base p50', 'delta', '90% interval', 'budget', ''],
        ...budgets.map((c) => [
          c.scenario,
          c.metric,
          cellKey(c.cell),
          String(c.p50),
          c.raw ? `${c.raw.p50}${c.against ? ` (${c.against})` : ''}` : '–',
          c.delta !== undefined ? `${c.delta >= 0 ? '+' : ''}${c.delta}` : '–',
          c.interval ? `${c.interval.lo} to ${c.interval.hi}` : '–',
          c.budget!,
          c.verdict === 'inconclusive' ? 'inconclusive' : c.verdict === 'over' ? 'OVER' : c.verdict === 'ok' ? 'ok' : c.pass === undefined ? '?' : c.pass ? 'ok' : 'OVER',
        ]),
      ])
    );
  }
  lines.push('', 'cells show p50/p90 in ms.');
  return lines.join('\n');
}

function renderTable(rows: string[][]): string[] {
  const widths = rows[0]!.map((_, i) => Math.max(...rows.map((r) => (r[i] ?? '').length)));
  return rows.map((r) => r.map((c, i) => (c ?? '').padEnd(widths[i]!)).join('  ').trimEnd());
}