Skip to content

Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World

6.1 KB
Raw Blame
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { collect, type Evaluation, evaluate, INVARIANTS, type Phase } from './invariants.ts';
import { KNOWN, knownFor, partition } from './known.ts';
import { type CellReport, type SessionReport, writeFragment } from './report.ts';
import type { Scenario, ScenarioRun } from './scenario.ts';
import { outDir, selectedWorlds, storageFor, type WorldName, worldSkipReason } from './worlds.ts';

/**
 * One test file per scenario: for each selected World and each of the
 * scenario's sessions, drive the session (open), inspect it with the CLI,
 * close it, inspect it again, and turn every invariant into a test. A known
 * issue (known.ts) becomes an `it.fails` naming its reason, next to a plain
 * test that the invariant has no other findings.
 */

const PHASES: Phase[] = ['open', 'closed'];
/** How long a session may take to drive and settle, and the CLI to inspect it twice. */
const SESSION_TIMEOUT_MS = 480_000;

export function defineCliCheck(scenario: Scenario): void {
  const nonBlockingOff = !scenario.blocking && process.env.CLI_CHECK_NONBLOCKING !== '1';
  for (const world of selectedWorlds()) {
    const skip =
      worldSkipReason(world) ??
      scenario.skip?.() ??
      (nonBlockingOff ? 'a non-blocking scenario: set CLI_CHECK_NONBLOCKING=1 to run it' : undefined);
    const cell: CellReport = { scenario: scenario.name, title: scenario.title, world, blocking: scenario.blocking, sessions: [] };
    if (skip) {
      describe.skip(`${scenario.title} on world-${world} (skipped: ${skip})`, () => {
        it('is skipped', () => {});
      });
      // Only a skip for this machine is worth reporting; a World nobody selected is not.
      cell.skipped = skip;
      writeFragment(outDir(), cell);
      continue;
    }
    describe(`${scenario.title} on world-${world}`, () => {
      afterAll(() => writeFragment(outDir(), cell));
      for (const spec of scenario.sessions) {
        const why = spec.skip?.();
        if (why) {
          describe.skip(`${spec.label} session (skipped: ${why})`, () => {
            it('is skipped', () => {});
          });
          cell.sessions.push({ label: spec.label, phase: 'open', findings: [], error: `skipped: ${why}` });
          continue;
        }
        defineSession(scenario, world, spec.label, spec.run, cell);
      }
    });
  }
}

function defineSession(scenario: Scenario, world: WorldName, label: string, drive: Scenario['sessions'][number]['run'], cell: CellReport) {
  describe(`${label} session`, () => {
    const results: Partial<Record<Phase, Evaluation>> = {};
    const reports: Partial<Record<Phase, SessionReport>> = {};
    let run: ScenarioRun | undefined;
    let failure: unknown;

    beforeAll(async () => {
      const storage = storageFor(world, `${scenario.name}-${label}`);
      const target = storage.target.world === 'local' ? storage.target.dataDir : storage.target.world;
      try {
        run = await drive(storage);
        const s = run.session;
        for (const phase of PHASES) {
          if (phase === 'closed') {
            if (!s.close) {
              results.closed = { findings: [{ invariant: 'status', code: 'no-close', message: 'the harness has no way to close a session (no run_completed)' }], observed: undefined as never };
              break;
            }
            await s.close();
          }
          const truth = await s.truth();
          const collected = await collect(storage.target, s.runId, phase, { workflowName: s.expect.workflowName, ...(truth.streamChunks ? { streamChunks: truth.streamChunks } : {}) });
          results[phase] = evaluate(collected, { ...s.expect, phase }, truth);
        }
      } catch (err) {
        failure = err;
      } finally {
        for (const phase of PHASES) {
          const r = results[phase];
          const report: SessionReport = {
            label,
            phase,
            ...(run ? { runId: run.session.runId } : {}),
            target,
            ...(r?.observed ? { observed: r.observed } : {}),
            findings: (r?.findings ?? []).map((f) => {
              const k = knownFor(f.invariant, { scenario: scenario.name, world, label, phase }).find((x) => x.codes.includes(f.code));
              return k ? { ...f, known: k.reason } : f;
            }),
            ...(failure ? { error: String(failure instanceof Error ? (failure.stack ?? failure.message) : failure).slice(0, 2000) } : {}),
          };
          reports[phase] = report;
          cell.sessions.push(report);
        }
      }
    }, SESSION_TIMEOUT_MS);

    afterAll(async () => {
      await run?.dispose().catch(() => {});
    });

    it('drives the session and inspects it with the CLI', () => {
      if (failure) throw failure;
    });

    for (const phase of PHASES) {
      for (const inv of INVARIANTS) {
        const known = knownFor(inv.id, { scenario: scenario.name, world, label, phase }, KNOWN);
        const name = `${phase}: ${inv.id}: ${inv.title}`;
        const findingsOf = () => {
          if (failure) throw new Error('the session did not run (see the first test)');
          const r = results[phase];
          if (!r) throw new Error(`no ${phase} inspection`);
          return r.findings.filter((f) => f.invariant === inv.id);
        };
        // A harness with no close path has no closed session to inspect (the `status` test says so).
        const noClosedSession = () => phase === 'closed' && inv.id !== 'status' && !!results.closed?.findings.some((f) => f.code === 'no-close');
        it(name, (ctx) => {
          if (noClosedSession()) ctx.skip();
          const { unexpected } = partition(findingsOf(), known);
          expect(unexpected.map((f) => `[${f.code}] ${f.message}`)).toEqual([]);
        });
        for (const k of known.filter((x) => !x.intermittent)) {
          it.fails(`${name}: known (${k.owner === 'workflow' ? 'Workflow' : 'ours'}): ${k.reason}`, (ctx) => {
            if (noClosedSession()) ctx.skip();
            const covered = findingsOf().filter((f) => k.codes.includes(f.code));
            expect(covered.map((f) => `[${f.code}] ${f.message}`)).toEqual([]);
          });
        }
      }
    }
  });
}