Skip to content

Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World

23.9 KB
Raw Blame
import { type ChildProcess, spawn } from 'node:child_process';
import { readdirSync } from 'node:fs';
import { mkdtemp } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { createInterface } from 'node:readline';
import { fileURLToPath } from 'node:url';
import type { World } from '@workflow/world';
import { createWorld } from '@workflow/world-local';
import { type DurableSessions, durableSessions, QUEUE_PREFIX, type SessionState } from 'harness-world';
import { filesFromState, readLedger, unsafeTools, workspaceTools } from 'harness-world-hands';
import { workspaceRuntime } from 'harness-world-hands/runtime';
import { journalRecords, transcriptOf } from 'harness-world-process';
import { afterEach, describe, expect, it } from 'vitest';
import { opencodeHarness } from '../src/index.ts';
import { postgresWorld } from './postgres.ts';
import { EVENTS, eventName, type OpenCodeEvent } from '../src/records.ts';

/**
 * OpenCode v2 for real: real model turns through the AI Gateway, on
 * world-local (and world-postgres when `WORKFLOW_POSTGRES_URL` is set),
 * with real crashes: a chaos crash that SIGKILLs the app's process during
 * `unsafe_make_payment` or a built-in's command, and a SIGKILL while the
 * model is streaming. Each crash is recovered in a fresh process: from the
 * log alone.
 *
 * Skipped without a Gateway credential (`AI_GATEWAY_API_KEY` or
 * `VERCEL_OIDC_TOKEN`).
 */

const credential = !!(process.env.AI_GATEWAY_API_KEY || process.env.VERCEL_OIDC_TOKEN);
const MODEL = process.env.MODEL ?? 'anthropic/claude-haiku-4.5';
const APP = fileURLToPath(new URL('./app.ts', import.meta.url));
/** The package's directory: `--import tsx` resolves from there (one process, so a SIGKILL kills the app itself). */
const CWD = fileURLToPath(new URL('..', import.meta.url));

const closers: (() => Promise<unknown> | unknown)[] = [];
afterEach(async () => {
  for (const c of closers.splice(0).reverse()) await c();
});

async function until<T>(fn: () => Promise<T | undefined> | T | undefined, timeoutMs = 120_000): Promise<T> {
  const start = Date.now();
  for (;;) {
    const v = await fn();
    if (v !== undefined) return v;
    if (Date.now() - start > timeoutMs) throw new Error('timed out');
    await new Promise((r) => setTimeout(r, 200));
  }
}

const turns = (s: SessionState) => transcriptOf(s).filter((r) => r.data.type === 'turn');
const waitTurns = (sessions: DurableSessions, id: string, n: number) =>
  until(async () => {
    const s = await sessions.inspect(id);
    return turns(s).length >= n ? s : undefined;
  });

async function local(dataDir: string, root: string, opts: { recover?: boolean; builtins?: boolean } = {}) {
  const world = createWorld({ dataDir, recoverActiveRuns: opts.recover ?? false });
  closers.push(() => world.close?.());
  return withWorld(world, root, opts.builtins);
}

async function withWorld(world: World & { registerHandler?: unknown }, root: string, builtins = false) {
  const sessions = durableSessions({
    world,
    harness: builtins
      ? // OpenCode's own tools, on a virtual workspace.
        opencodeHarness({ workspace: { kind: 'virtual', runtime: workspaceRuntime() }, tools: unsafeTools({ confirmMs: 4000 }), root, model: MODEL })
      : opencodeHarness({ tools: workspaceTools(), root, model: MODEL }),
    // The log holds the crash arming of a test below; it has fired, and must never kill this runner.
    chaos: true,
    crash: () => {
      throw new Error('a chaos crash in the test runner');
    },
  });
  (world as unknown as { registerHandler?(prefix: string, h: unknown): void }).registerHandler?.(QUEUE_PREFIX, sessions.handler);
  await world.start?.();
  return { world, sessions };
}

/** The app's process (test/app.ts), whose stdout lines are JSON. */
function app(dataDir: string, root: string, mode: string, arg: string, env: Record<string, string> = {}) {
  const child: ChildProcess = spawn(process.execPath, ['--import', 'tsx', APP, dataDir, root, mode, arg], {
    cwd: CWD,
    env: { ...process.env, ...env },
    stdio: ['ignore', 'pipe', 'pipe'],
  });
  closers.push(() => {
    if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL');
  });
  const lines: any[] = [];
  let stderr = '';
  child.stderr!.on('data', (d) => (stderr = (stderr + d).slice(-4000)));
  createInterface({ input: child.stdout! }).on('line', (l) => {
    try {
      lines.push(JSON.parse(l));
    } catch {}
  });
  const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((r) => child.once('exit', (code, signal) => r({ code, signal })));
  return { child, lines, exited, stderr: () => stderr };
}

const events = (s: SessionState) => journalRecords<OpenCodeEvent>(s, EVENTS).map((r) => r.data);
/** A session's drive directories on `root` (each drive's: removed when it ends; a killed process's, by the next drive). */
const driveDirs = (root: string, sessionId: string) => readdirSync(join(root, sessionId), { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name);

describe.skipIf(!credential)('OpenCode v2, durable, for real', () => {
  it('runs a turn with a tool call; another machine continues the session from the log', { timeout: 240_000 }, async () => {
    const dataDir = await mkdtemp(join(tmpdir(), 'oc-e2e-'));
    const rootA = await mkdtemp(join(tmpdir(), 'oc-root-a-'));
    const a = await local(dataDir, rootA);
    const { sessionId } = await a.sessions.start({
      prompt: 'Use write_file to write the word "lighthouse" to word.txt. Then reply with just: done.',
    });
    let state = await waitTurns(a.sessions, sessionId, 1);
    // The drive's database is a cache it removes when it ends.
    await until(() => (driveDirs(rootA, sessionId).length ? undefined : true), 10_000);
    const write = state.effectOrder.find((e) => e.name === 'tool.write_file')!;
    expect(write).toMatchObject({ status: 'completed', key: expect.stringMatching(/^tool:toolu_/) });
    expect(filesFromState(state)['/workspace/word.txt']).toContain('lighthouse');
    // The tool's effect is keyed by the call OpenCode committed before running it.
    const called = events(state).find((e) => eventName(e) === 'session.tool.called')!;
    expect(write.key).toBe(`tool:${called.data.id}`);
    expect(turns(state).map((r) => r.data)).toEqual([{ type: 'turn', status: 'completed' }]);
    await a.world.close?.();

    // A fresh root: no OpenCode database, no session files. Only the log.
    const b = await local(dataDir, await mkdtemp(join(tmpdir(), 'oc-root-b-')));
    await b.sessions.send(sessionId, 'Use read_file to read word.txt, then reply with exactly what it says.');
    state = await waitTurns(b.sessions, sessionId, 2);
    expect(state.effectOrder.filter((e) => e.name === 'tool.read_file' && e.status === 'completed').length).toBeGreaterThanOrEqual(1);
    const last = transcriptOf(state).map((r) => r.data).filter((e) => e.type === 'message').at(-1) as { text: string };
    expect(last.text.toLowerCase()).toContain('lighthouse');
    // OpenCode's seq is contiguous across both machines.
    expect(events(state).map((e) => e.seq)).toEqual(events(state).map((_, i) => i));
  });

  it('a steer joins the running turn', { timeout: 180_000 }, async () => {
    const dataDir = await mkdtemp(join(tmpdir(), 'oc-e2e-'));
    const a = await local(dataDir, await mkdtemp(join(tmpdir(), 'oc-root-')));
    const { sessionId } = await a.sessions.start();
    const started = (async () => {
      for await (const ev of a.sessions.live(sessionId) as AsyncIterable<any>) if (ev?.type === 'tool_start') return;
    })();
    await a.sessions.send(sessionId, 'Use bash to run `sleep 3; echo first`. Then reply with what it printed.');
    await started;
    const steer = await a.sessions.send(sessionId, 'Also use write_file to write the word "steer" to s.txt before you reply.', 'steer');
    const state = await until(async () => {
      const s = await a.sessions.inspect(sessionId);
      return filesFromState(s)['/workspace/s.txt'] && s.placed.has(steer) && transcriptOf(s).at(-1)?.data.type === 'turn' ? s : undefined;
    });
    // Placed, admitted under its own id with OpenCode's steer delivery, and acted on in the same session.
    const enq = events(state).find((e) => eventName(e) === 'session.inbox.enqueued' && e.data.inboxID === `msg_${steer}`);
    expect(enq?.data.item).toMatchObject({ type: 'user', delivery: 'steer' });
    expect(filesFromState(state)['/workspace/s.txt']).toContain('steer');
  });

  it('a crash during unsafe_make_payment: recovered from the log in a fresh process; the payment happened once', { timeout: 240_000 }, async () => {
    const dataDir = await mkdtemp(join(tmpdir(), 'oc-e2e-'));
    // The app arms a crash at the next tool and sends the prompt; the crash SIGKILLs it mid-payment.
    const rootA = await mkdtemp(join(tmpdir(), 'oc-root-a-'));
    const a = app(dataDir, rootA, 'chaos-tool', "Pay [email protected] 5 USD with unsafe_make_payment, memo 'order 7'. Pay exactly once: never retry a payment.");
    const exit = await Promise.race([a.exited, new Promise<never>((_, no) => setTimeout(() => no(new Error(`the app did not crash: ${a.stderr()}`)), 120_000))]);
    expect(exit.signal, a.stderr()).toBe('SIGKILL');
    const sessionId = a.lines[0].sessionId as string;
    expect(a.lines.some((l) => l.type === 'chaos' && l.effect === 'tool.unsafe_make_payment')).toBe(true);

    const rootB = await mkdtemp(join(tmpdir(), 'oc-root-b-'));
    const b = await local(dataDir, rootB, { recover: true });
    const state = await waitTurns(b.sessions, sessionId, 1);
    // Rebuilt on the empty root, and its drive's directory removed when it ended (just after the turn's last record).
    await until(() => (driveDirs(rootB, sessionId).length ? undefined : true), 10_000);
    const pays = state.effectOrder.filter((e) => e.name === 'tool.unsafe_make_payment');
    expect(pays[0]).toMatchObject({ status: 'failed', error: { interrupted: true } });
    // At most one side effect: the interrupted call was never rerun (the model may not repeat it either).
    const ledger = await readLedger(b.world, sessionId);
    expect(ledger.length).toBe(1);
    expect(ledger[0]!.callKey).toBe(pays[0]!.key);
    const entries = transcriptOf(state).map((r) => r.data);
    const note = entries.find((e) => e.type === 'message' && e.role === 'user') as { text: string; synthetic: string };
    expect(note).toMatchObject({ synthetic: 'recovery' });
    expect(note.text).toContain('may have taken effect');
    expect(entries.at(-1)).toEqual({ type: 'turn', status: 'completed' });
    // OpenCode's own sweep resumed it: its restart notice and the interrupted call's settlement are in the log.
    const names = events(state).map(eventName);
    expect(names).toContain('session.synthetic');
    expect(events(state).find((e) => eventName(e) === 'session.tool.failed')?.data.error?.message).toContain('interrupted');
  });

  it('a SIGKILL while the model streams: the next process on the same machine resumes the turn from the log', { timeout: 240_000 }, async () => {
    const dataDir = await mkdtemp(join(tmpdir(), 'oc-e2e-'));
    const rootA = await mkdtemp(join(tmpdir(), 'oc-root-a-'));
    const a = app(dataDir, rootA, 'stream', 'Write a 250-word story about a lighthouse keeper, in plain text. Do not use any tools.');
    await until(() => (a.lines.some((l) => l.type === 'text') ? true : undefined), 90_000).catch((err) => {
      throw new Error(`${err.message}: ${a.stderr()}`);
    });
    a.child.kill('SIGKILL');
    expect((await a.exited).signal).toBe('SIGKILL');
    const sessionId = a.lines[0].sessionId as string;
    expect(driveDirs(rootA, sessionId).length).toBeGreaterThan(0);

    // The same machine (the same session root): the dead drive's files swept.
    const b = await local(dataDir, rootA, { recover: true });
    const state = await waitTurns(b.sessions, sessionId, 1);
    await until(() => (driveDirs(rootA, sessionId).length ? undefined : true), 10_000);
    const entries = transcriptOf(state).map((r) => r.data);
    expect(entries.find((e) => e.type === 'message' && e.role === 'user')).toMatchObject({ synthetic: 'recovery' });
    expect(entries.at(-1)).toEqual({ type: 'turn', status: 'completed' });
    const story = entries.filter((e) => e.type === 'message' && e.role === 'assistant').map((e) => (e as { text: string }).text).join('\n');
    expect(story.length).toBeGreaterThan(400);
    // The step the kill cut off was started in the log before the kill (the barrier), and OpenCode resumed it.
    expect(events(state).filter((e) => eventName(e) === 'session.execution.started').length).toBeGreaterThanOrEqual(2);
  });

  it('"Crash during: model" after unsafe_send_email: the chaos crash kills the app in the model request for its reply, a fresh process recovers, the model call is made again, the email went out once', { timeout: 300_000 }, async () => {
    const dataDir = await mkdtemp(join(tmpdir(), 'oc-e2e-'));
    const rootA = await mkdtemp(join(tmpdir(), 'oc-root-a-'));
    // Armed when the email starts: it fires 1.5 s into the next model request, the reply's.
    const a = app(dataDir, rootA, 'chaos-model-at-tool', 'Send an email with unsafe_send_email to [email protected], subject "status", body "all good". Then, in your reply, write a 20-line poem about a lighthouse keeper. Never send the email twice.');
    const exit = await Promise.race([a.exited, new Promise<never>((_, no) => setTimeout(() => no(new Error(`the app did not crash: ${a.stderr()}`)), 180_000))]);
    expect(exit.signal, a.stderr()).toBe('SIGKILL');
    const sessionId = a.lines[0].sessionId as string;
    const armedAt = a.lines.findIndex((l) => l.armed);
    expect(armedAt).toBeGreaterThan(0);
    expect(a.lines.some((l) => l.type === 'chaos' && l.effect === 'model')).toBe(true);
    const partial = a.lines.slice(armedAt).filter((l) => l.type === 'text').map((l) => l.delta).join('');

    const b = await local(dataDir, await mkdtemp(join(tmpdir(), 'oc-root-b-')), { recover: true });
    const state = await waitTurns(b.sessions, sessionId, 1);
    const chaos = state.effectOrder.find((e) => e.name === 'chaos')!;
    expect(chaos).toMatchObject({ status: 'completed', output: { v: 2, effect: 'model' } });
    expect(state.effects.get((chaos.output as { crashedDuring: string }).crashedDuring)).toMatchObject({ name: 'model', status: 'completed', attempt: 2 });
    expect(state.effectOrder.filter((e) => e.status === 'running' || e.status === 'pending')).toEqual([]);
    const emails = state.effectOrder.filter((e) => e.name === 'tool.unsafe_send_email');
    expect(emails).toHaveLength(1);
    expect(emails[0]).toMatchObject({ status: 'completed' });
    expect((await readLedger(b.world, sessionId)).filter((l) => l.tool === 'unsafe_send_email')).toHaveLength(1);
    const entries = transcriptOf(state).map((r) => r.data);
    const note = entries.find((e) => e.type === 'message' && e.role === 'user') as { text: string };
    expect(note.text).toContain(`call ${emails[0]!.key.slice('tool:'.length)}: completed`);
    expect(entries.at(-1)).toEqual({ type: 'turn', status: 'completed' });
    console.log('[e2e] text streamed before the crash:', JSON.stringify(partial.slice(0, 200)));
    const said = entries.filter((e) => e.type === 'message' && e.role === 'assistant').map((e) => (e as { text: string }).text).join('\n');
    console.log('[e2e] reply after recovery:', JSON.stringify(said.slice(-300)));
  });

  it('"Crash during: model" armed before the prompt: the first model request dies, the same machine resumes and finishes the turn', { timeout: 300_000 }, async () => {
    const dataDir = await mkdtemp(join(tmpdir(), 'oc-e2e-'));
    const rootA = await mkdtemp(join(tmpdir(), 'oc-root-a-'));
    const a = app(dataDir, rootA, 'chaos-model', 'Write a 250-word story about a lighthouse keeper, in plain text. Do not use any tools.');
    const exit = await Promise.race([a.exited, new Promise<never>((_, no) => setTimeout(() => no(new Error(`the app did not crash: ${a.stderr()}`)), 180_000))]);
    expect(exit.signal, a.stderr()).toBe('SIGKILL');
    const sessionId = a.lines[0].sessionId as string;
    const partial = a.lines.filter((l) => l.type === 'text').map((l) => l.delta).join('');

    const b = await local(dataDir, rootA, { recover: true });
    const state = await waitTurns(b.sessions, sessionId, 1);
    const chaos = state.effectOrder.find((e) => e.name === 'chaos')!;
    expect(chaos).toMatchObject({ status: 'completed', output: { effect: 'model' } });
    expect(state.effects.get((chaos.output as { crashedDuring: string }).crashedDuring)).toMatchObject({ name: 'model', status: 'completed', attempt: 2 });
    const entries = transcriptOf(state).map((r) => r.data);
    expect(entries.find((e) => e.type === 'message' && e.role === 'user')).toMatchObject({ synthetic: 'recovery' });
    expect(entries.at(-1)).toEqual({ type: 'turn', status: 'completed' });
    const story = entries.filter((e) => e.type === 'message' && e.role === 'assistant').map((e) => (e as { text: string }).text).join('\n');
    expect(story.length).toBeGreaterThan(400);
    console.log('[e2e] text streamed before the crash:', partial.length, 'characters');
  });

  it("built-ins on a virtual workspace: OpenCode's own write, shell, edit and read run on its files, each call in the log; another machine reads them from the log", { timeout: 300_000 }, async () => {
    const dataDir = await mkdtemp(join(tmpdir(), 'oc-e2e-'));
    const a = await local(dataDir, await mkdtemp(join(tmpdir(), 'oc-root-a-')), { builtins: true });
    const { sessionId } = await a.sessions.start({
      prompt: 'Use your write tool to write "hello" to greet.txt. Then use your shell tool to run `cat greet.txt`. Then use your edit tool to change hello to durable in greet.txt. Then reply with one short sentence.',
    });
    let state = await waitTurns(a.sessions, sessionId, 1);
    const calls = state.effectOrder.filter((e) => e.name.startsWith('tool.'));
    console.log('[e2e] built-in calls:', JSON.stringify(calls.map((e) => [e.name, e.replay, e.status, e.input])));
    expect(calls.map((e) => e.name)).toEqual(expect.arrayContaining(['tool.write', 'tool.shell', 'tool.edit']));
    expect(calls.filter((e) => ['tool.write', 'tool.shell', 'tool.edit'].includes(e.name)).every((e) => e.status === 'completed' && e.replay === 'safe')).toBe(true);
    expect(calls.find((e) => e.name === 'tool.shell')?.output).toContain('hello');
    // Each call's intent came after OpenCode's record of it (its event) in the log.
    for (const c of calls) {
      const called = journalRecords<OpenCodeEvent>(state, EVENTS).find((r) => eventName(r.data) === 'session.tool.called' && `tool:${r.data.data.id}` === c.key);
      expect(called?.at ?? Infinity).toBeLessThan(state.effectOrder.indexOf(c));
    }
    // The files are the session's, in its log, not a directory of this machine.
    expect(filesFromState(state)['/workspace/greet.txt']).toMatch(/^durable/);
    await a.world.close?.();

    // Another machine: a fresh root, only the log.
    const b = await local(dataDir, await mkdtemp(join(tmpdir(), 'oc-root-b-')), { builtins: true });
    await b.sessions.send(sessionId, 'Read greet.txt with your read tool and reply with exactly what it says.');
    state = await waitTurns(b.sessions, sessionId, 2);
    expect(state.effectOrder.filter((e) => e.name === 'tool.read' && e.status === 'completed').length).toBeGreaterThanOrEqual(1);
    const last = transcriptOf(state).map((r) => r.data).filter((e) => e.type === 'message').at(-1) as { text: string };
    expect(last.text.toLowerCase()).toContain('durable');
  });

  it('built-ins on: a crash 1.5 s into a built-in shell command, recovered in a fresh process: the call is cut off with everything it changed, reported interrupted and safe to repeat, never rerun automatically', { timeout: 300_000 }, async () => {
    const dataDir = await mkdtemp(join(tmpdir(), 'oc-e2e-'));
    const root = await mkdtemp(join(tmpdir(), 'oc-root-'));
    const a = app(dataDir, root, 'chaos-tool', 'Use your shell tool to run exactly `echo started >> runs.txt && sleep 8 && date > slow.txt`, then show slow.txt.', {
      BUILTINS: '1',
      CRASH_AFTER_MS: '1500',
    });
    const exit = await Promise.race([a.exited, new Promise<never>((_, no) => setTimeout(() => no(new Error(`the app did not crash: ${a.stderr()}`)), 150_000))]);
    expect(exit.signal, a.stderr()).toBe('SIGKILL');
    const sessionId = a.lines[0].sessionId as string;
    expect(a.lines.some((l) => l.type === 'chaos' && l.effect === 'tool.shell')).toBe(true);

    const b = await local(dataDir, root, { recover: true, builtins: true });
    const state = await waitTurns(b.sessions, sessionId, 1);
    const shell = state.effectOrder.filter((e) => e.name === 'tool.shell');
    console.log('[e2e] shell calls:', JSON.stringify(shell.map((e) => [e.status, e.input])));
    // The cut-off call: its intent in the log, no outcome, never run again by itself.
    expect(shell[0]).toMatchObject({ replay: 'safe' });
    expect(shell[0]!.status).not.toBe('completed');
    // What it wrote went with the process: only a call the model made again (after the note) left runs.txt.
    const later = (part: string) => shell.slice(1).filter((e) => e.status === 'completed' && String((e.input as { command?: string })?.command ?? '').includes(part)).length;
    const files = filesFromState(state);
    expect(files['/workspace/runs.txt'] ?? '').toBe('started\n'.repeat(later('echo started >> runs.txt')));
    if (!later('slow.txt')) expect(files['/workspace/slow.txt']).toBeUndefined();
    const entries = transcriptOf(state).map((r) => r.data);
    expect(entries.find((e) => e.type === 'message' && e.role === 'user')).toMatchObject({ synthetic: 'recovery', text: expect.stringContaining('safe to repeat') });
    expect(entries.at(-1)).toEqual({ type: 'turn', status: 'completed' });
    console.log('[e2e] reply after recovery:', JSON.stringify((entries.filter((e) => e.type === 'message').at(-1) as { text: string }).text));
  });

  it.skipIf(!process.env.WORKFLOW_POSTGRES_URL)('runs a turn on world-postgres', { timeout: 180_000 }, async () => {
    const pg = await postgresWorld();
    closers.push(pg.close);
    const a = await withWorld(pg.world, await mkdtemp(join(tmpdir(), 'oc-root-pg-')));
    pg.serve(a.sessions.handler);
    const { sessionId } = await a.sessions.start({ prompt: 'Use write_file to write "postgres" to db.txt, then reply with just: done.' });
    const state = await waitTurns(a.sessions, sessionId, 1);
    expect(filesFromState(state)['/workspace/db.txt']).toContain('postgres');
    expect(turns(state).at(-1)!.data).toEqual({ type: 'turn', status: 'completed' });
  });

  it.skipIf(!process.env.WORKFLOW_POSTGRES_URL)('a crash during unsafe_make_payment on world-postgres: another process recovers it; the payment happened once', { timeout: 240_000 }, async () => {
    const a = app('-', await mkdtemp(join(tmpdir(), 'oc-root-pg-a-')), 'chaos-tool', "Pay [email protected] 5 USD with unsafe_make_payment, memo 'order 7'. Pay exactly once: never retry a payment.", {
      HARNESS_TEST_WORLD: 'postgres',
    });
    const exit = await Promise.race([a.exited, new Promise<never>((_, no) => setTimeout(() => no(new Error(`the app did not crash: ${a.stderr()}`)), 120_000))]);
    expect(exit.signal, a.stderr()).toBe('SIGKILL');
    const sessionId = a.lines[0].sessionId as string;

    // Another process (this one), another session root: the crash's wake reaches its flow route.
    const pg = await postgresWorld();
    closers.push(pg.close);
    const b = await withWorld(pg.world, await mkdtemp(join(tmpdir(), 'oc-root-pg-b-')));
    pg.serve(b.sessions.handler);
    const state = await waitTurns(b.sessions, sessionId, 1);
    const pays = state.effectOrder.filter((e) => e.name === 'tool.unsafe_make_payment');
    expect(pays[0]).toMatchObject({ status: 'failed', error: { interrupted: true } });
    const ledger = await readLedger(b.world, sessionId);
    expect(ledger.length).toBe(1);
    expect(ledger[0]!.callKey).toBe(pays[0]!.key);
    const entries = transcriptOf(state).map((r) => r.data);
    expect(entries.find((e) => e.type === 'message' && e.role === 'user')).toMatchObject({ synthetic: 'recovery' });
    expect(entries.at(-1)).toEqual({ type: 'turn', status: 'completed' });
  });
});