Skip to content

Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World

15.0 KB
Raw Blame
import { readFileSync } from 'node:fs';
import { mkdtemp } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { createWorld, type LocalWorld } from '@workflow/world-local';
import { type DurableSessions, durableSessions, QUEUE_PREFIX, type SessionState } from 'harness-world';
import { filesFromState, readLedger, workspaceTools } from 'harness-world-hands';
import { afterEach, describe, expect, it } from 'vitest';
import { type HostedProcess, isAlive, journalRecords, ProcessHost, type SpawnOptions, transcriptOf } from '../src/index.ts';
import { type FakeModel, fakeModel } from './fake-model.ts';
import { referenceHarness } from './harness.ts';

/**
 * The shared core end to end, on world-local, with a real child process:
 * the reference harness drives `fake-cli.mjs` through a turn with a tool,
 * survives a crash in the middle of a `replay: 'never'` tool (recovered on a
 * fresh session dir, and on the same one, where the stale child is killed
 * first), and stops its child when a journal write is refused.
 */

const worlds: LocalWorld[] = [];
const models: FakeModel[] = [];
const strays = new Set<number>();
afterEach(async () => {
  for (const w of worlds.splice(0)) await w.close?.();
  for (const m of models.splice(0)) await m.close();
  for (const pid of strays) {
    try {
      process.kill(-pid, 'SIGKILL');
    } catch {}
  }
  strays.clear();
});

/** A host that remembers what it spawned (to find the children in tests). */
class WatchedHost extends ProcessHost {
  readonly spawned: HostedProcess[] = [];
  override async spawn(runId: string, options: SpawnOptions): Promise<HostedProcess> {
    const proc = await super.spawn(runId, options);
    this.spawned.push(proc);
    strays.add(proc.pid);
    return proc;
  }
}

async function until<T>(fn: () => Promise<T | undefined>, timeoutMs = 20_000): Promise<T> {
  const start = Date.now();
  for (;;) {
    const value = await fn();
    if (value !== undefined) return value;
    if (Date.now() - start > timeoutMs) throw new Error('timed out');
    await new Promise((r) => setTimeout(r, 50));
  }
}

const turnsEnded = (state: SessionState) => transcriptOf(state).filter((r) => r.data.type === 'turn').length;

async function session(dataDir: string, root: string, opts: { chaos?: boolean; recover?: boolean; crash?: () => void; confirmMs?: number; modelUrl?: string } = {}) {
  const world = createWorld({ dataDir, recoverActiveRuns: opts.recover ?? false });
  worlds.push(world);
  const host = new WatchedHost({ root });
  const sessions = durableSessions({
    world,
    harness: referenceHarness({ tools: workspaceTools({ confirmMs: opts.confirmMs ?? 3000 }), host, ...(opts.modelUrl ? { modelUrl: opts.modelUrl } : {}) }),
    ...(opts.chaos ? { chaos: true, crash: opts.crash ?? (() => {}) } : {}),
  });
  world.registerHandler(QUEUE_PREFIX, sessions.handler);
  await world.start?.();
  return { world, host, sessions };
}

const nativeLines = (state: SessionState) => journalRecords<string>(state, 'native').map((r) => r.data);
const fileLines = (path: string) => readFileSync(path, 'utf8').split('\n').filter(Boolean);
const waitTurns = (sessions: DurableSessions, id: string, n: number) =>
  until(async () => {
    const s = await sessions.inspect(id);
    return turnsEnded(s) >= n ? s : undefined;
  });

describe('the reference process harness', () => {
  it('runs a turn with a tool, and a fresh machine resumes the session from the log', async () => {
    const dataDir = await mkdtemp(join(tmpdir(), 'hwp-ref-'));
    const rootA = await mkdtemp(join(tmpdir(), 'hwp-root-a-'));
    const a = await session(dataDir, rootA);
    const { sessionId } = await a.sessions.start({ prompt: 'call write_file {"path":"notes.txt","content":"hi"}' });
    let state = await waitTurns(a.sessions, sessionId, 1);

    // The tool ran once, keyed by the CLI's own call id.
    expect(state.effects.get('tool:call_1')).toMatchObject({ name: 'tool.write_file', status: 'completed' });
    expect(filesFromState(state)).toEqual({ '/workspace/notes.txt': 'hi' });
    // The normalized transcript: the call, the reply, the turn's end.
    expect(transcriptOf(state).map((r) => r.data)).toEqual([
      { type: 'message', role: 'assistant', text: 'Calling write_file.', toolCalls: [{ id: 'call_1', name: 'write_file' }] },
      { type: 'message', role: 'assistant', text: expect.stringContaining('write_file returned: wrote 2 bytes') },
      { type: 'turn', status: 'completed' },
    ]);
    // The native journal is the CLI's session file, line for line.
    const fileA = join(rootA, sessionId, 'session.jsonl');
    expect(nativeLines(state)).toEqual(fileLines(fileA));
    expect(nativeLines(state).map((l) => JSON.parse(l).type)).toEqual(['user', 'assistant', 'tool_result', 'assistant', 'turn_end']);
    // The prompt was placed before the CLI saw it.
    const place = state.effectOrder.find((e) => e.name === 'place')!;
    expect(place.placed).toEqual([state.submissions[0]!.id]);
    expect(state.effectOrder.indexOf(place)).toBeLessThan(state.effectOrder.findIndex((e) => e.name === 'journal'));

    await a.sessions.send(sessionId, 'hello');
    state = await waitTurns(a.sessions, sessionId, 2);
    expect(transcriptOf(state).at(-2)?.data).toEqual({ type: 'message', role: 'assistant', text: 'echo: hello' });
    // Each child exited at the end of its drive.
    await until(async () => (a.host.spawned.every((p) => !p.alive()) ? true : undefined));
    await a.world.close?.();

    // Another machine: an empty root. The session file comes back from the
    // log, and the workspace the tools see too.
    const rootB = await mkdtemp(join(tmpdir(), 'hwp-root-b-'));
    const b = await session(dataDir, rootB);
    await b.sessions.send(sessionId, 'call read_file {"path":"notes.txt"}');
    state = await waitTurns(b.sessions, sessionId, 3);
    expect(state.effects.get('tool:call_2')).toMatchObject({ status: 'completed', output: 'hi' });
    const fileB = join(rootB, sessionId, 'session.jsonl');
    expect(fileLines(fileB)).toEqual(nativeLines(state));
    expect(fileLines(fileB).slice(0, fileLines(fileA).length)).toEqual(fileLines(fileA));
  });

  it('a crash mid never-tool: recovered from the log on a fresh machine, with the recovery note, and the side effect happened once', async () => {
    const dataDir = await mkdtemp(join(tmpdir(), 'hwp-ref-'));
    const rootA = await mkdtemp(join(tmpdir(), 'hwp-root-a-'));
    let crashed!: () => void;
    const died = new Promise<void>((r) => (crashed = r));
    const a = await session(dataDir, rootA, { chaos: true, crash: () => crashed() });
    const { sessionId } = await a.sessions.start();
    await a.sessions.chaos(sessionId, { at: 'tool', afterMs: 300 });
    await a.sessions.send(sessionId, 'call unsafe_make_payment {"to":"acme","amount_usd":5,"memo":"order 7"}');
    await died;
    // The process is gone: its World with it. Its child is not (its own
    // process group), as after a real SIGKILL of the parent.
    await a.world.close?.();
    const stale = a.host.spawned.at(-1)!;
    expect(stale.alive()).toBe(true);

    const rootB = await mkdtemp(join(tmpdir(), 'hwp-root-b-'));
    const b = await session(dataDir, rootB, { chaos: true, recover: true });
    const state = await waitTurns(b.sessions, sessionId, 1);

    // The payment's intent was committed, its outcome lost: interrupted, never rerun.
    expect(state.effects.get('tool:call_1')).toMatchObject({ name: 'tool.unsafe_make_payment', status: 'failed', error: { interrupted: true } });
    expect(state.effectOrder.filter((e) => e.name === 'tool.unsafe_make_payment')).toHaveLength(1);
    const ledger = await readLedger(b.world, sessionId);
    expect(ledger.map((l) => [l.tool, l.callKey])).toEqual([['unsafe_make_payment', 'tool:call_1']]);

    // The recovery note told the model what the log knows.
    const entries = transcriptOf(state).map((r) => r.data);
    const note = entries.find((e) => e.type === 'message' && e.role === 'user');
    expect(note).toMatchObject({ synthetic: 'recovery' });
    expect((note as { text: string }).text).toContain('unsafe_make_payment');
    expect((note as { text: string }).text).toContain('may have taken effect');
    expect(entries.at(-2)).toEqual({ type: 'message', role: 'assistant', text: expect.stringContaining('Recovered from the log') });
    // The CLI's own resume answered the dangling call; that is in the journal too.
    const native = nativeLines(state).map((l) => JSON.parse(l));
    expect(native.find((r) => r.type === 'tool_result')).toMatchObject({ id: 'call_1', synthetic: true, isError: true });
    expect(fileLines(join(rootB, sessionId, 'session.jsonl'))).toEqual(nativeLines(state));
    stale.kill();
  });

  it('a crash on the same machine: the stale child is killed before the session resumes', async () => {
    const dataDir = await mkdtemp(join(tmpdir(), 'hwp-ref-'));
    const root = await mkdtemp(join(tmpdir(), 'hwp-root-'));
    let crashed!: () => void;
    const died = new Promise<void>((r) => (crashed = r));
    const a = await session(dataDir, root, { chaos: true, crash: () => crashed() });
    const { sessionId } = await a.sessions.start();
    await a.sessions.chaos(sessionId, { at: 'tool', afterMs: 300 });
    await a.sessions.send(sessionId, 'call unsafe_send_email {"to":"[email protected]","subject":"hi","body":"x"}');
    await died;
    await a.world.close?.();
    const stale = a.host.spawned.at(-1)!;
    expect(isAlive(stale.pid)).toBe(true);

    const b = await session(dataDir, root, { chaos: true, recover: true });
    const state = await waitTurns(b.sessions, sessionId, 1);
    await until(async () => (isAlive(stale.pid) ? undefined : true), 5000);
    expect(state.effects.get('tool:call_1')).toMatchObject({ status: 'failed', error: { interrupted: true } });
    expect((await readLedger(b.world, sessionId)).length).toBe(1);
    expect(fileLines(join(root, sessionId, 'session.jsonl'))).toEqual(nativeLines(state));
  });

  it('a refused journal write halts the drive and kills the child', async () => {
    const dataDir = await mkdtemp(join(tmpdir(), 'hwp-ref-'));
    const root = await mkdtemp(join(tmpdir(), 'hwp-root-'));
    const a = await session(dataDir, root);
    const { sessionId } = await a.sessions.start();
    await a.sessions.send(sessionId, 'hello', 'prompt', { wake: 'backstop' });
    const create = a.world.events.create.bind(a.world.events);
    const refused = new Error('the World refused the write');
    a.world.events.create = (async (runId: string, request: any, params?: any) => {
      if (request?.eventType === 'step_started' && String(request.eventData?.stepName).endsWith('//journal')) throw refused;
      return create(runId, request, params);
    }) as typeof a.world.events.create;
    await expect(a.sessions.drive(sessionId)).rejects.toBe(refused);
    const child = a.host.spawned[0]!;
    await until(async () => (child.alive() ? undefined : true), 5000);
    const state = await a.sessions.inspect(sessionId);
    expect(state.effectOrder.some((e) => e.name === 'journal')).toBe(false);
  });

  it('with its model behind the gateway proxy, a turn writes no model step (no drill is armed)', async () => {
    const model = await fakeModel();
    models.push(model);
    const a = await session(await mkdtemp(join(tmpdir(), 'hwp-ref-')), await mkdtemp(join(tmpdir(), 'hwp-root-')), { modelUrl: model.url });
    const { sessionId } = await a.sessions.start({ prompt: 'call write_file {"path":"notes.txt","content":"hi"}' });
    const state = await waitTurns(a.sessions, sessionId, 1);
    // The CLI's replies streamed from the model, through the proxy.
    expect(model.requests.map((r) => r.text)).toEqual(['Calling write_file.', expect.stringContaining('write_file returned: wrote 2 bytes')]);
    expect(transcriptOf(state).at(-2)?.data).toEqual({ type: 'message', role: 'assistant', text: expect.stringContaining('write_file returned') });
    expect(state.effectOrder.filter((e) => e.name === 'model')).toEqual([]);
  });

  it('a crash while the model streams, after a never tool: the chaos step names the model step, the recovered turn makes the call again, and the email went out once', async () => {
    // The model types the reply to the tool's result slowly: that is the stream the crash cuts.
    const model = await fakeModel({ pieceMs: (text) => (text.startsWith('unsafe_send_email returned') ? 150 : 5) });
    models.push(model);
    const dataDir = await mkdtemp(join(tmpdir(), 'hwp-ref-'));
    let crashed!: () => void;
    const died = new Promise<void>((r) => (crashed = r));
    const a = await session(dataDir, await mkdtemp(join(tmpdir(), 'hwp-root-a-')), { chaos: true, crash: () => crashed(), modelUrl: model.url, confirmMs: 500 });
    const { sessionId } = await a.sessions.start();
    await a.sessions.chaos(sessionId, { at: 'model', afterMs: 400 });
    await a.sessions.send(sessionId, 'call unsafe_send_email {"to":"[email protected]","subject":"hi","body":"x"}');
    await died;
    let state = await a.sessions.inspect(sessionId);
    // The first model request ("Calling ...") ended before the crash was due: it fired in the second.
    const steps = state.effectOrder.filter((e) => e.name === 'model');
    expect(steps).toHaveLength(2);
    expect(steps[0]).toMatchObject({ status: 'completed', output: { requests: 1, status: 200 } });
    const chaos = state.effectOrder.find((e) => e.name === 'chaos')!;
    expect(chaos).toMatchObject({ status: 'completed', output: { v: 2, effect: 'model', crashedDuring: steps[1]!.key } });
    expect(steps[1]).toMatchObject({ status: 'running' });
    expect(state.effects.get('tool:call_1')).toMatchObject({ name: 'tool.unsafe_send_email', status: 'completed' });
    expect(turnsEnded(state)).toBe(0);
    // The process is gone, its World with it.
    await a.world.close?.();

    const b = await session(dataDir, await mkdtemp(join(tmpdir(), 'hwp-root-b-')), { chaos: true, recover: true, modelUrl: model.url });
    state = await waitTurns(b.sessions, sessionId, 1);
    // The killed step was retried by the recovered turn's model request: attempt 2, completed. Nothing is left open.
    expect(state.effects.get(steps[1]!.key)).toMatchObject({ name: 'model', status: 'completed', attempt: 2 });
    expect(state.effectOrder.filter((e) => e.name === 'model')).toHaveLength(2);
    expect(state.effectOrder.filter((e) => e.status === 'running' || e.status === 'pending')).toEqual([]);
    // The email ran once, and the note told the model it had completed.
    expect(state.effectOrder.filter((e) => e.name === 'tool.unsafe_send_email')).toHaveLength(1);
    expect((await readLedger(b.world, sessionId)).map((l) => l.callKey)).toEqual(['tool:call_1']);
    const entries = transcriptOf(state).map((r) => r.data);
    const note = entries.find((e) => e.type === 'message' && e.role === 'user') as { text: string; synthetic?: string };
    expect(note).toMatchObject({ synthetic: 'recovery' });
    expect(note.text).toMatch(/unsafe_send_email\(.*\), call call_1: completed/);
    expect(entries.at(-2)).toEqual({ type: 'message', role: 'assistant', text: expect.stringContaining('Recovered from the log') });
    expect(entries.at(-1)).toEqual({ type: 'turn', status: 'completed' });
  });
});