Skip to content

Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World

14.8 KB
Raw Blame
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { createWorld, type LocalWorld } from '@workflow/world-local';
import { type DriveContext, durableSessions, type Harness } from 'harness-world';
import { afterEach, describe, expect, it } from 'vitest';
import { BuiltinCalls, replayBy } from './builtins.ts';
import { placeSubmissions } from './inbox.ts';
import { NativeJournal } from './journal.ts';
import { recoveryNote } from './recovery.ts';
import { journalRecords } from './records.ts';
import { openWorkspace, WORKSPACE_STREAM, workspaceNote } from './workspace.ts';

const worlds: LocalWorld[] = [];
afterEach(async () => {
  for (const w of worlds.splice(0)) await w.close?.();
});

/** One session, driven by `bodies[n]` on its n-th drive (each drive places what is pending first). */
async function session(bodies: ((ctx: DriveContext) => Promise<void>)[]) {
  const world = createWorld({ dataDir: await mkdtemp(join(tmpdir(), 'hwp-builtins-')), recoverActiveRuns: false });
  worlds.push(world);
  let n = 0;
  const lives: unknown[] = [];
  const harness: Harness = {
    name: 'builtins',
    async drive(ctx) {
      const body = bodies[n++];
      if (!body) return;
      if (ctx.pending().length) await placeSubmissions(ctx, ctx.pending());
      const live = ctx.live.bind(ctx);
      (ctx as { live: DriveContext['live'] }).live = (e) => {
        lives.push(e);
        live(e);
      };
      await body(ctx);
    },
  };
  const sessions = durableSessions({ world, harness });
  const { sessionId } = await sessions.start();
  await sessions.send(sessionId, 'go', 'prompt', { wake: 'backstop' });
  return {
    lives,
    drive: () => sessions.drive(sessionId).catch((err: unknown) => err),
    send: (text: string) => sessions.send(sessionId, text, 'prompt', { wake: 'backstop' }),
    inspect: () => sessions.inspect(sessionId),
  };
}

describe('BuiltinCalls', () => {
  it('commits the intent before the call runs, and the outcome when the harness reports it', async () => {
    const order: string[] = [];
    const s = await session([
      async (ctx) => {
        const calls = new BuiltinCalls(ctx, { replay: replayBy(['Read']) });
        const start = await calls.begin({ callId: 'toolu_1', name: 'Bash', input: { command: 'date > d.txt' } });
        expect(start).toEqual({ run: true });
        // The intent is in the log (born running) before the hook lets the call run.
        expect(ctx.state().effects.get('tool:toolu_1')).toMatchObject({ name: 'tool.Bash', status: 'running', replay: 'never', input: { command: 'date > d.txt' } });
        order.push('ran');
        await calls.end('toolu_1', { text: 'ok' });
        expect(ctx.state().effects.get('tool:toolu_1')).toMatchObject({ status: 'completed', output: 'ok' });
        // A read is `safe`; a failed call is a failed step with the harness's text.
        await calls.begin({ callId: 'toolu_2', name: 'Read', input: { file_path: 'x' } });
        await calls.end('toolu_2', { text: 'ENOENT: no such file', isError: true });
        expect(calls.size).toBe(0);
      },
    ]);
    await s.drive();
    const state = await s.inspect();
    expect(order).toEqual(['ran']);
    expect(state.effects.get('tool:toolu_2')).toMatchObject({ name: 'tool.Read', replay: 'safe', status: 'failed', error: { message: 'ENOENT: no such file' } });
    expect(s.lives).toEqual([
      { type: 'tool_start', name: 'Bash', args: { command: 'date > d.txt' }, callId: 'toolu_1' },
      { type: 'tool_end', name: 'Bash', callId: 'toolu_1' },
      { type: 'tool_start', name: 'Read', args: { file_path: 'x' }, callId: 'toolu_2' },
      { type: 'tool_end', name: 'Read', callId: 'toolu_2', isError: true },
    ]);
  });

  it('a call id the log already finished is never run again: begin answers from the log', async () => {
    const s = await session([
      async (ctx) => {
        const calls = new BuiltinCalls(ctx);
        await calls.begin({ callId: 'c1', name: 'Bash', input: {} });
        await calls.end('c1', { text: 'paid once' });
        // The same call dispatched again (a harness that re-issues it after a resume).
        expect(await calls.begin({ callId: 'c1', name: 'Bash', input: {} })).toEqual({ run: false, result: { text: 'paid once', isError: false } });
      },
    ]);
    await s.drive();
    expect((await s.inspect()).effectOrder.filter((e) => e.key === 'tool:c1')).toHaveLength(1);
  });

  it('the barrier runs before a never call, not before a safe one', async () => {
    const s = await session([
      async (ctx) => {
        const barriers: string[] = [];
        const calls = new BuiltinCalls(ctx, {
          replay: replayBy(['Grep']),
          beforeNever: async (c) => {
            barriers.push(c.callId);
            await ctx.effect(`barrier:${c.callId}`, { name: 'barrier', replay: 'safe', input: null }, async () => true);
          },
        });
        await calls.begin({ callId: 'g1', name: 'Grep', input: {} });
        await calls.end('g1', { text: 'none' });
        await calls.begin({ callId: 'w1', name: 'Write', input: {} });
        await calls.end('w1', { text: 'written' });
        expect(barriers).toEqual(['w1']);
      },
    ]);
    await s.drive();
    expect((await s.inspect()).effectOrder.map((e) => e.key)).toEqual([expect.stringMatching(/^place:/), 'tool:g1', 'barrier:w1', 'tool:w1']);
  });

  it('the harness died mid-call (the drive lives): interruptAll commits it interrupted, and the note says it may have taken effect', async () => {
    const s = await session([
      async (ctx) => {
        const calls = new BuiltinCalls(ctx, { replay: replayBy(['Read']) });
        await calls.begin({ callId: 'b1', name: 'Bash', input: { command: 'pay' } });
        await calls.begin({ callId: 'r1', name: 'Read', input: { file_path: 'a' } });
        await calls.interruptAll();
        const note = recoveryNote(ctx.state())!;
        expect(note.calls.map((c) => [c.callId, c.status])).toEqual([
          ['b1', 'interrupted'],
          ['r1', 'retryable'],
        ]);
        expect(note.text).toContain('Bash({"command":"pay"}), call b1: interrupted before it reported a result. It is not safe to repeat, and it may have taken effect');
        expect(note.text).toContain('Read({"file_path":"a"}), call r1: interrupted before it reported a result. It is safe to repeat.');
      },
    ]);
    await s.drive();
    const state = await s.inspect();
    expect(state.effects.get('tool:b1')).toMatchObject({ status: 'failed', error: { interrupted: true } });
  });

  it('the drive stops mid-call: the next delivery resolves the intent, interrupted (never) or for a retry (safe), and nothing reruns it', async () => {
    let notes: string[] = [];
    const s = await session([
      async (ctx) => {
        const calls = new BuiltinCalls(ctx, { replay: replayBy(['Glob']) });
        await calls.begin({ callId: 'n1', name: 'Edit', input: {} });
        await calls.begin({ callId: 's1', name: 'Glob', input: {} });
        // The drive goes away with both calls open: nothing more is written by it.
        ctx.halt?.(new Error('the process went away'));
        await calls.settled(5000);
      },
      async (ctx) => {
        // The next drive: the kernel has resolved the dead drive's intents by now.
        notes = recoveryNote(ctx.state())!.calls.map((c) => `${c.callId}:${c.status}`);
      },
    ]);
    await s.drive();
    expect((await s.inspect()).effects.get('tool:n1')).toMatchObject({ status: 'running' });
    await s.send('again');
    await s.drive();
    const state = await s.inspect();
    expect(state.effects.get('tool:n1')).toMatchObject({ status: 'failed', error: { interrupted: true } });
    expect(state.effects.get('tool:s1')).toMatchObject({ status: 'pending' });
    expect(notes).toEqual(['n1:interrupted', 's1:retryable']);
  });

  it('an end without its begin (a hook that fired alone) is recorded after the fact', async () => {
    const s = await session([
      async (ctx) => {
        const calls = new BuiltinCalls(ctx);
        await calls.end('late1', { name: 'commandExecution', input: { command: 'ls' }, text: 'a.txt' });
      },
    ]);
    await s.drive();
    expect((await s.inspect()).effects.get('tool:late1')).toMatchObject({ name: 'tool.commandExecution', replay: 'safe', status: 'completed', output: 'a.txt' });
  });

  it('around: the call runs inside the wrapper, begin resolving only once the wrapper lets it run, and the wrapper gives the outcome', async () => {
    const order: string[] = [];
    let stepId: string | undefined;
    const s = await session([
      async (ctx) => {
        const calls = new BuiltinCalls(ctx, {
          around: async (call, run, io) => {
            stepId = io.stepId;
            expect(io.ctx).toBe(ctx);
            // The effect's documents, so a wrapper can commit what the call changed with its outcome.
            expect(io.docs).toBeDefined();
            order.push(`before ${call.callId}`);
            // Setup the call must see (a workspace synced in) before the harness lets it run.
            await new Promise((r) => setTimeout(r, 20));
            order.push('synced');
            const text = await run();
            order.push(`after ${call.callId}: ${text}`);
            return `${text} (committed with the files)`;
          },
        });
        const start = await calls.begin({ callId: 'oc_1', name: 'write', input: { filePath: 'a.txt' } });
        expect(start).toEqual({ run: true });
        order.push('ran');
        await calls.end('oc_1', { text: 'wrote a.txt' });
        order.push('ended');
      },
    ]);
    await s.drive();
    expect(order).toEqual(['before oc_1', 'synced', 'ran', 'after oc_1: wrote a.txt', 'ended']);
    const state = await s.inspect();
    expect(state.effects.get('tool:oc_1')).toMatchObject({ status: 'completed', output: 'wrote a.txt (committed with the files)' });
    expect(stepId).toBe(state.effects.get('tool:oc_1')!.stepId);
  });

  it('around: a wrapper that fails before it lets the call run fails the call, and begin says not to run it', async () => {
    const s = await session([
      async (ctx) => {
        const calls = new BuiltinCalls(ctx, {
          around: async () => {
            throw new Error('the workspace could not be synced');
          },
        });
        const start = await calls.begin({ callId: 'oc_2', name: 'edit', input: {} });
        expect(start).toEqual({ run: false, result: { text: expect.stringContaining('the workspace could not be synced'), isError: true } });
        expect(calls.size).toBe(0);
      },
    ]);
    await s.drive();
    expect((await s.inspect()).effects.get('tool:oc_2')).toMatchObject({ status: 'failed', error: { message: 'the workspace could not be synced' } });
    // It never started.
    expect(s.lives).toEqual([]);
  });

  it('around: a wrapper that throws after the call ran fails the step, ends it on the live stream as an error, and end() resolves', async () => {
    const s = await session([
      async (ctx) => {
        const calls = new BuiltinCalls(ctx, {
          around: async (_call, run) => {
            await run();
            throw new Error('the workspace commit failed');
          },
        });
        expect(await calls.begin({ callId: 'oc_3', name: 'write', input: {} })).toEqual({ run: true });
        await calls.end('oc_3', { text: 'wrote b.txt' });
        expect(calls.size).toBe(0);
      },
    ]);
    await s.drive();
    expect((await s.inspect()).effects.get('tool:oc_3')).toMatchObject({ status: 'failed', error: { message: 'the workspace commit failed' } });
    expect(s.lives).toEqual([expect.objectContaining({ type: 'tool_start', callId: 'oc_3' }), expect.objectContaining({ type: 'tool_end', callId: 'oc_3', isError: true })]);
  });

  it('around: a wrapper that returns without letting the call run decides its outcome; begin answers with it, and nothing goes on the live stream', async () => {
    const s = await session([
      async (ctx) => {
        const calls = new BuiltinCalls(ctx, { around: async () => 'nothing to do: a.txt is already there' });
        expect(await calls.begin({ callId: 'oc_4', name: 'write', input: {} })).toEqual({ run: false, result: { text: 'nothing to do: a.txt is already there', isError: false } });
        expect(calls.size).toBe(0);
      },
    ]);
    await s.drive();
    expect((await s.inspect()).effects.get('tool:oc_4')).toMatchObject({ status: 'completed', output: 'nothing to do: a.txt is already there' });
    expect(s.lives).toEqual([]);
  });

  it('around: a wrapper that calls run twice runs the call once', async () => {
    const s = await session([
      async (ctx) => {
        const calls = new BuiltinCalls(ctx, {
          around: async (_call, run) => {
            const [a, b] = await Promise.all([run(), run()]);
            return `${a} / ${b}`;
          },
        });
        expect(await calls.begin({ callId: 'oc_5', name: 'write', input: {} })).toEqual({ run: true });
        await calls.end('oc_5', { text: 'wrote c.txt' });
      },
    ]);
    await s.drive();
    expect((await s.inspect()).effects.get('tool:oc_5')).toMatchObject({ status: 'completed', output: 'wrote c.txt / wrote c.txt' });
    expect(s.lives.filter((e) => (e as { type: string }).type === 'tool_start')).toHaveLength(1);
  });
});

describe('openWorkspace', () => {
  it('records the copy a drive used, and says when a later drive has another one (a fresh directory, another machine)', async () => {
    const root = await mkdtemp(join(tmpdir(), 'hwp-ws-'));
    const path = join(root, 'workspace');
    const marker = join(root, '.harness', 'workspace.json');
    const seen: { created: boolean; moved: boolean }[] = [];
    const open = async (ctx: DriveContext) => {
      const journal = new NativeJournal(ctx);
      const ws = await openWorkspace(journal, { path, marker });
      await journal.flush();
      seen.push({ created: ws.created, moved: ws.moved });
    };
    const s = await session([
      open,
      // The same machine: the same copy.
      open,
      // Another machine: no directory, no marker.
      async (ctx) => {
        await rm(root, { recursive: true, force: true });
        await open(ctx);
      },
      // The same machine as the last drive.
      open,
    ]);
    for (let i = 0; i < 4; i++) {
      if (i > 0) await s.send(`turn ${i}`);
      await s.drive();
    }
    expect(seen).toEqual([
      { created: true, moved: false },
      { created: false, moved: false },
      { created: true, moved: true },
      { created: false, moved: false },
    ]);
    const records = journalRecords<{ id: string; previous?: string }>(await s.inspect(), WORKSPACE_STREAM).map((r) => r.data);
    // Written only when the copy changes.
    expect(records).toHaveLength(2);
    expect(records[1]!.previous).toBe(records[0]!.id);
    expect(JSON.parse(await readFile(marker, 'utf8')).id).toBe(records[1]!.id);
    expect(workspaceNote(path)).toContain(`working directory (${path})`);
  });
});