Skip to content

Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World

14.9 KB
Raw Blame
// The Cloudflare lane's drills, with a real model, on local workerd
// (`wrangler dev` with local Durable Object storage), or on a deployed
// Worker with --url (no crash drill there: the process cannot be killed).
//
//   node scripts/drill.mjs [--harness pi|ai-sdk|fx] [--durability strict|run-ahead] [--port 8791] [--only turn,safe,never,second,steer]
//
// Per harness, in one persisted state directory:
//   turn    a tool turn (write a file, read it back)
//   safe    SIGKILL the whole wrangler process group while `bash`
//           (replay: 'safe') runs, restart on the same state, and check the
//           command reran once from the workspace the log had before it
//   never   SIGKILL while `unsafe_make_payment` (replay: 'never') waits for
//           its confirmation, after the payment went out; check the ledger
//           holds one payment and the model got an interrupted result
//   second  another turn on the same session
//   steer   a steer sent while a tool runs, placed at the next model call
//
// After each restart nothing touches the session for --quiet seconds, so
// the recovery is the run object's own alarm, not a request waking it.
// Needs .dev.vars with the AI Gateway credential (see the app's README).
import { mkdirSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { parseArgs } from 'node:util';
import { appDir, startDev } from './dev-server.mjs';

const { values: args } = parseArgs({
  options: {
    harness: { type: 'string', default: 'pi' },
    durability: { type: 'string', default: 'strict' },
    port: { type: 'string', default: String(8790 + Math.floor(Math.random() * 100)) },
    only: { type: 'string', default: 'turn,safe,never,second,steer' },
    quiet: { type: 'string', default: '25' },
    url: { type: 'string' },
    headers: { type: 'string' },
    // Non-secret Worker vars for this run, e.g. HARNESS_DEBUG=1 (repeatable).
    var: { type: 'string', multiple: true, default: [] },
  },
});
const steps = new Set(args.only.split(','));
const runDir = join(appDir, '.drill', `${args.harness}-${args.durability}-${Date.now()}`);
mkdirSync(runDir, { recursive: true });
const persistTo = join(runDir, 'state');
const report = { harness: args.harness, durability: args.durability, where: args.url ?? 'local workerd (wrangler dev)', steps: {} };
const extraHeaders = args.headers ? JSON.parse(args.headers) : {};

let server;
let base = args.url;
const start = async () => {
  if (args.url) return;
  const vars = Object.fromEntries(args.var.map((kv) => kv.split(/=(.*)/s).slice(0, 2)));
  server = await startDev({ port: Number(args.port), persistTo, vars, log: (s) => writeFileSync(join(runDir, 'dev.log'), s, { flag: 'a' }) });
  base = server.url;
};
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const call = async (path, body) => {
  const res = await fetch(base + path, { method: body ? 'POST' : 'GET', body: body ? JSON.stringify(body) : undefined, headers: extraHeaders });
  const json = await res.json();
  if (!res.ok) throw new Error(`${path}: ${res.status} ${JSON.stringify(json)}`);
  return json;
};
const say = (...a) => console.log(...a);
const brief = (s) => s.effects.map((e) => `${e.name}:${e.interrupted ? 'interrupted' : e.status}${e.retryOf ? '(rerun)' : ''}`);

async function waitIdle(id, submissionId, timeoutMs = 240_000) {
  const t0 = Date.now();
  for (;;) {
    const s = await call(`/sessions/${id}`);
    const sub = s.submissions.find((x) => x.id === submissionId);
    if (s.idle && (sub?.placed || sub?.withdrawn)) return { s, ms: Date.now() - t0 };
    if (Date.now() - t0 > timeoutMs) throw new Error(`turn ${submissionId} did not finish: ${JSON.stringify(brief(s))}`);
    await sleep(500);
  }
}

/** Wait until `tool` runs in a step after the first `from` effects (the ones before this turn). */
async function waitRunning(id, tool, from, submissionId, timeoutMs = 120_000) {
  const t0 = Date.now();
  for (;;) {
    const s = await call(`/sessions/${id}`);
    const fresh = s.effects.slice(from);
    const e = fresh.find((x) => x.name === tool);
    if (e?.status === 'running') return s;
    const other = fresh.find((x) => x.name.startsWith('tool.') && x.name !== tool && x.status === 'running');
    if (other) throw new Error(`the model picked ${other.name}, not ${tool}`);
    const placed = s.submissions.find((x) => x.id === submissionId)?.placed;
    if (s.idle && placed && !e) throw new Error(`the turn ended without ${tool}: ${JSON.stringify(brief(s))}`);
    if (Date.now() - t0 > timeoutMs) throw new Error(`${tool} never started`);
    await sleep(150);
  }
}

async function turn(id, text, kind = 'prompt') {
  const { submissionId } = await call(`/sessions/${id}/messages`, { text, kind });
  return { submissionId, ...(await waitIdle(id, submissionId)) };
}

/** SIGKILL everything, restart on the same state, and leave the session alone for --quiet seconds. */
async function killAndRestart(tool, eventsPath) {
  await sleep(1500);
  const killedAt = Date.now();
  const signal = await server.kill();
  say(`   killed wrangler dev (${signal}) 1.5 s into ${tool}`);
  await start();
  const restartedAt = Date.now();
  await sleep(Math.max(0, Number(args.quiet) * 1000 - (restartedAt - killedAt)));
  const pokedAt = Date.now();
  const events = await call(eventsPath);
  const beforePoke = events.filter((e) => Date.parse(e.createdAt) > restartedAt && Date.parse(e.createdAt) < pokedAt);
  return {
    killedAt,
    restartMs: restartedAt - killedAt,
    eventsByAlarmBeforeFirstRequest: beforePoke.length,
    firstRecoveredEvent: beforePoke[0] ? { type: beforePoke[0].eventType, afterKillMs: Date.parse(beforePoke[0].createdAt) - killedAt } : null,
  };
}

/** When the session's last event committed, from the log (not when we looked). */
const finishedAfter = async (eventsPath, killedAt) => Date.parse((await call(eventsPath)).at(-1).createdAt) - killedAt;

/** Kill everything mid-tool, restart on the same state, and let the alarm recover it. */
async function crashDrill(id, prompt, tool) {
  const from = (await call(`/sessions/${id}`)).effects.length;
  const { submissionId } = await call(`/sessions/${id}/messages`, { text: prompt });
  await waitRunning(id, tool, from, submissionId);
  const crash = await killAndRestart(tool, `/sessions/${id}/events`);
  const { s } = await waitIdle(id, submissionId);
  return { ...crash, finishedAfterKillMs: await finishedAfter(`/sessions/${id}/events`, crash.killedAt), submissionId, s };
}

// ---------------------------------------------------------------------------
// Pi variant A (pi-durable): its sessions are Pi journals, read through /pi-durable.

async function piATurn(id, text, kind = 'prompt') {
  const before = (await call(`/pi-durable/${id}`)).entries.length;
  await call(`/pi-durable/${id}/messages`, { text, kind });
  return piAIdle(id, before);
}

async function piAIdle(id, before, timeoutMs = 240_000) {
  const t0 = Date.now();
  for (;;) {
    const s = await call(`/pi-durable/${id}`);
    if (s.entries.length > before && !s.unfinished && s.running.length === 0 && s.entries.at(-1) === 'pi.assistant') return { s, ms: Date.now() - t0 };
    if (Date.now() - t0 > timeoutMs) throw new Error(`the Pi turn did not finish: ${JSON.stringify(s.tasks)}`);
    await sleep(500);
  }
}

async function piARunning(id, tool, timeoutMs = 120_000) {
  const t0 = Date.now();
  for (;;) {
    const s = await call(`/pi-durable/${id}`);
    if (s.running.includes(tool)) return s;
    if (Date.now() - t0 > timeoutMs) throw new Error(`${tool} never started: ${JSON.stringify(s.tasks)}`);
    await sleep(150);
  }
}

async function piACrash(id, prompt, tool) {
  const before = (await call(`/pi-durable/${id}`)).entries.length;
  await call(`/pi-durable/${id}/messages`, { text: prompt });
  await piARunning(id, tool);
  const crash = await killAndRestart(tool, `/pi-durable/${id}/events`);
  const { s } = await piAIdle(id, before);
  const runs = s.tasks.filter((t) => t.kind === 'pi.tool' && t.tool === tool);
  return {
    ...crash,
    finishedAfterKillMs: await finishedAfter(`/pi-durable/${id}/events`, crash.killedAt),
    // One start per process that ran the call: 2 is a rerun after the crash.
    toolTaskStarts: runs.at(-1)?.starts,
    toolResult: s.tools.filter((t) => t.name === tool.replace('tool.', '')).at(-1),
    s,
  };
}

async function drillPiA() {
  const { sessionId: id } = await call('/pi-durable', {});
  report.sessionId = id;
  say(`pi-durable (Pi variant A) on the Cloudflare World (${report.where}): ${id}`);
  if (steps.has('turn')) {
    const r = await piATurn(id, 'Write a haiku about Durable Objects to /workspace/haiku.txt, then read it back with bash cat.');
    report.steps.turn = { ms: r.ms, tools: r.s.tools.map((t) => t.name), text: r.s.text, haiku: r.s.files['/workspace/haiku.txt'] };
    say('1. tool turn', JSON.stringify(report.steps.turn, null, 1));
  }
  if (steps.has('safe') && server) {
    const cmd = 'echo started >> /workspace/log.txt && sleep 6 && echo finished >> /workspace/log.txt';
    const r = await piACrash(id, `Run exactly this bash command once, then say done: ${cmd}`, 'bash');
    const { s, ...rest } = r;
    report.steps.safe = { ...rest, logTxt: s.files['/workspace/log.txt'], text: s.text };
    say('2. crash in bash (safe)', JSON.stringify(report.steps.safe, null, 1));
  }
  if (steps.has('never') && server) {
    const r = await piACrash(id, 'Pay $5 to [email protected] for lunch with unsafe_make_payment.', 'unsafe_make_payment');
    const { s, ...rest } = r;
    report.steps.never = { ...rest, text: s.text };
    say('3. crash in unsafe_make_payment (never)', JSON.stringify(report.steps.never, null, 1));
  }
  if (steps.has('second')) {
    const r = await piATurn(id, 'Which files are in /workspace now? Answer in one line.');
    report.steps.second = { ms: r.ms, tools: r.s.tools.slice(-2).map((t) => t.name), text: r.s.text };
    say('4. second turn', JSON.stringify(report.steps.second, null, 1));
  }
  if (steps.has('steer')) {
    const before = (await call(`/pi-durable/${id}`)).entries.length;
    await call(`/pi-durable/${id}/messages`, { text: 'Run this bash command: sleep 5 && echo slept. Then report its output.' });
    await piARunning(id, 'bash');
    await call(`/pi-durable/${id}/messages`, { text: 'Also: end your answer with the word PINEAPPLE.', kind: 'steer' });
    const { s } = await piAIdle(id, before);
    report.steps.steer = { honored: /PINEAPPLE/i.test(s.text ?? ''), entries: s.entries.slice(before), text: s.text };
    say('5. steer', JSON.stringify(report.steps.steer, null, 1));
  }
  report.events = (await call(`/pi-durable/${id}/events`)).length;
  report.object = await call(`/sessions/${id}/object`);
}

try {
  await start();
  if (args.harness === 'pi-durable') {
    await drillPiA();
    report.ok = true;
  } else await drillKernel();
} catch (error) {
  report.ok = false;
  report.error = String(error?.stack ?? error);
  console.error(error);
  process.exitCode = 1;
} finally {
  await server?.kill();
  writeFileSync(join(runDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`);
  say(`report: ${join(runDir, 'report.json')}`);
}

async function drillKernel() {
  // fx's command tool is libfx's `shell` ({ request: { action: 'run', command } }); Pi's and the AI SDK's is `bash`.
  const shell = args.harness === 'fx' ? 'shell' : 'bash';
  const commandOf = (input) => String(input?.command ?? input?.request?.command ?? '');
  const { sessionId: id } = await call('/sessions', { harness: args.harness, durability: args.durability });
  report.sessionId = id;
  say(`${args.harness} on the Cloudflare World (${report.where}), Durable ${args.durability === 'strict' ? 'Strict' : 'On'}: ${id}`);

  if (steps.has('turn')) {
    const r = await turn(id, `Write a haiku about Durable Objects to /workspace/haiku.txt, then read it back with ${shell} cat.`);
    const files = await call(`/sessions/${id}/files`);
    report.steps.turn = { ms: r.ms, effects: brief(r.s), text: r.s.text, haiku: files['/workspace/haiku.txt'] };
    say('1. tool turn', JSON.stringify(report.steps.turn, null, 1));
  }

  if (steps.has('safe') && server) {
    const cmd = 'echo started >> /workspace/log.txt && sleep 6 && echo finished >> /workspace/log.txt';
    const r = await crashDrill(id, `Run exactly this command once with ${shell}, then say done: ${cmd}`, `tool.${shell}`);
    const files = await call(`/sessions/${id}/files`);
    const bash = r.s.effects.filter((e) => e.name === `tool.${shell}` && commandOf(e.input).includes('sleep 6'));
    report.steps.safe = {
      restartMs: r.restartMs,
      finishedAfterKillMs: r.finishedAfterKillMs,
      eventsByAlarmBeforeFirstRequest: r.eventsByAlarmBeforeFirstRequest,
      firstRecoveredEvent: r.firstRecoveredEvent,
      bashAttempts: bash.map((e) => `${e.interrupted ? 'interrupted' : e.status}${e.retryOf ? ' (rerun)' : ''}`),
      logTxt: files['/workspace/log.txt'],
      effects: brief(r.s),
      text: r.s.text,
    };
    say(`2. crash in ${shell} (safe)`, JSON.stringify(report.steps.safe, null, 1));
  }

  if (steps.has('never') && server) {
    const r = await crashDrill(id, 'Pay $5 to [email protected] for lunch with unsafe_make_payment.', 'tool.unsafe_make_payment');
    const ledger = await call(`/sessions/${id}/ledger`);
    const payments = r.s.effects.filter((e) => e.name === 'tool.unsafe_make_payment');
    report.steps.never = {
      restartMs: r.restartMs,
      finishedAfterKillMs: r.finishedAfterKillMs,
      eventsByAlarmBeforeFirstRequest: r.eventsByAlarmBeforeFirstRequest,
      firstRecoveredEvent: r.firstRecoveredEvent,
      paymentAttempts: payments.map((e) => `${e.interrupted ? 'interrupted' : e.status}${e.retryOf ? ' (rerun)' : ''}`),
      ledgerRecords: ledger.length,
      effects: brief(r.s),
      text: r.s.text,
    };
    say('3. crash in unsafe_make_payment (never)', JSON.stringify(report.steps.never, null, 1));
  }

  if (steps.has('second')) {
    const r = await turn(id, 'Which files are in /workspace now? Answer in one line.');
    report.steps.second = { ms: r.ms, effects: brief(r.s).slice(-4), text: r.s.text };
    say('4. second turn', JSON.stringify(report.steps.second, null, 1));
  }

  if (steps.has('steer')) {
    const from = (await call(`/sessions/${id}`)).effects.length;
    const { submissionId } = await call(`/sessions/${id}/messages`, { text: `Run this command with ${shell}: sleep 5 && echo slept. Then report its output.` });
    await waitRunning(id, `tool.${shell}`, from, submissionId);
    const { submissionId: steerId } = await call(`/sessions/${id}/messages`, { text: 'Also: end your answer with the word PINEAPPLE.', kind: 'steer' });
    const { s } = await waitIdle(id, steerId);
    await waitIdle(id, submissionId);
    const placedBy = s.effects.find((e) => e.placed.includes(steerId));
    report.steps.steer = { placedBy: placedBy && `${placedBy.name} ${placedBy.key}`, honored: /PINEAPPLE/i.test(s.text ?? ''), text: s.text };
    say('5. steer', JSON.stringify(report.steps.steer, null, 1));
  }

  report.object = await call(`/sessions/${id}/object`);
  report.events = (await call(`/sessions/${id}/events`)).length;
  report.ok = true;
}