Skip to content

Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World

15.8 KB
Raw Blame
import { spawn } from 'node:child_process';
import { appendFileSync, existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
import { mkdtemp, readFile, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { Yield } from 'harness-world';
import { afterEach, describe, expect, it } from 'vitest';
import { groupAlive, groupOf, isAlive, ProcessHost } from './host.ts';
import { FileTail } from './tail.ts';

const strays = new Set<number>();
afterEach(() => {
  for (const pid of strays) {
    try {
      process.kill(-pid, 'SIGKILL');
    } catch {}
  }
  strays.clear();
});

async function until(fn: () => boolean, timeoutMs = 5000) {
  const start = Date.now();
  while (!fn()) {
    if (Date.now() - start > timeoutMs) throw new Error('timed out');
    await new Promise((r) => setTimeout(r, 20));
  }
}

/**
 * Whether a shell registered `name`: the entry exists and holds its registration time. The shell
 * creates the file before `date` writes into it, and off Linux the host reads that time to tell
 * the group is the session's, so an empty entry is not registered yet.
 */
function registered(registry: string, name: string): boolean {
  try {
    return readFileSync(join(registry, name), 'utf8').trim() !== '';
  } catch {
    return false;
  }
}

/** A node child that runs `code`. */
const node = (code: string) => ({ command: process.execPath, args: ['-e', code] });

/** Spawns `sleep 30` in its own group (a grandchild that stays in it), prints both pids, then waits. */
const withGrandchild = node(`
  const { spawn } = require('node:child_process');
  const g = spawn('sleep', ['30'], { stdio: 'ignore' });
  console.log(JSON.stringify({ pid: process.pid, grandchild: g.pid }));
  setInterval(() => {}, 1000);
`);

describe('ProcessHost', () => {
  it('gives the child an explicit environment and a per-session home', async () => {
    const host = new ProcessHost({ root: await mkdtemp(join(tmpdir(), 'hwp-host-')) });
    process.env.HWP_SECRET_FOR_TEST = 'do-not-leak';
    const proc = await host.spawn('wrun_env', { ...node('console.log(JSON.stringify(process.env))'), env: { GIVEN: 'yes' } });
    strays.add(proc.pid);
    delete process.env.HWP_SECRET_FOR_TEST;
    let out = '';
    for await (const line of proc.lines()) out += line;
    const env = JSON.parse(out) as Record<string, string>;
    const dir = await host.dir('wrun_env');
    expect(env.GIVEN).toBe('yes');
    expect(env.HWP_SECRET_FOR_TEST).toBeUndefined();
    expect(env.HOME).toBe(dir.home);
    expect(env.TMPDIR).toBe(dir.tmp);
    expect(env.HARNESS_PROCESS_SESSION).toBe('wrun_env');
    // Nothing else of this process's environment (the OS may add its own, such as macOS's __CF_USER_TEXT_ENCODING).
    const os = (k: string) => k.startsWith('__CF') || k === 'PWD' || k === 'SHLVL' || k === '_';
    expect(Object.keys(env).filter((k) => !os(k)).sort()).toEqual([
      'BASH_ENV',
      'ENV',
      'GIVEN',
      'HARNESS_PROCESS_GROUPS',
      'HARNESS_PROCESS_SESSION',
      'HARNESS_PROCESS_TOKEN',
      'HOME',
      'PATH',
      'SHELL',
      'TMPDIR',
    ]);
    expect(env.HARNESS_PROCESS_GROUPS).toBe(dir.groups);
    expect((await proc.exited).code).toBe(0);
    // A clean exit removes the pidfile: nothing is stale.
    expect(await host.pidRecord('wrun_env')).toBeUndefined();
  });

  it('keeps a default root (`.sessions`) out of git', async () => {
    const root = join(await mkdtemp(join(tmpdir(), 'hwp-app-')), '.sessions');
    await new ProcessHost({ root }).dir('wrun_ignored');
    expect(await readFile(join(root, '.gitignore'), 'utf8')).toBe('*\n');
    const other = await mkdtemp(join(tmpdir(), 'hwp-root-'));
    await new ProcessHost({ root: other }).dir('wrun_ignored');
    expect(existsSync(join(other, '.gitignore'))).toBe(false);
  });

  it('kills the whole process group when the drive stops', async () => {
    const host = new ProcessHost({ root: await mkdtemp(join(tmpdir(), 'hwp-host-')) });
    const stop = new AbortController();
    const proc = await host.spawn('wrun_abort', { ...withGrandchild, signal: stop.signal });
    strays.add(proc.pid);
    const first = (await proc.lines()[Symbol.asyncIterator]().next()).value as string;
    const { grandchild } = JSON.parse(first) as { grandchild: number };
    expect(isAlive(grandchild)).toBe(true);
    stop.abort(new Error('halted'));
    expect((await proc.exited).signal).toBe('SIGKILL');
    await until(() => !isAlive(grandchild));
  });

  it('kills a stale child (and its group) a dead drive left, before the next one starts', async () => {
    const root = await mkdtemp(join(tmpdir(), 'hwp-host-'));
    // The first "process": spawns a child, then dies without killing it.
    const first = new ProcessHost({ root });
    const stale = await first.spawn('wrun_stale', withGrandchild);
    strays.add(stale.pid);
    const { grandchild } = JSON.parse((await stale.lines()[Symbol.asyncIterator]().next()).value as string) as { grandchild: number };
    expect((await first.pidRecord('wrun_stale'))?.pid).toBe(stale.pid);

    // The next drive, in another process on the same machine.
    const second = new ProcessHost({ root });
    const fresh = await second.spawn('wrun_stale', node('setInterval(() => {}, 1000)'));
    strays.add(fresh.pid);
    await until(() => !isAlive(stale.pid) && !isAlive(grandchild));
    expect(fresh.alive()).toBe(true);
    expect((await second.pidRecord('wrun_stale'))?.pid).toBe(fresh.pid);
    fresh.kill();
    await fresh.exited;
  });

  it('leaves a reused pid alone', async () => {
    const root = await mkdtemp(join(tmpdir(), 'hwp-host-'));
    const host = new ProcessHost({ root });
    const dir = await host.dir('wrun_reused');
    // Something unrelated now holds the recorded pid (its own group leader, too).
    const other = spawn('sleep', ['30'], { detached: true, stdio: 'ignore' });
    strays.add(other.pid!);
    await writeFile(join(dir.state, 'pid.json'), JSON.stringify({ pid: other.pid, token: 'x', argv: [process.execPath, 'fake-cli.mjs'], startedAt: Date.now() - 60_000, groups: [] }));
    expect(await host.killStale('wrun_reused')).toEqual([]);
    expect(isAlive(other.pid!)).toBe(true);
    expect(groupAlive(other.pid!)).toBe(true);
  });

  it('will not kill a child its own live drive still runs: the overlapping delivery yields', async () => {
    const host = new ProcessHost({ root: await mkdtemp(join(tmpdir(), 'hwp-host-')) });
    const proc = await host.spawn('wrun_live', node('setInterval(() => {}, 1000)'));
    strays.add(proc.pid);
    await expect(host.spawn('wrun_live', node('0'))).rejects.toBeInstanceOf(Yield);
    expect(proc.alive()).toBe(true);
    proc.kill();
    await proc.exited;
    // Once it exited, the session's next drive starts normally.
    const next = await host.spawn('wrun_live', node('0'));
    strays.add(next.pid);
    expect((await next.exited).code).toBe(0);
  });

  it('claims the session for the whole start: a concurrent start yields, and prepare runs before the child', async () => {
    const host = new ProcessHost({ root: await mkdtemp(join(tmpdir(), 'hwp-host-')) });
    const order: string[] = [];
    const first = host.spawn('wrun_race', {
      ...node('setInterval(() => {}, 1000)'),
      prepare: async (dir) => {
        await new Promise((r) => setTimeout(r, 50));
        await writeFile(join(dir.path, 'session.jsonl'), 'rebuilt\n');
        order.push('prepared');
      },
    });
    // The other delivery starts while the first is still preparing.
    await expect(host.spawn('wrun_race', node('0'))).rejects.toBeInstanceOf(Yield);
    const proc = await first;
    strays.add(proc.pid);
    order.push('spawned');
    expect(order).toEqual(['prepared', 'spawned']);
    expect(await readFile(join((await host.dir('wrun_race')).path, 'session.jsonl'), 'utf8')).toBe('rebuilt\n');
    proc.kill();
    await proc.exited;
  });

  it('records adopted groups and kills them with a stale child', async () => {
    const root = await mkdtemp(join(tmpdir(), 'hwp-host-'));
    const first = new ProcessHost({ root });
    const proc = await first.spawn('wrun_adopt', node('setInterval(() => {}, 1000)'));
    strays.add(proc.pid);
    const escaped = spawn('sleep', ['30'], { detached: true, stdio: 'ignore' });
    strays.add(escaped.pid!);
    await proc.adopt(escaped.pid!);
    expect(JSON.parse(await readFile(join((await first.dir('wrun_adopt')).state, 'pid.json'), 'utf8')).groups).toEqual([escaped.pid]);
    const killed = await new ProcessHost({ root }).killStale('wrun_adopt');
    expect(killed.sort()).toEqual([proc.pid, escaped.pid].sort());
    await until(() => !isAlive(proc.pid) && !isAlive(escaped.pid!));
  });
});

describe('the group registry', () => {
  /** A child that starts `command` the way a harness's shell tool does: detached (a group of its own), under its env. */
  const toolStarter = (shell: string, stdin: 'pipe' | 'ignore' = 'pipe') =>
    node(`
      const t = require('node:child_process').spawn(${JSON.stringify(shell)}, ['-c', 'sleep 30'], { detached: true, stdio: [${JSON.stringify(stdin)}, 'ignore', 'ignore'] });
      console.log(JSON.stringify({ tool: t.pid }));
      setInterval(() => {}, 1000);
    `);
  // The shells the harnesses' tools use. A non-interactive `sh -c` reads no startup file, so it
  // registers nothing (it stays in whatever group started it, which is then the one killed).
  const shells = ['/bin/bash', '/bin/zsh'].filter((s) => existsSync(s));

  it.each(shells.flatMap((sh) => [[sh, 'pipe'] as const, [sh, 'ignore'] as const]))(
    'a command %s starts in a group of its own (stdin %s) registers it, and a stale child is killed with it',
    async (shell, stdin) => {
      const root = await mkdtemp(join(tmpdir(), 'hwp-reg-'));
      const first = new ProcessHost({ root });
      const proc = await first.spawn('wrun_reg', toolStarter(shell, stdin));
      strays.add(proc.pid);
      const { tool } = JSON.parse((await proc.lines()[Symbol.asyncIterator]().next()).value as string) as { tool: number };
      strays.add(tool);
      const dir = await first.dir('wrun_reg');
      await until(() => registered(dir.groups, String(tool)));
      // The child's own group does not reach the command.
      expect(groupAlive(tool)).toBe(true);
      // The next drive on this machine, in another process.
      const killed = await new ProcessHost({ root }).killStale('wrun_reg');
      expect(killed).toContain(tool);
      await until(() => !groupAlive(tool) && !isAlive(proc.pid));
      expect(readdirSync(dir.groups)).toEqual([]);
    }
  );

  it('a command stays in the registered group when its shell turns job control on (as Claude Code\'s shell snapshot does)', async () => {
    // Claude Code's Bash sources a snapshot with `set -o monitor` after BASH_ENV: with job control on,
    // the command would get a group of its own that nothing registered, and outlive a crash.
    const job = "set -o monitor; /bin/sh -c 'echo $$; exec sleep 30'";
    const root = await mkdtemp(join(tmpdir(), 'hwp-reg-'));
    const first = new ProcessHost({ root });
    const proc = await first.spawn('wrun_jobs', node(`
      const t = require('node:child_process').spawn('/bin/bash', ['-c', ${JSON.stringify(job)}], { detached: true, stdio: ['ignore', 'pipe', 'ignore'] });
      t.stdout.once('data', (d) => console.log(JSON.stringify({ tool: t.pid, job: Number(String(d).trim()) })));
      setInterval(() => {}, 1000);
    `));
    strays.add(proc.pid);
    const { tool, job: pid } = JSON.parse((await proc.lines()[Symbol.asyncIterator]().next()).value as string) as { tool: number; job: number };
    strays.add(tool);
    strays.add(pid);
    expect(groupOf(pid)).toBe(tool);
    const dir = await first.dir('wrun_jobs');
    await until(() => registered(dir.groups, String(tool)));
    expect(await new ProcessHost({ root }).killStale('wrun_jobs')).toContain(tool);
    await until(() => !isAlive(pid));
  });

  it('a shell that cannot read its group (no ps: a sandbox) registers its pid, which the host resolves', async () => {
    const root = await mkdtemp(join(tmpdir(), 'hwp-reg-'));
    // A PATH with date and nothing else: `ps` is not there (on Linux /proc answers, so this checks macOS's path).
    const bin = await mkdtemp(join(tmpdir(), 'hwp-bin-'));
    const { symlinkSync } = await import('node:fs');
    // One link: on a merged-/usr system (Ubuntu, CI) both paths exist.
    const date = ['/bin/date', '/usr/bin/date'].find((d) => existsSync(d));
    if (date) symlinkSync(date, join(bin, 'date'));
    const first = new ProcessHost({ root });
    const proc = await first.spawn('wrun_pid', {
      ...node(`
        const t = require('node:child_process').spawn('/bin/bash', ['-c', '/bin/sleep 30'], { detached: true, stdio: 'ignore', env: { ...process.env, PATH: ${JSON.stringify(bin)} } });
        console.log(JSON.stringify({ tool: t.pid }));
        setInterval(() => {}, 1000);
      `),
    });
    strays.add(proc.pid);
    const { tool } = JSON.parse((await proc.lines()[Symbol.asyncIterator]().next()).value as string) as { tool: number };
    strays.add(tool);
    const dir = await first.dir('wrun_pid');
    const entry = existsSync('/proc/self/stat') ? String(tool) : `pid-${tool}`;
    await until(() => registered(dir.groups, entry));
    expect(await new ProcessHost({ root }).killStale('wrun_pid')).toContain(tool);
    await until(() => !groupAlive(tool));
  });

  it('a child that exits takes its tools with it; killGroups clears the registry', async () => {
    const host = new ProcessHost({ root: await mkdtemp(join(tmpdir(), 'hwp-reg-')) });
    const proc = await host.spawn('wrun_exit', node(`
      const t = require('node:child_process').spawn('bash', ['-c', 'sleep 30'], { detached: true, stdio: 'ignore' });
      console.log(JSON.stringify({ tool: t.pid }));
      setTimeout(() => process.exit(0), 300);
    `));
    const { tool } = JSON.parse((await proc.lines()[Symbol.asyncIterator]().next()).value as string) as { tool: number };
    strays.add(tool);
    expect((await proc.exited).code).toBe(0);
    await until(() => !groupAlive(tool));
    expect(await host.killGroups('wrun_exit')).toEqual([]);
  });

  it('leaves a registered group alone once its id belongs to something else', async () => {
    const host = new ProcessHost({ root: await mkdtemp(join(tmpdir(), 'hwp-reg-')) });
    const dir = await host.dir('wrun_reused_group');
    const other = spawn('sleep', ['30'], { detached: true, stdio: 'ignore' });
    strays.add(other.pid!);
    await new Promise((r) => setTimeout(r, 100));
    // Registered long before this process group started.
    await writeFile(join(dir.groups, String(other.pid)), String(Math.floor(Date.now() / 1000) - 600));
    expect(await host.killGroups('wrun_reused_group')).toEqual([]);
    expect(groupAlive(other.pid!)).toBe(true);
  });

  it('the watchdog: a SIGKILL of the drive process takes the child and its tools with it', async () => {
    const parent = spawn(process.execPath, ['--import', 'tsx', new URL('./fixtures/watchdog-parent.ts', import.meta.url).pathname, await mkdtemp(join(tmpdir(), 'hwp-watch-'))], {
      stdio: ['ignore', 'pipe', 'inherit'],
    });
    const { pid, tool } = await new Promise<{ pid: number; tool: number }>((r) => parent.stdout!.once('data', (d: Buffer) => r(JSON.parse(d.toString()))));
    strays.add(pid);
    strays.add(tool);
    expect(groupAlive(pid) && groupAlive(tool)).toBe(true);
    parent.kill('SIGKILL');
    await until(() => !groupAlive(pid) && !groupAlive(tool), 5000);
  });
});

describe('FileTail', () => {
  it('returns complete lines only, and refuses a file that shrank', async () => {
    const file = join(await mkdtemp(join(tmpdir(), 'hwp-tail-')), 'session.jsonl');
    const tail = new FileTail(file);
    expect(tail.read()).toEqual([]);
    writeFileSync(file, '{"a":1}\n{"b":');
    expect(tail.read()).toEqual(['{"a":1}']);
    appendFileSync(file, '2}\n');
    expect(tail.read()).toEqual(['{"b":2}']);
    expect(tail.read()).toEqual([]);
    writeFileSync(file, 'x\n');
    expect(() => tail.read()).toThrow(/shrank/);
  });
});