Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
15.2 KB
Raw Blame
/**
 * issue #5 — `runJobInChild` against REAL child processes (.mjs harnesses run
 * by process.execPath — the child-worker-supervisor.test.ts pattern).
 *
 * Paths pinned:
 *   - success outcome file → resolves with the handler result
 *   - error outcome file → throws the reconstructed error class (exit 0!)
 *   - exit 1 with no file → generic throw naming the exit (attempt burned)
 *   - SIGTERM-ignoring child + aborted signal → group SIGKILL at the injected
 *     grace; "terminated after abort" classification
 *   - pre-aborted signal → child killed promptly
 *   - spawn ENOENT → ChildSpawnInfraError (release, no attempt burned)
 *   - worker-shutdown: child finishes + reports during the drain window →
 *     normal success; child that can't report → ChildWorkerShutdownError
 *   - child env contract (result path, lock token, parent pid, pool bounds)
 */

import { describe, expect, test, beforeAll, afterAll } from 'bun:test';
import { mkdtempSync, writeFileSync, existsSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
  runJobInChild,
  ChildSpawnInfraError,
  ChildWorkerShutdownError,
  ChildNotClaimedError,
} from '../src/core/minions/child-job-runner.ts';
import { UnrecoverableError } from '../src/core/minions/types.ts';
import { RateLeaseUnavailableError } from '../src/core/minions/handlers/subagent.ts';
import { LocalConfigurationError } from '../src/core/minions/configuration-error.ts';

const TEST_TIMEOUT_MS = 30_000;

let harnessDir: string;

function makeHarness(name: string, body: string): string {
  const path = join(harnessDir, `${name}.mjs`);
  writeFileSync(
    path,
    `import { writeFileSync, renameSync } from 'node:fs';\n` +
    `const RESULT = process.env.GBRAIN_JOB_RESULT_PATH;\n` +
    `const writeOutcome = (o) => { writeFileSync(RESULT + '.tmp', JSON.stringify(o)); renameSync(RESULT + '.tmp', RESULT); };\n` +
    body +
    // Readiness handshake LAST (after the body installed its signal
    // handlers): fixed sleeps raced child startup on loaded CI runners —
    // a SIGTERM landing before process.on('SIGTERM') installs takes the
    // default disposition and flips shutdown-semantics assertions
    // (testing specialist). The runner's outcome dir is internal, so the
    // ready path comes from a TEST-provided env var.
    `\nif (process.env.HARNESS_READY_PATH) writeFileSync(process.env.HARNESS_READY_PATH, '1');\n`,
    'utf8',
  );
  return path;
}

/** Make a per-test ready path + the env to hand runJobInChild. */
function readiness(name: string): { env: Record<string, string | undefined>; wait: () => Promise<void> } {
  const readyPath = join(harnessDir, `${name}.ready`);
  return {
    env: { ...process.env, HARNESS_READY_PATH: readyPath },
    wait: async () => {
      const deadline = Date.now() + 10_000;
      while (Date.now() < deadline) {
        if (existsSync(readyPath)) return;
        await new Promise((r) => setTimeout(r, 25));
      }
      throw new Error('harness never signalled readiness');
    },
  };
}

beforeAll(() => {
  harnessDir = mkdtempSync(join(tmpdir(), 'gbrain-cjr-harness-'));
});

afterAll(() => {
  rmSync(harnessDir, { recursive: true, force: true });
});

function baseOpts(harnessPath: string) {
  return {
    jobId: 77,
    jobName: 'subagent',
    lockToken: 'tok-cjr',
    abortSignal: new AbortController().signal,
    shutdownSignal: new AbortController().signal,
    invocation: { cmd: process.execPath, argsPrefix: [harnessPath] },
    tiniPath: '', // direct spawn in tests; group signaling covers both shapes
  };
}

describe('runJobInChild (real children)', () => {
  test('success outcome resolves with the result; env contract honored', async () => {
    const harness = makeHarness(
      'success',
      `writeOutcome({ outcome: 'success', result: {
         echoedToken: process.env.GBRAIN_JOB_LOCK_TOKEN,
         isChild: process.env.GBRAIN_JOB_CHILD,
         parentPid: process.env.GBRAIN_JOB_PARENT_PID,
         poolSize: process.env.GBRAIN_POOL_SIZE,
         directPoolSize: process.env.GBRAIN_DIRECT_POOL_SIZE,
         argv: process.argv.slice(2),
       }});\n` +
      `process.exit(0);\n`,
    );
    const result = (await runJobInChild(baseOpts(harness))) as Record<string, unknown>;
    expect(result.echoedToken).toBe('tok-cjr');
    expect(result.isChild).toBe('1');
    expect(result.parentPid).toBe(String(process.pid));
    expect(result.poolSize).toBe('3');
    expect(result.directPoolSize).toBe('1'); // codex-2 #6: child direct pool bounded
    expect(result.argv).toEqual(['jobs', 'run-child', '--job-id', '77']);
  }, TEST_TIMEOUT_MS);

  // The shell opt-in travels to the isolated child as a FLAG (buildChildArgs):
  // the child re-runs the cwd-.env quarantine in the worker's cwd, so an
  // env-only handoff of GBRAIN_ALLOW_SHELL_JOBS would be dropped again there.
  test('--allow-shell-jobs pass-through: opts.env GBRAIN_ALLOW_SHELL_JOBS=1 appends the flag; any other value does not', async () => {
    const harness = makeHarness(
      'argv-echo',
      `writeOutcome({ outcome: 'success', result: { argv: process.argv.slice(2) } });\n` +
      `process.exit(0);\n`,
    );
    const on = (await runJobInChild({ ...baseOpts(harness), env: { ...process.env, GBRAIN_ALLOW_SHELL_JOBS: '1' } })) as { argv: string[] };
    expect(on.argv).toEqual(['jobs', 'run-child', '--job-id', '77', '--allow-shell-jobs']);
    const off = (await runJobInChild({ ...baseOpts(harness), env: { ...process.env, GBRAIN_ALLOW_SHELL_JOBS: 'true' } })) as { argv: string[] };
    expect(off.argv).toEqual(['jobs', 'run-child', '--job-id', '77']);
  }, TEST_TIMEOUT_MS);

  test('error outcome (exit 0) throws the reconstructed class', async () => {
    const harness = makeHarness(
      'error-unrecoverable',
      `writeOutcome({ outcome: 'error', errorKind: 'unrecoverable', message: 'bad schema, never retry' });\n` +
      `process.exit(0);\n`,
    );
    await expect(runJobInChild(baseOpts(harness))).rejects.toBeInstanceOf(UnrecoverableError);
  }, TEST_TIMEOUT_MS);

  test('rate-lease outcome rebuilds RateLeaseUnavailableError with fields', async () => {
    const harness = makeHarness(
      'error-lease',
      `writeOutcome({ outcome: 'error', errorKind: 'rate_lease', message: 'lease full', lease: { key: 'anthropic', active: 4, max: 4 } });\n` +
      `process.exit(0);\n`,
    );
    try {
      await runJobInChild(baseOpts(harness));
      throw new Error('should have thrown');
    } catch (e) {
      expect(e).toBeInstanceOf(RateLeaseUnavailableError);
      expect((e as RateLeaseUnavailableError).key).toBe('anthropic');
    }
  }, TEST_TIMEOUT_MS);

  test('exit 1 with no outcome file → generic throw naming the exit code', async () => {
    const harness = makeHarness('crash', `process.exit(1);\n`);
    await expect(runJobInChild(baseOpts(harness))).rejects.toThrow(/exit code=1/);
  }, TEST_TIMEOUT_MS);

  test('SIGTERM-ignoring child: abort → group SIGKILL at the injected grace', async () => {
    const harness = makeHarness(
      'stubborn',
      `process.on('SIGTERM', () => {});\n` +
      `setInterval(() => {}, 1000);\n`, // never exits voluntarily
    );
    const abort = new AbortController();
    const ready = readiness('stubborn');
    const opts = { ...baseOpts(harness), abortSignal: abort.signal, killGraceMs: 400, env: ready.env };
    const p = runJobInChild(opts);
    await ready.wait();
    abort.abort(new Error('timeout'));
    const started = Date.now();
    await expect(p).rejects.toThrow(/terminated after abort/);
    // Died via the SIGKILL escalation, not the 30s force-evict scale.
    expect(Date.now() - started).toBeLessThan(5_000);
  }, TEST_TIMEOUT_MS);

  test('pre-aborted signal: child is terminated promptly', async () => {
    const harness = makeHarness(
      'prekilled',
      `setInterval(() => {}, 1000);\n`,
    );
    const abort = new AbortController();
    abort.abort(new Error('cancel'));
    const opts = { ...baseOpts(harness), abortSignal: abort.signal, killGraceMs: 400 };
    await expect(runJobInChild(opts)).rejects.toThrow(/terminated after abort/);
  }, TEST_TIMEOUT_MS);

  test('spawn ENOENT → typed executable configuration failure', async () => {
    const opts = {
      ...baseOpts('/nonexistent'),
      invocation: { cmd: '/nonexistent/gbrain-binary', argsPrefix: [] },
    };
    await expect(runJobInChild(opts)).rejects.toBeInstanceOf(LocalConfigurationError);
  }, TEST_TIMEOUT_MS);

  test('worker shutdown: child finishes + reports during the drain window → normal success', async () => {
    const harness = makeHarness(
      'graceful-drain',
      `let done = false;\n` +
      `process.on('SIGTERM', () => {\n` +
      `  writeOutcome({ outcome: 'success', result: { finishedDuringDrain: true } });\n` +
      `  done = true; process.exit(0);\n` +
      `});\n` +
      `setInterval(() => {}, 1000);\n`,
    );
    const shutdown = new AbortController();
    const ready = readiness('graceful-drain');
    const opts = { ...baseOpts(harness), shutdownSignal: shutdown.signal, killGraceMs: 5_000, env: ready.env };
    const p = runJobInChild(opts);
    await ready.wait();
    shutdown.abort(new Error('worker-shutdown'));
    const result = (await p) as Record<string, unknown>;
    expect(result.finishedDuringDrain).toBe(true);
  }, TEST_TIMEOUT_MS);

  test('worker shutdown: child that cannot report → ChildWorkerShutdownError (no attempt burned)', async () => {
    const harness = makeHarness(
      'shutdown-stubborn',
      `process.on('SIGTERM', () => {});\n` +
      `setInterval(() => {}, 1000);\n`,
    );
    const shutdown = new AbortController();
    const ready = readiness('shutdown-stubborn');
    const opts = { ...baseOpts(harness), shutdownSignal: shutdown.signal, killGraceMs: 400, env: ready.env };
    const p = runJobInChild(opts);
    await ready.wait();
    shutdown.abort(new Error('worker-shutdown'));
    await expect(p).rejects.toBeInstanceOf(ChildWorkerShutdownError);
  }, TEST_TIMEOUT_MS);

  test('ERROR outcome during worker shutdown → ChildWorkerShutdownError, not a burned attempt (adversarial P2)', async () => {
    // A cooperative handler that bails on shutdown and reports an error must
    // be RELEASED — punishing exactly the well-behaved handlers on every
    // deploy inverts the no-burn guarantee.
    const harness = makeHarness(
      'shutdown-error-report',
      `process.on('SIGTERM', () => {\n` +
      `  writeOutcome({ outcome: 'error', errorKind: 'generic', message: 'aborted: shutdown' });\n` +
      `  process.exit(0);\n` +
      `});\n` +
      `setInterval(() => {}, 1000);\n`,
    );
    const shutdown = new AbortController();
    const ready = readiness('shutdown-error-report');
    const opts = { ...baseOpts(harness), shutdownSignal: shutdown.signal, killGraceMs: 5_000, env: ready.env };
    const p = runJobInChild(opts);
    await ready.wait();
    shutdown.abort(new Error('worker-shutdown'));
    await expect(p).rejects.toBeInstanceOf(ChildWorkerShutdownError);
  }, TEST_TIMEOUT_MS);

  test('UnrecoverableError outcome during worker shutdown dead-letters as usual (W9F item 6, Decision 7)', async () => {
    // A deterministic failure that coincides with a deploy must not be handed
    // back for a second (possibly paid) run with its real error lost.
    const harness = makeHarness(
      'shutdown-unrecoverable-report',
      `process.on('SIGTERM', () => {\n` +
      `  writeOutcome({ outcome: 'error', errorKind: 'unrecoverable', message: 'invalid params: slug is required' });\n` +
      `  process.exit(0);\n` +
      `});\n` +
      `setInterval(() => {}, 1000);\n`,
    );
    const shutdown = new AbortController();
    const ready = readiness('shutdown-unrecoverable-report');
    const opts = { ...baseOpts(harness), shutdownSignal: shutdown.signal, killGraceMs: 5_000, env: ready.env };
    const p = runJobInChild(opts);
    await ready.wait();
    shutdown.abort(new Error('worker-shutdown'));
    const err = await p.then(() => null, (e: unknown) => e);
    expect(err).toBeInstanceOf(UnrecoverableError);
    expect((err as Error).message).toBe('invalid params: slug is required');
  }, TEST_TIMEOUT_MS);

  test('watchdog drain (BOTH signals aborted, non-per-job reason) → shutdown class, not a burned attempt (adversarial P3)', async () => {
    const harness = makeHarness(
      'watchdog-stubborn',
      `process.on('SIGTERM', () => {});\n` +
      `setInterval(() => {}, 1000);\n`,
    );
    const abort = new AbortController();
    const shutdown = new AbortController();
    const ready = readiness('watchdog-stubborn');
    const opts = {
      ...baseOpts(harness),
      abortSignal: abort.signal,
      shutdownSignal: shutdown.signal,
      killGraceMs: 400,
      env: ready.env,
    };
    const p = runJobInChild(opts);
    await ready.wait();
    // gracefulShutdown('watchdog') aborts BOTH — shutdown classification must win.
    shutdown.abort(new Error('watchdog'));
    abort.abort(new Error('watchdog'));
    await expect(p).rejects.toBeInstanceOf(ChildWorkerShutdownError);
  }, TEST_TIMEOUT_MS);

  test('per-job reason (timeout) wins over a concurrent shutdown — attempt semantics preserved', async () => {
    const harness = makeHarness(
      'timeout-during-shutdown',
      `process.on('SIGTERM', () => {});\n` +
      `setInterval(() => {}, 1000);\n`,
    );
    const abort = new AbortController();
    const shutdown = new AbortController();
    const ready = readiness('timeout-during-shutdown');
    const opts = {
      ...baseOpts(harness),
      abortSignal: abort.signal,
      shutdownSignal: shutdown.signal,
      killGraceMs: 400,
      env: ready.env,
    };
    const p = runJobInChild(opts);
    await ready.wait();
    shutdown.abort(new Error('worker-shutdown'));
    abort.abort(new Error('timeout')); // the JOB was targeted — not shutdown class
    await expect(p).rejects.toThrow(/terminated after abort/);
  }, TEST_TIMEOUT_MS);

  test('bootstrap exit codes: 13 → ChildSpawnInfraError, 14 → ChildNotClaimedError (no burned attempts)', async () => {
    const usage = makeHarness('exit13', `process.exit(13);\n`);
    await expect(runJobInChild(baseOpts(usage))).rejects.toBeInstanceOf(ChildSpawnInfraError);

    const notClaimed = makeHarness('exit14', `process.exit(14);\n`);
    await expect(runJobInChild(baseOpts(notClaimed))).rejects.toBeInstanceOf(ChildNotClaimedError);
  }, TEST_TIMEOUT_MS);

  test('child pool-size env: a STRICTER user GBRAIN_POOL_SIZE is respected; explicit child override wins; invalid falls back', async () => {
    const harness = makeHarness(
      'pool-echo',
      `writeOutcome({ outcome: 'success', result: { poolSize: process.env.GBRAIN_POOL_SIZE } });\n` +
      `process.exit(0);\n`,
    );
    const run = (env: Record<string, string | undefined>) =>
      runJobInChild({ ...baseOpts(harness), env: { ...process.env, ...env } }) as Promise<{ poolSize: string }>;

    expect((await run({ GBRAIN_POOL_SIZE: '2' })).poolSize).toBe('2'); // stricter user tuning respected
    expect((await run({ GBRAIN_POOL_SIZE: '10' })).poolSize).toBe('3'); // never raised above the child default
    expect((await run({ GBRAIN_POOL_SIZE: '2', GBRAIN_JOB_CHILD_POOL_SIZE: '5' })).poolSize).toBe('5'); // explicit knob wins
    expect((await run({ GBRAIN_JOB_CHILD_POOL_SIZE: 'abc' })).poolSize).toBe('3'); // invalid → default
  }, TEST_TIMEOUT_MS);
});