Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.3 KB
Raw Blame
/**
 * issue #5 — full parent path with isolation on: claim → spawn child →
 * decode outcome → completeJob / failJob / release. Real in-memory PGLite
 * worker + the fake-run-child.mjs fixture standing in for the compiled CLI
 * (reads the isolation env contract, writes canned outcomes).
 *
 * Pins:
 *   - success: job completes with the CHILD's result (fenced completeJob)
 *   - error outcome: failJob path — attempt burned, delayed/dead per policy
 *   - crash (exit 1, no file): attempt burned
 *   - spawn failure (bad child CLI): RELEASED — status stays 'active',
 *     attempts NOT burned (infra class; stall sweeper would requeue)
 */

import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { MinionQueue } from '../src/core/minions/queue.ts';
import { MinionWorker } from '../src/core/minions/worker.ts';
import { withEnv } from './helpers/with-env.ts';

const FIXTURE = join(import.meta.dir, 'fixtures', 'fake-run-child.mjs');

let engine: PGLiteEngine;
let queue: MinionQueue;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({ database_url: '' });
  await engine.initSchema();
  queue = new MinionQueue(engine);
});

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await engine.executeRaw('DELETE FROM minion_jobs');
});

function makeWorker(invocationCmd = process.execPath, argsPrefix = [FIXTURE]) {
  const worker = new MinionWorker(engine, {
    queue: 'default',
    concurrency: 1,
    pollInterval: 25,
    healthCheckInterval: 0,
    maxRssMb: 0,
    jobIsolation: 'process',
    childCliInvocation: { cmd: invocationCmd, argsPrefix },
  });
  // Handler must exist in the parent registry (name-scoped claiming); its
  // body never runs in isolation mode.
  worker.register('isotest', async () => {
    throw new Error('parent-side handler must not run when isolated');
  });
  return worker;
}

async function runWorkerUntil(
  worker: MinionWorker,
  done: () => Promise<boolean>,
  timeoutMs = 10_000,
): Promise<void> {
  const run = worker.start();
  const deadline = Date.now() + timeoutMs;
  try {
    while (Date.now() < deadline) {
      if (await done()) return;
      await new Promise((r) => setTimeout(r, 50));
    }
    throw new Error('runWorkerUntil timed out');
  } finally {
    worker.stop();
    await run;
  }
}

async function jobRow(id: number): Promise<{ status: string; attempts_made: number; result: unknown; error_text: string | null }> {
  const rows = await engine.executeRaw<{
    status: string;
    attempts_made: number;
    result: unknown;
    error_text: string | null;
  }>('SELECT status, attempts_made, result, error_text FROM minion_jobs WHERE id = $1', [id]);
  if (!rows[0]) throw new Error('job row missing');
  return rows[0];
}

describe('worker with jobIsolation=process (PGLite + fake child)', () => {
  test("construction throws for jobIsolation 'process' without childCliInvocation (predicate-mismatch guard)", () => {
    // Red-team finding: 'process' without an invocation silently executed
    // handlers INLINE while the evict path believed it was isolated.
    expect(
      () => new MinionWorker(engine, { queue: 'default', jobIsolation: 'process' }),
    ).toThrow(/childCliInvocation/);
  });

  test('spawn-failure circuit breaker: 3 consecutive bootstrap failures emit unhealthy(child_spawn_failing)', async () => {
    // Red-team finding: a deterministic child-bootstrap failure looped
    // claim/release forever, invisible to the stall detector (every settle
    // refreshes the progress clock).
    const j1 = await queue.add('isotest', {});
    const j2 = await queue.add('isotest', {});
    const j3 = await queue.add('isotest', {});
    const worker = makeWorker(process.execPath, ['-e', 'process.exit(13)']);
    const unhealthy: unknown[] = [];
    worker.on('unhealthy', (i) => unhealthy.push(i));
    const run = worker.start();
    const deadline = Date.now() + 10_000;
    try {
      while (Date.now() < deadline && unhealthy.length === 0) {
        await new Promise((r) => setTimeout(r, 50));
      }
    } finally {
      worker.stop();
      await run;
    }
    expect(unhealthy.length).toBeGreaterThan(0);
    const info = unhealthy[0] as { reason: string; consecutiveFailures: number };
    expect(info.reason).toBe('child_spawn_failing');
    expect(info.consecutiveFailures).toBeGreaterThanOrEqual(3);
    // No attempts burned anywhere — all three rows released, not failed.
    for (const j of [j1, j2, j3]) {
      const row = await jobRow(j.id);
      expect(row.status).toBe('active');
      expect(row.attempts_made).toBe(0);
    }
  }, 20_000);

  test('success: child result lands via the fenced completeJob', async () => {
    await withEnv({ FAKE_RUN_CHILD_MODE: 'success' }, async () => {
      const job = await queue.add('isotest', { prompt: 'hi' });
      const worker = makeWorker();
      await runWorkerUntil(worker, async () => (await jobRow(job.id)).status === 'completed');
      const row = await jobRow(job.id);
      expect(row.status).toBe('completed');
      const result = typeof row.result === 'string' ? JSON.parse(row.result) : row.result;
      expect((result as { fromChild?: boolean }).fromChild).toBe(true);
      // The child got the REAL claim token through the env contract.
      expect((result as { token?: string }).token).toMatch(/^.+:.+$/);
    });
  }, 20_000);

  test('error outcome: failJob path, attempt burned', async () => {
    await withEnv({ FAKE_RUN_CHILD_MODE: 'error' }, async () => {
      const job = await queue.add('isotest', {}, { max_attempts: 1 });
      const worker = makeWorker();
      await runWorkerUntil(worker, async () => {
        const s = (await jobRow(job.id)).status;
        return s === 'dead' || s === 'failed';
      });
      const row = await jobRow(job.id);
      expect(row.status).toBe('dead'); // maxAttempts 1 → attempt burned → dead
      expect(row.error_text).toContain('fake child handler failure');
    });
  }, 20_000);

  test('crash (exit 1, no outcome file): attempt burned', async () => {
    await withEnv({ FAKE_RUN_CHILD_MODE: 'crash' }, async () => {
      const job = await queue.add('isotest', {}, { max_attempts: 1 });
      const worker = makeWorker();
      await runWorkerUntil(worker, async () => {
        const s = (await jobRow(job.id)).status;
        return s === 'dead' || s === 'failed';
      });
      const row = await jobRow(job.id);
      expect(row.status).toBe('dead');
      expect(row.error_text).toContain('exit code=1');
    });
  }, 20_000);

  test('serialization parity (codex-2 #8): unreportable results fail in BOTH modes, never complete', async () => {
    // Inline: a circular result blows up in completeJob's serialization →
    // failJob (attempt burned).
    const inlineWorker = new MinionWorker(engine, {
      queue: 'default', concurrency: 1, pollInterval: 25, healthCheckInterval: 0, maxRssMb: 0,
    });
    inlineWorker.register('isotest', async () => {
      const a: Record<string, unknown> = {};
      a.self = a; // circular — not JSONB-serializable
      return a;
    });
    const j1 = await queue.add('isotest', {}, { max_attempts: 1 });
    await runWorkerUntil(inlineWorker, async () => {
      const s = (await jobRow(j1.id)).status;
      return s !== 'waiting' && s !== 'active';
    });
    expect((await jobRow(j1.id)).status).not.toBe('completed');

    // Process mode: a child that cannot persist its outcome (exit 15) lands
    // in the same terminal class — failed loudly, never falsely completed.
    await withEnv({ FAKE_RUN_CHILD_MODE: 'exit15' }, async () => {
      const j2 = await queue.add('isotest', {}, { max_attempts: 1 });
      const worker = makeWorker();
      await runWorkerUntil(worker, async () => (await jobRow(j2.id)).status === 'dead');
      expect((await jobRow(j2.id)).error_text).toContain('outcome file');
    });
  }, 30_000);

  test('spawn failure: RELEASED — still active, attempts NOT burned (infra class)', async () => {
    const job = await queue.add('isotest', {});
    const worker = makeWorker(process.execPath, ['-e', 'process.exit(13)']);
    // The claim happens, the spawn fails, the job is released (stays
    // 'active' until lock expiry — the stall sweeper's requeue territory).
    const run = worker.start();
    await new Promise((r) => setTimeout(r, 1_500));
    worker.stop();
    await run;
    const row = await jobRow(job.id);
    expect(row.status).toBe('active'); // NOT dead, NOT failed
    expect(row.attempts_made).toBe(0); // release = no failJob = no attempt burned
  }, 20_000);

  test('missing executable blocks once and releases only the owned claim', async () => {
    const first = await queue.add('isotest', {});
    const second = await queue.add('isotest', {});
    const worker = makeWorker('/nonexistent/gbrain-binary', []);
    const events: unknown[] = [];
    worker.on('unhealthy', event => events.push(event));
    await worker.start();
    expect(events).toHaveLength(1);
    expect(events[0]).toMatchObject({ reason: 'client_misconfigured', error: { reasonCode: 'child_executable_invalid' } });
    expect(worker.configurationError?.reasonCode).toBe('child_executable_invalid');
    expect((await jobRow(first.id)).status).toBe('delayed');
    expect((await jobRow(first.id)).attempts_made).toBe(0);
    expect((await jobRow(second.id)).status).toBe('waiting');
    expect((await jobRow(second.id)).attempts_made).toBe(0);
    expect(worker.configurationReleaseResults.map(result => result.outcome)).toEqual(['released']);
  });

  test('typed child outcomes block before cleanup and retain unconfirmed ownership', async () => {
    const first = await queue.add('isotest', {});
    const second = await queue.add('isotest', {});
    const outcome = JSON.stringify({
      outcome: 'error', errorKind: 'local_configuration', protocolVersion: 1,
      reasonCode: 'postgres_cancellation_unavailable', message: 'fixture fault',
    });
    const dir = mkdtempSync(join(tmpdir(), 'gbrain-typed-unconfirmed-'));
    const grandchildPid = join(dir, 'grandchild.pid');
    const grandchild = join(dir, 'grandchild.cjs');
    const parent = join(dir, 'parent.cjs');
    writeFileSync(grandchild, `process.on('SIGTERM', () => {}); require('node:fs').writeFileSync(${JSON.stringify(grandchildPid)}, String(process.pid)); setInterval(() => {}, 1000);`);
    writeFileSync(parent, `const fs = require('node:fs');
      require('node:child_process').spawn(process.execPath, [${JSON.stringify(grandchild)}], { stdio: 'ignore' }).unref();
      while (!fs.existsSync(${JSON.stringify(grandchildPid)})) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5);
      fs.writeFileSync(process.env.GBRAIN_JOB_RESULT_PATH, ${JSON.stringify(outcome)});`);
    try {
      const worker = makeWorker(process.execPath, [parent]);
      const internals = worker as unknown as {
        configurationDeadline: number;
        executions: Map<string, { promise: Promise<void> }>;
      };
      let execution: Promise<void> | undefined;
      worker.on('unhealthy', event => {
        if (event.reason !== 'client_misconfigured') return;
        internals.configurationDeadline = performance.now() + 100;
        execution = [...internals.executions.values()][0]?.promise;
      });
      const began = performance.now();
      await worker.start();
      expect(performance.now() - began).toBeLessThan(5000);
      expect(worker.configurationError?.reasonCode).toBe('postgres_cancellation_unavailable');
      expect((await jobRow(first.id)).status).toBe('active');
      expect((await jobRow(first.id)).attempts_made).toBe(0);
      expect((await jobRow(second.id)).status).toBe('waiting');
      expect(worker.configurationReleaseResults.map(result => result.outcome)).toEqual(['unconfirmed']);
      await execution;
      expect((await jobRow(first.id)).status).toBe('active');
      expect((await jobRow(first.id)).attempts_made).toBe(0);
    } finally {
      if (existsSync(grandchildPid)) { try { process.kill(Number(readFileSync(grandchildPid, 'utf8')), 'SIGKILL'); } catch {} }
      rmSync(dir, { recursive: true, force: true });
    }
  }, 35_000);

  test.skipIf(process.platform !== 'linux')('typed child outcome with confirmed group cleanup releases the claim inside the configuration deadline', async () => {
    const first = await queue.add('isotest', {});
    const second = await queue.add('isotest', {});
    const outcome = JSON.stringify({
      outcome: 'error', errorKind: 'local_configuration', protocolVersion: 1,
      reasonCode: 'postgres_cancellation_unavailable', message: 'fixture fault',
    });
    const program = `require('node:fs').writeFileSync(process.env.GBRAIN_JOB_RESULT_PATH, ${JSON.stringify(outcome)});`;
    const worker = makeWorker(process.execPath, ['-e', program]);
    const began = performance.now();
    await worker.start();
    expect(performance.now() - began).toBeLessThan(5000);
    expect(worker.configurationError?.reasonCode).toBe('postgres_cancellation_unavailable');
    expect(worker.configurationReleaseResults.map(result => result.outcome)).toEqual(['released']);
    expect((await jobRow(first.id)).status).toBe('delayed');
    expect((await jobRow(first.id)).attempts_made).toBe(0);
    expect((await jobRow(second.id)).status).toBe('waiting');
  }, 35_000);
});