Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.2 KB
Raw Blame
/**
 * W9F item 6 (Decision 7): which handler errors during a worker shutdown burn
 * an attempt, for an in-process worker.
 *
 * Protects: during `requestShutdown()` a handler error is presumed
 * shutdown-induced (cooperative handlers throw plain errors when they bail:
 * subagent, shell), so the claim goes back to `waiting` with no attempt burned
 * and the handler's own text kept after the `worker_shutdown` marker; an
 * `UnrecoverableError` is deterministic and dead-letters on attempt 1 with its
 * original text. The watchdog and configuration-block rows pin today's
 * behavior so a later change to them is deliberate.
 * Fails when: an UnrecoverableError thrown during shutdown is requeued as
 * `worker_shutdown` (the pre-fix behavior: a second, possibly paid, run of a
 * deterministic failure with its real error lost), or a shutdown hand-back
 * overwrites the handler's error text.
 *
 * The forced watchdog case pins the claim round trip: the claim UPDATE has
 * committed (`active`) but its result has not reached the poll loop, so the
 * job is not in `inFlight` when the watchdog fires. Before the fix the job
 * launched unaborted and took the requestShutdown path (`worker_shutdown`
 * hand-back, or the handler's own text for an UnrecoverableError) instead of
 * the watchdog abort; this is the window the un-forced watchdog test hit on a
 * CI Postgres. Since GBRA-130 A-30 the watchdog charges only a job that ran
 * alone when it tripped, so a claim that registers after the drain began goes
 * back to `waiting` uncharged and marked to run alone
 * (watchdog-attribution.ts).
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import type { BrainEngine } from '../src/core/engine.ts';
import { isolatedSharedSkillsEngine } from './helpers/shared-skills-engine.ts';
import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';
import { MinionQueue } from '../src/core/minions/queue.ts';
import { WATCHDOG_DRAIN_RELEASE } from '../src/core/minions/watchdog-attribution.ts';
import { MinionWorker } from '../src/core/minions/worker.ts';
import { UnrecoverableError, type MinionJobContext } from '../src/core/minions/types.ts';
import { LocalConfigurationError } from '../src/core/minions/configuration-error.ts';

type Trigger = 'requestShutdown' | 'watchdog' | 'configuration';
const ERRORS: Record<string, (ctx: MinionJobContext) => unknown> = {
  reason: ctx => ctx.shutdownSignal.reason ?? ctx.signal.reason,
  abortErrorWithCause: ctx => Object.assign(new Error('The operation was aborted'), { name: 'AbortError', cause: ctx.shutdownSignal.reason }),
  wrappedAbort: ctx => new Error('subagent aborted during tool dispatch', { cause: ctx.shutdownSignal.reason }),
  unrecoverable: () => new UnrecoverableError('invalid params: slug is required'),
  plain: () => new Error('handler failed: upstream returned 500'),
};

type WorkerInternals = { queue: MinionQueue; inFlight: Map<number, unknown>; gracefulShutdown(reason: string): void };

/**
 * Without `holdClaimResult` the trigger fires once the handler is running: a
 * row already `active` in the database may still be in the claim round trip,
 * where the watchdog would see a late claim instead of a job running alone.
 * `holdClaimResult`: park the committed claim's result until the trigger has fired, so the trigger lands before launchJob.
 */
async function runCase(engine: BrainEngine, trigger: Trigger, kind: keyof typeof ERRORS, holdClaimResult = false) {
  const queue = new MinionQueue(engine);
  const worker = new MinionWorker(engine, { pollInterval: 10, lockDuration: 60_000, healthCheckInterval: 0, stalledInterval: 600_000 });
  const internals = worker as unknown as WorkerInternals;
  let release!: () => void;
  const gate = new Promise<void>(resolve => { release = resolve; });
  const name = `matrix-${trigger}-${kind}${holdClaimResult ? '-held' : ''}`;
  let handlerStarted!: () => void;
  const started = new Promise<void>(resolve => { handlerStarted = resolve; });
  worker.register(name, async (ctx) => { handlerStarted(); await gate; throw ERRORS[kind]!(ctx); });
  const job = await queue.add(name, {}, { max_attempts: 3 }, { allowProtectedSubmit: true });
  let releaseClaim = () => {};
  let claimCommitted: Promise<void> | null = null;
  if (holdClaimResult) {
    const claim = internals.queue.claim.bind(internals.queue);
    const held = new Promise<void>(resolve => { releaseClaim = resolve; });
    let committed!: () => void;
    claimCommitted = new Promise<void>(resolve => { committed = resolve; });
    internals.queue.claim = (async (...args: Parameters<typeof claim>) => {
      const row = await claim(...args);
      if (row) { committed(); await held; }
      return row;
    }) as typeof claim;
  }
  const running = worker.start();
  if (claimCommitted) {
    await claimCommitted;
    expect((await queue.getJob(job.id))?.status).toBe('active');
    expect(internals.inFlight.size).toBe(0);
  } else {
    await started;
  }
  if (trigger === 'requestShutdown') worker.requestShutdown();
  else if (trigger === 'watchdog') internals.gracefulShutdown('watchdog');
  else worker.blockForConfiguration(new LocalConfigurationError('postgres_cancellation_unavailable', 'fixture cancellation unavailable'));
  releaseClaim();
  release();
  let row = await queue.getJob(job.id);
  for (let i = 0; i < 500 && row?.status === 'active'; i++) { await Bun.sleep(10); row = await queue.getJob(job.id); }
  worker.stop();
  await running;
  row = await queue.getJob(job.id);
  return { status: row?.status, attempts: row?.attempts_made, error: row?.error_text };
}

for (const backend of testBackends()) describe(`worker shutdown error matrix (W9F item 6, in-process, ${backend})`, () => {
  let engine: BrainEngine;
  let close: () => Promise<void>;
  beforeAll(async () => {
    ({ engine, close } = await isolatedSharedSkillsEngine(backend === 'postgres' ? requirePostgresTestDatabase() : undefined));
  }, 120_000);
  afterAll(async () => { await close(); });

  test('requestShutdown: only UnrecoverableError burns an attempt; every hand-back keeps the handler text', async () => {
    const reasonText = 'shutdown';
    expect(await runCase(engine, 'requestShutdown', 'reason')).toEqual({ status: 'waiting', attempts: 0, error: `worker_shutdown: ${reasonText}` });
    expect(await runCase(engine, 'requestShutdown', 'abortErrorWithCause')).toEqual({ status: 'waiting', attempts: 0, error: 'worker_shutdown: The operation was aborted' });
    expect(await runCase(engine, 'requestShutdown', 'wrappedAbort')).toEqual({ status: 'waiting', attempts: 0, error: 'worker_shutdown: subagent aborted during tool dispatch' });
    expect(await runCase(engine, 'requestShutdown', 'plain')).toEqual({ status: 'waiting', attempts: 0, error: 'worker_shutdown: handler failed: upstream returned 500' });
    expect(await runCase(engine, 'requestShutdown', 'unrecoverable')).toEqual({ status: 'dead', attempts: 1, error: 'invalid params: slug is required' });
  }, 60_000);

  test('watchdog: the per-job abort fires, so the job takes the abort path (attempt burned; UnrecoverableError dead)', async () => {
    for (const kind of ['reason', 'abortErrorWithCause', 'wrappedAbort', 'plain'] as const) {
      expect(await runCase(engine, 'watchdog', kind)).toEqual({ status: 'delayed', attempts: 1, error: 'aborted: watchdog' });
    }
    expect(await runCase(engine, 'watchdog', 'unrecoverable')).toEqual({ status: 'dead', attempts: 1, error: 'aborted: watchdog' });
  }, 60_000);

  test('watchdog inside the claim round trip: a claim committed before the shutdown is aborted when it registers and goes back uncharged (it did not run alone when the watchdog tripped)', async () => {
    expect(await runCase(engine, 'watchdog', 'plain', true)).toEqual({ status: 'waiting', attempts: 0, error: WATCHDOG_DRAIN_RELEASE });
    expect(await runCase(engine, 'watchdog', 'unrecoverable', true)).toEqual({ status: 'waiting', attempts: 0, error: WATCHDOG_DRAIN_RELEASE });
  }, 60_000);

  test('configuration block: every error releases through the configuration path with no attempt burned', async () => {
    for (const kind of Object.keys(ERRORS)) {
      const row = await runCase(engine, 'configuration', kind);
      expect(row.attempts).toBe(0);
      expect(row.status).not.toBe('dead');
    }
  }, 60_000);
});