Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
7.4 KB
Raw Blame
/**
 * issue #1801 fix #2 — the worker's DB-liveness probe runs EVEN under a
 * supervisor (GBRAIN_SUPERVISED=1), so a supervised worker whose own pool dies
 * self-exits (db_dead → CLI process.exit(1) → supervisor respawns a fresh pool)
 * instead of sitting alive-but-wedged forever. Stall detection STAYS gated to
 * non-supervised (the supervisor's progress watchdog owns forward-progress).
 *
 * Behavioral: real PGLite engine with `executeRaw('SELECT 1')` monkeypatched to
 * throw, so the probe fails and emitUnhealthy('db_dead') fires after
 * dbFailExitAfter probes. Structural: pins the gating split in worker.ts so a
 * refactor can't silently re-disable the probe under supervision.
 */

import { describe, it, expect, beforeAll, afterAll } from 'bun:test';
import { readFileSync } from 'fs';
import { join } from 'path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { MinionWorker, type UnhealthyReason } from '../src/core/minions/worker.ts';
import { withEnv } from './helpers/with-env.ts';

// No resetPgliteState/beforeEach: these tests never insert jobs (empty queue),
// and resetPgliteState TRUNCATEs the `config` table that carries the minion
// schema `version` key the worker's ensureSchema() reads.
let engine: PGLiteEngine;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
});

afterAll(async () => {
  await engine.disconnect();
});

/** Make the DB-liveness probe (`SELECT 1`) throw; delegate every other query.
 *  Returns a restore fn that removes the instance override (falls back to the
 *  prototype method). Also records the probe's `opts` so tests can pin that
 *  the worker passes a cancellation signal (issue #6: a hung probe must be
 *  cancelled, not abandoned on a checked-out pool slot). */
const probeOptsSeen: Array<{ signal?: AbortSignal } | undefined> = [];
function breakLivenessProbe(eng: PGLiteEngine): () => void {
  const real = eng.executeRaw.bind(eng);
  (eng as { executeRaw: unknown }).executeRaw = async (
    sql: string,
    params?: unknown[],
    opts?: { signal?: AbortSignal },
  ) => {
    if (typeof sql === 'string' && sql.trim() === 'SELECT 1') {
      probeOptsSeen.push(opts);
      throw new Error('probe boom (simulated dead pool)');
    }
    return real(sql, params as never);
  };
  return () => { delete (eng as { executeRaw?: unknown }).executeRaw; };
}

async function runUntilUnhealthy(supervised: boolean): Promise<UnhealthyReason | null> {
  const restore = breakLivenessProbe(engine);
  try {
    return await withEnv(
      supervised ? { GBRAIN_SUPERVISED: '1' } : { GBRAIN_SUPERVISED: undefined },
      async () => {
        const worker = new MinionWorker(engine, {
          queue: 'default',
          concurrency: 1,
          pollInterval: 25,
          maxRssMb: 0,
          healthCheckInterval: 20,
          dbFailExitAfter: 3,
          dbProbeTimeoutMs: 200,
        });
        worker.register('noop', async () => {});

        const got = new Promise<UnhealthyReason>((resolve) => {
          worker.on('unhealthy', (i) => resolve(i));
        });
        const runPromise = worker.start();
        const captured = await Promise.race([
          got,
          new Promise<null>((r) => setTimeout(() => r(null), 3000)),
        ]);
        worker.stop();
        await runPromise;
        return captured;
      },
    );
  } finally {
    restore();
  }
}

describe('issue #1801 fix #2 — supervised DB self-defense', () => {
  it('a SUPERVISED worker with a dead pool emits db_dead (probe runs under supervision)', async () => {
    const info = await runUntilUnhealthy(true);
    expect(info).not.toBeNull();
    expect(info?.reason).toBe('db_dead');
  }, 10_000);

  it('an UNSUPERVISED worker with a dead pool also emits db_dead (back-compat)', async () => {
    const info = await runUntilUnhealthy(false);
    expect(info).not.toBeNull();
    expect(info?.reason).toBe('db_dead');
  }, 10_000);

  it('the probe passes a cancellation AbortSignal so a hung SELECT 1 releases its slot (issue #6)', () => {
    // Populated by the two runUntilUnhealthy() calls above (>= 3 probes each).
    expect(probeOptsSeen.length).toBeGreaterThan(0);
    for (const opts of probeOptsSeen) {
      expect(opts?.signal).toBeInstanceOf(AbortSignal);
    }
  });

  it('dual-pool probe gating: probeDirect is wired ONLY when isDualPoolActive() is true', async () => {
    // The load-bearing guard the adapter comments on: a kill-switched
    // executeRawDirect would probe the same starved read pool twice and
    // fabricate a verdict (testing specialist coverage gap).
    const directCalls: string[] = [];
    const runCase = async (dualPool: boolean): Promise<void> => {
      const cmStub = { isDualPoolActive: () => dualPool };
      (engine as unknown as { connectionManager?: unknown }).connectionManager = cmStub;
      const realDirect = engine.executeRawDirect.bind(engine);
      (engine as unknown as { executeRawDirect: unknown }).executeRawDirect = async (
        sql: string,
        params?: unknown[],
        opts?: { signal?: AbortSignal },
      ) => {
        if (typeof sql === 'string' && sql.trim() === 'SELECT 1') {
          directCalls.push(`dual=${dualPool}`);
          return [];
        }
        return realDirect(sql, params as never, opts);
      };
      const restore = breakLivenessProbe(engine);
      try {
        await withEnv({ GBRAIN_SUPERVISED: '1' }, async () => {
          const worker = new MinionWorker(engine, {
            queue: 'default', concurrency: 1, pollInterval: 25, maxRssMb: 0,
            healthCheckInterval: 20, dbFailExitAfter: 2, dbProbeTimeoutMs: 200,
          });
          worker.register('noop', async () => {});
          const got = new Promise<UnhealthyReason>((resolve) => {
            worker.on('unhealthy', (i) => resolve(i));
          });
          const runPromise = worker.start();
          const info = await Promise.race([
            got,
            new Promise<null>((r) => setTimeout(() => r(null), 3000)),
          ]);
          worker.stop();
          await runPromise;
          expect(info).not.toBeNull();
          if (dualPool) {
            // Direct lane succeeded -> verdict names the pooler path.
            expect(info!.reason === 'db_dead' && info!.verdict).toBe('pool_starved');
          } else {
            expect(info!.reason === 'db_dead' && info!.verdict).toBe('unknown');
          }
        });
      } finally {
        restore();
        delete (engine as unknown as { executeRawDirect?: unknown }).executeRawDirect;
        delete (engine as unknown as { connectionManager?: unknown }).connectionManager;
      }
    };

    await runCase(false);
    expect(directCalls.length).toBe(0); // single-pool: NEVER probes direct

    await runCase(true);
    expect(directCalls.filter((c) => c === 'dual=true').length).toBeGreaterThan(0);
  }, 15_000);

  it('structural: DB probe is NOT gated on !isSupervisedChild; stall detection IS', () => {
    const src = readFileSync(
      join(import.meta.dir, '..', 'src', 'core', 'minions', 'worker.ts'),
      'utf8',
    );
    // Outer self-health guard must NOT require non-supervised anymore.
    expect(src).toContain('if (this.opts.healthCheckInterval > 0) {');
    expect(src).not.toContain('if (!isSupervisedChild && this.opts.healthCheckInterval > 0)');
    // Stall detection must still be wrapped in a non-supervised guard.
    expect(src).toMatch(/Stall detection \(NON-supervised only\)[\s\S]*?if \(!isSupervisedChild\)/);
  });
});