Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
16.2 KB
Raw Blame
/**
 * #6317 (B1, T1b unit half): the consumer mode machine in
 * `consumer-election.ts`, driven through its seams (the host's consumer rows,
 * the switch, the ceiling) with a stub full consumer, so every transition is
 * pinned without a database: probing settles to full when the switch is off
 * or no live owner exists, to waiter-only beside a live full resident owner
 * (the whole external surface answers inertly meanwhile); a waiter promotes
 * when its owner lapses 60 s, reports `restart_required`, carries a root
 * barrier past the ceiling or disappears, and stays put while the owner is
 * merely stale; only a promoted consumer drains back, after three healthy
 * ticks of a `full` owner, never for another promoted peer; `stop()` during
 * an unanswered probe returns and the late answer is dropped; the forced
 * own-consumer flag wins; a probe the database cannot answer falls back to a
 * full consumer with one log line; the heartbeat row carries the mode. The
 * Postgres half (real rows, a spawned `serve`) is
 * test/e2e/persistence-two-consumers-postgres.test.ts.
 */
import { afterEach, describe, expect, test } from 'bun:test';
import type { BrainEngine } from '../src/core/engine.ts';
import { LANE_BUSY, type PersistenceConsumer } from '../src/core/persistence/consumer.ts';
import { DRAIN_BACK_HEALTHY_TICKS, electOwner, forceOwnConsumer, PROBE_TIMEOUT_MISSES, resetOwnConsumerForTest, WaiterOnlyConsumer } from '../src/core/persistence/consumer-election.ts';
import { CONSUMER_LAPSED_MS, CONSUMER_LIVE_MS, consumerIdentity, type ListedConsumer } from '../src/core/persistence/consumer-heartbeat.ts';

const CEILING = 600_000;

function fakeEngine(kind: 'postgres' | 'pglite' = 'postgres') {
  const statements: Array<{ sql: string; params: unknown[] }> = [];
  const engine = { kind, executeRaw: async (sql: string, params: unknown[] = []) => { statements.push({ sql, params }); return []; },
    executeRawDirect: async (sql: string, params: unknown[] = []) => { statements.push({ sql, params }); return []; } } as unknown as BrainEngine;
  return { engine, statements, heartbeatModes: () => statements.filter(s => s.sql.includes('INSERT INTO persistence_consumers')).map(s => s.params[5]) };
}

function stubConsumer() {
  const calls: string[] = [];
  let stopped = false;
  const consumer = {
    start() { calls.push('start'); }, async stop() { calls.push('stop'); stopped = true; }, wake(own?: boolean) { calls.push(`wake:${own ?? false}`); },
    async tick() { calls.push('tick'); }, holds: () => false, foregroundCompletions: () => 0, async onLane() { return LANE_BUSY; },
    restartRequired: () => false, oldestRootBarrierAgeMs: () => null,
    status: () => ({ accepting: !stopped, active_preparations: 0, active_worktrees: 0, restart_required: false, from_inner: true }),
  };
  return { consumer: consumer as unknown as PersistenceConsumer, calls };
}

function row(over: Partial<ListedConsumer> = {}): ListedConsumer {
  return { host_id: 'h', pid: 4242, nonce: 'owner-nonce', pid_ns: null, kind: 'serve', mode: 'full', started_at: '2026-10-08T00:00:00Z', renewed_at: '2026-10-08T00:10:00Z',
    restart_required: false, root_barrier_age_ms: null, pool: null, host_json_path: '/owner/.gbrain/persistence/host.json', persistence_home: '/owner/.gbrain/persistence', minted_under: null,
    version: '0.60.200.0', renewed_age_ms: 1_000, age_ms: 600_000, liveness: 'live', self: false, ...over };
}

function harness(opts: { rows?: () => ListedConsumer[] | Promise<ListedConsumer[]>; switchOn?: boolean; kind?: string; engineKind?: 'postgres' | 'pglite' } = {}) {
  const fake = fakeEngine(opts.engineKind);
  const made: ReturnType<typeof stubConsumer>[] = [];
  const logs: string[] = [];
  const consumer = new WaiterOnlyConsumer(fake.engine, { engine: 'postgres' }, () => { const stub = stubConsumer(); made.push(stub); return stub.consumer; },
    { kind: opts.kind ?? 'jobs', hostId: 'h', pollMs: 1_000_000, idleMaxMs: 1_000_000, phaseMs: 200, readConsumers: async () => opts.rows ? opts.rows() : [],
      singleConsumer: async () => opts.switchOn ?? true, ceilingMs: async () => CEILING, heartbeatEveryMs: 1_000_000, log: line => logs.push(line) });
  return { consumer, made, logs, fake };
}

afterEach(() => resetOwnConsumerForTest());

describe('electOwner', () => {
  const self = consumerIdentity();
  test('a live full resident row that is not this process wins; put/cli rows, this process, wedged and stale rows do not', () => {
    expect(electOwner([row()], null, CEILING, self)?.pid).toBe(4242);
    expect(electOwner([row({ kind: 'put' })], null, CEILING, self)).toBeNull();
    expect(electOwner([row({ pid: self.pid, nonce: self.nonce })], null, CEILING, self)).toBeNull();
    expect(electOwner([row({ pid: self.pid, nonce: 'another-process-with-my-pid' })], null, CEILING, self)?.pid).toBe(self.pid);
    expect(electOwner([row({ restart_required: true })], null, CEILING, self)).toBeNull();
    expect(electOwner([row({ root_barrier_age_ms: CEILING })], null, CEILING, self)).toBeNull();
    expect(electOwner([row({ root_barrier_age_ms: CEILING - 1 })], null, CEILING, self)?.pid).toBe(4242);
    expect(electOwner([row({ renewed_age_ms: CONSUMER_LIVE_MS })], null, CEILING, self)).toBeNull();
    expect(electOwner([row({ mode: 'promoted', kind: 'sync' })], null, CEILING, self)?.kind).toBe('sync');
  });
  test('the current owner is kept while merely stale (under 60 s) and dropped once lapsed', () => {
    const current = { pid: 4242, nonce: 'owner-nonce' };
    expect(electOwner([row({ renewed_age_ms: 45_000 })], current, CEILING, self)?.pid).toBe(4242);
    expect(electOwner([row({ renewed_age_ms: CONSUMER_LAPSED_MS })], current, CEILING, self)).toBeNull();
    expect(electOwner([row({ renewed_age_ms: 45_000 }), row({ pid: 7, nonce: 'n7', kind: 'sync' })], null, CEILING, self)?.pid).toBe(7);
  });
});

describe('WaiterOnlyConsumer mode machine', () => {
  test('switch off: full on the first tick without reading the rows; the heartbeat row says full', async () => {
    let reads = 0;
    const h = harness({ switchOn: false, rows: () => { reads++; return [row()]; } });
    h.consumer.start();
    await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('full');
    expect(reads).toBe(0);
    expect(h.made).toHaveLength(1);
    expect(h.made[0]!.calls).toEqual(['start']);
    expect(h.fake.heartbeatModes()).toEqual(['full']);
    await h.consumer.stop();
    expect(h.made[0]!.calls).toContain('stop');
  });
  test('PGLite: full at start, no probe, no heartbeat statement', async () => {
    const h = harness({ engineKind: 'pglite', rows: () => { throw new Error('never read'); } });
    h.consumer.start();
    expect(h.consumer.mode).toBe('full');
    await h.consumer.stop();
    expect(h.fake.statements).toEqual([]);
  });
  test('no live owner: full after the probe; a put-kind row beside it is not an owner', async () => {
    const h = harness({ rows: () => [row({ kind: 'put', pid: 9 }), row({ renewed_age_ms: CONSUMER_LIVE_MS + 1 })] });
    h.consumer.start();
    await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('full');
    await h.consumer.stop();
  });
  test('a live full serve: waiter-only with the consumer surface answering inertly and the owner in status', async () => {
    const h = harness({ rows: () => [row()] });
    h.consumer.start();
    await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('waiter_only');
    expect(h.made).toHaveLength(0);
    expect(h.consumer.holds('x')).toBe(false);
    expect(h.consumer.foregroundCompletions('w')).toBe(0);
    expect(await h.consumer.onLane(async () => 'ran')).toBe(LANE_BUSY);
    expect(h.consumer.restartRequired()).toBe(false);
    h.consumer.wake(true);
    expect(h.made).toHaveLength(0);
    const status = h.consumer.status();
    expect(status).toMatchObject({ accepting: true, active_preparations: 0, active_worktrees: 0, consumer_election: { mode: 'waiter_only', kind: 'jobs', owner: { kind: 'serve', pid: 4242, nonce: 'owner-nonce' } } });
    expect(h.consumer.electedOwner()?.pid).toBe(4242);
    expect(h.fake.heartbeatModes()).toEqual([]);
    expect(h.logs.some(line => line.includes('reason=waiter_only') && line.includes('pid 4242'))).toBe(true);
    await h.consumer.stop();
    expect(h.made).toHaveLength(0);
  });
  for (const [label, lapsedRows] of [
    ['lapsed 60 s', () => [row({ renewed_age_ms: CONSUMER_LAPSED_MS })]],
    ['restart_required while its heartbeat is live', () => [row({ restart_required: true })]],
    ['a root barrier past the ceiling while its heartbeat is live', () => [row({ root_barrier_age_ms: CEILING + 1 })]],
    ['gone', () => []],
    ['no longer full', () => [row({ mode: 'waiter_only' })]],
  ] as const) {
    test(`a waiter promotes on its next tick when the owner is ${label}; the heartbeat row says promoted`, async () => {
      let rows: ListedConsumer[] = [row()];
      const h = harness({ rows: () => rows });
      h.consumer.start();
      await h.consumer.probeTick();
      expect(h.consumer.mode).toBe('waiter_only');
      rows = lapsedRows();
      await h.consumer.probeTick();
      expect(h.consumer.mode).toBe('promoted');
      expect(h.made).toHaveLength(1);
      expect(h.made[0]!.calls).toEqual(['start']);
      expect(h.fake.heartbeatModes()).toEqual(['promoted']);
      expect(h.logs.some(line => line.includes('owner_lapsed promoted_to_consumer'))).toBe(true);
      await h.consumer.stop();
    });
  }
  test('a stale owner (30-60 s) keeps its waiter; a live replacement owner is adopted without a promotion', async () => {
    let rows: ListedConsumer[] = [row()];
    const h = harness({ rows: () => rows });
    h.consumer.start();
    await h.consumer.probeTick();
    rows = [row({ renewed_age_ms: 45_000 })];
    await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('waiter_only');
    rows = [row({ renewed_age_ms: 45_000 }), row({ pid: 7, nonce: 'n7', kind: 'mcp' })];
    await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('waiter_only');
    rows = [row({ pid: 7, nonce: 'n7', kind: 'mcp' })];
    await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('waiter_only');
    expect(h.consumer.electedOwner()?.pid).toBe(7);
    expect(h.made).toHaveLength(0);
    await h.consumer.stop();
  });
  test('a promoted consumer drains back after three healthy ticks of a full owner, then defers again; a promoted peer never drains it back', async () => {
    let rows: ListedConsumer[] = [row()];
    const h = harness({ rows: () => rows });
    h.consumer.start();
    await h.consumer.probeTick();
    rows = [];
    await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('promoted');
    // Another waiter that promoted at the same time is not an elected owner: both stay promoted (doctor names the overlap).
    rows = [row({ mode: 'promoted', kind: 'sync', pid: 8, nonce: 'n8' })];
    for (let i = 0; i < DRAIN_BACK_HEALTHY_TICKS + 1; i++) await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('promoted');
    expect(h.made[0]!.calls).not.toContain('stop');
    // The elected owner recovers: two healthy ticks are not enough, the third drains back; in-flight work finishes through stop().
    rows = [row()];
    await h.consumer.probeTick(); await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('promoted');
    await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('waiter_only');
    expect(h.made[0]!.calls).toContain('stop');
    expect(h.consumer.electedOwner()?.pid).toBe(4242);
    // A heartbeat outage resets the count: healthy, unhealthy, healthy ticks do not add up.
    rows = [];
    await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('promoted');
    expect(h.made).toHaveLength(2);
    rows = [row()]; await h.consumer.probeTick(); await h.consumer.probeTick();
    rows = [row({ renewed_age_ms: CONSUMER_LAPSED_MS })]; await h.consumer.probeTick();
    rows = [row()]; await h.consumer.probeTick(); await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('promoted');
    await h.consumer.probeTick();
    expect(h.consumer.mode).toBe('waiter_only');
    await h.consumer.stop();
    expect(h.fake.heartbeatModes()).toEqual(['promoted', 'promoted']);
    expect(h.fake.statements.filter(s => s.sql.startsWith('DELETE FROM persistence_consumers'))).toHaveLength(2);
  });
  test('stop() during an unanswered probe returns cleanly and the late answer is dropped', async () => {
    const pending = Promise.withResolvers<ListedConsumer[]>();
    const h = harness({ rows: () => pending.promise });
    h.consumer.start();
    const tick = h.consumer.probeTick();
    const stopped = h.consumer.stop();
    pending.resolve([]);
    await tick;
    await stopped;
    expect(h.consumer.mode).toBe('probing');
    expect(h.made).toHaveLength(0);
    expect(h.fake.statements).toEqual([]);
  });
  test('the forced own-consumer flag: full at start, and a waiter already made promotes to full (never drains back)', async () => {
    let rows: ListedConsumer[] = [row()];
    const waiter = harness({ rows: () => rows });
    waiter.consumer.start();
    await waiter.consumer.probeTick();
    expect(waiter.consumer.mode).toBe('waiter_only');
    forceOwnConsumer('no_delegate');
    await waiter.consumer.probeTick();
    expect(waiter.consumer.mode).toBe('full');
    for (let i = 0; i < DRAIN_BACK_HEALTHY_TICKS + 1; i++) await waiter.consumer.probeTick();
    expect(waiter.consumer.mode).toBe('full');
    expect(waiter.made[0]!.calls).not.toContain('stop');
    await waiter.consumer.stop();
    const forced = harness({ rows: () => [row()] });
    forced.consumer.start();
    expect(forced.consumer.mode).toBe('full');
    await forced.consumer.stop();
  });
  test('#6543: a forced consumer logs two_consumers_on_host once when a live serve of this host consumes beside it, and nothing when none does', async () => {
    forceOwnConsumer('opted_out');
    const beside = harness({ kind: 'sync', rows: () => [row({ kind: 'serve', pid: 4242 })] });
    beside.consumer.start();
    await beside.consumer.stop();
    expect(beside.consumer.mode).toBe('full');
    expect(beside.logs.filter(line => line.includes('reason=two_consumers_on_host'))).toEqual([
      'phase=consumer_probe reason=two_consumers_on_host message="this sync process consumes beside serve pid 4242 on this host (forced:opted_out); both claim writes until one exits"']);
    const alone = harness({ kind: 'sync', rows: () => [row({ kind: 'serve', renewed_age_ms: CONSUMER_LAPSED_MS + 1 }), row({ kind: 'jobs', pid: 7 })] });
    alone.consumer.start();
    await alone.consumer.stop();
    expect(alone.logs.filter(line => line.includes('two_consumers_on_host'))).toEqual([]);
  });
  test('promote() turns a waiter into a full consumer in place', async () => {
    const h = harness({ rows: () => [row()] });
    h.consumer.start();
    await h.consumer.probeTick();
    await h.consumer.promote('skew_fallback');
    expect(h.consumer.mode).toBe('full');
    expect(h.made).toHaveLength(1);
    await h.consumer.stop();
  });
  test('a probe the database cannot answer: full with one log line; a probe that times out stays probing until the third miss', async () => {
    const failing = harness({ rows: () => { throw new Error('relation "persistence_consumers" does not exist'); } });
    failing.consumer.start();
    await failing.consumer.probeTick();
    expect(failing.consumer.mode).toBe('full');
    expect(failing.logs.filter(line => line.includes('reason=probe_failed'))).toHaveLength(1);
    await failing.consumer.stop();
    const slow = harness({ rows: () => new Promise<ListedConsumer[]>(() => {}) });
    slow.consumer.start();
    for (let i = 1; i < PROBE_TIMEOUT_MISSES; i++) { await slow.consumer.probeTick(); expect(slow.consumer.mode).toBe('probing'); }
    await slow.consumer.probeTick();
    expect(slow.consumer.mode).toBe('full');
    expect(slow.logs.filter(line => line.includes('reason=probe_timeout'))).toHaveLength(1);
    await slow.consumer.stop();
  });
  test('tick() forwards to the full consumer once one exists', async () => {
    const h = harness({ switchOn: false });
    h.consumer.start();
    await h.consumer.tick();
    await h.consumer.tick(true);
    expect(h.made[0]!.calls).toEqual(['start', 'tick']);
    h.consumer.wake(true);
    expect(h.made[0]!.calls).toContain('wake:true');
    expect(h.consumer.status()).toMatchObject({ from_inner: true, consumer_election: { mode: 'full' } });
    await h.consumer.stop();
  });
});