Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
20.0 KB
Raw Blame
/**
 * #6317 (T1, T1b on Postgres): two processes, one brain. A real `gbrain serve
 * --http` subprocess (a full consumer with its heartbeat row) and this
 * process's managed drain (`performSync` as the `sync` kind) share one
 * isolated Postgres, a 100-page git fixture and lanes 4.
 *
 * With `persistence.single_consumer` on: the drain finishes, exactly one
 * full-mode heartbeat row exists on the host (the serve's), the CLI found the
 * owner through the table alone (its consumer reports `waiter_only` with the
 * serve's identity) and only the serve's pid appears in the members'
 * `claim_phase.owner`. With it off (the shipped default): both processes
 * run full consumers, both rows exist, the group still finishes. T1b: two
 * starters within one renewal both start full and doctor reports the
 * persistent overlap once both are older than 30 s; a waiter whose owner
 * disappears promotes within two leases and drains back after the owner has
 * been live for three ticks. Fails on the base branch: no heartbeat table,
 * no election, and the CLI's own pid stamps the claims whatever the switch.
 * #6543: with the switch on and a serve whose heartbeat renews while it
 * claims nothing, `--no-delegate` runs the sync's own consumer and drains its
 * rows (before: waiter-only behind the wedged serve, nothing claimed).
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { execFileSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { createServer } from 'node:net';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../../src/core/engine.ts';
import { PersistenceConsumer } from '../../src/core/persistence/consumer.ts';
import { resetOwnConsumerForTest, WaiterOnlyConsumer } from '../../src/core/persistence/consumer-election.ts';
import { maybeDelegateManagedSyncToServe } from '../../src/commands/sync-delegate.ts';
import { consumerIdentity, listHostConsumers, resetConsumerIdentityForTest, startConsumerHeartbeat, type ListedConsumer } from '../../src/core/persistence/consumer-heartbeat.ts';
import { claimOwner, setClaimOwnerForTest } from '../../src/core/persistence/claim-phase.ts';
import { localHostIdentity } from '../../src/core/persistence/identity.ts';
import { claimWorktree } from '../../src/core/persistence/ownership.ts';
import { disposePersistenceConsumer, persistenceConsumerStatus, preparePersistedMutation, startPersistenceConsumer } from '../../src/core/persistence/service.ts';
import { resetWriteSwitches } from '../../src/core/persistence/switches.ts';
import { performSync } from '../../src/commands/sync/perform.ts';
import { twoConsumersOnHostCheck } from '../../src/commands/doctor/checks/persistence-consumers.ts';
import { isolatedPersistencePostgres } from '../helpers/persistence-postgres.ts';
import { hasDatabase } from './helpers.ts';
import { withEnv } from '../helpers/with-env.ts';
import { waitFor } from '../helpers/wait-for.ts';

const REPO = join(import.meta.dir, '..', '..');
const home = mkdtempSync(join(tmpdir(), 'gbrain-two-consumers-'));
let engine: BrainEngine;
let databaseUrl = '';
let close: (() => Promise<void>) | undefined;
let hostId = '';
const original = consumerIdentity();
const originalOwner = claimOwner();
const PROVIDER_KEYS = /(ANTHROPIC|OPENAI|GEMINI|GOOGLE|VOYAGE|TYPESAFE)_API_KEY|^GBRAIN_(DATABASE_URL|DIRECT_DATABASE_URL)$|^DATABASE_URL$/;

function git(root: string, ...args: string[]): string { return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); }
const pad = (i: number) => String(i).padStart(3, '0');
async function fixture(count: number) {
  const id = `two-${randomUUID().replace(/-/g, '').slice(0, 12)}`;
  const root = join(home, id); mkdirSync(root); git(root, 'init', '-q');
  for (let i = 0; i < count; i++) {
    const full = join(root, 'notes', `n${pad(i)}.md`); mkdirSync(join(full, '..'), { recursive: true });
    writeFileSync(full, `---\ntitle: Note ${i}\n---\nA durable observation number ${i} about acme-example, see [[notes/n${pad((i + 7) % count)}]].\n`);
  }
  git(root, 'add', '.'); git(root, '-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qm', 'notes');
  await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
  await engine.executeRaw("INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,'{}')", [id, root]);
  await claimWorktree(engine, id, root);
  await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
  return { id, root };
}
function freePort(): Promise<number> {
  return new Promise(resolve => { const server = createServer(); server.listen(0, '127.0.0.1', () => { const port = (server.address() as { port: number }).port; server.close(() => resolve(port)); }); });
}
function serveEnv(): Record<string, string> {
  const env: Record<string, string> = {};
  for (const [k, v] of Object.entries(process.env)) if (v !== undefined && !PROVIDER_KEYS.test(k)) env[k] = v;
  for (const k of ['GBRAIN_TEST_ALLOW_DATABASE_URL', 'GBRAIN_SOURCE', 'CLAUDECODE', 'CODEX_HOME']) delete env[k];
  Object.assign(env, { HOME: home, GBRAIN_HOME: home, GBRAIN_SKIP_STARTUP_HOOKS: '1', NO_COLOR: '1', GBRAIN_NO_RETRY_CONNECT: '1', GBRAIN_ADMIN_BOOTSTRAP_TOKEN: 'two-consumers-test-token-0123456789abcdef' });
  return env;
}
/** A real `gbrain serve --http` against the isolated brain; resolves once /health answers. */
async function spawnServe() {
  const port = await freePort();
  const child = Bun.spawn([process.execPath, '--no-env-file', join(REPO, 'src', 'cli.ts'), 'serve', '--http', '--port', String(port)], { cwd: REPO, env: serveEnv(), stdout: 'pipe', stderr: 'pipe' });
  let stderr = '';
  void new Response(child.stderr).text().then(text => { stderr += text; });
  await waitFor(async () => { try { return (await fetch(`http://127.0.0.1:${port}/health`)).status < 500; } catch { return false; } }, { timeoutMs: 90_000, intervalMs: 250, label: `serve on :${port} answers /health` });
  return { pid: child.pid, port, stderr: () => stderr, async stop() { if (child.exitCode === null) child.kill('SIGTERM'); await Promise.race([child.exited, Bun.sleep(20_000).then(() => { if (child.exitCode === null) child.kill('SIGKILL'); })]); } };
}
const imports = (source: string) => engine.executeRaw<{ slug: string; state: string; owner_pid: number | null; owner_kind: string | null; owner_nonce: string | null }>(
  `SELECT slug,state,(claim_phase->'owner'->>'pid')::int AS owner_pid,claim_phase->'owner'->>'kind' AS owner_kind,claim_phase->'owner'->>'nonce' AS owner_nonce
   FROM persistence_requests WHERE source_id=$1 AND intent->>'kind'='managed_sync_import' ORDER BY sequence`, [source]);

beforeAll(async () => {
  if (!hasDatabase()) return;
  const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!, 'instance', 12);
  engine = pg.engine; close = pg.close; databaseUrl = pg.databaseUrl;
  mkdirSync(join(home, '.gbrain'), { recursive: true });
  writeFileSync(join(home, '.gbrain', 'config.json'), JSON.stringify({ engine: 'postgres', database_url: databaseUrl, embedding_disabled: true }));
  await withEnv({ GBRAIN_HOME: home }, () => { hostId = localHostIdentity().id; });
  // This process plays the sync CLI: a resident kind that defers to a live owner.
  setClaimOwnerForTest({ ...originalOwner, kind: 'sync' });
}, 180_000);
afterAll(async () => {
  setClaimOwnerForTest(undefined);
  resetConsumerIdentityForTest(original);
  if (engine) await withEnv({ GBRAIN_HOME: home }, async () => { await disposePersistenceConsumer(engine); });
  resetWriteSwitches();
  await close?.();
  rmSync(home, { recursive: true, force: true });
});

describe.skipIf(!hasDatabase())('two consumers on one host (Postgres, #6317)', () => {
  for (const single of [true, false]) {
    test(`a serve subprocess plus the CLI drain, lanes 4, 100 pages, persistence.single_consumer=${single}: the group finishes${single ? ' and only the serve publishes' : ' with both consumers full'}`, async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_FAILURES_DIR: home, GBRAIN_SYNC_BULK_SIZE: '8' }, async () => {
      await engine.setConfig('persistence.single_consumer', String(single));
      resetWriteSwitches();
      await disposePersistenceConsumer(engine);
      await engine.executeRaw('DELETE FROM persistence_consumers');
      const serve = await spawnServe();
      try {
        await waitFor(async () => (await listHostConsumers(engine, hostId)).some(row => row.pid === serve.pid && row.kind === 'serve' && row.mode === 'full'), { timeoutMs: 30_000, intervalMs: 200, label: 'the serve writes its heartbeat row' });
        const f = await fixture(100);
        // The CLI's consumer settles before the drain admits anything: while it still probes it claims nothing, and a
        // fast serve could publish all 100 first. Its election is what this test pins, not that race.
        startPersistenceConsumer(engine, { engine: 'postgres' });
        const settled = single ? 'waiter_only' : 'full';
        await waitFor(async () => (persistenceConsumerStatus(engine) as { consumer_election?: { mode: string } }).consumer_election?.mode === settled,
          { timeoutMs: 30_000, intervalMs: 50, label: `the CLI consumer settles to ${settled}` });
        const result = await performSync(engine, { sourceId: f.id, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes: 4 });
        expect(result.drain?.outcome).toBe('synced');
        const rows = await imports(f.id);
        expect(rows).toHaveLength(100);
        expect(rows.every(row => row.state === 'committed')).toBe(true);
        const status = persistenceConsumerStatus(engine) as { consumer_election?: { mode: string; owner: { pid: number; kind: string } | null } };
        const consumers = await listHostConsumers(engine, hostId);
        const full = consumers.filter(row => row.mode === 'full' || row.mode === 'promoted');
        const stampedPids = [...new Set(rows.map(row => row.owner_pid))];
        if (single) {
          // The CLI found the owner through the table alone (no socket on Postgres) and claimed nothing.
          expect(status.consumer_election).toMatchObject({ mode: 'waiter_only', owner: { pid: serve.pid, kind: 'serve' } });
          expect(full.map(row => ({ pid: row.pid, kind: row.kind }))).toEqual([{ pid: serve.pid, kind: 'serve' }]);
          expect(stampedPids).toEqual([serve.pid]);
          expect(rows.every(row => row.owner_kind === 'serve' && row.owner_nonce)).toBe(true);
        } else {
          expect(status.consumer_election).toMatchObject({ mode: 'full' });
          expect(full.map(row => row.kind).sort()).toEqual(['serve', 'sync']);
          expect(full.map(row => row.pid).sort()).toEqual([process.pid, serve.pid].sort());
          expect(stampedPids.every(pid => pid === serve.pid || pid === process.pid)).toBe(true);
          expect(stampedPids).toContain(process.pid);
        }
        expect(await engine.getPage('notes/n042', { sourceId: f.id })).not.toBeNull();
      } finally {
        await disposePersistenceConsumer(engine);
        await serve.stop();
      }
      // The serve's row leaves with it (stop() deletes it); the CLI's, if any, left with dispose.
      await waitFor(async () => (await listHostConsumers(engine, hostId)).every(row => row.pid !== serve.pid), { timeoutMs: 10_000, label: 'the serve row is gone' });
    }), 300_000);
  }

  test('#6543: --no-delegate with single_consumer on, beside a heartbeating serve that claims nothing: the sync runs its own consumer and drains its own rows', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_FAILURES_DIR: home, GBRAIN_SYNC_BULK_SIZE: '8' }, async () => {
    await engine.setConfig('persistence.single_consumer', 'true');
    resetWriteSwitches();
    await disposePersistenceConsumer(engine);
    await engine.executeRaw('DELETE FROM persistence_consumers');
    // The #6543 wedge: a serve whose heartbeat keeps renewing (not restart_required) while it claims nothing.
    const wedged = startConsumerHeartbeat(engine, hostId, { kind: 'serve', mode: 'full', identity: { pid: process.pid + 300_000, nonce: 'wedged-serve', pid_ns: original.pid_ns },
      everyMs: 2_000, report: () => ({ restart_required: false, root_barrier_age_ms: null }) });
    await wedged.renew();
    const stderr: string[] = [];
    const write = process.stderr.write.bind(process.stderr);
    process.stderr.write = ((chunk: string | Uint8Array, ...rest: unknown[]) => { stderr.push(String(chunk)); return (write as (...a: unknown[]) => boolean)(chunk, ...rest); }) as typeof process.stderr.write;
    try {
      const f = await fixture(20);
      const outcome = await maybeDelegateManagedSyncToServe(engine, ['--source', f.id, '--no-pull', '--no-delegate'], { sourceId: f.id, noPull: true });
      expect(outcome).toEqual({ kind: 'own_consumer', reason: 'opted_out', line: null });
      startPersistenceConsumer(engine, { engine: 'postgres' });
      await waitFor(async () => (persistenceConsumerStatus(engine) as { consumer_election?: { mode: string } }).consumer_election?.mode === 'full',
        { timeoutMs: 15_000, intervalMs: 50, label: 'the opted-out sync runs its own full consumer (base branch: waiter_only behind the wedged serve)' });
      await waitFor(async () => stderr.some(line => line.includes('reason=two_consumers_on_host') && line.includes(`serve pid ${process.pid + 300_000}`)),
        { timeoutMs: 10_000, intervalMs: 50, label: 'the forced consumer names the serve beside it' });
      const result = await performSync(engine, { sourceId: f.id, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes: 2 });
      expect(result.drain?.outcome).toBe('synced');
      const rows = await imports(f.id);
      expect(rows).toHaveLength(20);
      expect(rows.every(row => row.state === 'committed' && row.owner_pid === process.pid)).toBe(true);
    } finally {
      process.stderr.write = write;
      resetOwnConsumerForTest();
      await disposePersistenceConsumer(engine);
      await wedged.stop();
    }
  }), 180_000);

  test('T1b: two starters within one renewal both start full; doctor reports the persistent overlap once both are over 30 s old', async () => withEnv({ GBRAIN_HOME: home }, async () => {
    await engine.setConfig('persistence.single_consumer', 'true');
    resetWriteSwitches();
    await engine.executeRaw('DELETE FROM persistence_consumers');
    const make = () => new PersistenceConsumer(engine, { engine: 'postgres' }, preparePersistedMutation, { hostId, pollMs: 1_000_000, onError: () => {} });
    // "Both probe before either has written a row" is the precondition, not a race to win: each starter's first read
    // returns only once the other's first read has returned too. Holding the issue alone was not enough: the starter
    // whose SELECT came back first started its heartbeat, whose first renewal writes its `full` row, and on a loaded
    // host the other starter's already-issued SELECT could still execute after that write and see a live owner.
    const firstReads: Promise<ListedConsumer[]>[] = [];
    const bothRead = Promise.withResolvers<void>();
    const readTogether = () => {
      let first = true;
      return async (signal: AbortSignal) => {
        if (!first) return listHostConsumers(engine, hostId, { signal });
        first = false;
        const mine = listHostConsumers(engine, hostId, { signal });
        if (firstReads.push(mine) === 2) bothRead.resolve();
        await bothRead.promise;
        await Promise.all(firstReads);
        return mine;
      };
    };
    const a = new WaiterOnlyConsumer(engine, { engine: 'postgres' }, make, { kind: 'jobs', hostId, readConsumers: readTogether(), pollMs: 1_000_000, idleMaxMs: 1_000_000, heartbeatEveryMs: 1_000_000, log: () => {} });
    resetConsumerIdentityForTest({ pid: process.pid + 100_000, nonce: 'second-starter' });
    const b = new WaiterOnlyConsumer(engine, { engine: 'postgres' }, make, { kind: 'sync', hostId, readConsumers: readTogether(), pollMs: 1_000_000, idleMaxMs: 1_000_000, heartbeatEveryMs: 1_000_000, log: () => {} });
    resetConsumerIdentityForTest(original);
    try {
      // Both probe before either has written a row: the known residual (a preference, not a fenced role).
      a.start(); b.start();
      await Promise.all([a.probeTick(), b.probeTick()]);
      expect([a.mode, b.mode]).toEqual(['full', 'full']);
      await waitFor(async () => (await listHostConsumers(engine, hostId)).filter(row => row.mode === 'full').length === 2, { timeoutMs: 10_000, label: 'both rows exist' });
      expect((await twoConsumersOnHostCheck(engine, hostId)).status).toBe('ok');
      await engine.executeRaw("UPDATE persistence_consumers SET started_at=now()-interval '45 seconds' WHERE host_id=$1::uuid", [hostId]);
      const check = await twoConsumersOnHostCheck(engine, hostId);
      expect(check.status).toBe('warn');
      expect(check.message).toContain(`jobs pid ${process.pid}`);
      expect(check.message).toContain(`sync pid ${process.pid + 100_000}`);
      // Neither drains back: only a promoted consumer does.
      for (let i = 0; i < 4; i++) { await a.probeTick(); await b.probeTick(); }
      expect([a.mode, b.mode]).toEqual(['full', 'full']);
    } finally { await a.stop(); await b.stop(); }
    expect(await listHostConsumers(engine, hostId)).toEqual([]);
  }), 60_000);

  test('T1b: a waiter beside a live owner claims nothing; when the owner goes it promotes within two leases, and drains back after the owner is live for three ticks', async () => withEnv({ GBRAIN_HOME: home }, async () => {
    await engine.setConfig('persistence.single_consumer', 'true');
    resetWriteSwitches();
    await engine.executeRaw('DELETE FROM persistence_consumers');
    const ownerIdentity = { pid: process.pid + 200_000, nonce: 'owner-serve', pid_ns: original.pid_ns };
    const owner = () => startConsumerHeartbeat(engine, hostId, { kind: 'serve', mode: 'full', identity: ownerIdentity, everyMs: 1_000_000, report: () => ({ restart_required: false, root_barrier_age_ms: null }) });
    let heartbeat = owner();
    await heartbeat.renew();
    const made: PersistenceConsumer[] = [];
    const waiter = new WaiterOnlyConsumer(engine, { engine: 'postgres' }, () => { const c = new PersistenceConsumer(engine, { engine: 'postgres' }, preparePersistedMutation, { hostId, pollMs: 1_000_000, onError: () => {} }); made.push(c); return c; },
      { kind: 'jobs', hostId, pollMs: 1_000_000, idleMaxMs: 1_000_000, heartbeatEveryMs: 1_000_000, log: () => {} });
    try {
      waiter.start();
      await waiter.probeTick();
      expect(waiter.mode).toBe('waiter_only');
      expect(waiter.electedOwner()).toMatchObject({ pid: ownerIdentity.pid, kind: 'serve' });
      expect(made).toHaveLength(0);
      expect((await listHostConsumers(engine, hostId)).map(row => row.pid)).toEqual([ownerIdentity.pid]);
      // The owner exits (its stop() removes the row): the waiter promotes on its next probe and writes a `promoted` row.
      await heartbeat.stop();
      await waiter.probeTick();
      expect(waiter.mode).toBe('promoted');
      expect(made).toHaveLength(1);
      await waitFor(async () => (await listHostConsumers(engine, hostId)).some(row => row.pid === process.pid && row.mode === 'promoted'), { timeoutMs: 10_000, label: 'the promoted row' });
      // The owner comes back: three live ticks later the promoted consumer finishes and defers again; its row goes.
      heartbeat = owner();
      await heartbeat.renew();
      await waiter.probeTick(); await waiter.probeTick();
      expect(waiter.mode).toBe('promoted');
      await waiter.probeTick();
      expect(waiter.mode).toBe('waiter_only');
      expect((await listHostConsumers(engine, hostId)).map(row => row.pid)).toEqual([ownerIdentity.pid]);
      // An owner whose heartbeat merely lapses (no stop) promotes the waiter too, once the row is 60 s stale.
      await engine.executeRaw("UPDATE persistence_consumers SET renewed_at=now()-interval '61 seconds' WHERE pid=$1", [ownerIdentity.pid]);
      await waiter.probeTick();
      expect(waiter.mode).toBe('promoted');
      expect(made).toHaveLength(2);
    } finally { await waiter.stop(); await heartbeat.stop(); }
  }), 60_000);
});