Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
5.2 KB
Raw Blame
/**
 * #6712 PR 2 (convergence mode f): a pooled connection that dies while a bulk group publishes.
 *
 * Protects: the owner that claimed the group releases every claim of the pass that never committed as soon as the
 * database answers again, so the drain re-claims them at once. Before, the error escaped the pass with each member
 * still `running` under the publication lease, and nothing freed them until that lease lapsed (about six minutes,
 * "claim lapsed: owner missing").
 *
 * Postgres only (bulk groups publish on Postgres).
 */
import { afterAll, beforeAll, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { execFileSync } from 'node:child_process';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { claimWorktree } from '../src/core/persistence/ownership.ts';
import { performSync } from '../src/commands/sync/perform.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
import { installFaultHook } from '../src/core/persistence/fault-points.ts';
import { withEnv } from './helpers/with-env.ts';

const home = mkdtempSync(join(tmpdir(), 'gbrain-connection-loss-'));
let engine: BrainEngine | undefined;
let closePostgres: (() => Promise<void>) | undefined;
const sources: string[] = [];
const git = (root: string, ...args: string[]) => execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
const note = (i: number, word: string) => `---\ntitle: Note ${i}\n---\nA ${word} observation number ${i}.\n`;

beforeAll(async () => {
  if (!process.env.DATABASE_URL) return;
  const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL); engine = pg.engine; closePostgres = pg.close;
}, 120_000);
afterAll(async () => {
  if (engine) await withEnv({ GBRAIN_HOME: home }, async () => {
    await disposePersistenceConsumer(engine!); await engine!.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
    for (const id of sources) await engine!.executeRaw('DELETE FROM sources WHERE id=$1', [id]);
  });
  await closePostgres?.();
  rmSync(home, { recursive: true, force: true });
});

test('a connection lost while a bulk group publishes frees the group\'s claims at once; the drain finishes without waiting out the publication lease', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_BULK_SIZE: '8' }, async () => {
  const e = engine;
  if (!e) return;
  const id = `loss-${randomUUID().replace(/-/g, '').slice(0, 16)}`; sources.push(id);
  const root = join(home, id); mkdirSync(join(root, 'notes'), { recursive: true }); git(root, 'init', '-q');
  for (let i = 0; i < 40; i++) writeFileSync(join(root, `notes/n${i}.md`), note(i, 'first'));
  git(root, 'add', '.'); git(root, '-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qm', 'notes');
  await e.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
  await e.executeRaw("INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,'{}')", [id, root]);
  await claimWorktree(e, id, root);
  await e.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
  const opts = { sourceId: id, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes: 1 };
  expect((await performSync(e, opts)).drain).toMatchObject({ outcome: 'synced' });
  for (let i = 0; i < 40; i++) writeFileSync(join(root, `notes/n${i}.md`), note(i, 'revised'));
  git(root, '-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qam', 'revise');

  // The pooler drops: from the publication's commit boundary on, every statement fails CONNECTION_CLOSED for two seconds.
  const connectionClosed = () => Object.assign(new Error('write CONNECTION_CLOSED 127.0.0.1:6543'), { code: 'CONNECTION_CLOSED' });
  let downUntil = 0, dropped = false;
  const executeRaw = e.executeRaw;
  e.executeRaw = (async function (this: BrainEngine, ...args: Parameters<BrainEngine['executeRaw']>) {
    if (Date.now() < downUntil) throw connectionClosed();
    return executeRaw.apply(this, args);
  }) as BrainEngine['executeRaw'];
  installFaultHook(async (point, detail) => {
    if (dropped || point !== 'publication:before_commit' || detail.sourceId !== id) return;
    dropped = true; downUntil = Date.now() + 2000;
    throw connectionClosed();
  });
  const started = Date.now();
  let result;
  try { result = await performSync(e, opts); }
  finally { installFaultHook(undefined); e.executeRaw = executeRaw; }
  expect(dropped).toBe(true);
  expect(result.drain).toMatchObject({ outcome: 'synced' });
  // The owner-kill bound the convergence gate holds a pooler drop to (mode f), far below the publication lease.
  expect(Date.now() - started).toBeLessThan(70_000);
  const stuck = await e.executeRaw("SELECT 1 FROM persistence_requests WHERE source_id=$1 AND state IN ('queued','running')", [id]);
  expect(stuck).toHaveLength(0);
  for (let i = 0; i < 40; i++) expect((await e.getPage(`notes/n${i}`, { sourceId: id }))?.compiled_truth).toContain(`revised observation number ${i}`);
}), 600_000);