Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #5370: an idle resident consumer must let its Postgres pool drain. Idle
* probes share one reserved connection, every other pooled connection closes
* through the engine's 20 s idle_timeout, and no LISTEN is involved, so the
* transaction-mode PgBouncer path behaves the same as a direct server.
*/
import { describe, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { existsSync, mkdtempSync, readFileSync, readdirSync, readlinkSync, rmSync } from 'node:fs';
import { lookup } from 'node:dns/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import postgres from '#postgres'
import { PostgresEngine } from '../../src/core/postgres-engine.ts';
import { PersistenceConsumer } from '../../src/core/persistence/consumer.ts';
import { admitWrite, getWriteRequestById } from '../../src/core/persistence/journal.ts';
import { admission, fixtures, initializeFixtures, prepared, selectFixtureHost, type HarnessConfig } from '../../scripts/persistence/harness.ts';
import { assertSafeE2eDatabaseUrl } from '../helpers/db-guard.ts';
import { withEnv } from '../helpers/with-env.ts';
import { waitFor } from '../helpers/wait-for.ts';
const direct = process.env.DATABASE_URL;
const pooled = process.env.GBRAIN_PGBOUNCER_URL;
const pooledAdmin = process.env.GBRAIN_PGBOUNCER_DIRECT_URL;
if (process.env.GBRAIN_CI_REQUIRE_PGBOUNCER === '1' && !(pooled && pooledAdmin)) throw new Error('Idle pool drain requires the configured CI PgBouncer fixture.');
const IDLE_DRAIN_MS = 28_000;
async function scratchDatabase(adminUrl: string, clientUrl: string) {
assertSafeE2eDatabaseUrl(adminUrl);
const name = `gbrain_test_idle_pool_${randomUUID().replaceAll('-', '')}`;
const admin = postgres(adminUrl, { max: 1, prepare: false });
await admin.unsafe(`CREATE DATABASE ${name}`);
const url = new URL(clientUrl);
url.pathname = `/${name}`;
return { name, admin, url: url.toString(),
drop: async () => { try { await admin.unsafe(`DROP DATABASE IF EXISTS ${name} WITH (FORCE)`); } finally { await admin.end(); } } };
}
async function withConsumer(url: string, run: (engine: PostgresEngine, consumer: PersistenceConsumer, config: HarnessConfig) => Promise<void>) {
const root = mkdtempSync(join(tmpdir(), 'gbrain-idle-pool-'));
const config: HarnessConfig = { kind: 'postgres', root, dataDir: join(root, 'unused'), hostId: randomUUID(),
seed: 5370, schedules: 0, operations: 0, sourceIds: ['idle-pool'], principalIds: [randomUUID()] };
const engine = new PostgresEngine();
try {
await withEnv({ GBRAIN_HOME: root, GBRAIN_PERSISTENCE_FIXTURE_HOME: root }, async () => {
await engine.connect({ database_url: url, poolSize: 4 });
await engine.initSchema();
selectFixtureHost(config.hostId);
await initializeFixtures(engine, config);
const sources = await fixtures(engine, config);
const consumer = new PersistenceConsumer(engine, { engine: 'postgres' }, async (_e, row) => prepared(row, sources),
{ hostId: config.hostId });
// Saturate the pool the way a busy tick does, then go idle.
await Promise.all(Array.from({ length: 4 }, () => engine.executeRaw('SELECT pg_sleep(0.2)')));
consumer.start();
try { await run(engine, consumer, config); } finally { await consumer.stop(); }
});
} finally {
await engine.disconnect();
rmSync(root, { recursive: true, force: true });
}
}
/** Established TCP sockets this process holds to host:port (Linux /proc). */
async function ownSockets(host: string, port: number): Promise<number> {
const inodes = new Set<string>();
for (const fd of readdirSync('/proc/self/fd')) {
try { const link = readlinkSync(`/proc/self/fd/${fd}`); if (link.startsWith('socket:[')) inodes.add(link.slice(8, -1)); } catch { /* closed */ }
}
const { address, family } = await lookup(host);
const want = family === 4
? address.split('.').map(part => Number(part).toString(16).padStart(2, '0')).reverse().join('').toUpperCase()
: null;
const portHex = port.toString(16).toUpperCase().padStart(4, '0');
let count = 0;
for (const table of ['/proc/self/net/tcp', '/proc/self/net/tcp6']) {
if (!existsSync(table)) continue;
for (const line of readFileSync(table, 'utf8').split('\n').slice(1)) {
const cols = line.trim().split(/\s+/);
if (cols.length < 10 || cols[3] !== '01' || !inodes.has(cols[9]!)) continue;
const [remoteIp, remotePort] = cols[2]!.split(':');
if (remotePort !== portHex) continue;
if (want === null || remoteIp === want || remoteIp!.endsWith(want)) count++;
}
}
return count;
}
describe.skipIf(!direct)('idle persistence consumer on PostgreSQL', () => {
test('an idle consumer drains a direct pool to one backend', async () => {
const db = await scratchDatabase(direct!, direct!);
try {
await withConsumer(db.url, async () => {
const backends = async () => Number((await db.admin.unsafe(
"SELECT count(*)::int AS n FROM pg_stat_activity WHERE datname=$1 AND backend_type='client backend'", [db.name]))[0]!.n);
expect(await backends()).toBeGreaterThanOrEqual(3);
const started = performance.now();
// The pool drains idle_timeout (20 s) after its last pooled statement. The consumer's start-up fan-out and its
// background workers are that last statement on a quiet box; a loaded runner can stretch them past the fixed
// 28 s mark this test used to sleep for, so the drain is awaited (bounded) and then must hold.
await waitFor(async () => (await backends()) === 1, { timeoutMs: IDLE_DRAIN_MS + 20_000, intervalMs: 500 }).catch(() => undefined);
const drainedAfterMs = Math.round(performance.now() - started);
const dump = async () => db.admin.unsafe(`SELECT pid, application_name, backend_type, state, wait_event,
to_char(backend_start, 'HH24:MI:SS.MS') AS backend_start, to_char(state_change, 'HH24:MI:SS.MS') AS state_change,
left(query, 160) AS query FROM pg_stat_activity WHERE datname=$1 ORDER BY backend_start`, [db.name]);
let count = await backends();
if (count !== 1) console.error(`[idle-pool] ${count} backend(s) after ${drainedAfterMs} ms:`, JSON.stringify(await dump(), null, 1));
expect(count).toBe(1);
// Drained means drained: a hold longer than one 5 s probe cycle shows the idle probe itself opens no pooled connection.
await Bun.sleep(6_000);
count = await backends();
if (count !== 1) console.error(`[idle-pool] ${count} backend(s) 6 s after the drain (drain took ${drainedAfterMs} ms):`, JSON.stringify(await dump(), null, 1));
expect(count).toBe(1);
});
} finally { await db.drop(); }
}, 120_000);
test('forced probe: a pooled statement 10 s into the idle window delays the drain by idle_timeout and nothing more', async () => {
// GBRA-60's reproduction of the flake: one ordinary-pool statement mid-window kept a backend alive past the fixed
// 28 s mark the previous test slept for (8 of 8 runs). The pool still drains to one backend; it does so 20 s after
// that statement, which is what the awaited drain above tolerates and the old fixed sleep did not.
const db = await scratchDatabase(direct!, direct!);
try {
await withConsumer(db.url, async engine => {
const backends = async () => Number((await db.admin.unsafe(
"SELECT count(*)::int AS n FROM pg_stat_activity WHERE datname=$1 AND backend_type='client backend'", [db.name]))[0]!.n);
await Bun.sleep(10_000);
await engine.executeRaw('SELECT 1');
const late = performance.now();
await Bun.sleep(IDLE_DRAIN_MS - 10_000);
// 28 s from start, 18 s after the late statement: the pooled backend it used is still inside idle_timeout.
expect(await backends()).toBeGreaterThanOrEqual(2);
await waitFor(async () => (await backends()) === 1, { timeoutMs: 30_000, intervalMs: 500 });
const drainedMs = performance.now() - late;
expect(drainedMs).toBeGreaterThanOrEqual(19_000);
expect(drainedMs).toBeLessThan(30_000);
});
} finally { await db.drop(); }
}, 120_000);
test('a write from another process is published within the idle cap; a same-process wake is immediate', async () => {
const db = await scratchDatabase(direct!, direct!);
try {
await withConsumer(db.url, async (engine, consumer, config) => {
const sources = await fixtures(engine, config);
const other = new PostgresEngine();
await other.connect({ database_url: db.url, poolSize: 2 });
try {
await Bun.sleep(12_000);
let started = performance.now();
const remote = await admitWrite(other, admission(config, sources[0], 'idle/cross-process', 'remote body'));
await waitFor(async () => (await getWriteRequestById(other, remote.id))?.state === 'committed', { timeoutMs: 10_000 });
expect(performance.now() - started).toBeLessThan(5_000 + 3_000);
await Bun.sleep(12_000);
started = performance.now();
const local = await admitWrite(engine, admission(config, sources[0], 'idle/same-process', 'local body'));
consumer.wake();
await waitFor(async () => (await getWriteRequestById(engine, local.id))?.state === 'committed', { timeoutMs: 10_000 });
expect(performance.now() - started).toBeLessThan(3_000);
} finally { await other.disconnect(); }
});
} finally { await db.drop(); }
}, 120_000);
});
describe.skipIf(!direct)('#5233 / #6317 idle lane with a direct/session route configured', () => {
// #6317 (reporter ask 2 on #6278): with a direct/session route configured the consumer's own statements take it, so an
// idle consumer's probes run on the direct pool and the ordinary (pooler) connections all drain; before #6317 (#5233)
// the idle probe reserved an ordinary-pool connection and opened no direct one.
test('an idle consumer probes on the direct route and lets every ordinary-pool connection drain', async () => {
const db = await scratchDatabase(direct!, direct!);
const directRoute = new URL(db.url);
directRoute.searchParams.set('application_name', 'gbrain_5233_direct_route');
try {
await withEnv({ GBRAIN_DIRECT_DATABASE_URL: directRoute.toString() }, () => withConsumer(db.url, async (engine, consumer) => {
expect(engine.connectionManager?.isDualPoolActive()).toBe(true);
expect(consumer.status().connection).toMatchObject({ lane: 'direct' });
await Bun.sleep(IDLE_DRAIN_MS);
const rows = await db.admin.unsafe<{ direct: number; total: number }[]>(`SELECT
count(*) FILTER (WHERE application_name = 'gbrain_5233_direct_route')::int AS direct, count(*)::int AS total
FROM pg_stat_activity WHERE datname = $1 AND backend_type = 'client backend'`, [db.name]);
expect(rows[0]!.direct).toBeGreaterThanOrEqual(1);
expect(rows[0]!.total - rows[0]!.direct).toBe(0);
expect(engine.getPoolDiagnostics()?.tracked.reserved ?? 0).toBe(0);
}));
} finally { await db.drop(); }
}, 120_000);
test('a partly busy pool skips the idle reservation and still publishes writes', async () => {
const db = await scratchDatabase(direct!, direct!);
try {
await withConsumer(db.url, async (engine, consumer, config) => {
const sources = await fixtures(engine, config);
const release = Promise.withResolvers<void>();
let held = 0;
const holding = Array.from({ length: 2 }, () => engine.transaction(async tx => {
await tx.executeRaw('SELECT 1'); held++; await release.promise;
}));
try {
await waitFor(() => held === 2);
await consumer.tick();
const write = await admitWrite(engine, admission(config, sources[0], 'idle/busy-pool', 'busy body'));
consumer.wake();
await waitFor(async () => (await getWriteRequestById(engine, write.id))?.state === 'committed', { timeoutMs: 10_000 });
expect(engine.getPoolDiagnostics()?.tracked.reserved).toBe(0);
} finally { release.resolve(); await Promise.all(holding); }
});
} finally { await db.drop(); }
}, 120_000);
});
describe.skipIf(!(pooled && pooledAdmin) || !existsSync('/proc/self/net/tcp'))('idle persistence consumer behind transaction-mode PgBouncer', () => {
test('an idle consumer drains its pooler client pool to one connection without LISTEN', async () => {
const db = await scratchDatabase(pooledAdmin!, pooled!);
const target = new URL(pooled!);
const port = Number(target.port || 5432);
try {
await withConsumer(db.url, async () => {
expect(await ownSockets(target.hostname, port)).toBeGreaterThanOrEqual(3);
await Bun.sleep(IDLE_DRAIN_MS);
expect(await ownSockets(target.hostname, port)).toBe(1);
});
} finally { await db.drop(); }
}, 120_000);
});