Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #6383: the vendored driver's in-flight watchdog and live pool numbers.
*
* A connection whose head statement has waited `inflight_timeout` seconds
* with nothing coming back from the server (a desynced pipeline, a
* transaction pooler that swallowed a CancelRequest, a dead peer) is retired:
* the statement and everything queued behind it reject with
* `CONNECTION_STUCK`, the pool reports it once through `onstuck`, the
* connection leaves rotation before its socket closes, and the next statement
* runs on a fresh connection. A statement the server answers in time is never
* retired, because the budget is at least the server's own statement timeout.
*
* `sql.pool` used to be a construction-time snapshot (Object.assign read the
* getter once), so `busy` was always 0; it is live now and carries the oldest
* in-flight age and the completed count the stall watch and `/health?deep=1`
* read. Regression that fails it on master: `busy` stays 0 during `pg_sleep`,
* and a statement the server never answers hangs forever instead of
* rejecting with `CONNECTION_STUCK`.
*
* #6543 (A4): a second clock that received bytes never restart retires a
* statement that keeps streaming past its budget (`reason: 'age'`), and the
* budget is per statement: a transaction that declared a longer
* statement_timeout, or a reserved connection that declared 0, keeps running.
* (A6): reconnects of connections retired by an incident are reported through
* `onrecovered` / `onrecoveryfailed`. Regression that fails it on master: the
* trickle runs to completion, and no recovery hook fires.
*/
import { afterEach, describe, expect, spyOn, test } from 'bun:test';
import { createServer, connect as tcpConnect, type Server, type Socket } from 'node:net';
import postgres from '#postgres';
import { PostgresEngine } from '../../src/core/postgres-engine.ts';
import { poolHealthSnapshot } from '../../src/core/postgres-engine/pool-stats.ts';
import { assertSafeE2eDatabaseUrl } from '../helpers/db-guard.ts';
const url = process.env.DATABASE_URL;
const opened: Array<{ end: (o?: { timeout?: number }) => Promise<void> }> = [];
afterEach(async () => {
while (opened.length) await opened.pop()!.end({ timeout: 1 }).catch(() => {});
});
type Settled = { rows: unknown[] } | { code: string } | { hung: true };
function settle(query: PromiseLike<unknown>, ms = 5000): Promise<Settled> {
return Promise.race([
Promise.resolve(query).then(rows => ({ rows: rows as unknown[] }), (e: { code?: string; message?: string }) => ({ code: e.code ?? e.message ?? 'error' })),
new Promise<Settled>(resolve => setTimeout(() => resolve({ hung: true }), ms)),
]);
}
function pool(max: number, inflightTimeout: number | null, stucks: unknown[], extra: Record<string, unknown> = {}, target = url!) {
assertSafeE2eDatabaseUrl(url!);
const sql = postgres(target, {
max,
prepare: false,
onnotice: () => {},
// No server-side statement_timeout on these connections: the watchdog is the only bound.
connection: { application_name: `gbrain-6383-stuck-${Math.random().toString(36).slice(2)}` },
inflight_timeout: inflightTimeout,
onstuck: (info: unknown) => { stucks.push(info); },
...extra,
} as Parameters<typeof postgres>[1]);
opened.push(sql);
return sql;
}
/** One 9 KB row (larger than the server's send buffer, so each is flushed) every `everyS` seconds. */
const trickle = (rows: number, everyS: number) => `SELECT g, repeat('x', 9000) AS pad, pg_sleep(${everyS}) FROM generate_series(1, ${rows}) g`;
/** A TCP relay to the test database that can stop accepting, so a reconnect is refused. */
function relay(): Promise<{ url: string; stop: () => void; server: Server }> {
const target = new URL(url!);
const sockets = new Set<Socket>();
const server = createServer(client => {
const upstream = tcpConnect(Number(target.port || 5432), target.hostname);
for (const s of [client, upstream]) { sockets.add(s); s.on('error', () => {}); s.on('close', () => sockets.delete(s)); }
client.pipe(upstream).pipe(client);
});
return new Promise(resolve => server.listen(0, '127.0.0.1', () => {
const port = (server.address() as { port: number }).port;
const relayed = new URL(url!);
relayed.hostname = '127.0.0.1';
relayed.port = String(port);
resolve({ url: relayed.toString(), server, stop: () => { server.close(); for (const s of sockets) s.destroy(); } });
}));
}
describe.skipIf(!url)('#6383 in-flight watchdog and live pool numbers', () => {
test('sql.pool is live: busy and inflight_oldest_ms move while a statement runs, completed counts settled statements', async () => {
const sql = pool(2, null, []);
await sql`SELECT 1`;
const before = sql.pool;
expect(before.open).toBe(1);
expect(before.busy).toBe(0);
const slow = sql`SELECT pg_sleep(0.4)`.execute();
await new Promise(r => setTimeout(r, 150));
const during = sql.pool;
expect(during.busy + during.full).toBe(1);
expect(during.inflight_oldest_ms).toBeGreaterThanOrEqual(100);
await slow;
const after = sql.pool;
expect(after.busy + after.full).toBe(0);
expect(after.inflight_oldest_ms).toBe(0);
expect(after.completed).toBeGreaterThan(before.completed);
});
test('a statement the server never answers in time rejects with CONNECTION_STUCK, its queued follower too, and the pool recovers on a fresh connection', async () => {
const stucks: Array<{ age_ms: number; queued: number; statement: string }> = [];
const sql = pool(1, 1, stucks);
await sql`SELECT 1`;
const started = Date.now();
const slow = settle(sql`SELECT pg_sleep(5), 'slow' AS who`);
await new Promise(r => setTimeout(r, 100));
const follower = settle(sql`SELECT 'follower' AS who`);
expect(await slow).toEqual({ code: 'CONNECTION_STUCK' });
expect(await follower).toEqual({ code: 'CONNECTION_STUCK' });
expect(Date.now() - started).toBeLessThan(3000);
expect(stucks).toHaveLength(1);
expect(stucks[0].queued).toBe(1);
expect(stucks[0].age_ms).toBeGreaterThanOrEqual(900);
expect(stucks[0].statement).toBe("SELECT pg_sleep(5), 'slow' AS who");
expect(sql.pool.busy + sql.pool.full).toBe(0);
expect(await settle(sql`SELECT 'fresh' AS who`)).toEqual({ rows: [{ who: 'fresh' }] });
expect(sql.pool.inflight_oldest_ms).toBe(0);
});
test('a stuck reserved connection rejects its holder and the pool serves the next caller', async () => {
const stucks: unknown[] = [];
const sql = pool(1, 1, stucks);
const reserved = await sql.reserve();
expect(await settle(reserved`SELECT pg_sleep(5)`)).toEqual({ code: 'CONNECTION_STUCK' });
expect(await settle(reserved`SELECT 1`)).toEqual({ code: 'CONNECTION_DESTROYED' });
reserved.release();
expect(await settle(sql`SELECT 'next' AS who`)).toEqual({ rows: [{ who: 'next' }] });
expect(stucks).toHaveLength(1);
});
test('a statement answered within the budget is never retired, and a null budget disables the watchdog', async () => {
const stucks: unknown[] = [];
const bounded = pool(1, 2, stucks);
expect(await settle(bounded`SELECT pg_sleep(0.5), 1 AS x`)).toEqual({ rows: [{ pg_sleep: '', x: 1 }] });
const unbounded = pool(1, null, stucks);
expect(await settle(unbounded`SELECT pg_sleep(1.2), 2 AS x`)).toEqual({ rows: [{ pg_sleep: '', x: 2 }] });
expect(stucks).toEqual([]);
});
test('#6543 A4: a statement that keeps streaming past its budget is retired with reason age, and the pool serves the next caller', async () => {
const stucks: Array<{ age_ms: number; queued: number; statement: string; reason: string; budget_ms: number }> = [];
// Idle budget 1 s (a row arrives every 0.4 s, so it never fires), age budget 2 s.
const sql = pool(1, 1, stucks, { inflight_max: 2 });
await sql`SELECT 1`;
const started = Date.now();
expect(await settle(sql.unsafe(trickle(15, 0.4)), 8000)).toEqual({ code: 'CONNECTION_STUCK' });
expect(Date.now() - started).toBeLessThan(4000);
expect(stucks).toHaveLength(1);
expect(stucks[0]).toMatchObject({ reason: 'age', budget_ms: 2000, queued: 0 });
expect(stucks[0].age_ms).toBeGreaterThanOrEqual(1900);
expect(await settle(sql`SELECT 'next' AS who`)).toEqual({ rows: [{ who: 'next' }] });
});
test('#6543 A4: a transaction that declared a longer statement_timeout streams past the pool budget unretired, and deep health does not call it over budget; its SET LOCAL ends with it', async () => {
const stucks: unknown[] = [];
const sql = pool(1, 1, stucks, { inflight_max: 2, inflight_grace: 1 });
await sql`SELECT 1`;
let midway: { inflight_oldest_ms: number; inflight_over_budget_ms: number } | null = null;
const probe = setTimeout(() => { midway = sql.pool as never; }, 2600);
const rows = await settle(sql.begin(async tx => {
await tx`SET LOCAL statement_timeout = '600s'`;
return tx.unsafe(trickle(8, 0.4));
}), 8000);
clearTimeout(probe);
expect('rows' in rows && rows.rows.length).toBe(8);
expect(stucks).toEqual([]);
expect(midway!.inflight_oldest_ms).toBeGreaterThan(2000);
expect(midway!.inflight_over_budget_ms).toBe(0);
expect(await settle(sql.unsafe(trickle(15, 0.4)), 8000)).toEqual({ code: 'CONNECTION_STUCK' });
expect(stucks).toHaveLength(1);
});
test('#6543 A4: a reserved connection that declared statement_timeout 0 is never retired until it resets the timeout', async () => {
const stucks: unknown[] = [];
const sql = pool(1, 1, stucks, { inflight_max: 1, inflight_grace: 1 });
const reserved = await sql.reserve();
await reserved`SET statement_timeout = 0`;
expect(await settle(reserved`SELECT pg_sleep(1.6), 1 AS x`)).toEqual({ rows: [{ pg_sleep: '', x: 1 }] });
await reserved`RESET statement_timeout`;
expect(await settle(reserved`SELECT pg_sleep(5)`)).toEqual({ code: 'CONNECTION_STUCK' });
reserved.release();
expect(stucks).toHaveLength(1);
});
test('#6543 A6: a connection closed under a statement reports onrecovered on its next startup, and a refused reconnect reports onrecoveryfailed', async () => {
const proxy = await relay();
const recovered: unknown[] = [];
const failed: Array<{ cause: string; code: string }> = [];
const sql = pool(1, null, [], { onrecovered: (i: unknown) => recovered.push(i), onrecoveryfailed: (i: { cause: string; code: string }) => failed.push(i), backoff: 0 }, proxy.url);
await sql`SELECT 1`;
const running = settle(sql`SELECT pg_sleep(5)`);
await new Promise(r => setTimeout(r, 200));
proxy.stop();
expect(await running).toEqual({ code: 'CONNECTION_CLOSED' });
expect(await settle(sql`SELECT 1`)).toEqual({ code: 'ECONNREFUSED' });
expect(failed).toHaveLength(1);
expect(failed[0]).toMatchObject({ cause: 'closed', code: 'ECONNREFUSED' });
expect(recovered).toEqual([]);
await new Promise<void>(resolve => proxy.server.listen(Number(new URL(proxy.url).port), '127.0.0.1', () => resolve()));
expect(await settle(sql`SELECT 'back' AS who`)).toEqual({ rows: [{ who: 'back' }] });
expect(recovered).toEqual([{ connection: expect.any(Number), cause: 'closed' }]);
expect(failed).toHaveLength(1);
proxy.stop();
});
test('#6543 A6 engine: pg_terminate_backend under a statement counts recovered once, in getPoolDiagnostics and the deep-health snapshot', async () => {
assertSafeE2eDatabaseUrl(url!);
const engine = new PostgresEngine();
const admin = pool(1, null, []);
try {
await engine.connect({ engine: 'postgres', database_url: url!, poolSize: 2 });
const marker = `pg_sleep(4.${Math.floor(Math.random() * 900) + 100})`;
const sleeping = settle(engine.executeRaw(`SELECT ${marker}`), 6000);
let pids: Array<{ pid: number }> = [];
for (let i = 0; i < 20 && pids.length === 0; i++) {
await new Promise(r => setTimeout(r, 100));
pids = await admin.unsafe<Array<{ pid: number }>>(`SELECT pid FROM pg_stat_activity WHERE query = 'SELECT ${marker}' AND pid <> pg_backend_pid()`);
}
expect(pids).toHaveLength(1);
await admin`SELECT pg_terminate_backend(${pids[0]!.pid})`;
expect('code' in (await sleeping)).toBe(true);
expect(engine.getPoolDiagnostics()?.recovered).toBe(0);
// Two statements at once need both connections, so the terminated one reconnects.
expect(await Promise.all([engine.executeRaw<{ x: number; pg_sleep: string }>('SELECT 1 AS x, pg_sleep(0.3)'), engine.executeRaw<{ x: number; pg_sleep: string }>('SELECT 1 AS x, pg_sleep(0.3)')]))
.toEqual([[{ x: 1, pg_sleep: '' }], [{ x: 1, pg_sleep: '' }]]);
expect(engine.getPoolDiagnostics()?.recovered).toBe(1);
expect(engine.getPoolDiagnostics()?.recoveryFailed).toBe(0);
expect(poolHealthSnapshot(engine)).toMatchObject({ recovered: 1, recovery_failed: 0 });
} finally {
await engine.disconnect();
}
});
test('engine: a stuck connection is counted once, logged as one pg_connection_stuck line, and seen by getPoolDiagnostics', async () => {
assertSafeE2eDatabaseUrl(url!);
const previous = { grace: process.env.GBRAIN_PG_STUCK_GRACE_MS, timeout: process.env.GBRAIN_STATEMENT_TIMEOUT };
// Budget = statement_timeout (500 ms) + grace (500 ms) = 1 s. The session below lifts its own server-side
// statement_timeout first, so the sleep is only ever ended by the watchdog. It lifts it inside a DO body, which
// the driver does not read (as a pooler that never applies the timeout would leave it): since #6543 a lift the
// driver reads (`SET statement_timeout = 0`) also lifts the watchdog for that connection.
process.env.GBRAIN_STATEMENT_TIMEOUT = '500ms';
process.env.GBRAIN_PG_STUCK_GRACE_MS = '500';
const engine = new PostgresEngine();
const warn = spyOn(console, 'warn').mockImplementation(() => {});
try {
await engine.connect({ engine: 'postgres', database_url: url!, poolSize: 2 });
const stuck = await settle(engine.withReservedConnection(async conn => {
await conn.executeRaw("DO $$ BEGIN PERFORM set_config('statement_timeout', '0', false); END $$");
await conn.executeRaw('SELECT pg_sleep(5)');
}), 6000);
expect(stuck).toEqual({ code: 'CONNECTION_STUCK' });
const diagnostics = engine.getPoolDiagnostics();
expect(diagnostics?.stuckDiscards).toBe(1);
expect(diagnostics?.pool?.inflight_oldest_ms).toBe(0);
const rows = await engine.executeRaw<{ x: number }>('SELECT 1 AS x');
expect(rows).toEqual([{ x: 1 }]);
const lines = warn.mock.calls.map(c => String(c[0])).filter(l => l.includes('pg_connection_stuck'));
expect(lines).toHaveLength(1);
expect(lines[0]).toContain('pool=read');
expect(lines[0]).toContain('sql=SELECT');
expect(lines[0]).toContain('reason=idle budget_ms=1000');
expect(lines[0]).not.toContain('pg_sleep');
expect(lines[0]).not.toContain(url!);
} finally {
warn.mockRestore();
await engine.disconnect();
if (previous.grace === undefined) delete process.env.GBRAIN_PG_STUCK_GRACE_MS; else process.env.GBRAIN_PG_STUCK_GRACE_MS = previous.grace;
if (previous.timeout === undefined) delete process.env.GBRAIN_STATEMENT_TIMEOUT; else process.env.GBRAIN_STATEMENT_TIMEOUT = previous.timeout;
}
});
});