Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.7 KB
Raw Blame
/**
 * #6355: two reconnect defects of the vendored driver that the crash robot's `pooler_disconnect` fault reached.
 *
 * A backend terminated while the driver is still starting the connection (the array-types fetch that
 * `fetch_types` runs before the caller's first statement) used to leave that internal query and its FATAL
 * `57P01` behind in the connection's state. The fresh connection then delivered the stale error to the caller's
 * query at its first ReadyForQuery (the crash robot's `put_page refused/57P01` on a live database), and the
 * internal query's promise rejected with nobody awaiting it, which Bun treats as fatal for the process (the
 * robot worker's exit 1). The vendored driver now settles the internal query and clears the saved error before
 * reconnecting, and the fetch swallows its own failure.
 *
 * A close that arrives while a statement's bytes wait for the connection's write immediate used to leave those
 * bytes and the cleared timer handle behind, so the next connection's StartupMessage was appended to them and never
 * scheduled: the socket stayed open until CONNECT_TIMEOUT and every statement the pool bound to that slot failed,
 * for as long as the slot kept being chosen (the robot's effects worker, 10 s per pass, for minutes).
 *
 * Forced probes against a scripted server: the first kills the first connection with FATAL 57P01 at its first
 * statement (before the fix the caller's `select 1` rejected with 57P01 or the process saw an unhandled rejection);
 * the second delivers a close to the driver while a statement's bytes wait for the write immediate (before the fix
 * the statement after it failed with CONNECT_TIMEOUT against a server that answers in a millisecond). Both serve
 * the second connection normally.
 */
import { afterEach, expect, test } from 'bun:test';
import { createServer, Socket, type Server } from 'node:net';
import postgres from '#postgres';

/** A client socket whose close the test can deliver to the driver synchronously, the way a FIN read in the same poll as a reply lands. */
class ScriptedSocket extends Socket {
  closeNow(): void {
    const listeners = this.listeners('close') as ((hadError: boolean) => void)[];
    this.removeAllListeners('close');
    this.destroy();
    for (const listener of listeners) listener(false);
  }
  /** A connection reset as Bun reports it: 'error' (ECONNRESET), then 'close' with hadError on a later turn. */
  async resetNow(): Promise<void> {
    const errors = this.listeners('error') as ((error: Error) => void)[], closes = this.listeners('close') as ((hadError: boolean) => void)[];
    this.removeAllListeners('error'); this.removeAllListeners('close'); this.on('error', () => undefined); this.destroy();
    for (const listener of errors) listener(Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }));
    await new Promise(resolve => setImmediate(resolve));
    for (const listener of closes) listener(true);
  }
}

/** A one-byte type plus int32 length frame of the wire protocol. */
const frame = (type: string, body: Buffer = Buffer.alloc(0)): Buffer => {
  const header = Buffer.alloc(5);
  header.write(type, 0, 'ascii'); header.writeInt32BE(body.length + 4, 1);
  return Buffer.concat([header, body]);
};
const cstr = (s: string): Buffer => Buffer.from(`${s}\0`, 'utf8');
const readyForQuery = frame('Z', Buffer.from('I'));
const startupReply = Buffer.concat([
  frame('R', Buffer.from([0, 0, 0, 0])),
  frame('S', Buffer.concat([cstr('server_version'), cstr('16.0')])),
  frame('S', Buffer.concat([cstr('client_encoding'), cstr('UTF8')])),
  frame('K', Buffer.from([0, 0, 0, 7, 0, 0, 0, 9])),
  readyForQuery,
]);
const fatalTerminated = frame('E', Buffer.concat([
  Buffer.from('S'), cstr('FATAL'), Buffer.from('V'), cstr('FATAL'), Buffer.from('C'), cstr('57P01'),
  Buffer.from('M'), cstr('terminating connection due to administrator command'), Buffer.from([0]),
]));

type Script = { kill?: boolean; killAt?: number; holdAt?: number };
/**
 * Answers the extended protocol with empty results (ParseComplete, BindComplete, NoData, CommandComplete, ReadyForQuery)
 * and a simple Query with CommandComplete. `kill` ends the connection with FATAL 57P01 at its first statement, `killAt`
 * at its n-th (1-based), `holdAt` never answers its n-th. ReadyForQuery reports T between a BEGIN and its COMMIT or ROLLBACK.
 */
function serve(socket: Socket, buffered: Buffer, mode: Script, onStatement: () => number, held = { on: false, tx: false }): Buffer {
  let rest: Buffer = buffered;
  for (;;) {
    if (rest.length < 5) return rest;
    const type = String.fromCharCode(rest[0]!), length = rest.readInt32BE(1);
    if (rest.length < length + 1) return rest;
    const body = rest.subarray(5, length + 1);
    rest = rest.subarray(length + 1);
    const statement = type === 'P' || type === 'Q' ? onStatement() : 0;
    if (statement > 0) {
      const text = (type === 'P' ? body.subarray(body.indexOf(0) + 1) : body).toString('utf8').toLowerCase();
      held.tx = /^\s*begin/.test(text) || (held.tx && !/^\s*(commit|rollback)/.test(text));
    }
    const ready = held.tx ? frame('Z', Buffer.from('T')) : readyForQuery;
    if (mode.kill || (statement > 0 && statement === mode.killAt)) { socket.write(fatalTerminated); socket.end(); return Buffer.alloc(0); }
    if (statement > 0) held.on = statement === mode.holdAt;
    if (held.on) continue;
    if (type === 'P') socket.write(frame('1'));
    else if (type === 'B') socket.write(frame('2'));
    else if (type === 'D') socket.write(frame('n'));
    else if (type === 'E') socket.write(frame('C', cstr('SELECT 0')));
    else if (type === 'S') socket.write(ready);
    else if (type === 'Q') socket.write(Buffer.concat([frame('C', cstr('SELECT 0')), ready]));
    else if (type === 'X') { socket.end(); return Buffer.alloc(0); }
  }
}

let server: Server | undefined;
afterEach(() => new Promise<void>(resolve => { server ? server.close(() => resolve()) : resolve(); server = undefined; }));

/** A scripted server; `script(n)` chooses how the n-th connection (1-based) behaves. */
async function listen(script: (connection: number) => Script): Promise<{ port: number; connections: { statements: number }[] }> {
  const connections: { statements: number }[] = [];
  server = createServer(socket => {
    const connection = { statements: 0 };
    connections.push(connection);
    const mode = script(connections.length);
    let buffered: Buffer = Buffer.alloc(0), started = false;
    const held = { on: false, tx: false };
    socket.on('data', (chunk: Buffer) => {
      buffered = Buffer.concat([buffered, chunk]);
      if (!started) {
        if (buffered.length < 8) return;
        const length = buffered.readInt32BE(0);
        if (buffered.length < length) return;
        buffered = buffered.subarray(length); started = true;
        socket.write(startupReply);
      }
      buffered = serve(socket, buffered, mode, () => ++connection.statements, held);
    });
    socket.on('error', () => undefined);
  });
  await new Promise<void>(resolve => server!.listen(0, '127.0.0.1', () => resolve()));
  return { port: (server.address() as { port: number }).port, connections };
}
function client(port: number, extra: Record<string, unknown> = {}): { sql: ReturnType<typeof postgres>; socket: () => ScriptedSocket } {
  let current: ScriptedSocket | undefined;
  // A socket option hands the driver a connecting socket (the driver then skips its own connect step); the option is not in the public types.
  const options = { host: '127.0.0.1', port, user: 'u', password: 'p', database: 'd', ssl: false, max: 1, prepare: false, fetch_types: true, connect_timeout: 2, onnotice() {},
    socket: () => (current = new ScriptedSocket()).connect(port, '127.0.0.1'), ...extra };
  const sql = postgres(options as unknown as Parameters<typeof postgres>[0]);
  return { sql, socket: () => current! };
}

test('forced probe: a backend terminated during startup neither fails the caller\'s first statement with the stale 57P01 nor leaves an unhandled rejection', async () => {
  const { port, connections } = await listen(connection => ({ kill: connection === 1 }));
  const unhandled: unknown[] = [];
  const onUnhandled = (reason: unknown) => { unhandled.push(reason); };
  process.on('unhandledRejection', onUnhandled);
  const { sql } = client(port);
  try {
    // The caller's first statement: the driver's own types fetch runs ahead of it on the first connection, which the server kills.
    const rows = await sql.unsafe('select 1');
    expect([...rows]).toEqual([]);
    expect(connections.length).toBe(2);
    expect(connections[0]).toEqual({ statements: 1 });
    expect(connections[1]!.statements).toBeGreaterThanOrEqual(2);
    await Bun.sleep(20);
    expect(unhandled).toEqual([]);
  } finally {
    process.off('unhandledRejection', onUnhandled);
    await sql.end({ timeout: 1 });
  }
}, 15_000);

test('forced probe: a close delivered while a statement\'s bytes wait for the write immediate does not leave the next connection\'s startup unsent', async () => {
  // The crash robot's stuck pool slot: a backend answered and was terminated at once; the caller's next statement was
  // buffered for that socket (its write immediate pending) when the close arrived. The cleared immediate left its
  // handle and bytes behind, so the reconnect's StartupMessage was appended and never flushed: the socket sat open
  // until CONNECT_TIMEOUT, and every statement the pool bound to that slot failed the same way, for minutes.
  const { port, connections } = await listen(() => ({}));
  const { sql, socket } = client(port);
  try {
    expect([...await sql.unsafe('select 1')]).toEqual([]);
    const second = sql.unsafe('select 2').then(() => null, (error: { code?: string }) => error.code);
    // Let the statement reach the connection (Query.handle awaits one tick before dispatching) without letting the
    // write immediate run, then deliver the close: the bytes are still buffered for the socket that just closed.
    for (let i = 0; i < 4; i++) await Promise.resolve();
    socket().closeNow();
    expect(await second).toBe('CONNECTION_CLOSED');
    // The next statement takes the slot the pool reconnects: before the fix, CONNECT_TIMEOUT against a server that answers at once.
    expect([...await sql.unsafe('select 3')]).toEqual([]);
    expect(connections.length).toBe(2);
    expect(connections[1]!.statements).toBeGreaterThanOrEqual(2);
  } finally { await sql.end({ timeout: 1 }); }
}, 15_000);

test('forced probe (#6543): a backend terminated under a running statement does not hand its 57P01 to the first statement of the replacement connection', async () => {
  // The serve wedge's `recovery_scan reason=57P01`: a FATAL that arrived before the close stayed in the connection's
  // state, and the reconnect's startup ReadyForQuery delivered it to the statement the pool had queued for the slot.
  // `max_pipeline: 0` keeps that statement in the pool's queue, so it is written only on the replacement connection.
  const { port, connections } = await listen(connection => (connection === 1 ? { killAt: 2 } : {}));
  const { sql } = client(port, { fetch_types: false, max_pipeline: 0 });
  const settled = (query: PromiseLike<unknown>) => Promise.resolve(query).then(rows => ({ rows: [...rows as unknown[]] }), (error: { code?: string }) => ({ code: error.code }));
  try {
    expect([...await sql.unsafe('select 1')]).toEqual([]);
    const head = settled(sql.unsafe('select 2'));
    const queued = settled(sql.unsafe('select 3'));
    expect(await head).toEqual({ code: 'CONNECTION_CLOSED' });
    expect(await queued).toEqual({ rows: [] });
    expect(connections.length).toBe(2);
    expect(connections[1]!.statements).toBe(1);
  } finally { await sql.end({ timeout: 1 }); }
}, 15_000);

test('forced probe (GBRA-133): a statement a transaction issues between a socket error and its close settles, and the reconnect serves the next reservation', async () => {
  // Managed-sync convergence mode (f): a pooler drop reset the transaction's connection under a running statement. Its
  // rejection ran the coordinated write's finally, which sent the settings restore on the same connection before the
  // socket's 'close' arrived. The close skipped rejecting it (hadError), the replacement socket's startup ReadyForQuery
  // made it the current statement without sending it, and the reservation waiting for that startup never got the
  // connection: the run stalled 330 s until the in-flight watchdog retired it.
  const { port, connections } = await listen(connection => (connection === 1 ? { holdAt: 2 } : {}));
  const { sql, socket } = client(port, { fetch_types: false });
  const within = <T>(promise: Promise<T>, label: string) => Promise.race([promise, Bun.sleep(3_000).then(() => `${label} never settled`)]);
  try {
    let restore: Promise<string | null> | undefined;
    let running!: () => void;
    const started = new Promise<void>(resolve => { running = resolve; });
    const transaction = sql.begin(async tx => {
      const held = tx.unsafe('select 1');
      running();
      await held.catch(() => {
        restore = Promise.resolve(tx.unsafe('select 2')).then(() => null, (error: { code?: string }) => error.code ?? 'error');
        return restore;
      });
    }).then(() => null, (error: { code?: string }) => error.code ?? 'error');
    await started;
    await Bun.sleep(20);
    await socket().resetNow();
    expect(await within(transaction, 'transaction')).toMatch(/^(ECONNRESET|CONNECTION_CLOSED)$/);
    expect(restore).toBeDefined();
    expect(await within(restore!, 'restore')).toBe('CONNECTION_CLOSED');
    const next = await within(sql.reserve().then(async reserved => { try { return [...await reserved.unsafe('select 3')]; } finally { reserved.release(); } }), 'next reservation');
    expect(next).toEqual([]);
    expect(connections.length).toBe(2);
  } finally { await sql.end({ timeout: 1 }); }
}, 15_000);