Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(postgres): bound a preparation read's pool wait and its trailing COMMIT (#8651)
garrytandcef01affed1
diff --git a/src/core/persistence/bounded-reads.ts b/src/core/persistence/bounded-reads.tsindex 11c2af69b8..3536d74be6 100644--- a/src/core/persistence/bounded-reads.ts+++ b/src/core/persistence/bounded-reads.ts@@ -39,7 +39,7 @@import type { BrainEngine, ReservedConnection } from '../engine.ts';import { PoolCapacityError } from '../pool-budget.ts';import { READ_WRITE_BEGIN } from '../postgres-engine/read-write-transactions.ts';-import { withReadDeadline } from '../read-deadline.ts';+import { ReadDeadlineError, withReadDeadline } from '../read-deadline.ts';import { recordClaimSql, type ClaimPhaseClock } from './claim-phase.ts';import { registerEngineView, viewedEngine } from './switches.ts';@@ -117,9 +117,10 @@ export function boundedReads(engine: BrainEngine, clock: ClaimPhaseClock | undefreturn (...args: unknown[]) => {const result = withReadDeadline(deadlineAt, () => value.apply(target, args));return result instanceof Promise ? result.catch(error => {- if (!isStatementTimeout(error) || isSharedReadFailure(error) || Date.now() < deadlineAt) throw error;+ const refused = error instanceof ReadDeadlineError;+ if (!refused && (!isStatementTimeout(error) || isSharedReadFailure(error) || Date.now() < deadlineAt)) throw error;if (clock!.signal?.aborted) throw clock!.signal.reason;- throw new PreparationDeadlineError(clock!.step, error.code, error);+ throw new PreparationDeadlineError(clock!.step, refused ? null : error.code, error);}) : result;};} }), viewedEngine(engine));diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.tsindex 9c9d976bd4..83c1ca033b 100644--- a/src/core/postgres-engine.ts+++ b/src/core/postgres-engine.ts@@ -371,7 +371,7 @@ export class PostgresEngine implements BrainEngine {// Flag off + no pre-existing transaction need: call through on the// shared pool exactly as master does. No tx round-trip, no pool slot// held for the duration of the read.- if (deadlineAt !== undefined && !this.rlsScopeBindingEnabled && !opts?.alwaysTransaction) return await runDeadlineRead(this.sql, deadlineAt, callback, this.checkoutGauge);+ if (deadlineAt !== undefined && !this.rlsScopeBindingEnabled && !opts?.alwaysTransaction) return await runDeadlineRead(this.sql, deadlineAt, callback, this.checkoutGauge, cancelSettleMs());if (!this.rlsScopeBindingEnabled && !opts?.alwaysTransaction) {return await callback(this.sql);}diff --git a/src/core/postgres-engine/bounded-statement.ts b/src/core/postgres-engine/bounded-statement.tsindex 452cb90c84..4d602c57c3 100644--- a/src/core/postgres-engine/bounded-statement.ts+++ b/src/core/postgres-engine/bounded-statement.ts@@ -23,6 +23,7 @@ import type postgres from '#postgres';import { reserveWithCancellation } from './cancellation.ts';import { READ_WRITE_BEGIN } from './read-write-transactions.ts';import type { CheckoutGauge } from '../pool-gauge.ts';+import { ReadDeadlineError } from '../read-deadline.ts';type Sql = ReturnType<typeof postgres>;export type RunStatement = <T>(conn: Sql, sql: string, params: unknown[] | undefined, opts: { signal?: AbortSignal; prepare?: boolean }) => Promise<T[]>;@@ -62,11 +63,28 @@ export async function runBoundedStatement<T>(conn: Sql, sql: string, params: unk* (one simple statement), the read's own statements and `COMMIT`, pipelined: the server ends a read still waiting on* a lock at the deadline (57014), through a transaction-mode pooler, and the answer returns without waiting for* COMMIT's round trip. A failed read leaves the transaction aborted, and COMMIT then rolls it back.+ * #8651: the deadline also ends the wait for a free connection; a read whose deadline passed before it had one is+ * never sent (`ReadDeadlineError`). A COMMIT still unsettled at the deadline, and at least `settleMs` after it was+ * sent, has its connection discarded so the connection and its gauge slot come back.*/// engine-sql-ok: transaction control around a caller's read (BEGIN READ ONLY / SET LOCAL statement_timeout / COMMIT), no storage-domain SQL-export async function runDeadlineRead<T>(conn: Sql, deadlineAt: number, run: (tx: Sql) => Promise<T>, gauge: CheckoutGauge): Promise<T> {+export async function runDeadlineRead<T>(conn: Sql, deadlineAt: number, run: (tx: Sql) => Promise<T>, gauge: CheckoutGauge, settleMs: number): Promise<T> {+ if (Date.now() >= deadlineAt) throw new ReadDeadlineError();gauge.acquire('reserved');- const reserved = await conn.reserve().catch(error => { gauge.release('reserved'); throw error; });+ const expiry = new AbortController();+ const timer = setTimeout(() => expiry.abort(), deadlineAt - Date.now());+ let reserved: postgres.ReservedSql;+ try {+ reserved = await reserveWithCancellation(reserve => conn.reserve(reserve), expiry.signal);+ } catch (error) {+ gauge.release('reserved');+ throw expiry.signal.aborted ? new ReadDeadlineError() : error;+ } finally { clearTimeout(timer); }+ if (Date.now() >= deadlineAt) {+ reserved.release();+ gauge.release('reserved');+ throw new ReadDeadlineError();+ }gauge.checkedOut();const ms = Math.max(1, Math.ceil(deadlineAt - Date.now()));const opened = reserved.unsafe(`BEGIN READ ONLY; SET LOCAL statement_timeout = ${ms}`).simple().execute().then(() => undefined, (error: unknown) => error);@@ -76,7 +94,16 @@ export async function runDeadlineRead<T>(conn: Sql, deadlineAt: number, run: (txif (failed) throw failed;return result;} finally {- void reserved.unsafe('COMMIT').execute().then(() => undefined, () => undefined).finally(() => { reserved.release(); gauge.release('reserved'); });+ let settled = false;+ const settle = () => {+ if (settled) return;+ settled = true;+ clearTimeout(stuck);+ reserved.release();+ gauge.release('reserved');+ };+ const stuck = setTimeout(() => { reserved.discard(); settle(); }, Math.max(deadlineAt - Date.now(), settleMs));+ void reserved.unsafe('COMMIT').execute().then(settle, settle);}}diff --git a/src/core/read-deadline.ts b/src/core/read-deadline.tsindex 688a0995fd..017878689a 100644--- a/src/core/read-deadline.ts+++ b/src/core/read-deadline.ts@@ -13,6 +13,15 @@ import { AsyncLocalStorage } from 'node:async_hooks';const store = new AsyncLocalStorage<number | undefined>();+/** #8651: a read refused because its deadline passed before it had a connection; nothing was sent. */+export class ReadDeadlineError extends Error {+ readonly code = 'read_deadline';+ constructor() {+ super('The read was not sent: its deadline passed while it waited for a database connection.');+ this.name = 'ReadDeadlineError';+ }+}+export function withReadDeadline<T>(deadlineAt: number, fn: () => T): T {return store.run(deadlineAt, fn);}diff --git a/test/e2e/persistence-preparation-pool-wait-postgres.test.ts b/test/e2e/persistence-preparation-pool-wait-postgres.test.tsnew file mode 100644index 0000000000..e68b3c0f82--- /dev/null+++ b/test/e2e/persistence-preparation-pool-wait-postgres.test.ts@@ -0,0 +1,56 @@+/**+ * #8651 on Postgres: a managed-sync preparation's engine read that is waiting for a pool connection ends at the+ * preparation's deadline. `runDeadlineRead` (postgres-engine/bounded-statement.ts) reserved its connection with no+ * bound, so with every pool connection held elsewhere the read settled only when a connection came back, long after the+ * budget, and then ran on it anyway.+ *+ * Value: protects=the deadline on the pool wait of a bounded engine read (bounded-statement.ts runDeadlineRead,+ * persistence/bounded-reads.ts); fails_when=the read settles only after the holder gives the sole connection back, or+ * settles as anything but preparation_deadline; seam=none (a one-connection pool held by a real transaction).+ */+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';+import { isolatedPersistencePostgres } from '../helpers/persistence-postgres.ts';+import { hasDatabase } from './helpers.ts';+import { boundedReads, PreparationDeadlineError } from '../../src/core/persistence/bounded-reads.ts';+import { startClaimPhase } from '../../src/core/persistence/claim-phase.ts';+import type { PostgresEngine } from '../../src/core/postgres-engine.ts';+import type { CheckoutGauge } from '../../src/core/pool-gauge.ts';++const BUDGET_MS = 300;+const HOLD_MS = 4_000;+let engine: PostgresEngine | undefined;+let close: (() => Promise<void>) | undefined;++beforeAll(async () => {+ if (!hasDatabase()) return;+ const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!, 'instance', 1);+ engine = pg.engine; close = pg.close;+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');+ await engine.putPage('pool-wait-probe', { title: 'Pool wait probe', type: 'note', compiled_truth: 'A synthetic page.' });+}, 120_000);+afterAll(async () => { await close?.(); });++describe.skipIf(!hasDatabase())('a bounded engine read waiting for a pool connection ends at its deadline (Postgres, #8651)', () => {+ test('with the only connection held, the page read settles as preparation_deadline near the budget, not when the holder lets go', async () => {+ const e = engine!;+ const gauge = (e as unknown as { checkoutGauge: CheckoutGauge }).checkoutGauge;+ const held = Promise.withResolvers<void>();+ const letGo = Promise.withResolvers<void>();+ const holder = e.transaction(async tx => { await tx.executeRaw('SELECT 1'); held.resolve(); await letGo.promise; });+ await held.promise;+ const fallback = setTimeout(() => letGo.resolve(), HOLD_MS);+ const started = performance.now();+ let outcome: unknown;+ let elapsed = 0;+ try {+ const clock = startClaimPhase(Date.now(), undefined, BUDGET_MS);+ outcome = await boundedReads(e, clock, { engineReads: true }).readPageSnapshot('pool-wait-probe', { sourceId: 'default' }).catch(error => error);+ elapsed = performance.now() - started;+ } finally { clearTimeout(fallback); letGo.resolve(); await holder; }+ expect(elapsed).toBeLessThan(BUDGET_MS + 1_500);+ expect(outcome).toBeInstanceOf(PreparationDeadlineError);+ expect((outcome as PreparationDeadlineError).code).toBe('preparation_deadline');+ expect(gauge.snapshot().reserved).toBe(0);+ expect((await e.readPageSnapshot('pool-wait-probe', { sourceId: 'default' }))?.page.title).toBe('Pool wait probe');+ }, 30_000);+});diff --git a/test/postgres-deadline-read.test.ts b/test/postgres-deadline-read.test.tsnew file mode 100644index 0000000000..c5c4ef8a0b--- /dev/null+++ b/test/postgres-deadline-read.test.ts@@ -0,0 +1,76 @@+/**+ * #8651: `runDeadlineRead` (postgres-engine/bounded-statement.ts), the reserved read-only transaction a bounded+ * preparation's engine reads run in. Its deadline covers the wait for a pool connection, a read whose deadline passed is+ * never sent, and a trailing COMMIT that never settles gives back its connection and gauge slot. The real pool-wait case+ * is test/e2e/persistence-preparation-pool-wait-postgres.test.ts.+ */+import { expect, test } from 'bun:test';+import type postgres from '#postgres';+import { runDeadlineRead } from '../src/core/postgres-engine/bounded-statement.ts';+import { CheckoutGauge } from '../src/core/pool-gauge.ts';++type Sql = ReturnType<typeof postgres>;+const never = () => new Promise<never>(() => {});++function pool(opts: { reserve?: (owner: unknown) => Promise<unknown>; commit?: () => Promise<unknown> } = {}) {+ const sent: string[] = [];+ const counts = { reserves: 0, releases: 0, discards: 0 };+ const owner = {+ release: () => { counts.releases++; },+ discard: () => { counts.discards++; },+ unsafe: (sql: string) => {+ sent.push(sql);+ const query = { simple: () => query, execute: () => sql === 'COMMIT' && opts.commit ? opts.commit() : Promise.resolve([]) };+ return query;+ },+ };+ const conn = { reserve: () => { counts.reserves++; return opts.reserve ? opts.reserve(owner) : Promise.resolve(owner); } } as unknown as Sql;+ return { conn, sent, counts, gauge: new CheckoutGauge() };+}++test('a read whose deadline already passed is refused before it reserves a connection', async () => {+ const p = pool();+ let ran = 0;+ const outcome = await runDeadlineRead(p.conn, Date.now() - 1, async () => { ran++; }, p.gauge, 50).catch(error => error);+ expect(outcome).toMatchObject({ code: 'read_deadline' });+ expect(ran).toBe(0);+ expect(p.counts.reserves).toBe(0);+ expect(p.gauge.snapshot().reserved).toBe(0);+});++test('a connection that arrives after the deadline is handed back without BEGIN or the read', async () => {+ const late = Promise.withResolvers<void>();+ const p = pool({ reserve: async owner => { await late.promise; return owner; } });+ let ran = 0;+ const fallback = setTimeout(() => late.resolve(), 1_500);+ const started = Date.now();+ const outcome = await runDeadlineRead(p.conn, Date.now() + 40, async () => { ran++; }, p.gauge, 50).catch(error => error);+ const settledAfter = Date.now() - started;+ clearTimeout(fallback);+ late.resolve();+ await Bun.sleep(10);+ expect(outcome).toMatchObject({ code: 'read_deadline' });+ expect(settledAfter).toBeLessThan(1_000);+ expect(ran).toBe(0);+ expect(p.sent).toEqual([]);+ expect(p.counts).toMatchObject({ releases: 1, discards: 0 });+ expect(p.gauge.snapshot().reserved).toBe(0);+});++test('a trailing COMMIT that never settles has its connection discarded and its gauge slot returned', async () => {+ const p = pool({ commit: never });+ const answer = await runDeadlineRead(p.conn, Date.now() + 30, async () => 'answer', p.gauge, 50);+ expect(answer).toBe('answer');+ await Bun.sleep(150);+ expect(p.sent.at(-1)).toBe('COMMIT');+ expect(p.counts).toMatchObject({ releases: 1, discards: 1 });+ expect(p.gauge.snapshot().reserved).toBe(0);+});++test('a COMMIT that settles releases the connection and never discards it', async () => {+ const p = pool();+ expect(await runDeadlineRead(p.conn, Date.now() + 30, async () => 'answer', p.gauge, 50)).toBe('answer');+ await Bun.sleep(150);+ expect(p.counts).toMatchObject({ reserves: 1, releases: 1, discards: 0 });+ expect(p.gauge.snapshot().reserved).toBe(0);+});