Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(postgres): bound a preparation read's pool wait and its trailing COMMIT (#8651)

garrytandcef01affed1
diff --git a/src/core/persistence/bounded-reads.ts b/src/core/persistence/bounded-reads.ts
index 11c2af69b8..3536d74be6 100644
--- a/src/core/persistence/bounded-reads.ts
+++ b/src/core/persistence/bounded-reads.ts
@@ -39,7 +39,7 @@
import type { BrainEngine, ReservedConnection } from '../engine.ts';
import { PoolCapacityError } from '../pool-budget.ts';
import { READ_WRITE_BEGIN } from '../postgres-engine/read-write-transactions.ts';
-import { withReadDeadline } from '../read-deadline.ts';
+import { ReadDeadlineError, withReadDeadline } from '../read-deadline.ts';
import { recordClaimSql, type ClaimPhaseClock } from './claim-phase.ts';
import { registerEngineView, viewedEngine } from './switches.ts';
@@ -117,9 +117,10 @@ export function boundedReads(engine: BrainEngine, clock: ClaimPhaseClock | undef
return (...args: unknown[]) => {
const result = withReadDeadline(deadlineAt, () => value.apply(target, args));
return result instanceof Promise ? result.catch(error => {
- if (!isStatementTimeout(error) || isSharedReadFailure(error) || Date.now() < deadlineAt) throw error;
+ const refused = error instanceof ReadDeadlineError;
+ if (!refused && (!isStatementTimeout(error) || isSharedReadFailure(error) || Date.now() < deadlineAt)) throw error;
if (clock!.signal?.aborted) throw clock!.signal.reason;
- throw new PreparationDeadlineError(clock!.step, error.code, error);
+ throw new PreparationDeadlineError(clock!.step, refused ? null : error.code, error);
}) : result;
};
} }), viewedEngine(engine));
diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts
index 9c9d976bd4..83c1ca033b 100644
--- a/src/core/postgres-engine.ts
+++ b/src/core/postgres-engine.ts
@@ -371,7 +371,7 @@ export class PostgresEngine implements BrainEngine {
// Flag off + no pre-existing transaction need: call through on the
// shared pool exactly as master does. No tx round-trip, no pool slot
// held for the duration of the read.
- if (deadlineAt !== undefined && !this.rlsScopeBindingEnabled && !opts?.alwaysTransaction) return await runDeadlineRead(this.sql, deadlineAt, callback, this.checkoutGauge);
+ if (deadlineAt !== undefined && !this.rlsScopeBindingEnabled && !opts?.alwaysTransaction) return await runDeadlineRead(this.sql, deadlineAt, callback, this.checkoutGauge, cancelSettleMs());
if (!this.rlsScopeBindingEnabled && !opts?.alwaysTransaction) {
return await callback(this.sql);
}
diff --git a/src/core/postgres-engine/bounded-statement.ts b/src/core/postgres-engine/bounded-statement.ts
index 452cb90c84..4d602c57c3 100644
--- a/src/core/postgres-engine/bounded-statement.ts
+++ b/src/core/postgres-engine/bounded-statement.ts
@@ -23,6 +23,7 @@ import type postgres from '#postgres';
import { reserveWithCancellation } from './cancellation.ts';
import { READ_WRITE_BEGIN } from './read-write-transactions.ts';
import type { CheckoutGauge } from '../pool-gauge.ts';
+import { ReadDeadlineError } from '../read-deadline.ts';
type Sql = ReturnType<typeof postgres>;
export type RunStatement = <T>(conn: Sql, sql: string, params: unknown[] | undefined, opts: { signal?: AbortSignal; prepare?: boolean }) => Promise<T[]>;
@@ -62,11 +63,28 @@ export async function runBoundedStatement<T>(conn: Sql, sql: string, params: unk
* (one simple statement), the read's own statements and `COMMIT`, pipelined: the server ends a read still waiting on
* a lock at the deadline (57014), through a transaction-mode pooler, and the answer returns without waiting for
* COMMIT's round trip. A failed read leaves the transaction aborted, and COMMIT then rolls it back.
+ * #8651: the deadline also ends the wait for a free connection; a read whose deadline passed before it had one is
+ * never sent (`ReadDeadlineError`). A COMMIT still unsettled at the deadline, and at least `settleMs` after it was
+ * sent, has its connection discarded so the connection and its gauge slot come back.
*/
// engine-sql-ok: transaction control around a caller's read (BEGIN READ ONLY / SET LOCAL statement_timeout / COMMIT), no storage-domain SQL
-export async function runDeadlineRead<T>(conn: Sql, deadlineAt: number, run: (tx: Sql) => Promise<T>, gauge: CheckoutGauge): Promise<T> {
+export async function runDeadlineRead<T>(conn: Sql, deadlineAt: number, run: (tx: Sql) => Promise<T>, gauge: CheckoutGauge, settleMs: number): Promise<T> {
+ if (Date.now() >= deadlineAt) throw new ReadDeadlineError();
gauge.acquire('reserved');
- const reserved = await conn.reserve().catch(error => { gauge.release('reserved'); throw error; });
+ const expiry = new AbortController();
+ const timer = setTimeout(() => expiry.abort(), deadlineAt - Date.now());
+ let reserved: postgres.ReservedSql;
+ try {
+ reserved = await reserveWithCancellation(reserve => conn.reserve(reserve), expiry.signal);
+ } catch (error) {
+ gauge.release('reserved');
+ throw expiry.signal.aborted ? new ReadDeadlineError() : error;
+ } finally { clearTimeout(timer); }
+ if (Date.now() >= deadlineAt) {
+ reserved.release();
+ gauge.release('reserved');
+ throw new ReadDeadlineError();
+ }
gauge.checkedOut();
const ms = Math.max(1, Math.ceil(deadlineAt - Date.now()));
const opened = reserved.unsafe(`BEGIN READ ONLY; SET LOCAL statement_timeout = ${ms}`).simple().execute().then(() => undefined, (error: unknown) => error);
@@ -76,7 +94,16 @@ export async function runDeadlineRead<T>(conn: Sql, deadlineAt: number, run: (tx
if (failed) throw failed;
return result;
} finally {
- void reserved.unsafe('COMMIT').execute().then(() => undefined, () => undefined).finally(() => { reserved.release(); gauge.release('reserved'); });
+ let settled = false;
+ const settle = () => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(stuck);
+ reserved.release();
+ gauge.release('reserved');
+ };
+ const stuck = setTimeout(() => { reserved.discard(); settle(); }, Math.max(deadlineAt - Date.now(), settleMs));
+ void reserved.unsafe('COMMIT').execute().then(settle, settle);
}
}
diff --git a/src/core/read-deadline.ts b/src/core/read-deadline.ts
index 688a0995fd..017878689a 100644
--- a/src/core/read-deadline.ts
+++ b/src/core/read-deadline.ts
@@ -13,6 +13,15 @@ import { AsyncLocalStorage } from 'node:async_hooks';
const store = new AsyncLocalStorage<number | undefined>();
+/** #8651: a read refused because its deadline passed before it had a connection; nothing was sent. */
+export class ReadDeadlineError extends Error {
+ readonly code = 'read_deadline';
+ constructor() {
+ super('The read was not sent: its deadline passed while it waited for a database connection.');
+ this.name = 'ReadDeadlineError';
+ }
+}
+
export function withReadDeadline<T>(deadlineAt: number, fn: () => T): T {
return store.run(deadlineAt, fn);
}
diff --git a/test/e2e/persistence-preparation-pool-wait-postgres.test.ts b/test/e2e/persistence-preparation-pool-wait-postgres.test.ts
new file mode 100644
index 0000000000..e68b3c0f82
--- /dev/null
+++ b/test/e2e/persistence-preparation-pool-wait-postgres.test.ts
@@ -0,0 +1,56 @@
+/**
+ * #8651 on Postgres: a managed-sync preparation's engine read that is waiting for a pool connection ends at the
+ * preparation's deadline. `runDeadlineRead` (postgres-engine/bounded-statement.ts) reserved its connection with no
+ * bound, so with every pool connection held elsewhere the read settled only when a connection came back, long after the
+ * budget, and then ran on it anyway.
+ *
+ * Value: protects=the deadline on the pool wait of a bounded engine read (bounded-statement.ts runDeadlineRead,
+ * persistence/bounded-reads.ts); fails_when=the read settles only after the holder gives the sole connection back, or
+ * settles as anything but preparation_deadline; seam=none (a one-connection pool held by a real transaction).
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { isolatedPersistencePostgres } from '../helpers/persistence-postgres.ts';
+import { hasDatabase } from './helpers.ts';
+import { boundedReads, PreparationDeadlineError } from '../../src/core/persistence/bounded-reads.ts';
+import { startClaimPhase } from '../../src/core/persistence/claim-phase.ts';
+import type { PostgresEngine } from '../../src/core/postgres-engine.ts';
+import type { CheckoutGauge } from '../../src/core/pool-gauge.ts';
+
+const BUDGET_MS = 300;
+const HOLD_MS = 4_000;
+let engine: PostgresEngine | undefined;
+let close: (() => Promise<void>) | undefined;
+
+beforeAll(async () => {
+ if (!hasDatabase()) return;
+ const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!, 'instance', 1);
+ engine = pg.engine; close = pg.close;
+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+ await engine.putPage('pool-wait-probe', { title: 'Pool wait probe', type: 'note', compiled_truth: 'A synthetic page.' });
+}, 120_000);
+afterAll(async () => { await close?.(); });
+
+describe.skipIf(!hasDatabase())('a bounded engine read waiting for a pool connection ends at its deadline (Postgres, #8651)', () => {
+ test('with the only connection held, the page read settles as preparation_deadline near the budget, not when the holder lets go', async () => {
+ const e = engine!;
+ const gauge = (e as unknown as { checkoutGauge: CheckoutGauge }).checkoutGauge;
+ const held = Promise.withResolvers<void>();
+ const letGo = Promise.withResolvers<void>();
+ const holder = e.transaction(async tx => { await tx.executeRaw('SELECT 1'); held.resolve(); await letGo.promise; });
+ await held.promise;
+ const fallback = setTimeout(() => letGo.resolve(), HOLD_MS);
+ const started = performance.now();
+ let outcome: unknown;
+ let elapsed = 0;
+ try {
+ const clock = startClaimPhase(Date.now(), undefined, BUDGET_MS);
+ outcome = await boundedReads(e, clock, { engineReads: true }).readPageSnapshot('pool-wait-probe', { sourceId: 'default' }).catch(error => error);
+ elapsed = performance.now() - started;
+ } finally { clearTimeout(fallback); letGo.resolve(); await holder; }
+ expect(elapsed).toBeLessThan(BUDGET_MS + 1_500);
+ expect(outcome).toBeInstanceOf(PreparationDeadlineError);
+ expect((outcome as PreparationDeadlineError).code).toBe('preparation_deadline');
+ expect(gauge.snapshot().reserved).toBe(0);
+ expect((await e.readPageSnapshot('pool-wait-probe', { sourceId: 'default' }))?.page.title).toBe('Pool wait probe');
+ }, 30_000);
+});
diff --git a/test/postgres-deadline-read.test.ts b/test/postgres-deadline-read.test.ts
new file mode 100644
index 0000000000..c5c4ef8a0b
--- /dev/null
+++ b/test/postgres-deadline-read.test.ts
@@ -0,0 +1,76 @@
+/**
+ * #8651: `runDeadlineRead` (postgres-engine/bounded-statement.ts), the reserved read-only transaction a bounded
+ * preparation's engine reads run in. Its deadline covers the wait for a pool connection, a read whose deadline passed is
+ * never sent, and a trailing COMMIT that never settles gives back its connection and gauge slot. The real pool-wait case
+ * is test/e2e/persistence-preparation-pool-wait-postgres.test.ts.
+ */
+import { expect, test } from 'bun:test';
+import type postgres from '#postgres';
+import { runDeadlineRead } from '../src/core/postgres-engine/bounded-statement.ts';
+import { CheckoutGauge } from '../src/core/pool-gauge.ts';
+
+type Sql = ReturnType<typeof postgres>;
+const never = () => new Promise<never>(() => {});
+
+function pool(opts: { reserve?: (owner: unknown) => Promise<unknown>; commit?: () => Promise<unknown> } = {}) {
+ const sent: string[] = [];
+ const counts = { reserves: 0, releases: 0, discards: 0 };
+ const owner = {
+ release: () => { counts.releases++; },
+ discard: () => { counts.discards++; },
+ unsafe: (sql: string) => {
+ sent.push(sql);
+ const query = { simple: () => query, execute: () => sql === 'COMMIT' && opts.commit ? opts.commit() : Promise.resolve([]) };
+ return query;
+ },
+ };
+ const conn = { reserve: () => { counts.reserves++; return opts.reserve ? opts.reserve(owner) : Promise.resolve(owner); } } as unknown as Sql;
+ return { conn, sent, counts, gauge: new CheckoutGauge() };
+}
+
+test('a read whose deadline already passed is refused before it reserves a connection', async () => {
+ const p = pool();
+ let ran = 0;
+ const outcome = await runDeadlineRead(p.conn, Date.now() - 1, async () => { ran++; }, p.gauge, 50).catch(error => error);
+ expect(outcome).toMatchObject({ code: 'read_deadline' });
+ expect(ran).toBe(0);
+ expect(p.counts.reserves).toBe(0);
+ expect(p.gauge.snapshot().reserved).toBe(0);
+});
+
+test('a connection that arrives after the deadline is handed back without BEGIN or the read', async () => {
+ const late = Promise.withResolvers<void>();
+ const p = pool({ reserve: async owner => { await late.promise; return owner; } });
+ let ran = 0;
+ const fallback = setTimeout(() => late.resolve(), 1_500);
+ const started = Date.now();
+ const outcome = await runDeadlineRead(p.conn, Date.now() + 40, async () => { ran++; }, p.gauge, 50).catch(error => error);
+ const settledAfter = Date.now() - started;
+ clearTimeout(fallback);
+ late.resolve();
+ await Bun.sleep(10);
+ expect(outcome).toMatchObject({ code: 'read_deadline' });
+ expect(settledAfter).toBeLessThan(1_000);
+ expect(ran).toBe(0);
+ expect(p.sent).toEqual([]);
+ expect(p.counts).toMatchObject({ releases: 1, discards: 0 });
+ expect(p.gauge.snapshot().reserved).toBe(0);
+});
+
+test('a trailing COMMIT that never settles has its connection discarded and its gauge slot returned', async () => {
+ const p = pool({ commit: never });
+ const answer = await runDeadlineRead(p.conn, Date.now() + 30, async () => 'answer', p.gauge, 50);
+ expect(answer).toBe('answer');
+ await Bun.sleep(150);
+ expect(p.sent.at(-1)).toBe('COMMIT');
+ expect(p.counts).toMatchObject({ releases: 1, discards: 1 });
+ expect(p.gauge.snapshot().reserved).toBe(0);
+});
+
+test('a COMMIT that settles releases the connection and never discards it', async () => {
+ const p = pool();
+ expect(await runDeadlineRead(p.conn, Date.now() + 30, async () => 'answer', p.gauge, 50)).toBe('answer');
+ await Bun.sleep(150);
+ expect(p.counts).toMatchObject({ reserves: 1, releases: 1, discards: 0 });
+ expect(p.gauge.snapshot().reserved).toBe(0);
+});