Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Consolidate PostgreSQL notification and cron connections (#1185)

Regan Belld32fd209de78
diff --git a/docs/postgres-connections.md b/docs/postgres-connections.md
new file mode 100644
index 000000000..39266639c
--- /dev/null
+++ b/docs/postgres-connections.md
@@ -0,0 +1,11 @@
+# PostgreSQL connection ownership
+
+Ordinary queries, including the pg-boss cron queue, share the process-wide query pool for a database URL. When DATABASE_POOL_URL is configured, these queries use transaction pooling. Queue shutdown releases its pool reference without closing other stores' connections. Queue migrations retain pg-boss's single-call transaction boundaries.
+
+Notification subscriptions share one direct connection per database URL per process. Session state, ledger events, run signals, and run streaming register their channels on that connection. Closing a consumer removes its subscription; the last unsubscribe closes the listener. Reconnection restores the active channel set before notifying consumers to resynchronize from durable state. Notifications remain wake-up hints, not durable storage.
+
+Session advisory locks and leader leases continue to use direct session connections. Query pool sizes, PgBouncer backend limits, and database placement are unchanged. Multiple processes and separate company databases still have separate listeners and pools.
+
+Validate notification delivery, reconnection, and teardown with test/postgres-listener.test.ts against a disposable DATABASE_URL. The race and queue lifecycle tests run with the standard module-mocking test command. scripts/test-pgbouncer.sh exercises queue startup, execution, shutdown, and restart through a one-backend transaction pool.
+
+This removes three notification connections when all four channels are in use, plus pg-boss's independent query pool. It does not promise a fixed total connection count: active queries, session locks, deployment overlap, and pooler replicas still determine server usage.
diff --git a/package.json b/package.json
index b89ff7d44..335adeccd 100644
--- a/package.json
+++ b/package.json
@@ -36,7 +36,7 @@
"sandbox:local:build": "bash scripts/local-sandbox-build.sh",
"sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"",
"deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",
- "test:pg": "node --test --test-concurrency=1 test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/postgres-instance-registry.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts",
+ "test:pg": "node --test --test-concurrency=1 test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/postgres-instance-registry.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts",
"livetest": "node scripts/api-livetest.ts",
"bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts",
"prepare": "node .husky/install.mjs",
diff --git a/src/cron/job-queue.ts b/src/cron/job-queue.ts
index e811ad791..49f4fc6aa 100644
--- a/src/cron/job-queue.ts
+++ b/src/cron/job-queue.ts
@@ -1,4 +1,5 @@
import { PgBoss } from "pg-boss";
+import { createPgPool, type PgPool } from "../persistence/pg-pool.ts";
import { createSweeper, type Sweeper } from "../util/sweeper.ts";
import { errMessage } from "../util/errors.ts";
@@ -31,15 +32,30 @@ export function createPgBossCronQueue(
schema: string = "pgboss",
fireConcurrency: number = 1,
): CronJobQueue {
- const boss = new PgBoss({ connectionString: databaseUrl, schema, max: 5 });
+ let pg: PgPool | null = null;
+ const boss = new PgBoss({
+ schema,
+ db: {
+ executeSql: async (text, values) => {
+ if (!pg) throw new Error("Cron queue database is closed");
+ return (await pg.pool()).query(text, values);
+ },
+ },
+ });
+ async function closePool() {
+ const previous = pg;
+ pg = null;
+ await previous?.close();
+ }
boss.on("error", (e) => console.error("[cron-queue] pg-boss error:", errMessage(e)));
let ticker: Sweeper | null = null;
let started = false;
let lastSendOkAt = 0;
return {
async start(handlers, tickIntervalMs) {
- await boss.start();
+ pg ??= createPgPool(databaseUrl, []);
try {
+ await boss.start();
await boss.createQueue(FIRE_QUEUE, { policy: "short", notify: true });
await boss.createQueue(TICK_QUEUE, { policy: "short", notify: true });
const localConcurrency = Math.min(32, Math.max(1, Math.trunc(fireConcurrency)));
@@ -53,6 +69,7 @@ export function createPgBossCronQueue(
await boss.work(TICK_QUEUE, { pollingIntervalSeconds: 1 }, () => handlers.onTick());
} catch (e) {
await boss.stop({ close: true, graceful: false }).catch(() => {});
+ await closePool();
throw e;
}
started = true;
@@ -83,7 +100,11 @@ export function createPgBossCronQueue(
async stop() {
started = false;
ticker?.stop();
- await boss.stop({ close: true, graceful: false });
+ try {
+ await boss.stop({ close: true, graceful: false });
+ } finally {
+ await closePool();
+ }
},
};
}
diff --git a/src/persistence/postgres-listener.ts b/src/persistence/postgres-listener.ts
new file mode 100644
index 000000000..8e7b35785
--- /dev/null
+++ b/src/persistence/postgres-listener.ts
@@ -0,0 +1,137 @@
+import { createPgPool, type PoolClient } from "./pg-pool.ts";
+import { swallow } from "../util/errors.ts";
+
+type Subscription = { channel: string; receive: (payload: string) => void; resync: () => void };
+const listeners = new Map<string, ReturnType<typeof createListener>>();
+
+function createListener(connectionString: string) {
+ const pg = createPgPool(connectionString, []);
+ const subscriptions = new Set<Subscription>();
+ const ready = new Set<Subscription>();
+ const channels = new Set<string>();
+ let client: PoolClient | null = null;
+ let pending = Promise.resolve();
+ let retry: ReturnType<typeof setTimeout> | undefined;
+ let closed = false;
+
+ function drop() {
+ const previous = client;
+ client = null;
+ channels.clear();
+ ready.clear();
+ previous?.release(true);
+ }
+
+ function scheduleRetry() {
+ if (closed || retry) return;
+ retry = setTimeout(() => {
+ retry = undefined;
+ sync();
+ }, 1_000);
+ retry.unref();
+ }
+
+ function invoke(callback: () => void) {
+ try {
+ callback();
+ } catch (error) {
+ swallow("postgres listener callback", error);
+ }
+ }
+
+ async function reconcile() {
+ if (closed) {
+ drop();
+ await pg.close();
+ return;
+ }
+ if (!client) {
+ const acquired = await (await pg.sessionPool()).connect();
+ if (closed) {
+ acquired.release(true);
+ return;
+ }
+ client = acquired;
+ const disconnected = () => {
+ if (client !== acquired) return;
+ drop();
+ scheduleRetry();
+ };
+ acquired.on("error", disconnected);
+ acquired.on("end", disconnected);
+ acquired.on("notification", (message) => {
+ if (client !== acquired || !message.payload) return;
+ for (const subscription of subscriptions) {
+ if (subscription.channel === message.channel) invoke(() => subscription.receive(message.payload!));
+ }
+ });
+ }
+ const current = client;
+ const wanted = new Set([...subscriptions].map(({ channel }) => channel));
+ for (const channel of channels) {
+ if (!wanted.has(channel)) {
+ await current.query(`UNLISTEN ${channel}`);
+ if (closed || client !== current) return;
+ channels.delete(channel);
+ }
+ }
+ for (const channel of wanted) {
+ if (!channels.has(channel)) {
+ await current.query(`LISTEN ${channel}`);
+ if (closed || client !== current) return;
+ channels.add(channel);
+ }
+ }
+ if (closed || client !== current) return;
+ for (const subscription of subscriptions) {
+ if (channels.has(subscription.channel) && !ready.has(subscription)) {
+ ready.add(subscription);
+ invoke(subscription.resync);
+ }
+ }
+ }
+
+ function sync() {
+ pending = pending.then(reconcile).catch((error) => {
+ swallow("postgres listener connection", error);
+ drop();
+ scheduleRetry();
+ });
+ return pending;
+ }
+
+ return {
+ subscribe(subscription: Subscription) {
+ subscriptions.add(subscription);
+ sync();
+ let removed = false;
+ return async () => {
+ if (removed) return;
+ removed = true;
+ subscriptions.delete(subscription);
+ ready.delete(subscription);
+ if (subscriptions.size === 0) {
+ closed = true;
+ listeners.delete(connectionString);
+ clearTimeout(retry);
+ }
+ await sync();
+ };
+ },
+ };
+}
+
+export function subscribePostgresChannel(
+ connectionString: string,
+ channel: string,
+ receive: (payload: string) => void,
+ resync: () => void,
+): () => Promise<void> {
+ if (!/^[a-z][a-z0-9_]*$/.test(channel)) throw new Error(`invalid NOTIFY channel name: ${channel}`);
+ let listener = listeners.get(connectionString);
+ if (!listener) {
+ listener = createListener(connectionString);
+ listeners.set(connectionString, listener);
+ }
+ return listener.subscribe({ channel, receive, resync });
+}
diff --git a/src/persistence/postgres-notify-bus.ts b/src/persistence/postgres-notify-bus.ts
index 1488945b6..97a4a0936 100644
--- a/src/persistence/postgres-notify-bus.ts
+++ b/src/persistence/postgres-notify-bus.ts
@@ -1,8 +1,8 @@
-import { createPgPool, type PoolClient } from "./pg-pool.ts";
-import { swallow, swallowAs } from "../util/errors.ts";
+import { createPgPool } from "./pg-pool.ts";
+import { swallowAs } from "../util/errors.ts";
import { createMemoryEventBus, type EventBus } from "../util/event-bus.ts";
-const RECONNECT_DELAY_MS = 1_000;
+import { subscribePostgresChannel } from "./postgres-listener.ts";
export const MAX_NOTIFY_PAYLOAD_BYTES = 7_500;
function encodeCapped<T>(event: T): string | null {
@@ -20,50 +20,24 @@ export function createPostgresNotifyBus<T>(
const pg = createPgPool(connectionString, []);
const local = createMemoryEventBus<T>(label);
- let listenClient: PoolClient | null = null;
- let connecting = false;
+ let stopListening: (() => Promise<void>) | null = null;
let closed = false;
+ let cleanup = Promise.resolve();
- function dropListenClient(): void {
- const client = listenClient;
- listenClient = null;
- if (client) client.release(true);
+ function dropListenClient() {
+ const stop = stopListening;
+ stopListening = null;
+ if (stop) cleanup = Promise.all([cleanup, stop()]).then(() => {});
}
- function ensureListening(): void {
- if (closed || connecting || listenClient || local.size() === 0) return;
- connecting = true;
- void (async () => {
- const client = await (await pg.sessionPool()).connect();
- client.on("notification", (msg) => {
- if (msg.channel !== channel || !msg.payload) return;
- try {
- local.emit(JSON.parse(msg.payload) as T);
- } catch (e) {
- swallow(`${label} notification parse`, e);
- }
- });
- client.on("error", () => {
- dropListenClient();
- setTimeout(() => ensureListening(), RECONNECT_DELAY_MS).unref?.();
- });
- try {
- await client.query(`LISTEN ${channel}`);
- } catch (e) {
- client.release(true);
- throw e;
- }
- listenClient = client;
- local.resync();
- })()
- .catch(swallowAs(`${label}: listen connect`, undefined))
- .finally(() => {
- connecting = false;
- if (closed) dropListenClient();
- else if (!listenClient && local.size() > 0) {
- setTimeout(() => ensureListening(), RECONNECT_DELAY_MS).unref?.();
- }
- });
+ function ensureListening() {
+ if (closed || stopListening || local.size() === 0) return;
+ stopListening = subscribePostgresChannel(
+ connectionString,
+ channel,
+ (payload) => local.emit(JSON.parse(payload) as T),
+ () => local.resync(),
+ );
}
return {
@@ -76,12 +50,16 @@ export function createPostgresNotifyBus<T>(
subscribe(cb, opts) {
const off = local.subscribe(cb, opts);
ensureListening();
- return off;
+ return () => {
+ off();
+ if (local.size() === 0) dropListenClient();
+ };
},
async close() {
closed = true;
dropListenClient();
+ await cleanup;
await pg.close();
},
};
diff --git a/src/runs/postgres-run-signal-store.ts b/src/runs/postgres-run-signal-store.ts
index a6e7632b5..294f21598 100644
--- a/src/runs/postgres-run-signal-store.ts
+++ b/src/runs/postgres-run-signal-store.ts
@@ -1,9 +1,8 @@
-import { createPgPool, type PoolClient, type Rows } from "../persistence/pg-pool.ts";
-import { swallowAs } from "../util/errors.ts";
+import { createPgPool, type Rows } from "../persistence/pg-pool.ts";
+import { subscribePostgresChannel } from "../persistence/postgres-listener.ts";
import type { RunSignal, RunSignalKind, RunSignalStore } from "./run-signal-store.ts";
const CHANNEL = "run_signals";
-const RECONNECT_DELAY_MS = 1_000;
function toSignals(rows: Rows): RunSignal[] {
return rows
@@ -47,46 +46,25 @@ export function createPostgresRunSignalStore(connectionString: string): RunSigna
const q = pg.query;
const listeners = new Map<string, Set<() => void>>();
- let listenClient: PoolClient | null = null;
- let connecting = false;
+ let stopListening: (() => Promise<void>) | null = null;
let closed = false;
+ let cleanup = Promise.resolve();
function ring(runId: string): void {
for (const cb of listeners.get(runId) ?? []) cb();
}
- function dropListenClient(): void {
- const client = listenClient;
- listenClient = null;
- if (client) client.release(true);
+ function dropListenClient() {
+ const stop = stopListening;
+ stopListening = null;
+ if (stop) cleanup = Promise.all([cleanup, stop()]).then(() => {});
}
- function ensureListening(): void {
- if (closed || connecting || listenClient || listeners.size === 0) return;
- connecting = true;
- void (async () => {
- const client = await (await pg.sessionPool()).connect();
- client.on("notification", (msg) => {
- if (msg.channel === CHANNEL && msg.payload) ring(msg.payload);
- });
- client.on("error", () => {
- dropListenClient();
- setTimeout(() => {
- ensureListening();
- for (const runId of listeners.keys()) ring(runId);
- }, RECONNECT_DELAY_MS).unref?.();
- });
- await client.query(`LISTEN ${CHANNEL}`);
- listenClient = client;
- })()
- .catch(swallowAs("run-signals: listen connect", undefined))
- .finally(() => {
- connecting = false;
- if (closed) dropListenClient();
- else if (!listenClient && listeners.size > 0) {
- setTimeout(() => ensureListening(), RECONNECT_DELAY_MS).unref?.();
- }
- });
+ function ensureListening() {
+ if (closed || stopListening || listeners.size === 0) return;
+ stopListening = subscribePostgresChannel(connectionString, CHANNEL, ring, () => {
+ for (const runId of listeners.keys()) ring(runId);
+ });
}
return {
@@ -162,12 +140,14 @@ export function createPostgresRunSignalStore(connectionString: string): RunSigna
return () => {
set.delete(cb);
if (set.size === 0) listeners.delete(runId);
+ if (listeners.size === 0) dropListenClient();
};
},
async close() {
closed = true;
dropListenClient();
+ await cleanup;
await pg.close();
},
};
diff --git a/test/cron-queue-lifecycle.test.ts b/test/cron-queue-lifecycle.test.ts
new file mode 100644
index 000000000..d0a5abbd0
--- /dev/null
+++ b/test/cron-queue-lifecycle.test.ts
@@ -0,0 +1,87 @@
+import assert from "node:assert/strict";
+import { mock, test } from "node:test";
+
+type Adapter = { executeSql(text: string, values?: unknown[]): Promise<unknown> };
+const pools: { closed: boolean }[] = [];
+let adapter: Adapter;
+let failStart = false;
+
+mock.module("../src/persistence/pg-pool.ts", {
+ namedExports: {
+ createPgPool: () => {
+ const state = { closed: false };
+ pools.push(state);
+ return {
+ pool: async () => {
+ if (state.closed) throw new Error("Postgres store is closed");
+ return { query: async () => ({ rows: [] }) };
+ },
+ close: async () => {
+ state.closed = true;
+ },
+ };
+ },
+ },
+});
+
+mock.module("pg-boss", {
+ namedExports: {
+ PgBoss: class {
+ constructor(options: { db: Adapter }) {
+ adapter = options.db;
+ }
+ on() {}
+ async start() {
+ await adapter.executeSql("SELECT 1");
+ if (failStart) throw new Error("startup failed");
+ }
+ async createQueue() {}
+ async work() {}
+ async send() {
+ await adapter.executeSql("SELECT 1");
+ }
+ async stop() {}
+ },
+ },
+});
+
+const { createPgBossCronQueue } = await import("../src/cron/job-queue.ts");
+const handlers = { onFire: async () => {}, onTick: async () => {} };
+
+test("late worker SQL after stop cannot reopen the queue database", async () => {
+ const queue = createPgBossCronQueue("postgres://unused");
+ await queue.start(handlers, 60_000);
+ await queue.stop();
+ const count = pools.length;
+ await assert.rejects(adapter.executeSql("SELECT 1"), /closed/);
+ assert.equal(pools.length, count);
+ assert.ok(pools.at(-1)?.closed);
+ await queue.start(handlers, 60_000);
+ try {
+ assert.equal(pools.length, count + 1);
+ await adapter.executeSql("SELECT 1");
+ assert.ok(queue.healthy());
+ } finally {
+ await queue.stop();
+ }
+});
+
+test("startup failure closes the database and permits a fresh start", async () => {
+ const queue = createPgBossCronQueue("postgres://unused");
+ failStart = true;
+ try {
+ await assert.rejects(queue.start(handlers, 60_000), /startup failed/);
+ assert.ok(pools.at(-1)?.closed);
+ assert.equal(queue.healthy(), false);
+ await assert.rejects(adapter.executeSql("SELECT 1"), /closed/);
+ } finally {
+ failStart = false;
+ }
+ await queue.start(handlers, 60_000);
+ try {
+ assert.equal(pools.at(-1)?.closed, false);
+ assert.ok(queue.healthy());
+ } finally {
+ await queue.stop();
+ }
+});
diff --git a/test/pg-pool-routing.test.ts b/test/pg-pool-routing.test.ts
index 62cce9ee9..2c16a94d1 100644
--- a/test/pg-pool-routing.test.ts
+++ b/test/pg-pool-routing.test.ts
@@ -130,3 +130,58 @@ test("dynamic migration waits for its prerequisite schema on a fresh database",
await a.close();
}
});
+
+test(
+ "cron queue shares transaction pooling and stopping it preserves other stores",
+ { skip, timeout: 30_000 },
+ async () => {
+ const { createPgBossCronQueue } = await import("../src/cron/job-queue.ts");
+ const store = createPgPool(direct!);
+ const queue = createPgBossCronQueue(direct!, "pooled_cron_test");
+ let fired = false;
+ try {
+ await store.q("SELECT 1");
+ await queue.start(
+ {
+ onFire: async () => {
+ fired = true;
+ },
+ onTick: async () => {},
+ },
+ 5_000,
+ );
+ await queue.enqueueFire({ cronId: "pool-test", scheduledAt: Date.now() });
+ const deadline = Date.now() + 10_000;
+ while (!fired && Date.now() < deadline) await new Promise((r) => setTimeout(r, 50));
+ assert.ok(fired);
+ const observer = await (await store.sessionPool()).connect();
+ try {
+ const result = await observer.query(
+ "SELECT count(*)::int n FROM pg_stat_activity WHERE datname=current_database() AND application_name='pgboss'",
+ );
+ assert.equal(result.rows[0].n, 0);
+ } finally {
+ observer.release();
+ }
+ await queue.stop();
+ assert.equal((await store.q("SELECT 42 AS answer"))[0]!.answer, 42);
+ fired = false;
+ await queue.start(
+ {
+ onFire: async () => {
+ fired = true;
+ },
+ onTick: async () => {},
+ },
+ 5_000,
+ );
+ await queue.enqueueFire({ cronId: "pool-test-restart", scheduledAt: Date.now() });
+ const restartedDeadline = Date.now() + 10_000;
+ while (!fired && Date.now() < restartedDeadline) await new Promise((r) => setTimeout(r, 50));
+ assert.ok(fired);
+ } finally {
+ await queue.stop();
+ await store.close();
+ }
+ },
+);
diff --git a/test/postgres-listener-race.test.ts b/test/postgres-listener-race.test.ts
new file mode 100644
index 000000000..a0c78c756
--- /dev/null
+++ b/test/postgres-listener-race.test.ts
@@ -0,0 +1,146 @@
+import { EventEmitter } from "node:events";
+import { mock, test } from "node:test";
+import assert from "node:assert/strict";
+
+class FakeClient extends EventEmitter {
+ queries: string[] = [];
+ releases = 0;
+ afterQuery?: () => void;
+
+ query(sql: string) {
+ this.queries.push(sql);
+ const result = Promise.resolve({ rows: [] });
+ this.afterQuery?.();
+ return result;
+ }
+
+ release() {
+ this.releases++;
+ }
+}
+
+interface FakePool {
+ connect(): Promise<FakeClient>;
+ closes: number;
+}
+
+const pools = new Map<string, FakePool>();
+mock.module("../src/persistence/pg-pool.ts", {
+ namedExports: {
+ createPgPool(url: string) {
+ const pool = pools.get(url)!;
+ return {
+ sessionPool: async () => pool,
+ close: async () => {
+ pool.closes++;
+ },
+ };
+ },
+ },
+});
+const { subscribePostgresChannel } = await import("../src/persistence/postgres-listener.ts");
+
+async function until(check: () => boolean) {
+ const deadline = Date.now() + 3_000;
+ while (!check()) {
+ assert.ok(Date.now() < deadline, "condition timed out");
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+}
+
+test("reconnect repeats LISTEN when the old client disconnects before its query continuation", async () => {
+ const clients = [new FakeClient(), new FakeClient()];
+ clients[0]!.afterQuery = () => clients[0]!.emit("error", new Error("connection lost"));
+ let attempts = 0;
+ const pool = { closes: 0, connect: async () => clients[attempts++]! };
+ pools.set("listen-race", pool);
+ let resyncs = 0;
+ const payloads: string[] = [];
+ const stop = subscribePostgresChannel(
+ "listen-race",
+ "events",
+ (value) => payloads.push(value),
+ () => resyncs++,
+ );
+ try {
+ await until(() => resyncs === 1);
+ assert.equal(attempts, 2);
+ assert.deepEqual(
+ clients.map((client) => client.queries),
+ [["LISTEN events"], ["LISTEN events"]],
+ );
+ clients[1]!.emit("notification", { channel: "events", payload: "recovered" });
+ assert.deepEqual(payloads, ["recovered"]);
+ } finally {
+ await stop();
+ }
+ assert.deepEqual(
+ clients.map((client) => client.releases),
+ [1, 1],
+ );
+ assert.equal(pool.closes, 1);
+});
+
+test("failed acquisition retries and closes the recovered connection", async () => {
+ const client = new FakeClient();
+ let attempts = 0;
+ const pool = {
+ closes: 0,
+ async connect() {
+ if (++attempts === 1) throw new Error("database unavailable");
+ return client;
+ },
+ };
+ pools.set("acquire-retry", pool);
+ let resyncs = 0;
+ const stop = subscribePostgresChannel(
+ "acquire-retry",
+ "events",
+ () => {},
+ () => resyncs++,
+ );
+ try {
+ await until(() => resyncs === 1);
+ assert.equal(attempts, 2);
+ assert.deepEqual(client.queries, ["LISTEN events"]);
+ } finally {
+ await stop();
+ }
+ assert.equal(client.releases, 1);
+ assert.equal(pool.closes, 1);
+});
+
+for (const fails of [false, true]) {
+ test(`unsubscribe during acquisition closes resources when acquisition ${fails ? "fails" : "succeeds"}`, async () => {
+ const acquired = Promise.withResolvers<FakeClient>();
+ const client = new FakeClient();
+ let attempts = 0;
+ const pool = {
+ closes: 0,
+ connect() {
+ attempts++;
+ return acquired.promise;
+ },
+ };
+ const url = `acquire-close-${fails}`;
+ pools.set(url, pool);
+ let resyncs = 0;
+ const stop = subscribePostgresChannel(
+ url,
+ "events",
+ () => {},
+ () => resyncs++,
+ );
+ await until(() => attempts === 1);
+ const closing = stop();
+ if (fails) acquired.reject(new Error("database unavailable"));
+ else acquired.resolve(client);
+ await closing;
+ assert.equal(resyncs, 0);
+ assert.deepEqual(client.queries, []);
+ assert.equal(client.releases, fails ? 0 : 1);
+ assert.equal(pool.closes, 1);
+ await stop();
+ assert.equal(pool.closes, 1);
+ });
+}
diff --git a/test/postgres-listener.test.ts b/test/postgres-listener.test.ts
new file mode 100644
index 000000000..d04ad38bd
--- /dev/null
+++ b/test/postgres-listener.test.ts
@@ -0,0 +1,126 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import pg from "pg";
+import { subscribePostgresChannel } from "../src/persistence/postgres-listener.ts";
+import { createPostgresNotifyBus } from "../src/persistence/postgres-notify-bus.ts";
+import { createPostgresRunSignalStore } from "../src/runs/postgres-run-signal-store.ts";
+
+const url = process.env.DATABASE_URL;
+const skip = !url;
+async function until(check: () => boolean | Promise<boolean>) {
+ const deadline = Date.now() + 8_000;
+ while (!(await check())) {
+ assert.ok(Date.now() < deadline, "condition timed out");
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ }
+}
+
+test("channels share a backend, recover after termination, and release it after unsubscribe", { skip }, async () => {
+ const observer = new pg.Client({ connectionString: url });
+ await observer.connect();
+ const seen: string[] = [];
+ let resyncs = 0;
+ const stops: (() => Promise<void>)[] = [];
+ const pids = async () =>
+ (
+ await observer.query(
+ "SELECT pid FROM pg_stat_activity WHERE datname=current_database() AND query LIKE 'LISTEN consolidation_%' AND pid<>pg_backend_pid()",
+ )
+ ).rows.map((r) => r.pid);
+ try {
+ stops.push(
+ subscribePostgresChannel(
+ url!,
+ "consolidation_a",
+ (v) => seen.push(`a:${v}`),
+ () => resyncs++,
+ ),
+ );
+ stops.push(
+ subscribePostgresChannel(
+ url!,
+ "consolidation_b",
+ (v) => seen.push(`b:${v}`),
+ () => resyncs++,
+ ),
+ );
+ await until(() => resyncs === 2);
+ assert.equal((await pids()).length, 1);
+ await observer.query("SELECT pg_notify('consolidation_a','one'),pg_notify('consolidation_b','two')");
+ await until(() => seen.length === 2);
+ assert.deepEqual(seen.sort(), ["a:one", "b:two"]);
+ await observer.query("SELECT pg_terminate_backend($1)", [(await pids())[0]]);
+ await until(() => resyncs === 4);
+ assert.equal((await pids()).length, 1);
+ await stops[0]!();
+ await observer.query("SELECT pg_notify('consolidation_a','ignored'),pg_notify('consolidation_b','three')");
+ await until(() => seen.includes("b:three"));
+ assert.equal(seen.length, 3);
+ await stops[1]!();
+ await until(async () => (await pids()).length === 0);
+ } finally {
+ await Promise.all(stops.map((stop) => stop()));
+ await observer.end();
+ }
+});
+
+test("notification buses and run signals share a listener without closing their siblings", { skip }, async () => {
+ const a = createPostgresNotifyBus<string>(url!, "consolidation_bus", "test");
+ const b = createPostgresNotifyBus<string>(url!, "consolidation_bus", "test");
+ const signals = createPostgresRunSignalStore(url!);
+ const observer = new pg.Client({ connectionString: url });
+ await observer.connect();
+ let ready = 0;
+ let messages = 0;
+ let signalsSeen = 0;
+ const offA = a.subscribe(() => messages++, { onResync: () => ready++ });
+ const offB = b.subscribe(() => messages++, { onResync: () => ready++ });
+ const offSignal = signals.onSignal("consolidation-run", () => signalsSeen++);
+ try {
+ await until(() => ready === 2 && signalsSeen > 0);
+ signalsSeen = 0;
+ await observer.query("SELECT pg_notify('run_signals','consolidation-run')");
+ await until(() => signalsSeen > 0);
+ a.emit("one");
+ await until(() => messages === 2);
+ offA();
+ await a.close!();
+ b.emit("two");
+ await until(() => messages === 3);
+ offSignal();
+ await signals.close!();
+ b.emit("three");
+ await until(() => messages === 4);
+ } finally {
+ offA();
+ offB();
+ offSignal();
+ await Promise.all([a.close!(), b.close!(), signals.close!()]);
+ await observer.end();
+ }
+});
+
+test("immediate unsubscribe and replacement do not leak a pending connection", { skip }, async () => {
+ const first = subscribePostgresChannel(
+ url!,
+ "consolidation_race",
+ () => {},
+ () => {},
+ );
+ const closing = first();
+ let ready = false;
+ const second = subscribePostgresChannel(
+ url!,
+ "consolidation_race",
+ () => {},
+ () => {
+ ready = true;
+ },
+ );
+ try {
+ await closing;
+ await until(() => ready);
+ } finally {
+ await second();
+ }
+});