Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: handle errors on checked-out pg clients (#12061)

Adam Eivy0b047088e7a2
diff --git a/server/lib/db.js b/server/lib/db.js
index bd91e6ff57..8e8ef54c74 100644
--- a/server/lib/db.js
+++ b/server/lib/db.js
@@ -58,6 +58,15 @@ pool.on('error', (err) => {
console.error(`🗄️ Database pool error: ${err.message}`);
});
+// pg-pool only installs its error handler while a client is idle. Every
+// checked-out client needs a temporary listener of its own because pg emits
+// unexpected socket errors while a transaction or other operation is active.
+// Keep one stable callback and remove it before release so client reuse cannot
+// accumulate listeners.
+function logCheckedOutClientError(err) {
+ console.error(`🗄️ Checked-out database client error: ${err.message}`);
+}
+
/**
* Is the configured database a DESIGNATED test database?
*
@@ -328,6 +337,7 @@ export async function withTransaction(fn) {
async function runTransaction(fn) {
const client = await pool.connect();
+ client.on('error', logCheckedOutClientError);
// Guard the client's row writes with the same backstop as query(). The raw
// pg client bypasses query() entirely, so without this wrapper a test-runner
// process pointed at the real `portos` DB writes through every store's
@@ -354,6 +364,7 @@ async function runTransaction(fn) {
if (began) await client.query('ROLLBACK').catch(() => {});
throw err;
} finally {
+ client.removeListener('error', logCheckedOutClientError);
client.release(releaseErr);
}
}
@@ -423,6 +434,7 @@ export async function verifyDatabaseMaintenanceTarget(id) {
const operation = readTarget();
if (isTestRunner() && !/_test$/.test(POOL_CONFIG.database) && process.env.TEST_DB_OK !== '1') throw refuse();
const client = await pool.connect();
+ client.on('error', logCheckedOutClientError);
let discard = false;
try {
await client.query({ text: 'BEGIN READ ONLY', query_timeout: 10_000 });
@@ -443,6 +455,7 @@ export async function verifyDatabaseMaintenanceTarget(id) {
return { id, stage: operation.stage, target: operation.target.mode, targetHealthy: true, fenced: true };
} finally {
await client.query({ text: 'ROLLBACK', query_timeout: 10_000 }).catch(() => { discard = true; });
+ client.removeListener('error', logCheckedOutClientError);
client.release(discard);
}
}
@@ -541,6 +554,7 @@ async function ensureSchemaImpl() {
// deliberately not wrapped in one either — see the comment block above), so
// nothing here may be wrapped in withTransaction()/BEGIN.
const client = await pool.connect();
+ client.on('error', logCheckedOutClientError);
try {
const { rows: [{ locked }] } = await client.query('SELECT pg_try_advisory_lock($1) AS locked', [SCHEMA_DDL_ADVISORY_LOCK_KEY]);
if (!locked) {
@@ -585,6 +599,7 @@ async function ensureSchemaImpl() {
// itself fails, destroy the connection instead: closing it releases the
// lock automatically, same as a hard kill mid-boot.
const unlocked = await client.query('SELECT pg_advisory_unlock($1)', [SCHEMA_DDL_ADVISORY_LOCK_KEY]).then(() => true, () => false);
+ client.removeListener('error', logCheckedOutClientError);
client.release(!unlocked);
}
}
diff --git a/server/lib/db.restoreRecovery.test.js b/server/lib/db.restoreRecovery.test.js
index 1059273f19..19d560884d 100644
--- a/server/lib/db.restoreRecovery.test.js
+++ b/server/lib/db.restoreRecovery.test.js
@@ -8,7 +8,13 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
-const fixture = vi.hoisted(() => ({ dataRoot: null, releaseFault: null, executionFault: false }));
+const fixture = vi.hoisted(() => ({
+ dataRoot: null,
+ releaseFault: null,
+ releaseJournalUnlinked: false,
+ releaseFaultTriggered: false,
+ executionFault: false,
+}));
vi.mock('node:fs', async (importOriginal) => {
const fs = await importOriginal();
return {
@@ -17,10 +23,14 @@ vi.mock('node:fs', async (importOriginal) => {
if (fixture.releaseFault === 'unlink' && path === join(fixture.dataRoot, 'database-restore-recovery.json')) {
throw Object.assign(new Error('synthetic unlink failure'), { code: 'EIO' });
}
- return fs.unlinkSync(path);
+ const result = fs.unlinkSync(path);
+ if (path === join(fixture.dataRoot, 'database-restore-recovery.json')) fixture.releaseJournalUnlinked = true;
+ return result;
}),
fsyncSync: vi.fn((fd) => {
- if (fixture.releaseFault === 'directory-sync' && fs.fstatSync(fd).isDirectory()) {
+ if (fixture.releaseFault === 'directory-sync' && fixture.releaseJournalUnlinked
+ && !fixture.releaseFaultTriggered && fs.fstatSync(fd).isDirectory()) {
+ fixture.releaseFaultTriggered = true;
throw Object.assign(new Error('synthetic directory sync failure'), { code: 'EIO' });
}
return fs.fsyncSync(fd);
@@ -60,7 +70,12 @@ vi.mock('./paths.js', async (importOriginal) => {
});
const pool = vi.hoisted(() => ({
- connect: vi.fn(async () => ({ query: vi.fn(async () => ({ rows: [] })), release: vi.fn() })),
+ connect: vi.fn(async () => ({
+ query: vi.fn(async () => ({ rows: [] })),
+ on: vi.fn(),
+ removeListener: vi.fn(),
+ release: vi.fn(),
+ })),
on: vi.fn(),
query: vi.fn(async () => ({ rows: [] })),
}));
@@ -86,6 +101,9 @@ const FENCED = { status: 503, code: 'DATABASE_RESTORE_RECOVERY' };
beforeEach(() => {
rmSync(databaseRestoreRecovery.path, { force: true });
+ fixture.releaseFault = null;
+ fixture.releaseJournalUnlinked = false;
+ fixture.releaseFaultTriggered = false;
fixture.executionFault = false;
vi.clearAllMocks();
});
diff --git a/server/lib/db.schemaLifecycle.test.js b/server/lib/db.schemaLifecycle.test.js
index 85ecd62175..cc7c4c2281 100644
--- a/server/lib/db.schemaLifecycle.test.js
+++ b/server/lib/db.schemaLifecycle.test.js
@@ -1,7 +1,7 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
const state = vi.hoisted(() => ({
- query: vi.fn(), release: vi.fn(), connect: vi.fn(),
+ query: vi.fn(), on: vi.fn(), removeListener: vi.fn(), release: vi.fn(), connect: vi.fn(),
}));
vi.mock('pg', async (importOriginal) => ({ default: {
Client: (await importOriginal()).default.Client,
@@ -20,7 +20,12 @@ beforeEach(() => {
vi.resetModules();
vi.clearAllMocks();
state.query.mockImplementation(async () => ({ rows: [{ locked: true }] }));
- state.connect.mockResolvedValue({ query: state.query, release: state.release });
+ state.connect.mockResolvedValue({
+ query: state.query,
+ on: state.on,
+ removeListener: state.removeListener,
+ release: state.release,
+ });
});
describe('schema readiness across store startup', () => {
@@ -34,6 +39,8 @@ describe('schema readiness across store startup', () => {
'repair catalog', 'SELECT pg_advisory_unlock($1)',
]);
expect(state.release).toHaveBeenCalledWith(false);
+ expect(state.on).toHaveBeenCalledWith('error', expect.any(Function));
+ expect(state.removeListener).toHaveBeenCalledWith('error', state.on.mock.calls[0][1]);
});
it('allows forced repair and retries after its failure without caching partial work', async () => {
diff --git a/server/lib/db.transaction.test.js b/server/lib/db.transaction.test.js
index 0ef7929d86..2ab68ba661 100644
--- a/server/lib/db.transaction.test.js
+++ b/server/lib/db.transaction.test.js
@@ -1,4 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { EventEmitter } from 'node:events';
const pool = vi.hoisted(() => ({
client: null,
@@ -18,10 +19,10 @@ vi.mock('pg', async (importOriginal) => ({
import { withTransaction, query, withDatabaseMaintenance } from './db.js';
function makeClient(query) {
- return {
- query,
- release: vi.fn(),
- };
+ const client = new EventEmitter();
+ client.query = query;
+ client.release = vi.fn();
+ return client;
}
afterEach(() => {
@@ -113,6 +114,31 @@ describe('withTransaction', () => {
expect(client.release).toHaveBeenCalledWith(dropError);
});
+
+ it('handles a checked-out client socket error and removes the listener before reuse', async () => {
+ const socketError = new Error('Connection terminated unexpectedly');
+ const client = makeClient(vi.fn()
+ .mockResolvedValueOnce({})
+ .mockRejectedValueOnce(socketError)
+ .mockRejectedValueOnce(new Error('ROLLBACK on closed socket')));
+ pool.client = client;
+ const log = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+ await expect(withTransaction(async (transaction) => {
+ expect(client.listenerCount('error')).toBe(1);
+ expect(() => client.emit('error', socketError)).not.toThrow();
+ await transaction.query('SELECT 1');
+ })).rejects.toBe(socketError);
+
+ expect(log).toHaveBeenCalledWith(`🗄️ Checked-out database client error: ${socketError.message}`);
+ expect(client.listenerCount('error')).toBe(0);
+ expect(client.release).toHaveBeenCalledWith(socketError);
+
+ client.query.mockResolvedValue({});
+ await expect(withTransaction(async () => 'reused')).resolves.toBe('reused');
+ expect(client.listenerCount('error')).toBe(0);
+ expect(client.release).toHaveBeenLastCalledWith(undefined);
+ });
});
// Cross-reference: db.guards.test.js covers assertWriteAllowed() directly; this