Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: handle errors on checked-out pg clients (#12061)
Adam Eivy0b047088e7a2
diff --git a/server/lib/db.js b/server/lib/db.jsindex bd91e6ff57..8e8ef54c74 100644--- a/server/lib/db.js+++ b/server/lib/db.js@@ -58,6 +58,15 @@ pool.on('error', (err) => {console.error(`🗄️ Database pool error: ${err.message}`);});+// pg-pool only installs its error handler while a client is idle. Every+// checked-out client needs a temporary listener of its own because pg emits+// unexpected socket errors while a transaction or other operation is active.+// Keep one stable callback and remove it before release so client reuse cannot+// accumulate listeners.+function logCheckedOutClientError(err) {+ console.error(`🗄️ Checked-out database client error: ${err.message}`);+}+/*** Is the configured database a DESIGNATED test database?*@@ -328,6 +337,7 @@ export async function withTransaction(fn) {async function runTransaction(fn) {const client = await pool.connect();+ client.on('error', logCheckedOutClientError);// Guard the client's row writes with the same backstop as query(). The raw// pg client bypasses query() entirely, so without this wrapper a test-runner// process pointed at the real `portos` DB writes through every store's@@ -354,6 +364,7 @@ async function runTransaction(fn) {if (began) await client.query('ROLLBACK').catch(() => {});throw err;} finally {+ client.removeListener('error', logCheckedOutClientError);client.release(releaseErr);}}@@ -423,6 +434,7 @@ export async function verifyDatabaseMaintenanceTarget(id) {const operation = readTarget();if (isTestRunner() && !/_test$/.test(POOL_CONFIG.database) && process.env.TEST_DB_OK !== '1') throw refuse();const client = await pool.connect();+ client.on('error', logCheckedOutClientError);let discard = false;try {await client.query({ text: 'BEGIN READ ONLY', query_timeout: 10_000 });@@ -443,6 +455,7 @@ export async function verifyDatabaseMaintenanceTarget(id) {return { id, stage: operation.stage, target: operation.target.mode, targetHealthy: true, fenced: true };} finally {await client.query({ text: 'ROLLBACK', query_timeout: 10_000 }).catch(() => { discard = true; });+ client.removeListener('error', logCheckedOutClientError);client.release(discard);}}@@ -541,6 +554,7 @@ async function ensureSchemaImpl() {// deliberately not wrapped in one either — see the comment block above), so// nothing here may be wrapped in withTransaction()/BEGIN.const client = await pool.connect();+ client.on('error', logCheckedOutClientError);try {const { rows: [{ locked }] } = await client.query('SELECT pg_try_advisory_lock($1) AS locked', [SCHEMA_DDL_ADVISORY_LOCK_KEY]);if (!locked) {@@ -585,6 +599,7 @@ async function ensureSchemaImpl() {// itself fails, destroy the connection instead: closing it releases the// lock automatically, same as a hard kill mid-boot.const unlocked = await client.query('SELECT pg_advisory_unlock($1)', [SCHEMA_DDL_ADVISORY_LOCK_KEY]).then(() => true, () => false);+ client.removeListener('error', logCheckedOutClientError);client.release(!unlocked);}}diff --git a/server/lib/db.restoreRecovery.test.js b/server/lib/db.restoreRecovery.test.jsindex 1059273f19..19d560884d 100644--- a/server/lib/db.restoreRecovery.test.js+++ b/server/lib/db.restoreRecovery.test.js@@ -8,7 +8,13 @@ import { tmpdir } from 'node:os';import { join } from 'node:path';import { fileURLToPath, pathToFileURL } from 'node:url';-const fixture = vi.hoisted(() => ({ dataRoot: null, releaseFault: null, executionFault: false }));+const fixture = vi.hoisted(() => ({+ dataRoot: null,+ releaseFault: null,+ releaseJournalUnlinked: false,+ releaseFaultTriggered: false,+ executionFault: false,+}));vi.mock('node:fs', async (importOriginal) => {const fs = await importOriginal();return {@@ -17,10 +23,14 @@ vi.mock('node:fs', async (importOriginal) => {if (fixture.releaseFault === 'unlink' && path === join(fixture.dataRoot, 'database-restore-recovery.json')) {throw Object.assign(new Error('synthetic unlink failure'), { code: 'EIO' });}- return fs.unlinkSync(path);+ const result = fs.unlinkSync(path);+ if (path === join(fixture.dataRoot, 'database-restore-recovery.json')) fixture.releaseJournalUnlinked = true;+ return result;}),fsyncSync: vi.fn((fd) => {- if (fixture.releaseFault === 'directory-sync' && fs.fstatSync(fd).isDirectory()) {+ if (fixture.releaseFault === 'directory-sync' && fixture.releaseJournalUnlinked+ && !fixture.releaseFaultTriggered && fs.fstatSync(fd).isDirectory()) {+ fixture.releaseFaultTriggered = true;throw Object.assign(new Error('synthetic directory sync failure'), { code: 'EIO' });}return fs.fsyncSync(fd);@@ -60,7 +70,12 @@ vi.mock('./paths.js', async (importOriginal) => {});const pool = vi.hoisted(() => ({- connect: vi.fn(async () => ({ query: vi.fn(async () => ({ rows: [] })), release: vi.fn() })),+ connect: vi.fn(async () => ({+ query: vi.fn(async () => ({ rows: [] })),+ on: vi.fn(),+ removeListener: vi.fn(),+ release: vi.fn(),+ })),on: vi.fn(),query: vi.fn(async () => ({ rows: [] })),}));@@ -86,6 +101,9 @@ const FENCED = { status: 503, code: 'DATABASE_RESTORE_RECOVERY' };beforeEach(() => {rmSync(databaseRestoreRecovery.path, { force: true });+ fixture.releaseFault = null;+ fixture.releaseJournalUnlinked = false;+ fixture.releaseFaultTriggered = false;fixture.executionFault = false;vi.clearAllMocks();});diff --git a/server/lib/db.schemaLifecycle.test.js b/server/lib/db.schemaLifecycle.test.jsindex 85ecd62175..cc7c4c2281 100644--- a/server/lib/db.schemaLifecycle.test.js+++ b/server/lib/db.schemaLifecycle.test.js@@ -1,7 +1,7 @@import { beforeEach, describe, expect, it, vi } from 'vitest';const state = vi.hoisted(() => ({- query: vi.fn(), release: vi.fn(), connect: vi.fn(),+ query: vi.fn(), on: vi.fn(), removeListener: vi.fn(), release: vi.fn(), connect: vi.fn(),}));vi.mock('pg', async (importOriginal) => ({ default: {Client: (await importOriginal()).default.Client,@@ -20,7 +20,12 @@ beforeEach(() => {vi.resetModules();vi.clearAllMocks();state.query.mockImplementation(async () => ({ rows: [{ locked: true }] }));- state.connect.mockResolvedValue({ query: state.query, release: state.release });+ state.connect.mockResolvedValue({+ query: state.query,+ on: state.on,+ removeListener: state.removeListener,+ release: state.release,+ });});describe('schema readiness across store startup', () => {@@ -34,6 +39,8 @@ describe('schema readiness across store startup', () => {'repair catalog', 'SELECT pg_advisory_unlock($1)',]);expect(state.release).toHaveBeenCalledWith(false);+ expect(state.on).toHaveBeenCalledWith('error', expect.any(Function));+ expect(state.removeListener).toHaveBeenCalledWith('error', state.on.mock.calls[0][1]);});it('allows forced repair and retries after its failure without caching partial work', async () => {diff --git a/server/lib/db.transaction.test.js b/server/lib/db.transaction.test.jsindex 0ef7929d86..2ab68ba661 100644--- a/server/lib/db.transaction.test.js+++ b/server/lib/db.transaction.test.js@@ -1,4 +1,5 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { EventEmitter } from 'node:events';const pool = vi.hoisted(() => ({client: null,@@ -18,10 +19,10 @@ vi.mock('pg', async (importOriginal) => ({import { withTransaction, query, withDatabaseMaintenance } from './db.js';function makeClient(query) {- return {- query,- release: vi.fn(),- };+ const client = new EventEmitter();+ client.query = query;+ client.release = vi.fn();+ return client;}afterEach(() => {@@ -113,6 +114,31 @@ describe('withTransaction', () => {expect(client.release).toHaveBeenCalledWith(dropError);});++ it('handles a checked-out client socket error and removes the listener before reuse', async () => {+ const socketError = new Error('Connection terminated unexpectedly');+ const client = makeClient(vi.fn()+ .mockResolvedValueOnce({})+ .mockRejectedValueOnce(socketError)+ .mockRejectedValueOnce(new Error('ROLLBACK on closed socket')));+ pool.client = client;+ const log = vi.spyOn(console, 'error').mockImplementation(() => {});++ await expect(withTransaction(async (transaction) => {+ expect(client.listenerCount('error')).toBe(1);+ expect(() => client.emit('error', socketError)).not.toThrow();+ await transaction.query('SELECT 1');+ })).rejects.toBe(socketError);++ expect(log).toHaveBeenCalledWith(`🗄️ Checked-out database client error: ${socketError.message}`);+ expect(client.listenerCount('error')).toBe(0);+ expect(client.release).toHaveBeenCalledWith(socketError);++ client.query.mockResolvedValue({});+ await expect(withTransaction(async () => 'reused')).resolves.toBe('reused');+ expect(client.listenerCount('error')).toBe(0);+ expect(client.release).toHaveBeenLastCalledWith(undefined);+ });});// Cross-reference: db.guards.test.js covers assertWriteAllowed() directly; this