Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(database): keep a dropped boot lock connection from crashing boot

Yudhi Armyndharis6229c578845d
diff --git a/src/database/pg-boot-migrations.spec.ts b/src/database/pg-boot-migrations.spec.ts
index ef496cdc5..1d8ad3bfe 100644
--- a/src/database/pg-boot-migrations.spec.ts
+++ b/src/database/pg-boot-migrations.spec.ts
@@ -1,3 +1,4 @@
+import { EventEmitter } from 'events';
import { DataSource, DataSourceOptions } from 'typeorm';
// Default import (not `import * as`) on purpose: it binds straight to pg's module.exports, so the
// constructor spy below reaches the same object pg-boot-migrations reads `Client` from at call time.
@@ -350,11 +351,11 @@ describe('createBootDataSource (postgres boot migrations)', () => {
// The production path. Only the spots that would touch the outside world are stubbed — the
// DataSource lifecycle methods (initialize would open a pool) and the pg Client constructor
// (connect would open a socket) — so both default factories run as shipped.
- const lockClient: AdvisoryLockClient = {
+ const lockClient: AdvisoryLockClient = Object.assign(new EventEmitter(), {
connect: jest.fn(() => Promise.resolve()),
query: jest.fn(() => Promise.resolve()),
end: jest.fn(() => Promise.resolve()),
- };
+ });
const clientCtor = jest.spyOn(pg, 'Client').mockImplementation(() => lockClient as unknown as PgClient);
const initialize = jest.spyOn(DataSource.prototype, 'initialize').mockImplementation(function (this: DataSource) {
return Promise.resolve(this);
@@ -398,6 +399,28 @@ describe('createBootDataSource (postgres boot migrations)', () => {
}
});
+ // pg emits 'error' on the client when its socket drops outside the client's own end() (failover,
+ // pg_terminate_backend, an idle-timeout on the silent wait inside pg_advisory_lock). With no
+ // listener that emit throws from the socket handler and kills the process before the failed lock
+ // query can reject into the factory's cleanup and Nest's retry loop.
+ it('listens for lock-client errors so a dropped connection cannot crash the process', async () => {
+ const lockClient = Object.assign(new EventEmitter(), {
+ connect: jest.fn(() => Promise.resolve()),
+ query: jest.fn(() => Promise.resolve()),
+ end: jest.fn(() => Promise.resolve()),
+ });
+ const clientCtor = jest.spyOn(pg, 'Client').mockImplementation(() => lockClient as unknown as PgClient);
+ const { deps } = makeFakes();
+ try {
+ await createBootDataSource(PG_OPTIONS, { createDataSource: deps.createDataSource });
+
+ expect(lockClient.listenerCount('error')).toBeGreaterThan(0);
+ expect(() => lockClient.emit('error', new Error('Connection terminated unexpectedly'))).not.toThrow();
+ } finally {
+ clientCtor.mockRestore();
+ }
+ });
+
it('refuses to migrate on a connection whose session is not on UTC', async () => {
// A pin that did not take (a pooler dropping the SET, a server-side default re-applied after it)
// would have the driver read every naive timestamp as UTC while the server keeps writing its own
diff --git a/src/database/pg-boot-migrations.ts b/src/database/pg-boot-migrations.ts
index 7270e0e64..b89505be9 100644
--- a/src/database/pg-boot-migrations.ts
+++ b/src/database/pg-boot-migrations.ts
@@ -1,7 +1,10 @@
import { Client, ClientConfig } from 'pg';
import { DataSource, DataSourceOptions } from 'typeorm';
+import { createLogger } from '../common/services/logger.service';
import { assertDataConnectionUtc, postgresUtcExtra } from './postgres-utc';
+const logger = createLogger('PgBootMigrations');
+
// The postgres data connection runs its boot migrations while holding a session-scoped Postgres
// advisory lock, so replicas that boot at the same time serialize instead of racing DDL against
// the shared migrations ledger: the lock holder applies the chain while every other process waits
@@ -46,7 +49,7 @@ export async function createBootDataSource(
deps: BootDataSourceDeps = {},
): Promise<DataSource> {
const createDataSource = deps.createDataSource ?? (opts => new DataSource(opts));
- const createLockClient = deps.createLockClient ?? (config => new Client(config));
+ const createLockClient = deps.createLockClient ?? createPgLockClient;
if (options?.type !== 'postgres') {
// useFactory always resolves a full options object; the optional parameter is the library's
@@ -109,6 +112,19 @@ export async function createBootDataSource(
return dataSource;
}
+// pg emits 'error' on the client when its socket drops while the client is not ending (a failover,
+// pg_terminate_backend, an idle-timeout on the silent wait inside pg_advisory_lock). Unheard, that
+// emit throws from the socket handler and exits the process. The in-flight query rejects on its own,
+// so the factory's cleanup and Nest's retry loop take it from there; a holder that loses the socket
+// has lost the lock with it, which is worth a warning.
+function createPgLockClient(config: ClientConfig): AdvisoryLockClient {
+ const client = new Client(config);
+ client.on('error', (error: Error) => {
+ logger.warn(`Boot migration lock connection lost: ${error.message}`);
+ });
+ return client;
+}
+
function lockClientConfig(options: PostgresOptions): ClientConfig {
const extra = (options.extra ?? {}) as { connectionTimeoutMillis?: number };
return {