Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(database): stop a boot migration whose lock connection drops
Yudhi Armyndharis922f5e84bca5
diff --git a/src/database/pg-boot-migrations.spec.ts b/src/database/pg-boot-migrations.spec.tsindex 1d8ad3bfe..a53ac008e 100644--- a/src/database/pg-boot-migrations.spec.ts+++ b/src/database/pg-boot-migrations.spec.ts@@ -194,6 +194,7 @@ describe('createBootDataSource (postgres boot migrations)', () => {connectionTimeoutMillis: 10000,options: '-c statement_timeout=0',}),+ expect.any(Function),);});@@ -330,6 +331,7 @@ describe('createBootDataSource (postgres boot migrations)', () => {expect(deps.createLockClient).toHaveBeenCalledWith(expect.objectContaining({ connectionTimeoutMillis: 10000, options: '-c statement_timeout=0' }),+ expect.any(Function),);});@@ -421,6 +423,34 @@ describe('createBootDataSource (postgres boot migrations)', () => {}});+ // The lock is session-scoped, so a holder whose lock connection drops has lost it: another replica+ // can start the same chain. The holder must stop migrating and fail the boot, which the retry loop+ // then reruns under a new lock.+ it('stops migrating and fails the boot when the held lock connection drops', async () => {+ let finishChain: () => void = () => undefined;+ const chain = new Promise<void>(resolve => (finishChain = resolve));+ const { calls, deps } = makeFakes(() => chain);+ let onLost: (error: Error) => void = () => undefined;+ const createLockClient = deps.createLockClient as jest.Mock;+ const lockClient = createLockClient.getMockImplementation()!() as AdvisoryLockClient;+ createLockClient.mockImplementation((_config: ClientConfig, lost: (error: Error) => void) => {+ onLost = lost;+ return lockClient;+ });++ const boot = createBootDataSource(PG_OPTIONS, deps);+ await new Promise(resolve => setImmediate(resolve));+ expect(calls).toContain('runMigrations');++ onLost(new Error('Connection terminated unexpectedly'));+ expect(calls.filter(c => c === 'destroy')).toHaveLength(1);+ finishChain();++ await expect(boot).rejects.toThrow(/lock connection lost while migrating/);+ // The runtime DataSource is never built on a chain that ran partly unlocked.+ expect(calls.filter(c => c === 'initialize')).toHaveLength(1);+ });+it('refuses to migrate on a connection whose session is not on UTC', async () => {// A pin that did not take (a pooler dropping the SET, a server-side default re-applied after it)// would have the driver read every naive timestamp as UTC while the server keeps writing its owndiff --git a/src/database/pg-boot-migrations.ts b/src/database/pg-boot-migrations.tsindex b89505be9..f3e8c6b3c 100644--- a/src/database/pg-boot-migrations.ts+++ b/src/database/pg-boot-migrations.ts@@ -28,7 +28,7 @@ export interface AdvisoryLockClient {// Test seams over the two constructions this module performs.export interface BootDataSourceDeps {createDataSource?: (options: DataSourceOptions) => DataSource;- createLockClient?: (config: ClientConfig) => AdvisoryLockClient;+ createLockClient?: (config: ClientConfig, onLost: (error: Error) => void) => AdvisoryLockClient;}type PostgresOptions = Extract<DataSourceOptions, { type: 'postgres' }>;@@ -77,14 +77,27 @@ export async function createBootDataSource(// Before any migration writes a row: a connection whose UTC pin did not take stores timestamps in// one zone and reads them in another, which nothing downstream can detect (see postgres-utc.ts).await assertDataConnectionUtc(migrator);- const lockClient = createLockClient(lockClientConfig(options));+ // A holder whose lock connection drops has lost the lock with it, so another replica can start+ // the same chain. Stop this one where it stands: tearing the migration pool down fails its+ // in-flight statement and rolls the current migration back, and the boot fails so the retry loop+ // reruns it under a new lock. A waiter needs nothing extra: its pending lock query rejects.+ let holding = false;+ let lockLost = false;+ const lockClient = createLockClient(lockClientConfig(options), () => {+ if (!holding) return;+ lockLost = true;+ void migrator.destroy().catch(() => undefined);+ });try {await lockClient.connect();await lockClient.query('SELECT pg_advisory_lock($1, $2)', [...POSTGRES_BOOT_MIGRATION_LOCK_KEYS]);+ holding = true;try {// Same transaction mode DataSource.initialize() passes for the built-in migrationsRun.await migrator.runMigrations({ transaction: options.migrationsTransactionMode });+ if (lockLost) throw new Error('Boot migration lock connection lost while migrating; retrying the boot');} finally {+ holding = false;// Session-scoped lock: even when the unlock call itself fails, end() below tears the// session — and with it the lock — down, so no crashed boot can leave it held.await lockClient@@ -114,13 +127,14 @@ export async function createBootDataSource(// pg emits 'error' on the client when its socket drops while the client is not ending (a failover,// pg_terminate_backend, an idle-timeout on the silent wait inside pg_advisory_lock). Unheard, that-// emit throws from the socket handler and exits the process. The in-flight query rejects on its own,-// so the factory's cleanup and Nest's retry loop take it from there; a holder that loses the socket-// has lost the lock with it, which is worth a warning.-function createPgLockClient(config: ClientConfig): AdvisoryLockClient {+// emit throws from the socket handler and exits the process. A waiter's pending lock query rejects on+// its own, so the factory's cleanup and Nest's retry loop take it from there; a holder is stopped by+// onLost, since it lost the lock with the socket.+function createPgLockClient(config: ClientConfig, onLost: (error: Error) => void): AdvisoryLockClient {const client = new Client(config);client.on('error', (error: Error) => {logger.warn(`Boot migration lock connection lost: ${error.message}`);+ onLost(error);});return client;}