Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(database): stop a boot migration whose lock connection drops

Yudhi Armyndharis922f5e84bca5
diff --git a/src/database/pg-boot-migrations.spec.ts b/src/database/pg-boot-migrations.spec.ts
index 1d8ad3bfe..a53ac008e 100644
--- a/src/database/pg-boot-migrations.spec.ts
+++ b/src/database/pg-boot-migrations.spec.ts
@@ -194,6 +194,7 @@ describe('createBootDataSource (postgres boot migrations)', () => {
connectionTimeoutMillis: 10000,
options: '-c statement_timeout=0',
}),
+ expect.any(Function),
);
});
@@ -330,6 +331,7 @@ describe('createBootDataSource (postgres boot migrations)', () => {
expect(deps.createLockClient).toHaveBeenCalledWith(
expect.objectContaining({ connectionTimeoutMillis: 10000, options: '-c statement_timeout=0' }),
+ expect.any(Function),
);
});
@@ -421,6 +423,34 @@ describe('createBootDataSource (postgres boot migrations)', () => {
}
});
+ // The lock is session-scoped, so a holder whose lock connection drops has lost it: another replica
+ // can start the same chain. The holder must stop migrating and fail the boot, which the retry loop
+ // then reruns under a new lock.
+ it('stops migrating and fails the boot when the held lock connection drops', async () => {
+ let finishChain: () => void = () => undefined;
+ const chain = new Promise<void>(resolve => (finishChain = resolve));
+ const { calls, deps } = makeFakes(() => chain);
+ let onLost: (error: Error) => void = () => undefined;
+ const createLockClient = deps.createLockClient as jest.Mock;
+ const lockClient = createLockClient.getMockImplementation()!() as AdvisoryLockClient;
+ createLockClient.mockImplementation((_config: ClientConfig, lost: (error: Error) => void) => {
+ onLost = lost;
+ return lockClient;
+ });
+
+ const boot = createBootDataSource(PG_OPTIONS, deps);
+ await new Promise(resolve => setImmediate(resolve));
+ expect(calls).toContain('runMigrations');
+
+ onLost(new Error('Connection terminated unexpectedly'));
+ expect(calls.filter(c => c === 'destroy')).toHaveLength(1);
+ finishChain();
+
+ await expect(boot).rejects.toThrow(/lock connection lost while migrating/);
+ // The runtime DataSource is never built on a chain that ran partly unlocked.
+ expect(calls.filter(c => c === 'initialize')).toHaveLength(1);
+ });
+
it('refuses to migrate on a connection whose session is not on UTC', async () => {
// A pin that did not take (a pooler dropping the SET, a server-side default re-applied after it)
// would have the driver read every naive timestamp as UTC while the server keeps writing its own
diff --git a/src/database/pg-boot-migrations.ts b/src/database/pg-boot-migrations.ts
index b89505be9..f3e8c6b3c 100644
--- a/src/database/pg-boot-migrations.ts
+++ b/src/database/pg-boot-migrations.ts
@@ -28,7 +28,7 @@ export interface AdvisoryLockClient {
// Test seams over the two constructions this module performs.
export interface BootDataSourceDeps {
createDataSource?: (options: DataSourceOptions) => DataSource;
- createLockClient?: (config: ClientConfig) => AdvisoryLockClient;
+ createLockClient?: (config: ClientConfig, onLost: (error: Error) => void) => AdvisoryLockClient;
}
type PostgresOptions = Extract<DataSourceOptions, { type: 'postgres' }>;
@@ -77,14 +77,27 @@ export async function createBootDataSource(
// Before any migration writes a row: a connection whose UTC pin did not take stores timestamps in
// one zone and reads them in another, which nothing downstream can detect (see postgres-utc.ts).
await assertDataConnectionUtc(migrator);
- const lockClient = createLockClient(lockClientConfig(options));
+ // A holder whose lock connection drops has lost the lock with it, so another replica can start
+ // the same chain. Stop this one where it stands: tearing the migration pool down fails its
+ // in-flight statement and rolls the current migration back, and the boot fails so the retry loop
+ // reruns it under a new lock. A waiter needs nothing extra: its pending lock query rejects.
+ let holding = false;
+ let lockLost = false;
+ const lockClient = createLockClient(lockClientConfig(options), () => {
+ if (!holding) return;
+ lockLost = true;
+ void migrator.destroy().catch(() => undefined);
+ });
try {
await lockClient.connect();
await lockClient.query('SELECT pg_advisory_lock($1, $2)', [...POSTGRES_BOOT_MIGRATION_LOCK_KEYS]);
+ holding = true;
try {
// Same transaction mode DataSource.initialize() passes for the built-in migrationsRun.
await migrator.runMigrations({ transaction: options.migrationsTransactionMode });
+ if (lockLost) throw new Error('Boot migration lock connection lost while migrating; retrying the boot');
} finally {
+ holding = false;
// Session-scoped lock: even when the unlock call itself fails, end() below tears the
// session — and with it the lock — down, so no crashed boot can leave it held.
await lockClient
@@ -114,13 +127,14 @@ export async function createBootDataSource(
// pg emits 'error' on the client when its socket drops while the client is not ending (a failover,
// pg_terminate_backend, an idle-timeout on the silent wait inside pg_advisory_lock). Unheard, that
-// emit throws from the socket handler and exits the process. The in-flight query rejects on its own,
-// so the factory's cleanup and Nest's retry loop take it from there; a holder that loses the socket
-// has lost the lock with it, which is worth a warning.
-function createPgLockClient(config: ClientConfig): AdvisoryLockClient {
+// emit throws from the socket handler and exits the process. A waiter's pending lock query rejects on
+// its own, so the factory's cleanup and Nest's retry loop take it from there; a holder is stopped by
+// onLost, since it lost the lock with the socket.
+function createPgLockClient(config: ClientConfig, onLost: (error: Error) => void): AdvisoryLockClient {
const client = new Client(config);
client.on('error', (error: Error) => {
logger.warn(`Boot migration lock connection lost: ${error.message}`);
+ onLost(error);
});
return client;
}