Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(database): run Postgres boot migrations without the runtime timeout

Yudhi Armyndharis60ea19ba9e8e
diff --git a/src/database/pg-boot-migrations.spec.ts b/src/database/pg-boot-migrations.spec.ts
index ad09c139c..ef496cdc5 100644
--- a/src/database/pg-boot-migrations.spec.ts
+++ b/src/database/pg-boot-migrations.spec.ts
@@ -82,6 +82,7 @@ describe('createBootDataSource (postgres boot migrations)', () => {
const returned = await createBootDataSource(PG_OPTIONS, deps);
expect(returned).toBe(dataSource);
+ // One fake stands in for both the migration pool (torn down after the chain) and the runtime one.
expect(calls).toEqual([
'initialize',
'connect',
@@ -89,6 +90,8 @@ describe('createBootDataSource (postgres boot migrations)', () => {
'runMigrations',
'SELECT pg_advisory_unlock($1, $2)',
'end',
+ 'destroy',
+ 'initialize',
]);
// Same key for acquire and release, in the (key1, key2) form.
expect(lockClient.query).toHaveBeenNthCalledWith(1, 'SELECT pg_advisory_lock($1, $2)', [
@@ -97,10 +100,10 @@ describe('createBootDataSource (postgres boot migrations)', () => {
expect(lockClient.query).toHaveBeenNthCalledWith(2, 'SELECT pg_advisory_unlock($1, $2)', [
...POSTGRES_BOOT_MIGRATION_LOCK_KEYS,
]);
- // Migration execution preserves the built-in migrationsRun transaction mode, and the boot
- // error path (destroy) never fires on success.
+ // Migration execution preserves the built-in migrationsRun transaction mode, and only the
+ // migration pool is torn down on success.
expect(dataSource.runMigrations).toHaveBeenCalledWith({ transaction: 'all' });
- expect(dataSource.destroy).not.toHaveBeenCalled();
+ expect(dataSource.destroy).toHaveBeenCalledTimes(1);
});
it('constructs the DataSource with migrationsRun disabled and never mutates the config', async () => {
@@ -124,6 +127,55 @@ describe('createBootDataSource (postgres boot migrations)', () => {
expect(PG_OPTIONS.extra).toEqual({ statement_timeout: 30000, connectionTimeoutMillis: 10000 });
});
+ it('runs the chain on its own pool without the runtime statement_timeout, then returns one that keeps it', async () => {
+ // pg sends statement_timeout in the startup packet, so every statement on a pool built with it
+ // inherits the limit: a backfill or index build over a large table was cancelled at 30 s, the
+ // whole 'all' transaction rolled back, and every retry of the boot failed the same way.
+ const migrator = makeFakes();
+ const runtime = makeFakes();
+ const createDataSource = jest
+ .fn<DataSource, [DataSourceOptions]>()
+ .mockReturnValueOnce(migrator.dataSource as unknown as DataSource)
+ .mockReturnValueOnce(runtime.dataSource as unknown as DataSource);
+
+ const returned = await createBootDataSource(PG_OPTIONS, {
+ createDataSource,
+ createLockClient: migrator.deps.createLockClient,
+ });
+
+ expect(returned).toBe(runtime.dataSource);
+ expect(migrator.calls).toEqual([
+ 'initialize',
+ 'connect',
+ 'SELECT pg_advisory_lock($1, $2)',
+ 'runMigrations',
+ 'SELECT pg_advisory_unlock($1, $2)',
+ 'end',
+ 'destroy',
+ ]);
+ expect(runtime.calls).toEqual(['initialize']);
+ const [[migratorOptions], [runtimeOptions]] = createDataSource.mock.calls;
+ expect(migratorOptions.extra).not.toHaveProperty('statement_timeout');
+ expect(migratorOptions.extra).toMatchObject({ connectionTimeoutMillis: 10000 });
+ expect(runtimeOptions.extra).toMatchObject({ statement_timeout: 30000, connectionTimeoutMillis: 10000 });
+ expect(runtime.dataSource.query).toHaveBeenCalled(); // the runtime pool's UTC pin is verified too
+ });
+
+ it('tears the runtime DataSource down when its session is not on UTC, after the chain is applied', async () => {
+ const migrator = makeFakes();
+ const runtime = makeFakes(jest.fn(), 25200);
+ const createDataSource = jest
+ .fn<DataSource, [DataSourceOptions]>()
+ .mockReturnValueOnce(migrator.dataSource as unknown as DataSource)
+ .mockReturnValueOnce(runtime.dataSource as unknown as DataSource);
+
+ await expect(
+ createBootDataSource(PG_OPTIONS, { createDataSource, createLockClient: migrator.deps.createLockClient }),
+ ).rejects.toThrow(/not on UTC/);
+
+ expect(runtime.calls).toEqual(['initialize', 'destroy']);
+ });
+
it('builds the lock client without a statement timeout (pg_advisory_lock must survive the wait)', async () => {
const { deps } = makeFakes();
@@ -217,8 +269,10 @@ describe('createBootDataSource (postgres boot migrations)', () => {
'runMigrations',
'SELECT pg_advisory_unlock($1, $2)',
'end',
+ 'destroy',
+ 'initialize',
]);
- expect(dataSource.destroy).not.toHaveBeenCalled();
+ expect(dataSource.destroy).toHaveBeenCalledTimes(1); // the migration pool only
});
it('still resolves when lock-client end() fails: the boot result does not depend on client teardown', async () => {
@@ -238,8 +292,10 @@ describe('createBootDataSource (postgres boot migrations)', () => {
'runMigrations',
'SELECT pg_advisory_unlock($1, $2)',
'end',
+ 'destroy',
+ 'initialize',
]);
- expect(dataSource.destroy).not.toHaveBeenCalled();
+ expect(dataSource.destroy).toHaveBeenCalledTimes(1); // the migration pool only
});
it('surfaces the migration error, not the teardown error, when destroy() rejects too', async () => {
@@ -330,7 +386,9 @@ describe('createBootDataSource (postgres boot migrations)', () => {
...POSTGRES_BOOT_MIGRATION_LOCK_KEYS,
]);
expect(runMigrations).toHaveBeenCalledWith({ transaction: 'all' });
- expect(destroy).not.toHaveBeenCalled();
+ // The migration pool is torn down; the returned runtime one keeps the statement timeout.
+ expect(destroy).toHaveBeenCalledTimes(1);
+ expect(returned.options.extra).toMatchObject({ statement_timeout: 30000 });
} finally {
clientCtor.mockRestore();
initialize.mockRestore();
diff --git a/src/database/pg-boot-migrations.ts b/src/database/pg-boot-migrations.ts
index 6699100d3..7270e0e64 100644
--- a/src/database/pg-boot-migrations.ts
+++ b/src/database/pg-boot-migrations.ts
@@ -58,23 +58,29 @@ export async function createBootDataSource(
// DataSource itself never starts them unsynchronized inside initialize(). The UTC pin is merged in
// at the same point, because this is the one place the runtime postgres data connection is built
// (the migration CLI's own data source carries it directly).
- const dataSource = createDataSource({
- ...options,
- migrationsRun: false,
- extra: { ...(options.extra as Record<string, unknown> | undefined), ...postgresUtcExtra() },
- });
+ const build = (extra: Record<string, unknown> | undefined): DataSource =>
+ createDataSource({ ...options, migrationsRun: false, extra: { ...extra, ...postgresUtcExtra() } });
+ const runtimeExtra = options.extra as Record<string, unknown> | undefined;
+
+ // statement_timeout bounds live runtime queries, and pg sends it in the startup packet, so every
+ // statement on a pool built with it inherits the limit. A backfill or index build over a large
+ // table can legitimately run longer, so the chain runs on its own short-lived pool without it (as
+ // the migration CLI does), and the runtime DataSource is built only once the chain is applied.
+ const migrationExtra = { ...runtimeExtra };
+ delete migrationExtra.statement_timeout;
+ const migrator = build(migrationExtra);
try {
- await dataSource.initialize();
+ await migrator.initialize();
// Before any migration writes a row: a connection whose UTC pin did not take stores timestamps in
// one zone and reads them in another, which nothing downstream can detect (see postgres-utc.ts).
- await assertDataConnectionUtc(dataSource);
+ await assertDataConnectionUtc(migrator);
const lockClient = createLockClient(lockClientConfig(options));
try {
await lockClient.connect();
await lockClient.query('SELECT pg_advisory_lock($1, $2)', [...POSTGRES_BOOT_MIGRATION_LOCK_KEYS]);
try {
// Same transaction mode DataSource.initialize() passes for the built-in migrationsRun.
- await dataSource.runMigrations({ transaction: options.migrationsTransactionMode });
+ await migrator.runMigrations({ transaction: options.migrationsTransactionMode });
} finally {
// Session-scoped lock: even when the unlock call itself fails, end() below tears the
// session — and with it the lock — down, so no crashed boot can leave it held.
@@ -85,10 +91,18 @@ export async function createBootDataSource(
} finally {
await lockClient.end().catch(() => undefined);
}
+ } finally {
+ // The migration pool never outlives this block: on success the runtime DataSource below replaces
+ // it, and on failure a half-open one would stack pools across the boot retry loop. The failure
+ // still fails boot via the factory's rejection; a teardown error never masks it.
+ await migrator.destroy().catch(() => undefined);
+ }
+
+ const dataSource = build(runtimeExtra);
+ try {
+ await dataSource.initialize();
+ await assertDataConnectionUtc(dataSource);
} catch (error) {
- // Same failure handling as DataSource.initialize()'s own migrate step: never leave a
- // half-open DataSource behind (the boot retry loop would stack their pools). The error still
- // fails boot via the factory's rejection.
await dataSource.destroy().catch(() => undefined);
throw error;
}