Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(database): bound the UTC pin on a silent server

Yudhi Armyndharisc1b7b67dadb0
diff --git a/src/database/data-source.ts b/src/database/data-source.ts
index aafcd11a6..93776f082 100644
--- a/src/database/data-source.ts
+++ b/src/database/data-source.ts
@@ -83,6 +83,7 @@ const sqliteDataSourceOptions: DataSourceOptions = {
export function buildPostgresDataSourceOptions(env: NodeJS.ProcessEnv = process.env): DataSourceOptions {
const schema = env.POSTGRES_SCHEMA || 'public';
const useCustomSearchPath = schema !== 'public';
+ const connectionTimeoutMillis = parseInt(env.DATABASE_CONNECTION_TIMEOUT_MS || '10000', 10);
return {
type: 'postgres',
schema,
@@ -104,12 +105,12 @@ export function buildPostgresDataSourceOptions(env: NodeJS.ProcessEnv = process.
extra: {
// Same UTC pin the runtime data connection carries (pg-boot-migrations.ts): a migration that
// rewrites a timestamp column must mean the same thing as the app that wrote it.
- ...postgresUtcExtra(),
+ ...postgresUtcExtra(connectionTimeoutMillis),
max: parseInt(env.DATABASE_POOL_SIZE || '10', 10),
// Pool resilience only. NO statement_timeout here: this connection runs migrations, and a
// long CREATE INDEX / backfill must not be aborted mid-flight.
idleTimeoutMillis: parseInt(env.DATABASE_IDLE_TIMEOUT_MS || '30000', 10),
- connectionTimeoutMillis: parseInt(env.DATABASE_CONNECTION_TIMEOUT_MS || '10000', 10),
+ connectionTimeoutMillis,
// Only set for a non-public schema (see above).
...(useCustomSearchPath ? { options: `-c search_path=${schema},public` } : {}),
},
diff --git a/src/database/pg-boot-migrations.spec.ts b/src/database/pg-boot-migrations.spec.ts
index 523b52681..96c202cc3 100644
--- a/src/database/pg-boot-migrations.spec.ts
+++ b/src/database/pg-boot-migrations.spec.ts
@@ -378,6 +378,20 @@ describe('createBootDataSource (postgres boot migrations)', () => {
expect(config.ssl).toBe(ssl);
});
+ it('bounds the UTC pin of the migration and the runtime pool with that connect timeout', async () => {
+ const { deps } = makeFakes();
+
+ await createBootDataSource({ ...PG_OPTIONS, extra: { connectionTimeoutMillis: 2500 } }, deps);
+
+ const built = (deps.createDataSource as jest.Mock<DataSource, [DataSourceOptions]>).mock.calls;
+ expect(built).toHaveLength(2);
+ for (const [options] of built) {
+ const query = jest.fn().mockResolvedValue({});
+ await (options.extra as { onConnect: (client: never) => Promise<unknown> }).onConnect({ query } as never);
+ expect(query).toHaveBeenCalledWith({ text: "SET TIME ZONE 'UTC'", query_timeout: 2500 });
+ }
+ });
+
it('falls back to the real DataSource and pg Client constructors when no deps are injected', async () => {
// The production path. Only the spots that would touch the outside world are stubbed — the
// DataSource lifecycle methods (initialize would open a pool) and the pg Client constructor
diff --git a/src/database/pg-boot-migrations.ts b/src/database/pg-boot-migrations.ts
index 703ba7c8a..0ed8a1682 100644
--- a/src/database/pg-boot-migrations.ts
+++ b/src/database/pg-boot-migrations.ts
@@ -91,9 +91,13 @@ export async function createBootDataSource(
// DataSource itself never starts them unsynchronized inside initialize(). The UTC pin is merged in
// at the same point, because this is the one place the runtime postgres data connection is built
// (the migration CLI's own data source carries it directly).
- const build = (extra: Record<string, unknown> | undefined): DataSource =>
- createDataSource({ ...options, migrationsRun: false, extra: { ...extra, ...postgresUtcExtra() } });
- const runtimeExtra = options.extra as Record<string, unknown> | undefined;
+ const build = (extra: ClientConfig | undefined): DataSource =>
+ createDataSource({
+ ...options,
+ migrationsRun: false,
+ extra: { ...extra, ...postgresUtcExtra(extra?.connectionTimeoutMillis) },
+ });
+ const runtimeExtra = options.extra as ClientConfig | undefined;
// statement_timeout bounds live runtime queries, and pg sends it in the startup packet, so every
// statement on a pool built with it inherits the limit. A backfill or index build over a large
diff --git a/src/database/postgres-utc.spec.ts b/src/database/postgres-utc.spec.ts
index 7ce3a606d..80beef8ea 100644
--- a/src/database/postgres-utc.spec.ts
+++ b/src/database/postgres-utc.spec.ts
@@ -1,5 +1,5 @@
import * as net from 'net';
-import { defaults as pgDefaults, Pool, types as pgTypes } from 'pg';
+import { ClientBase, defaults as pgDefaults, Pool, types as pgTypes } from 'pg';
import { DataSource } from 'typeorm';
import { assertDataConnectionUtc, parseTimestampAsUtc, postgresUtcExtra, utcTimestampTypes } from './postgres-utc';
import { buildPostgresDataSourceOptions } from './data-source';
@@ -46,8 +46,8 @@ interface FakeServer {
close: () => Promise<void>;
}
-/** Accepts a connection, then either cuts the socket or refuses the first statement it receives. */
-const startFakeServer = async (onFirstQuery: 'cut' | 'refuse'): Promise<FakeServer> => {
+/** Accepts a connection, then cuts the socket, refuses the first statement it receives or never answers it. */
+const startFakeServer = async (onFirstQuery: 'cut' | 'refuse' | 'silent'): Promise<FakeServer> => {
const sockets = new Set<net.Socket>();
const queries: string[] = [];
const server = net.createServer(socket => {
@@ -60,6 +60,7 @@ const startFakeServer = async (onFirstQuery: 'cut' | 'refuse'): Promise<FakeServ
return;
}
queries.push(chunk.subarray(5, chunk.length - 1).toString('utf8'));
+ if (onFirstQuery === 'silent') return;
if (onFirstQuery === 'cut') socket.destroy();
else socket.write(Buffer.concat([errorResponse('SET TIME ZONE is not supported'), READY_FOR_QUERY]));
});
@@ -69,12 +70,22 @@ const startFakeServer = async (onFirstQuery: 'cut' | 'refuse'): Promise<FakeServ
port: (server.address() as net.AddressInfo).port,
queries,
openSockets: () => sockets.size,
- close: () => new Promise<void>(resolve => server.close(() => resolve())),
+ close: () => {
+ for (const socket of sockets) socket.destroy();
+ return new Promise<void>(resolve => server.close(() => resolve()));
+ },
};
};
-/** One acquire against a fake server, with the process's own uncaught-exception handlers stood aside. */
-const acquireThrough = async (fake: FakeServer): Promise<{ rejection?: string; uncaught: Error[] }> => {
+/**
+ * One acquire against a fake server, with the process's own uncaught-exception handlers stood aside. An
+ * acquire still pending after twice the connect budget is reported as a rejection of its own, so a hang
+ * fails the assertion instead of the suite.
+ */
+const acquireThrough = async (
+ fake: FakeServer,
+ connectionTimeoutMillis = 5000,
+): Promise<{ rejection?: string; uncaught: Error[]; socketsOnRejection: number }> => {
const uncaught: Error[] = [];
const installed = process.listeners('uncaughtException');
process.removeAllListeners('uncaughtException');
@@ -85,23 +96,33 @@ const acquireThrough = async (fake: FakeServer): Promise<{ rejection?: string; u
port: fake.port,
user: 'openwa',
database: 'openwa',
- connectionTimeoutMillis: 5000,
- ...postgresUtcExtra(),
+ connectionTimeoutMillis,
+ ...postgresUtcExtra(connectionTimeoutMillis),
});
pool.on('error', () => undefined);
let rejection: string | undefined;
+ let deadline: NodeJS.Timeout | undefined;
try {
- (await pool.connect()).release();
+ await Promise.race([
+ pool.connect().then(client => client.release()),
+ new Promise<never>((_resolve, reject) => {
+ deadline = setTimeout(() => reject(new Error('acquire still pending')), connectionTimeoutMillis * 2);
+ }),
+ ]);
} catch (error) {
rejection = (error as Error).message;
+ } finally {
+ clearTimeout(deadline);
}
// An unhandled 'error' event lands a turn after the acquire settles.
await new Promise(resolve => setTimeout(resolve, 100));
+ const socketsOnRejection = fake.openSockets();
process.removeAllListeners('uncaughtException');
for (const listener of installed) process.on('uncaughtException', listener as never);
- await pool.end().catch(() => undefined);
- return { rejection, uncaught };
+ // A client stuck in its connect hook is never released, so pool.end() would wait for it forever.
+ await Promise.race([pool.end().catch(() => undefined), new Promise(resolve => setTimeout(resolve, 500))]);
+ return { rejection, uncaught, socketsOnRejection };
};
describe('postgres UTC pin', () => {
@@ -203,6 +224,35 @@ describe('postgres UTC pin', () => {
expect(fake.openSockets()).toBe(0);
await fake.close();
});
+
+ it('rejects the acquire, and closes the connection, when the server never answers the pin', async () => {
+ // A database or pooler that accepts the connection and then goes quiet. pg-pool stops its connect
+ // timer before it calls the hook, so without a bound of its own the acquire (and a boot waiting on
+ // it) never ends.
+ const fake = await startFakeServer('silent');
+ const { rejection, uncaught, socketsOnRejection } = await acquireThrough(fake, 250);
+
+ expect(fake.queries).toEqual(["SET TIME ZONE 'UTC'"]);
+ expect(rejection).toMatch(/Query read timeout/);
+ expect(socketsOnRejection).toBe(0);
+ expect(uncaught).toEqual([]);
+ await fake.close();
+ });
+
+ it.each([undefined, 0, -1, NaN, Infinity])('bounds the pin with the default budget for %s', async value => {
+ const query = jest.fn().mockResolvedValue({});
+ await postgresUtcExtra(value).onConnect({ query } as unknown as ClientBase);
+ expect(query).toHaveBeenCalledWith({ text: "SET TIME ZONE 'UTC'", query_timeout: 10000 });
+ });
+
+ it('bounds the pin with the connection budget the data source was given', async () => {
+ const query = jest.fn().mockResolvedValue({});
+ const extra = buildPostgresDataSourceOptions({ DATABASE_CONNECTION_TIMEOUT_MS: '2500' }).extra as {
+ onConnect: (client: ClientBase) => Promise<unknown>;
+ };
+ await extra.onConnect({ query } as unknown as ClientBase);
+ expect(query).toHaveBeenCalledWith({ text: "SET TIME ZONE 'UTC'", query_timeout: 2500 });
+ });
});
describe('boot assertion', () => {
diff --git a/src/database/postgres-utc.ts b/src/database/postgres-utc.ts
index 539a183d9..b72e65114 100644
--- a/src/database/postgres-utc.ts
+++ b/src/database/postgres-utc.ts
@@ -63,6 +63,8 @@ export const utcTimestampTypes = {
};
const SET_SESSION_UTC = "SET TIME ZONE 'UTC'";
+/** The pin's budget when no connect timeout is configured, or when it is 0 (the pool's timer off). */
+const DEFAULT_CONNECTION_TIMEOUT_MS = 10_000;
/**
* Pin a pooled session to UTC, through pg-pool's own connect hook.
@@ -82,9 +84,14 @@ const SET_SESSION_UTC = "SET TIME ZONE 'UTC'";
*
* `connectionTimeoutMillis` does NOT cover this statement: pg-pool clears that timer once the socket
* is connected, before it invokes the hook, so a server that accepts the connection and then never
- * answers holds the acquire until the statement's own failure. Nothing else in the pool bounds it.
+ * answers would hold the acquire, and a boot waiting on it, for good. The statement therefore carries its
+ * own `query_timeout`: its rejection makes the pool end the client, and no other statement is limited by it.
*/
-const pinSessionToUtc = (client: ClientBase): Promise<unknown> => client.query(SET_SESSION_UTC);
+const pinSessionToUtc = (client: ClientBase, timeoutMs = DEFAULT_CONNECTION_TIMEOUT_MS): Promise<unknown> => {
+ // Held in a variable: @types/pg omits `query_timeout`, which the driver reads, so a literal fails the excess check.
+ const query = { text: SET_SESSION_UTC, query_timeout: timeoutMs };
+ return client.query(query);
+};
/**
* The `extra` block that pins a postgres data connection to UTC, for both entry points (the runtime
@@ -95,9 +102,20 @@ const pinSessionToUtc = (client: ClientBase): Promise<unknown> => client.query(S
* form. Setting it here keeps it next to the two halves it belongs with, and ahead of any connection
* built from the returned block.
*/
-export function postgresUtcExtra(): { types: typeof utcTimestampTypes; onConnect: typeof pinSessionToUtc } {
+export function postgresUtcExtra(connectionTimeoutMillis?: number): {
+ types: typeof utcTimestampTypes;
+ onConnect: (client: ClientBase) => Promise<unknown>;
+} {
pgDefaults.parseInputDatesAsUTC = true;
- return { types: utcTimestampTypes, onConnect: pinSessionToUtc };
+ const timeoutMs =
+ connectionTimeoutMillis && Number.isFinite(connectionTimeoutMillis) && connectionTimeoutMillis > 0
+ ? connectionTimeoutMillis
+ : DEFAULT_CONNECTION_TIMEOUT_MS;
+ return {
+ types: utcTimestampTypes,
+ onConnect:
+ timeoutMs === DEFAULT_CONNECTION_TIMEOUT_MS ? pinSessionToUtc : client => pinSessionToUtc(client, timeoutMs),
+ };
}
/**