Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(database): bound the UTC pin on a silent server
Yudhi Armyndharisc1b7b67dadb0
diff --git a/src/database/data-source.ts b/src/database/data-source.tsindex aafcd11a6..93776f082 100644--- a/src/database/data-source.ts+++ b/src/database/data-source.ts@@ -83,6 +83,7 @@ const sqliteDataSourceOptions: DataSourceOptions = {export function buildPostgresDataSourceOptions(env: NodeJS.ProcessEnv = process.env): DataSourceOptions {const schema = env.POSTGRES_SCHEMA || 'public';const useCustomSearchPath = schema !== 'public';+ const connectionTimeoutMillis = parseInt(env.DATABASE_CONNECTION_TIMEOUT_MS || '10000', 10);return {type: 'postgres',schema,@@ -104,12 +105,12 @@ export function buildPostgresDataSourceOptions(env: NodeJS.ProcessEnv = process.extra: {// Same UTC pin the runtime data connection carries (pg-boot-migrations.ts): a migration that// rewrites a timestamp column must mean the same thing as the app that wrote it.- ...postgresUtcExtra(),+ ...postgresUtcExtra(connectionTimeoutMillis),max: parseInt(env.DATABASE_POOL_SIZE || '10', 10),// Pool resilience only. NO statement_timeout here: this connection runs migrations, and a// long CREATE INDEX / backfill must not be aborted mid-flight.idleTimeoutMillis: parseInt(env.DATABASE_IDLE_TIMEOUT_MS || '30000', 10),- connectionTimeoutMillis: parseInt(env.DATABASE_CONNECTION_TIMEOUT_MS || '10000', 10),+ connectionTimeoutMillis,// Only set for a non-public schema (see above)....(useCustomSearchPath ? { options: `-c search_path=${schema},public` } : {}),},diff --git a/src/database/pg-boot-migrations.spec.ts b/src/database/pg-boot-migrations.spec.tsindex 523b52681..96c202cc3 100644--- a/src/database/pg-boot-migrations.spec.ts+++ b/src/database/pg-boot-migrations.spec.ts@@ -378,6 +378,20 @@ describe('createBootDataSource (postgres boot migrations)', () => {expect(config.ssl).toBe(ssl);});+ it('bounds the UTC pin of the migration and the runtime pool with that connect timeout', async () => {+ const { deps } = makeFakes();++ await createBootDataSource({ ...PG_OPTIONS, extra: { connectionTimeoutMillis: 2500 } }, deps);++ const built = (deps.createDataSource as jest.Mock<DataSource, [DataSourceOptions]>).mock.calls;+ expect(built).toHaveLength(2);+ for (const [options] of built) {+ const query = jest.fn().mockResolvedValue({});+ await (options.extra as { onConnect: (client: never) => Promise<unknown> }).onConnect({ query } as never);+ expect(query).toHaveBeenCalledWith({ text: "SET TIME ZONE 'UTC'", query_timeout: 2500 });+ }+ });+it('falls back to the real DataSource and pg Client constructors when no deps are injected', async () => {// The production path. Only the spots that would touch the outside world are stubbed — the// DataSource lifecycle methods (initialize would open a pool) and the pg Client constructordiff --git a/src/database/pg-boot-migrations.ts b/src/database/pg-boot-migrations.tsindex 703ba7c8a..0ed8a1682 100644--- a/src/database/pg-boot-migrations.ts+++ b/src/database/pg-boot-migrations.ts@@ -91,9 +91,13 @@ export async function createBootDataSource(// DataSource itself never starts them unsynchronized inside initialize(). The UTC pin is merged in// at the same point, because this is the one place the runtime postgres data connection is built// (the migration CLI's own data source carries it directly).- const build = (extra: Record<string, unknown> | undefined): DataSource =>- createDataSource({ ...options, migrationsRun: false, extra: { ...extra, ...postgresUtcExtra() } });- const runtimeExtra = options.extra as Record<string, unknown> | undefined;+ const build = (extra: ClientConfig | undefined): DataSource =>+ createDataSource({+ ...options,+ migrationsRun: false,+ extra: { ...extra, ...postgresUtcExtra(extra?.connectionTimeoutMillis) },+ });+ const runtimeExtra = options.extra as ClientConfig | undefined;// statement_timeout bounds live runtime queries, and pg sends it in the startup packet, so every// statement on a pool built with it inherits the limit. A backfill or index build over a largediff --git a/src/database/postgres-utc.spec.ts b/src/database/postgres-utc.spec.tsindex 7ce3a606d..80beef8ea 100644--- a/src/database/postgres-utc.spec.ts+++ b/src/database/postgres-utc.spec.ts@@ -1,5 +1,5 @@import * as net from 'net';-import { defaults as pgDefaults, Pool, types as pgTypes } from 'pg';+import { ClientBase, defaults as pgDefaults, Pool, types as pgTypes } from 'pg';import { DataSource } from 'typeorm';import { assertDataConnectionUtc, parseTimestampAsUtc, postgresUtcExtra, utcTimestampTypes } from './postgres-utc';import { buildPostgresDataSourceOptions } from './data-source';@@ -46,8 +46,8 @@ interface FakeServer {close: () => Promise<void>;}-/** Accepts a connection, then either cuts the socket or refuses the first statement it receives. */-const startFakeServer = async (onFirstQuery: 'cut' | 'refuse'): Promise<FakeServer> => {+/** Accepts a connection, then cuts the socket, refuses the first statement it receives or never answers it. */+const startFakeServer = async (onFirstQuery: 'cut' | 'refuse' | 'silent'): Promise<FakeServer> => {const sockets = new Set<net.Socket>();const queries: string[] = [];const server = net.createServer(socket => {@@ -60,6 +60,7 @@ const startFakeServer = async (onFirstQuery: 'cut' | 'refuse'): Promise<FakeServreturn;}queries.push(chunk.subarray(5, chunk.length - 1).toString('utf8'));+ if (onFirstQuery === 'silent') return;if (onFirstQuery === 'cut') socket.destroy();else socket.write(Buffer.concat([errorResponse('SET TIME ZONE is not supported'), READY_FOR_QUERY]));});@@ -69,12 +70,22 @@ const startFakeServer = async (onFirstQuery: 'cut' | 'refuse'): Promise<FakeServport: (server.address() as net.AddressInfo).port,queries,openSockets: () => sockets.size,- close: () => new Promise<void>(resolve => server.close(() => resolve())),+ close: () => {+ for (const socket of sockets) socket.destroy();+ return new Promise<void>(resolve => server.close(() => resolve()));+ },};};-/** One acquire against a fake server, with the process's own uncaught-exception handlers stood aside. */-const acquireThrough = async (fake: FakeServer): Promise<{ rejection?: string; uncaught: Error[] }> => {+/**+ * One acquire against a fake server, with the process's own uncaught-exception handlers stood aside. An+ * acquire still pending after twice the connect budget is reported as a rejection of its own, so a hang+ * fails the assertion instead of the suite.+ */+const acquireThrough = async (+ fake: FakeServer,+ connectionTimeoutMillis = 5000,+): Promise<{ rejection?: string; uncaught: Error[]; socketsOnRejection: number }> => {const uncaught: Error[] = [];const installed = process.listeners('uncaughtException');process.removeAllListeners('uncaughtException');@@ -85,23 +96,33 @@ const acquireThrough = async (fake: FakeServer): Promise<{ rejection?: string; uport: fake.port,user: 'openwa',database: 'openwa',- connectionTimeoutMillis: 5000,- ...postgresUtcExtra(),+ connectionTimeoutMillis,+ ...postgresUtcExtra(connectionTimeoutMillis),});pool.on('error', () => undefined);let rejection: string | undefined;+ let deadline: NodeJS.Timeout | undefined;try {- (await pool.connect()).release();+ await Promise.race([+ pool.connect().then(client => client.release()),+ new Promise<never>((_resolve, reject) => {+ deadline = setTimeout(() => reject(new Error('acquire still pending')), connectionTimeoutMillis * 2);+ }),+ ]);} catch (error) {rejection = (error as Error).message;+ } finally {+ clearTimeout(deadline);}// An unhandled 'error' event lands a turn after the acquire settles.await new Promise(resolve => setTimeout(resolve, 100));+ const socketsOnRejection = fake.openSockets();process.removeAllListeners('uncaughtException');for (const listener of installed) process.on('uncaughtException', listener as never);- await pool.end().catch(() => undefined);- return { rejection, uncaught };+ // A client stuck in its connect hook is never released, so pool.end() would wait for it forever.+ await Promise.race([pool.end().catch(() => undefined), new Promise(resolve => setTimeout(resolve, 500))]);+ return { rejection, uncaught, socketsOnRejection };};describe('postgres UTC pin', () => {@@ -203,6 +224,35 @@ describe('postgres UTC pin', () => {expect(fake.openSockets()).toBe(0);await fake.close();});++ it('rejects the acquire, and closes the connection, when the server never answers the pin', async () => {+ // A database or pooler that accepts the connection and then goes quiet. pg-pool stops its connect+ // timer before it calls the hook, so without a bound of its own the acquire (and a boot waiting on+ // it) never ends.+ const fake = await startFakeServer('silent');+ const { rejection, uncaught, socketsOnRejection } = await acquireThrough(fake, 250);++ expect(fake.queries).toEqual(["SET TIME ZONE 'UTC'"]);+ expect(rejection).toMatch(/Query read timeout/);+ expect(socketsOnRejection).toBe(0);+ expect(uncaught).toEqual([]);+ await fake.close();+ });++ it.each([undefined, 0, -1, NaN, Infinity])('bounds the pin with the default budget for %s', async value => {+ const query = jest.fn().mockResolvedValue({});+ await postgresUtcExtra(value).onConnect({ query } as unknown as ClientBase);+ expect(query).toHaveBeenCalledWith({ text: "SET TIME ZONE 'UTC'", query_timeout: 10000 });+ });++ it('bounds the pin with the connection budget the data source was given', async () => {+ const query = jest.fn().mockResolvedValue({});+ const extra = buildPostgresDataSourceOptions({ DATABASE_CONNECTION_TIMEOUT_MS: '2500' }).extra as {+ onConnect: (client: ClientBase) => Promise<unknown>;+ };+ await extra.onConnect({ query } as unknown as ClientBase);+ expect(query).toHaveBeenCalledWith({ text: "SET TIME ZONE 'UTC'", query_timeout: 2500 });+ });});describe('boot assertion', () => {diff --git a/src/database/postgres-utc.ts b/src/database/postgres-utc.tsindex 539a183d9..b72e65114 100644--- a/src/database/postgres-utc.ts+++ b/src/database/postgres-utc.ts@@ -63,6 +63,8 @@ export const utcTimestampTypes = {};const SET_SESSION_UTC = "SET TIME ZONE 'UTC'";+/** The pin's budget when no connect timeout is configured, or when it is 0 (the pool's timer off). */+const DEFAULT_CONNECTION_TIMEOUT_MS = 10_000;/*** Pin a pooled session to UTC, through pg-pool's own connect hook.@@ -82,9 +84,14 @@ const SET_SESSION_UTC = "SET TIME ZONE 'UTC'";** `connectionTimeoutMillis` does NOT cover this statement: pg-pool clears that timer once the socket* is connected, before it invokes the hook, so a server that accepts the connection and then never- * answers holds the acquire until the statement's own failure. Nothing else in the pool bounds it.+ * answers would hold the acquire, and a boot waiting on it, for good. The statement therefore carries its+ * own `query_timeout`: its rejection makes the pool end the client, and no other statement is limited by it.*/-const pinSessionToUtc = (client: ClientBase): Promise<unknown> => client.query(SET_SESSION_UTC);+const pinSessionToUtc = (client: ClientBase, timeoutMs = DEFAULT_CONNECTION_TIMEOUT_MS): Promise<unknown> => {+ // Held in a variable: @types/pg omits `query_timeout`, which the driver reads, so a literal fails the excess check.+ const query = { text: SET_SESSION_UTC, query_timeout: timeoutMs };+ return client.query(query);+};/*** The `extra` block that pins a postgres data connection to UTC, for both entry points (the runtime@@ -95,9 +102,20 @@ const pinSessionToUtc = (client: ClientBase): Promise<unknown> => client.query(S* form. Setting it here keeps it next to the two halves it belongs with, and ahead of any connection* built from the returned block.*/-export function postgresUtcExtra(): { types: typeof utcTimestampTypes; onConnect: typeof pinSessionToUtc } {+export function postgresUtcExtra(connectionTimeoutMillis?: number): {+ types: typeof utcTimestampTypes;+ onConnect: (client: ClientBase) => Promise<unknown>;+} {pgDefaults.parseInputDatesAsUTC = true;- return { types: utcTimestampTypes, onConnect: pinSessionToUtc };+ const timeoutMs =+ connectionTimeoutMillis && Number.isFinite(connectionTimeoutMillis) && connectionTimeoutMillis > 0+ ? connectionTimeoutMillis+ : DEFAULT_CONNECTION_TIMEOUT_MS;+ return {+ types: utcTimestampTypes,+ onConnect:+ timeoutMs === DEFAULT_CONNECTION_TIMEOUT_MS ? pinSessionToUtc : client => pinSessionToUtc(client, timeoutMs),+ };}/**