Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(infra): drain an import's writes before it takes a client
Yudhi Armyndharise9dfd7a25880
diff --git a/src/modules/infra/infra-data.import.pg.spec.ts b/src/modules/infra/infra-data.import.pg.spec.tsindex 3fa7e4939..f9b57fdf5 100644--- a/src/modules/infra/infra-data.import.pg.spec.ts+++ b/src/modules/infra/infra-data.import.pg.spec.ts@@ -23,6 +23,7 @@ import { StatusUpdate } from '../status-store/entities/status-update.entity';import { SendIdempotencyKey } from '../message/entities/send-idempotency-key.entity';import { AutomationRule } from '../automation/entities/automation-rule.entity';import { postgresUtcExtra } from '../../database/postgres-utc';+import { captureSessionWrites } from '../../common/utils/session-write-scope';/*** The replace-all import trims send_idempotency_keys with a statement that compares the claim's@@ -172,4 +173,100 @@ describePostgres('InfraDataController.importData (postgres)', () => {await strict.destroy();}});++ // Before its transaction opens, the import waits for the writes still in flight for the sessions it+ // removes. A write that has yet to get a client from the pool needs the one the import would hold, so on a+ // pool of one the import must not take it until the wait is over.+ it.each([0, 10_000])(+ 'waits out a write of a session it removes on a pool of one client (connection timeout %i ms)',+ async connectionTimeoutMillis => {+ const sessions = ds.getRepository(Session);+ const removed = randomUUID();+ await sessions.save(+ sessions.create({+ id: removed,+ name: `removed-${removed.slice(0, 8)}`,+ status: SessionStatus.READY,+ config: {},+ }),+ );+ const dump = await controller.exportData();+ const withoutRemoved = {+ tables: {+ ...dump.tables,+ sessions: (dump.tables.sessions as unknown as Array<{ id: string }>).filter(row => row.id !== removed),+ },+ };+ const applicationName = `${schema}_pool_of_one`;+ const single = await new DataSource({+ type: 'postgres',+ host: process.env.DATABASE_HOST || 'localhost',+ port: Number(process.env.DATABASE_PORT || 5432),+ username: process.env.DATABASE_USERNAME || 'openwa',+ password: process.env.DATABASE_PASSWORD || 'openwa',+ database: process.env.DATABASE_NAME || 'openwa',+ schema,+ extra: {+ ...postgresUtcExtra(),+ options: `-c search_path=${schema},public`,+ max: 1,+ connectionTimeoutMillis,+ application_name: applicationName,+ },+ entities: ds.options.entities,+ synchronize: false,+ }).initialize();+ const cfg = { get: (key: string, def?: unknown) => (key === 'dataDatabase.type' ? 'postgres' : def) };+ const service = new InfraDataService(cfg as never, single, undefined, undefined, undefined);+ // The write is let go once the import reads the sessions it removes, so it asks the pool for its client+ // after the import has begun.+ let reading!: () => void;+ const read = new Promise<void>(resolve => (reading = resolve));+ let letGo!: () => void;+ const go = new Promise<void>(resolve => (letGo = resolve));+ const realCreate = single.createQueryRunner.bind(single);+ jest.spyOn(single, 'createQueryRunner').mockImplementation((...args: Parameters<typeof realCreate>) => {+ const runner = realCreate(...args);+ const query = runner.query.bind(runner);+ runner.query = ((...queryArgs: Parameters<typeof query>) => {+ if (/^SELECT id FROM sessions/.test(queryArgs[0])) reading();+ return query(...queryArgs);+ }) as typeof runner.query;+ return runner;+ });+ const scope = captureSessionWrites(single, removed);+ const writing = scope.write(async () => {+ await go;+ await single+ .getRepository(Message)+ .insert({ sessionId: removed, chatId: '[email protected]', from: 'me', to: '[email protected]', body: 'm' });+ });++ const importing = service.importData(withoutRemoved as never);+ await read;+ letGo();+ const outcome = await Promise.race([+ importing.then(result => result.imported),+ new Promise(resolve => setTimeout(() => resolve('still waiting after 8 s'), 8_000)),+ ]);+ try {+ expect(outcome).toBe(true);+ await expect(writing).resolves.toBeUndefined();+ expect(await sessions.countBy({ id: removed })).toBe(0);+ expect(await ds.getRepository(Message).countBy({ sessionId: removed })).toBe(0);+ } finally {+ jest.restoreAllMocks();+ scope.close();+ // Only when the import is stuck: cut its connection so the pool can close.+ if (outcome !== true) {+ await admin.query('SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name = $1', [+ applicationName,+ ]);+ }+ await Promise.allSettled([importing, writing]);+ await single.destroy();+ }+ },+ 30_000,+ );});diff --git a/src/modules/infra/infra-data.service.ts b/src/modules/infra/infra-data.service.tsindex 9c9d6f5fd..137ce99d8 100644--- a/src/modules/infra/infra-data.service.ts+++ b/src/modules/infra/infra-data.service.ts@@ -735,7 +735,6 @@ export class InfraDataService {// silently, which is strictly worse than the teardown the suspension prevents.try {const queryRunner = this.dataDataSource.createQueryRunner();- await queryRunner.connect();// Refuse BEFORE a single row is deleted, not after. On better-sqlite3 every query runner is a// driver SINGLETON, so if anything else already holds a transaction on this DataSource — a@@ -766,12 +765,20 @@ export class InfraDataService {};try {refuseNestedTransaction();- // Drained before BEGIN: on SQLite a write still in flight would otherwise join this transaction.- const existing = (await queryRunner.query('SELECT id FROM sessions')) as Array<{ id: string }>;+ // Drained before BEGIN: on SQLite a write still in flight would otherwise join this transaction. And+ // before this runner takes a pooled client on PostgreSQL, where the write may still need one: with a+ // pool of one it would wait for the import while the import waits for it. So the ids are read on a+ // client that goes back at once there; on SQLite this runner is the one connection anyway.+ const existing = await this.dataDataSource.query<Array<{ id: string }>>(+ 'SELECT id FROM sessions',+ [],+ sharesOneConnection ? queryRunner : undefined,+ );retired = existing.filter(({ id }) => !importedSessionIds.has(id) && !leftRunning.has(id)).map(({ id }) => ({ id, writes: closeSessionWrites(this.dataDataSource, id) }));await Promise.all(retired.map(({ writes }) => writes.drain()));+ await queryRunner.connect();// Again after the drain's await: BEGIN must follow the check with nothing in between.refuseNestedTransaction();await queryRunner.startTransaction();