Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(infra): drain an import's writes before it takes a client

Yudhi Armyndharise9dfd7a25880
diff --git a/src/modules/infra/infra-data.import.pg.spec.ts b/src/modules/infra/infra-data.import.pg.spec.ts
index 3fa7e4939..f9b57fdf5 100644
--- a/src/modules/infra/infra-data.import.pg.spec.ts
+++ b/src/modules/infra/infra-data.import.pg.spec.ts
@@ -23,6 +23,7 @@ import { StatusUpdate } from '../status-store/entities/status-update.entity';
import { SendIdempotencyKey } from '../message/entities/send-idempotency-key.entity';
import { AutomationRule } from '../automation/entities/automation-rule.entity';
import { postgresUtcExtra } from '../../database/postgres-utc';
+import { captureSessionWrites } from '../../common/utils/session-write-scope';
/**
* The replace-all import trims send_idempotency_keys with a statement that compares the claim's
@@ -172,4 +173,100 @@ describePostgres('InfraDataController.importData (postgres)', () => {
await strict.destroy();
}
});
+
+ // Before its transaction opens, the import waits for the writes still in flight for the sessions it
+ // removes. A write that has yet to get a client from the pool needs the one the import would hold, so on a
+ // pool of one the import must not take it until the wait is over.
+ it.each([0, 10_000])(
+ 'waits out a write of a session it removes on a pool of one client (connection timeout %i ms)',
+ async connectionTimeoutMillis => {
+ const sessions = ds.getRepository(Session);
+ const removed = randomUUID();
+ await sessions.save(
+ sessions.create({
+ id: removed,
+ name: `removed-${removed.slice(0, 8)}`,
+ status: SessionStatus.READY,
+ config: {},
+ }),
+ );
+ const dump = await controller.exportData();
+ const withoutRemoved = {
+ tables: {
+ ...dump.tables,
+ sessions: (dump.tables.sessions as unknown as Array<{ id: string }>).filter(row => row.id !== removed),
+ },
+ };
+ const applicationName = `${schema}_pool_of_one`;
+ const single = await new DataSource({
+ type: 'postgres',
+ host: process.env.DATABASE_HOST || 'localhost',
+ port: Number(process.env.DATABASE_PORT || 5432),
+ username: process.env.DATABASE_USERNAME || 'openwa',
+ password: process.env.DATABASE_PASSWORD || 'openwa',
+ database: process.env.DATABASE_NAME || 'openwa',
+ schema,
+ extra: {
+ ...postgresUtcExtra(),
+ options: `-c search_path=${schema},public`,
+ max: 1,
+ connectionTimeoutMillis,
+ application_name: applicationName,
+ },
+ entities: ds.options.entities,
+ synchronize: false,
+ }).initialize();
+ const cfg = { get: (key: string, def?: unknown) => (key === 'dataDatabase.type' ? 'postgres' : def) };
+ const service = new InfraDataService(cfg as never, single, undefined, undefined, undefined);
+ // The write is let go once the import reads the sessions it removes, so it asks the pool for its client
+ // after the import has begun.
+ let reading!: () => void;
+ const read = new Promise<void>(resolve => (reading = resolve));
+ let letGo!: () => void;
+ const go = new Promise<void>(resolve => (letGo = resolve));
+ const realCreate = single.createQueryRunner.bind(single);
+ jest.spyOn(single, 'createQueryRunner').mockImplementation((...args: Parameters<typeof realCreate>) => {
+ const runner = realCreate(...args);
+ const query = runner.query.bind(runner);
+ runner.query = ((...queryArgs: Parameters<typeof query>) => {
+ if (/^SELECT id FROM sessions/.test(queryArgs[0])) reading();
+ return query(...queryArgs);
+ }) as typeof runner.query;
+ return runner;
+ });
+ const scope = captureSessionWrites(single, removed);
+ const writing = scope.write(async () => {
+ await go;
+ await single
+ .getRepository(Message)
+ .insert({ sessionId: removed, chatId: '[email protected]', from: 'me', to: '[email protected]', body: 'm' });
+ });
+
+ const importing = service.importData(withoutRemoved as never);
+ await read;
+ letGo();
+ const outcome = await Promise.race([
+ importing.then(result => result.imported),
+ new Promise(resolve => setTimeout(() => resolve('still waiting after 8 s'), 8_000)),
+ ]);
+ try {
+ expect(outcome).toBe(true);
+ await expect(writing).resolves.toBeUndefined();
+ expect(await sessions.countBy({ id: removed })).toBe(0);
+ expect(await ds.getRepository(Message).countBy({ sessionId: removed })).toBe(0);
+ } finally {
+ jest.restoreAllMocks();
+ scope.close();
+ // Only when the import is stuck: cut its connection so the pool can close.
+ if (outcome !== true) {
+ await admin.query('SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name = $1', [
+ applicationName,
+ ]);
+ }
+ await Promise.allSettled([importing, writing]);
+ await single.destroy();
+ }
+ },
+ 30_000,
+ );
});
diff --git a/src/modules/infra/infra-data.service.ts b/src/modules/infra/infra-data.service.ts
index 9c9d6f5fd..137ce99d8 100644
--- a/src/modules/infra/infra-data.service.ts
+++ b/src/modules/infra/infra-data.service.ts
@@ -735,7 +735,6 @@ export class InfraDataService {
// silently, which is strictly worse than the teardown the suspension prevents.
try {
const queryRunner = this.dataDataSource.createQueryRunner();
- await queryRunner.connect();
// Refuse BEFORE a single row is deleted, not after. On better-sqlite3 every query runner is a
// driver SINGLETON, so if anything else already holds a transaction on this DataSource — a
@@ -766,12 +765,20 @@ export class InfraDataService {
};
try {
refuseNestedTransaction();
- // Drained before BEGIN: on SQLite a write still in flight would otherwise join this transaction.
- const existing = (await queryRunner.query('SELECT id FROM sessions')) as Array<{ id: string }>;
+ // Drained before BEGIN: on SQLite a write still in flight would otherwise join this transaction. And
+ // before this runner takes a pooled client on PostgreSQL, where the write may still need one: with a
+ // pool of one it would wait for the import while the import waits for it. So the ids are read on a
+ // client that goes back at once there; on SQLite this runner is the one connection anyway.
+ const existing = await this.dataDataSource.query<Array<{ id: string }>>(
+ 'SELECT id FROM sessions',
+ [],
+ sharesOneConnection ? queryRunner : undefined,
+ );
retired = existing
.filter(({ id }) => !importedSessionIds.has(id) && !leftRunning.has(id))
.map(({ id }) => ({ id, writes: closeSessionWrites(this.dataDataSource, id) }));
await Promise.all(retired.map(({ writes }) => writes.drain()));
+ await queryRunner.connect();
// Again after the drain's await: BEGIN must follow the check with nothing in between.
refuseNestedTransaction();
await queryRunner.startTransaction();