Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(infra): fail a refused import's batches outside other work

Yudhi Armyndharis27143cc79b5b
diff --git a/src/modules/infra/infra-data.service.ts b/src/modules/infra/infra-data.service.ts
index 137ce99d8..a42e21dbf 100644
--- a/src/modules/infra/infra-data.service.ts
+++ b/src/modules/infra/infra-data.service.ts
@@ -1,3 +1,4 @@
+import { setTimeout } from 'node:timers/promises';
import { BadRequestException, ConflictException, Injectable, Optional } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { ConfigService } from '@nestjs/config';
@@ -155,6 +156,13 @@ export function toSqliteDatetime(value: unknown): unknown {
*/
const SHARED_CONNECTION_DIALECTS = new Set(['better-sqlite3', 'sqlite']);
+/**
+ * How long an import refused because another transaction opened on the shared connection waits for that
+ * transaction to end before it fails the batches of the sessions it meant to remove. Those transactions are
+ * short (a session create, the closing transaction of a delete); the bound is for one that never ends.
+ */
+const FOREIGN_TRANSACTION_WAIT_MS = 10_000;
+
/**
* Aggregate budget for the inline base64 media ONE export may carry, counted in the encoded bytes
* that actually land in the JSON body. Override with EXPORT_INLINE_MEDIA_BUDGET_BYTES; 0 omits every
@@ -726,6 +734,8 @@ export class InfraDataService {
const leftRunning = new Set([...heldElsewhere, ...(data.stopOrphans ? [] : orphanedEngines)]);
let retired: Array<{ id: string; writes: ReturnType<typeof closeSessionWrites> }> = [];
let committed = false;
+ // Set when another transaction opened on the shared connection while the writes drained.
+ let foreignTransaction: QueryRunner | undefined;
// Everything from here is wrapped so the token cannot outlive this request. It is not enough to
// release it in the transaction's own finally: createQueryRunner/connect/startTransaction sit
@@ -780,6 +790,7 @@ export class InfraDataService {
await Promise.all(retired.map(({ writes }) => writes.drain()));
await queryRunner.connect();
// Again after the drain's await: BEGIN must follow the check with nothing in between.
+ if (queryRunner.isTransactionActive) foreignTransaction = queryRunner;
refuseNestedTransaction();
await queryRunner.startTransaction();
} catch (error) {
@@ -1131,7 +1142,18 @@ export class InfraDataService {
} finally {
// A restore that did not commit keeps those sessions, but a batch run that captured their writes before
// the close stays stopped, so its row is failed before the release, as a failed delete does. Their
- // engines are gone, so the batch could not have sent anyway.
+ // engines are gone, so the batch could not have sent anyway. Not inside a transaction another caller
+ // opened on the shared connection while the writes drained: the fail would roll back with it.
+ if (foreignTransaction && retired.length > 0) {
+ const settleBy = Date.now() + FOREIGN_TRANSACTION_WAIT_MS;
+ while (foreignTransaction.isTransactionActive && Date.now() < settleBy) await setTimeout(5);
+ if (foreignTransaction.isTransactionActive) {
+ this.logger.warn('Failing the unfinished batches of a refused import inside another open transaction', {
+ action: 'infra_import_foreign_transaction_open',
+ sessionIds: retired.map(({ id }) => id),
+ });
+ }
+ }
for (const { id } of committed ? [] : retired) {
await failUnfinishedSessionBatches(this.dataDataSource, id).catch((error: unknown) =>
this.logger.warn(
diff --git a/src/modules/infra/infra-data.write-scopes.spec.ts b/src/modules/infra/infra-data.write-scopes.spec.ts
index fb29514c1..8e18ca7ee 100644
--- a/src/modules/infra/infra-data.write-scopes.spec.ts
+++ b/src/modules/infra/infra-data.write-scopes.spec.ts
@@ -299,17 +299,18 @@ describe('import-data and the writes of the sessions it removes', () => {
'another transaction opened while the writes drained',
() => {
onRunner(runner => {
- let read = false;
+ let opened = false;
const query = runner.query.bind(runner);
- runner.query = ((...args: Parameters<typeof query>) => {
- if (/^SELECT id FROM sessions/.test(args[0])) read = true;
- return query(...args);
+ runner.query = (async (...args: Parameters<typeof query>) => {
+ const rows: unknown = await query(...args);
+ if (!opened && /^SELECT id FROM sessions/.test(args[0])) {
+ opened = true;
+ // Someone else's transaction on the one connection, rolled back once the import has refused.
+ await runner.startTransaction();
+ setTimeout(() => void runner.rollbackTransaction(), 20);
+ }
+ return rows;
}) as typeof runner.query;
- Object.defineProperty(runner, 'isTransactionActive', {
- get: () => read,
- set: () => undefined,
- configurable: true,
- });
});
return importNow({ stopOrphans: true });
},
@@ -332,6 +333,45 @@ describe('import-data and the writes of the sessions it removes', () => {
});
});
+ // On SQLite a transaction someone else opens while the import drains is open on the import's connection
+ // too. Failing the batch inside it would tie the fail to that transaction's outcome.
+ it.each([['rolls back'], ['commits']])(
+ 'fails the batch of an import refused for a transaction opened while it drained, which then %s',
+ async outcome => {
+ const entered = deferred();
+ const finish = deferred();
+ const messages = ds.getRepository(Message);
+ const save = messages.save.bind(messages);
+ jest.spyOn(messages, 'save').mockImplementationOnce(async message => {
+ entered.resolve();
+ await finish.promise;
+ return save(message);
+ });
+ const sending = sender.sendText('s1', { chatId: '[email protected]', text: 'hello' }).catch((error: unknown) => error);
+ await entered.promise;
+ const batch = await parkBatch();
+ const importing = importNow({ stopOrphans: true }).catch((error: unknown) => error);
+ await new Promise<void>(setImmediate);
+ const outer = ds.createQueryRunner();
+ await outer.connect();
+ await outer.startTransaction();
+ finish.resolve();
+ // The drained write has landed and the import has refused by the time a timer runs.
+ await new Promise(resolve => setTimeout(resolve, 20));
+ await (outcome === 'commits' ? outer.commitTransaction() : outer.rollbackTransaction());
+
+ expect(await importing).toMatchObject({ response: { code: 'IMPORT_NESTED_TRANSACTION' } });
+ await batch.finish();
+ await sending;
+ const row = await batchRow(batch.id);
+ expect(row.status).toBe(BatchStatus.FAILED);
+ expect(JSON.stringify(row.messages)).not.toContain('MEDIA-PAYLOAD');
+ expect(engine.sendTextMessage).not.toHaveBeenCalled();
+ expect(await ds.getRepository(Session).count()).toBe(2);
+ expect(current('s1')).toBe(true);
+ },
+ );
+
describe.each<[string, () => Promise<unknown>]>([
['would orphan a running engine', () => importNow({})],
[