Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): keep a delete's last transaction out of another's

Yudhi Armyndharis5df613458358
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.md
index b47457f1f..b644c22dd 100644
--- a/docs/06-api-specification.md
+++ b/docs/06-api-specification.md
@@ -1288,7 +1288,7 @@ Delete a session.
The delete first removes the session's messages, webhook outbox rows and webhook delivery-failure records in batches (on SQLite up to 2000 rows, each sized to about 150 ms; 500 rows on PostgreSQL) that are not atomic with the rest: a failure part-way (a database error, a timeout, a restart) leaves the session in place with fewer rows and stopped, as `POST /stop` leaves it, and repeating the delete finishes it. It then removes, in one transaction, the session's message batches, chat states, received statuses, webhooks, templates, stored Baileys messages, automation rules, integration dead-letter rows and send idempotency claims together with the session itself, then purges its on-disk auth data. It keeps the audit log, the LID mappings, and the integration conversation mappings (so a re-paired session keeps its provider conversations); ingress dedup rows age out on their own after `INGRESS_DEDUP_RETENTION_DAYS` (default 7).
-**Errors:** `401` missing/invalid key · `403` key not scoped to this session, or key role below OPERATOR · `404` session not found · `409` credential teardown for the same session name still in flight (retryable; body carries `code: 'SESSION_NAME_TEARDOWN_PENDING'`; on a `409` the row is **not** deleted and no hook/auth-purge runs — retry after cleanup settles), or, on a multi-node deployment, another node currently holds this session's live engine (no `code`; nothing is deleted; send the delete to the owning node) · `503` the session could not be claimed for the delete because the database did not answer (nothing is deleted; retry)
+**Errors:** `401` missing/invalid key · `403` key not scoped to this session, or key role below OPERATOR · `404` session not found · `409` credential teardown for the same session name still in flight (retryable; body carries `code: 'SESSION_NAME_TEARDOWN_PENDING'`; on a `409` the row is **not** deleted and no hook/auth-purge runs — retry after cleanup settles), or, on a multi-node deployment, another node currently holds this session's live engine (no `code`; nothing is deleted; send the delete to the owning node) · `503` the session could not be claimed for the delete because the database did not answer (nothing is deleted; retry), or, on SQLite, another database transaction stayed open for 10 s before the final transaction (the session stays, stopped, possibly with fewer rows; retry)
### 6.4.2 Messages
diff --git a/src/common/utils/shared-connection.spec.ts b/src/common/utils/shared-connection.spec.ts
new file mode 100644
index 000000000..40a7a0a91
--- /dev/null
+++ b/src/common/utils/shared-connection.spec.ts
@@ -0,0 +1,55 @@
+import { DataSource } from 'typeorm';
+import { awaitNoOpenTransaction } from './shared-connection';
+
+describe('awaitNoOpenTransaction', () => {
+ let db: DataSource;
+
+ beforeEach(async () => {
+ db = await new DataSource({ type: 'better-sqlite3', database: ':memory:' }).initialize();
+ });
+ afterEach(async () => {
+ jest.useRealTimers();
+ await db.destroy();
+ });
+
+ it('resolves at once where each runner has a connection of its own, or none reports its state', async () => {
+ jest.useFakeTimers();
+ await expect(awaitNoOpenTransaction({ options: { type: 'postgres' } } as DataSource)).resolves.toBe(true);
+ await expect(awaitNoOpenTransaction({ options: { type: 'better-sqlite3' } } as DataSource)).resolves.toBe(true);
+ });
+
+ it('waits for a transaction open on the shared connection to end', async () => {
+ const runner = db.createQueryRunner();
+ await runner.startTransaction();
+ let ended = false;
+ setTimeout(() => {
+ ended = true;
+ void runner.rollbackTransaction();
+ }, 30);
+
+ await expect(awaitNoOpenTransaction(db)).resolves.toBe(true);
+ expect(ended).toBe(true);
+ });
+
+ it('gives up with false when the transaction outlasts the bound', async () => {
+ const runner = db.createQueryRunner();
+ await runner.startTransaction();
+ try {
+ await expect(awaitNoOpenTransaction(db, 20)).resolves.toBe(false);
+ } finally {
+ await runner.rollbackTransaction();
+ }
+ });
+
+ // TypeORM sets isTransactionActive before its BEGIN and does not clear it when the BEGIN fails.
+ it('does not wait on a runner left marked active by a BEGIN that failed', async () => {
+ const runner = db.createQueryRunner();
+ const connection = (db.driver as unknown as { databaseConnection: { exec(sql: string): void } }).databaseConnection;
+ connection.exec('BEGIN');
+ await expect(runner.startTransaction()).rejects.toThrow();
+ connection.exec('ROLLBACK');
+ expect(runner.isTransactionActive).toBe(true);
+
+ await expect(awaitNoOpenTransaction(db, 1000)).resolves.toBe(true);
+ });
+});
diff --git a/src/common/utils/shared-connection.ts b/src/common/utils/shared-connection.ts
new file mode 100644
index 000000000..559e68347
--- /dev/null
+++ b/src/common/utils/shared-connection.ts
@@ -0,0 +1,46 @@
+import { setImmediate, setTimeout } from 'timers/promises';
+import type { DataSource } from 'typeorm';
+
+/**
+ * Dialects where every TypeORM query runner shares ONE connection, so an open transaction is visible
+ * to anything else querying in the same process. A positive list on purpose: a dialect nobody has
+ * classified must not silently opt into suspending a safety mechanism.
+ */
+export const SHARED_CONNECTION_DIALECTS = new Set(['better-sqlite3', 'sqlite']);
+
+/**
+ * How long a caller waits for another transaction on the shared connection to end. Those transactions are
+ * short (a session create, the closing transaction of a delete); the bound is for one that never ends.
+ */
+export const FOREIGN_TRANSACTION_WAIT_MS = 10_000;
+
+/**
+ * Wait until no transaction is open on a connection that every query runner shares. A transaction opened in the
+ * same synchronous step as the resolve is then the caller's own: TypeORM would otherwise open it as a SAVEPOINT
+ * inside the other one, and whether its writes last would depend on how that one ends. Resolves true at once on
+ * a dialect that gives each runner its own connection, and false when the bound ran out with a transaction open.
+ *
+ * The check runs on a turn of the event loop of its own. better-sqlite3 runs each statement synchronously, so a
+ * transaction that awaits nothing but its statements, as every one in this code base does, begins and ends within
+ * one turn: none is half done when the check runs, and none can begin inside the caller's before it ends.
+ *
+ * It reads the connection's own state rather than the runner's isTransactionActive, which stays set for good after
+ * a BEGIN that failed.
+ */
+export async function awaitNoOpenTransaction(
+ dataSource: DataSource,
+ boundMs = FOREIGN_TRANSACTION_WAIT_MS,
+): Promise<boolean> {
+ if (!SHARED_CONNECTION_DIALECTS.has(dataSource.options.type)) return true;
+ const connection = (dataSource.driver as { databaseConnection?: { inTransaction?: unknown } } | undefined)
+ ?.databaseConnection;
+ // A connection that does not report its state (node-sqlite3, or none at all) has nothing to wait on.
+ if (typeof connection?.inTransaction !== 'boolean') return true;
+ const settleBy = Date.now() + boundMs;
+ await setImmediate();
+ while (connection.inTransaction) {
+ if (Date.now() >= settleBy) return false;
+ await setTimeout(5);
+ }
+ return true;
+}
diff --git a/src/modules/infra/infra-data.service.ts b/src/modules/infra/infra-data.service.ts
index a42e21dbf..7c9b01b83 100644
--- a/src/modules/infra/infra-data.service.ts
+++ b/src/modules/infra/infra-data.service.ts
@@ -1,4 +1,3 @@
-import { setTimeout } from 'node:timers/promises';
import { BadRequestException, ConflictException, Injectable, Optional } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { ConfigService } from '@nestjs/config';
@@ -7,6 +6,7 @@ import { InjectDataSource } from '@nestjs/typeorm';
import { createLogger } from '../../common/services/logger.service';
import { isMissingTableError } from '../../common/utils/db-errors';
import { closeSessionWrites } from '../../common/utils/session-write-scope';
+import { awaitNoOpenTransaction, SHARED_CONNECTION_DIALECTS } from '../../common/utils/shared-connection';
import { failUnfinishedSessionBatches } from '../message/batch-media.util';
import { AuditService } from '../audit/audit.service';
import { AuditAction } from '../audit/entities/audit-log.entity';
@@ -149,20 +149,6 @@ export function toSqliteDatetime(value: unknown): unknown {
return Number.isNaN(date.getTime()) ? value : DateUtils.mixedDateToUtcDatetimeString(date);
}
-/**
- * Dialects where every TypeORM query runner shares ONE connection, so an open transaction is visible
- * to anything else querying in the same process. A positive list on purpose: a dialect nobody has
- * classified must not silently opt into suspending a safety mechanism.
- */
-const SHARED_CONNECTION_DIALECTS = new Set(['better-sqlite3', 'sqlite']);
-
-/**
- * How long an import refused because another transaction opened on the shared connection waits for that
- * transaction to end before it fails the batches of the sessions it meant to remove. Those transactions are
- * short (a session create, the closing transaction of a delete); the bound is for one that never ends.
- */
-const FOREIGN_TRANSACTION_WAIT_MS = 10_000;
-
/**
* Aggregate budget for the inline base64 media ONE export may carry, counted in the encoded bytes
* that actually land in the JSON body. Override with EXPORT_INLINE_MEDIA_BUDGET_BYTES; 0 omits every
@@ -735,7 +721,7 @@ export class InfraDataService {
let retired: Array<{ id: string; writes: ReturnType<typeof closeSessionWrites> }> = [];
let committed = false;
// Set when another transaction opened on the shared connection while the writes drained.
- let foreignTransaction: QueryRunner | undefined;
+ let foreignTransaction = false;
// Everything from here is wrapped so the token cannot outlive this request. It is not enough to
// release it in the transaction's own finally: createQueryRunner/connect/startTransaction sit
@@ -790,7 +776,7 @@ export class InfraDataService {
await Promise.all(retired.map(({ writes }) => writes.drain()));
await queryRunner.connect();
// Again after the drain's await: BEGIN must follow the check with nothing in between.
- if (queryRunner.isTransactionActive) foreignTransaction = queryRunner;
+ if (queryRunner.isTransactionActive) foreignTransaction = true;
refuseNestedTransaction();
await queryRunner.startTransaction();
} catch (error) {
@@ -1144,15 +1130,11 @@ export class InfraDataService {
// the close stays stopped, so its row is failed before the release, as a failed delete does. Their
// engines are gone, so the batch could not have sent anyway. Not inside a transaction another caller
// opened on the shared connection while the writes drained: the fail would roll back with it.
- if (foreignTransaction && retired.length > 0) {
- const settleBy = Date.now() + FOREIGN_TRANSACTION_WAIT_MS;
- while (foreignTransaction.isTransactionActive && Date.now() < settleBy) await setTimeout(5);
- if (foreignTransaction.isTransactionActive) {
- this.logger.warn('Failing the unfinished batches of a refused import inside another open transaction', {
- action: 'infra_import_foreign_transaction_open',
- sessionIds: retired.map(({ id }) => id),
- });
- }
+ if (foreignTransaction && retired.length > 0 && !(await awaitNoOpenTransaction(this.dataDataSource))) {
+ this.logger.warn('Failing the unfinished batches of a refused import inside another open transaction', {
+ action: 'infra_import_foreign_transaction_open',
+ sessionIds: retired.map(({ id }) => id),
+ });
}
for (const { id } of committed ? [] : retired) {
await failUnfinishedSessionBatches(this.dataDataSource, id).catch((error: unknown) =>
diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.ts
index ec0cd62d0..ff593e9d0 100644
--- a/src/modules/session/session-content-lifetime.spec.ts
+++ b/src/modules/session/session-content-lifetime.spec.ts
@@ -1,7 +1,7 @@
-import { ConflictException } from '@nestjs/common';
+import { ConflictException, ServiceUnavailableException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { randomUUID } from 'crypto';
-import { DataSource } from 'typeorm';
+import { DataSource, QueryRunner } from 'typeorm';
import { Session } from './entities/session.entity';
import { Message } from '../message/entities/message.entity';
import { BatchStatus, MessageBatch } from '../message/entities/message-batch.entity';
@@ -35,6 +35,7 @@ import { StatusStoreService } from '../status-store/status-store.service';
import { StorageService } from '../../common/storage/storage.service';
import { EngineNotReadyError } from '../../common/errors/engine-not-ready.error';
import { captureSessionWrites, closeSessionWrites } from '../../common/utils/session-write-scope';
+import { FOREIGN_TRANSACTION_WAIT_MS } from '../../common/utils/shared-connection';
function deferred<T = void>() {
let resolve!: (value: T | PromiseLike<T>) => void;
@@ -695,4 +696,124 @@ describe('Session content lifetime', () => {
expect(await statuses()).toEqual([]);
});
});
+
+ // On SQLite every query runner is the one connection, and a transaction opened while another is open there
+ // becomes a savepoint inside it: the rows the delete removed would come back when that one rolls back, after
+ // the delete had answered. On PostgreSQL the other transaction holds a pooled client of its own.
+ describe('a delete beside another transaction on the data connection', () => {
+ const gone = { sessions: 0, messages: 0, batches: 0 };
+
+ async function seedContent(name: string): Promise<string> {
+ const session = await db.getRepository(Session).save({ name, config: {} });
+ await db
+ .getRepository(Message)
+ .insert({ sessionId: session.id, chatId: '[email protected]', from: 'me', to: '[email protected]', body: 'm' });
+ await db.getRepository(MessageBatch).save({
+ batchId: `b-${name}`,
+ sessionId: session.id,
+ status: BatchStatus.PENDING,
+ messages: [],
+ options: { delayBetweenMessages: 0, randomizeDelay: false, stopOnError: false },
+ progress: { total: 0, sent: 0, failed: 0, pending: 0, cancelled: 0 },
+ results: [],
+ });
+ return session.id;
+ }
+
+ const rowsOf = async (sessionId: string) => ({
+ sessions: await db.getRepository(Session).countBy({ id: sessionId }),
+ messages: await db.getRepository(Message).countBy({ sessionId }),
+ batches: await db.getRepository(MessageBatch).countBy({ sessionId }),
+ });
+
+ /** A transaction someone else opened on the connection, held until the test ends it. */
+ async function openTransaction(): Promise<QueryRunner> {
+ const runner = db.createQueryRunner();
+ await runner.connect();
+ await runner.startTransaction();
+ return runner;
+ }
+
+ it.each(['rolls back', 'commits'])(
+ 'removes every row of the session when a transaction open beside it %s',
+ async ending => {
+ const id = await seedContent('doomed');
+ const other = await openTransaction();
+
+ const deletion = controls.delete(id);
+ await new Promise(resolve => setTimeout(resolve, 50));
+ if (ending === 'rolls back') await other.rollbackTransaction();
+ else await other.commitTransaction();
+ await other.release();
+ await deletion;
+
+ expect(await rowsOf(id)).toEqual(gone);
+ },
+ );
+
+ // Begun from the session:deleted hook a number of promise steps later, in the same turn of the event loop, as a
+ // session create does when another session holds its name. Over this range it begins before, during and after
+ // the delete's closing transaction.
+ it('keeps its closing transaction its own against one begun in the same turn', async () => {
+ await db.getRepository(Session).save({ name: 'taken', config: {} });
+ let steps = 0;
+ let create: Promise<unknown> = Promise.resolve();
+ hooks.register('test', 'session:deleted', () => {
+ create = (async () => {
+ for (let i = 0; i < steps; i += 1) await Promise.resolve();
+ await db.transaction(manager => manager.save(manager.create(Session, { name: 'taken', config: {} })));
+ })().catch(() => undefined);
+ return Promise.resolve({ continue: true });
+ });
+
+ for (; steps <= 100; steps += 1) {
+ const id = await seedContent(`doomed-${steps}`);
+ const outcome = await controls.delete(id).then(
+ () => 'deleted',
+ (error: unknown) => String(error),
+ );
+ await create;
+ expect({ steps, outcome, ...(await rowsOf(id)) }).toEqual({ steps, outcome: 'deleted', ...gone });
+ }
+ }, 60000);
+
+ it('fails the batches of a failed delete outside a transaction opened beside it', async () => {
+ const id = await seedContent('doomed');
+ let ended: Promise<void> = Promise.resolve();
+ jest.spyOn(db, 'transaction').mockImplementationOnce(async () => {
+ const other = await openTransaction();
+ ended = new Promise(resolve => setTimeout(resolve, 20))
+ .then(() => other.rollbackTransaction())
+ .then(() => other.release());
+ throw new Error('closing transaction failed');
+ });
+
+ await expect(controls.delete(id)).rejects.toThrow('closing transaction failed');
+ await ended;
+
+ expect((await db.getRepository(MessageBatch).findOneByOrFail({ sessionId: id })).status).toBe(BatchStatus.FAILED);
+ });
+
+ (process.env.DATABASE_TYPE === 'postgres' ? it.skip : it)(
+ 'refuses with 503 and removes nothing more when a transaction beside it stays open',
+ async () => {
+ const id = await seedContent('doomed');
+ const other = await openTransaction();
+ const start = Date.now();
+ let calls = 0;
+ const clock = jest.spyOn(Date, 'now').mockImplementation(() => start + FOREIGN_TRANSACTION_WAIT_MS * calls++);
+
+ await expect(controls.delete(id)).rejects.toThrow(ServiceUnavailableException);
+ clock.mockRestore();
+ await other.rollbackTransaction();
+ await other.release();
+
+ expect(await rowsOf(id)).toEqual({ sessions: 1, messages: 1, batches: 1 });
+ expect(logger.warn).toHaveBeenCalledWith(
+ expect.stringContaining('inside another open transaction'),
+ expect.objectContaining({ action: 'session_delete_foreign_transaction_open' }),
+ );
+ },
+ );
+ });
});
diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.ts
index 977ac83dd..c2f5d3d65 100644
--- a/src/modules/session/session-engine-controls.ts
+++ b/src/modules/session/session-engine-controls.ts
@@ -22,6 +22,7 @@ import { BaileysStoredMessage } from '../../engine';
import { ChatState } from '../../engine/adapters/baileys-chat-state.entity';
import { StatusUpdate } from '../status-store/entities/status-update.entity';
import { closeSessionWrites } from '../../common/utils/session-write-scope';
+import { awaitNoOpenTransaction } from '../../common/utils/shared-connection';
import { EngineFactory } from '../../engine/engine.factory';
import { EngineRegistry } from '../../engine/engine-registry.service';
import { IWhatsAppEngine } from '../../engine/interfaces/whatsapp-engine.interface';
@@ -891,7 +892,15 @@ export class SessionEngineControls {
// instance-keyed inbound dedup record, payload already dropped after dispatch, always pruned
// after INGRESS_DEDUP_RETENTION_DAYS), lid_mappings (a global cache by design) and audit_logs (the audit trail).
await this.deleteTrafficRowsInBatches(id);
- await this.host.dataSource().transaction(async manager => {
+ // On SQLite every caller shares one connection: opened inside another caller's transaction, this one
+ // would only be a savepoint in it, undone if that one rolls back after this delete has answered.
+ const dataSource = this.host.dataSource();
+ if (!(await awaitNoOpenTransaction(dataSource))) {
+ throw new ServiceUnavailableException(
+ `Session ${id} was not deleted: another database transaction stayed open. Retry the delete.`,
+ );
+ }
+ await dataSource.transaction(async manager => {
await manager.delete(Message, { sessionId: id });
await manager.delete(MessageBatch, { sessionId: id });
await manager.delete(ChatState, { sessionId: id });
@@ -971,7 +980,15 @@ export class SessionEngineControls {
*/
private async failUnfinishedBatches(sessionId: string): Promise<void> {
try {
- await failUnfinishedSessionBatches(this.host.dataSource(), sessionId);
+ const dataSource = this.host.dataSource();
+ // Not inside another caller's transaction on the shared connection, where the fail would roll back with it.
+ if (!(await awaitNoOpenTransaction(dataSource))) {
+ this.logger.warn(`Failing the unfinished batches of session ${sessionId} inside another open transaction`, {
+ sessionId,
+ action: 'session_delete_foreign_transaction_open',
+ });
+ }
+ await failUnfinishedSessionBatches(dataSource, sessionId);
} catch (error) {
this.logger.warn(`Could not fail the unfinished batches of session ${sessionId} after a failed delete`, {
sessionId,