Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(infra): drain in-flight writes of sessions an import removes
Yudhi Armyndharis4529c08f9fc2
diff --git a/src/modules/infra/infra-data.service.ts b/src/modules/infra/infra-data.service.tsindex c16332ef3..9c9d6f5fd 100644--- a/src/modules/infra/infra-data.service.ts+++ b/src/modules/infra/infra-data.service.ts@@ -5,6 +5,8 @@ import { DataSource, QueryRunner } from 'typeorm';import { InjectDataSource } from '@nestjs/typeorm';import { createLogger } from '../../common/services/logger.service';import { isMissingTableError } from '../../common/utils/db-errors';+import { closeSessionWrites } from '../../common/utils/session-write-scope';+import { failUnfinishedSessionBatches } from '../message/batch-media.util';import { AuditService } from '../audit/audit.service';import { AuditAction } from '../audit/entities/audit-log.entity';import { SessionService } from '../session/session.service';@@ -717,6 +719,14 @@ export class InfraDataService {const sharesOneConnection = SHARED_CONNECTION_DIALECTS.has(this.dataDataSource.options.type);const resumeLossDetection = sharesOneConnection ? this.ownership?.suspendLossDetection() : undefined;+ // The replace deletes every session the archive does not hold, rows and all, and messages has no foreign+ // key to refuse a send or batch row that lands after it. So their write scopes are closed and drained+ // before the transaction opens, as SessionEngineControls.delete does. Not for an engine force left+ // running, which keeps writing (restartRequired says so), nor for one another node holds.+ const leftRunning = new Set([...heldElsewhere, ...(data.stopOrphans ? [] : orphanedEngines)]);+ let retired: Array<{ id: string; writes: ReturnType<typeof closeSessionWrites> }> = [];+ let committed = false;+// Everything from here is wrapped so the token cannot outlive this request. It is not enough to// release it in the transaction's own finally: createQueryRunner/connect/startTransaction sit// before that finally exists, so any throw there — a driver change, a broadcaster subscriber, a@@ -742,7 +752,7 @@ export class InfraDataService {// The try/finally that releases the runner opens only after BEGIN, so the refusal and a failing// BEGIN give the runner back here: a pooled PostgreSQL client would otherwise stay checked out for// the life of the process.- try {+ const refuseNestedTransaction = (): void => {if (queryRunner.isTransactionActive) {throw new ConflictException({statusCode: 409,@@ -753,6 +763,17 @@ export class InfraDataService {code: 'IMPORT_NESTED_TRANSACTION',});}+ };+ try {+ refuseNestedTransaction();+ // Drained before BEGIN: on SQLite a write still in flight would otherwise join this transaction.+ const existing = (await queryRunner.query('SELECT id FROM sessions')) as Array<{ id: string }>;+ retired = existing+ .filter(({ id }) => !importedSessionIds.has(id) && !leftRunning.has(id))+ .map(({ id }) => ({ id, writes: closeSessionWrites(this.dataDataSource, id) }));+ await Promise.all(retired.map(({ writes }) => writes.drain()));+ // Again after the drain's await: BEGIN must follow the check with nothing in between.+ refuseNestedTransaction();await queryRunner.startTransaction();} catch (error) {await queryRunner.release().catch(() => undefined);@@ -1044,6 +1065,7 @@ export class InfraDataService {}await queryRunner.commitTransaction();+ committed = true;// Runtime reconciliation, part 2 (post-commit): the in-memory lid->phone mirror was warmed from// the OLD lid_mappings rows and is write-through only, so the just-restored table would never@@ -1100,6 +1122,22 @@ export class InfraDataService {await queryRunner.release();}} finally {+ // A restore that did not commit keeps those sessions, but a batch run that captured their writes before+ // the close stays stopped, so its row is failed before the release, as a failed delete does. Their+ // engines are gone, so the batch could not have sent anyway.+ for (const { id } of committed ? [] : retired) {+ await failUnfinishedSessionBatches(this.dataDataSource, id).catch((error: unknown) =>+ this.logger.warn(+ `Could not fail the unfinished batches of session ${id} after an import that did not commit`,+ {+ sessionId: id,+ action: 'infra_import_batch_fail_failed',+ error: error instanceof Error ? error.message : String(error),+ },+ ),+ );+ }+ for (const { writes } of retired) writes.release();// Its own finally, so a throwing release() cannot strand the token either.resumeLossDetection?.();}diff --git a/src/modules/infra/infra-data.write-scopes.spec.ts b/src/modules/infra/infra-data.write-scopes.spec.tsnew file mode 100644index 000000000..fb29514c1--- /dev/null+++ b/src/modules/infra/infra-data.write-scopes.spec.ts@@ -0,0 +1,397 @@+import { ConfigService } from '@nestjs/config';+import { DataSource, QueryRunner } from 'typeorm';+import { InfraDataService } from './infra-data.service';+import type { MigrationTables } from './migration-tables.types';+import { Session, SessionStatus } from '../session/entities/session.entity';+import { Webhook } from '../webhook/entities/webhook.entity';+import { Message } from '../message/entities/message.entity';+import { BatchStatus, MessageBatch } from '../message/entities/message-batch.entity';+import { Template } from '../template/entities/template.entity';+import { BaileysStoredMessage } from '../../engine/adapters/baileys-stored-message.entity';+import { LidMapping } from '../../engine/identity/lid-mapping.entity';+import { ChatState } from '../../engine/adapters/baileys-chat-state.entity';+import { PluginInstance } from '../integration/entities/plugin-instance.entity';+import { ConversationMapping } from '../integration/entities/conversation-mapping.entity';+import { IngressEvent } from '../integration/entities/ingress-event.entity';+import { WebhookDeliveryFailure } from '../webhook/entities/webhook-delivery-failure.entity';+import { WebhookOutboxEvent } from '../webhook/entities/webhook-outbox-event.entity';+import { IntegrationDeliveryFailure } from '../integration/entities/integration-delivery-failure.entity';+import { StatusUpdate } from '../status-store/entities/status-update.entity';+import { SendIdempotencyKey } from '../message/entities/send-idempotency-key.entity';+import { AutomationRule } from '../automation/entities/automation-rule.entity';+import { EngineRegistry } from '../../engine/engine-registry.service';+import type { IWhatsAppEngine } from '../../engine/interfaces/whatsapp-engine.interface';+import { HookManager } from '../../core/hooks';+import { MessageSendService } from '../message/message-send.service';+import { BulkMessageService } from '../message/bulk-message.service';+import { SendPacingService } from '../message/send-pacing.service';+import type { MessageService } from '../message/message.service';+import type { SessionService } from '../session/session.service';+import { captureSessionWrites } from '../../common/utils/session-write-scope';++const deferred = () => {+ let resolve!: () => void;+ const promise = new Promise<void>(res => (resolve = res));+ return { promise, resolve };+};++type ImportOptions = { force?: boolean; stopOrphans?: boolean };++// The replace removes every session the archive does not hold, rows and all, in one transaction. A send or+// batch row still being written for such a session must not land after it, as it must not after a delete,+// and an import that does not commit keeps those sessions, so it must not leave their batches unfinished.+describe('import-data and the writes of the sessions it removes', () => {+ let ds: DataSource;+ let engines: EngineRegistry;+ let hooks: HookManager;+ let sender: MessageSendService;+ let bulk: BulkMessageService;+ // An archive that holds only s2, so the replace removes s1.+ let onlyS2: Partial<MigrationTables>;+ const engine = { sendTextMessage: jest.fn(), sendImageMessage: jest.fn(), sendChatState: jest.fn() };+ const sessionService = {+ getActiveSessionIds: () => ['s1', 's2'].filter(id => engines.has(id)),+ stopOrphanEngines: (ids: string[]) => {+ for (const id of ids) engines.delete(id);+ return Promise.resolve({ stopped: ids, notRunning: [], failed: [] });+ },+ };+ const cfg = { get: (key: string, fallback?: unknown) => (key === 'dataDatabase.type' ? 'sqlite' : fallback) };+ const newService = (ownership?: object) =>+ new InfraDataService(cfg as never, ds, undefined, sessionService as never, undefined, ownership as never);+ const importNow = (options: ImportOptions, tables: Partial<MigrationTables> = onlyS2, ownership?: object) =>+ newService(ownership).importData({ tables, ...options });+ // A message row without its non-null columns: the restore cannot insert it and rolls everything back.+ const withBadRow = (): Partial<MigrationTables> => ({+ ...onlyS2,+ messages: [+ { id: 'mX', sessionId: 's2', chatId: 'c', type: 'text', direction: 'incoming', status: 'sent' } as never,+ ],+ });+ const current = (id: string) => {+ const scope = captureSessionWrites(ds, id);+ try {+ scope.assertCurrent();+ return true;+ } catch {+ return false;+ } finally {+ scope.close();+ }+ };+ const batchRow = (id: string) => ds.getRepository(MessageBatch).findOneByOrFail({ id });+ // Every runner this test sees: better-sqlite3 hands out one per DataSource, so prepare it once.+ const onRunner = (prepare: (runner: QueryRunner) => void) => {+ const prepared = new WeakSet<object>();+ const realCreate = ds.createQueryRunner.bind(ds);+ jest.spyOn(ds, 'createQueryRunner').mockImplementation((...args: Parameters<typeof realCreate>) => {+ const runner = realCreate(...args);+ if (!prepared.has(runner)) {+ prepared.add(runner);+ prepare(runner);+ }+ return runner;+ });+ };++ // A bulk batch for s1 whose first item waits in its message:sending hook until released. Its second item+ // carries a media payload, so a terminal write shows whether it stripped it.+ const parkBatch = async () => {+ const entered = deferred();+ const finish = deferred();+ hooks.register('moderation', 'message:sending', async () => {+ entered.resolve();+ await finish.promise;+ return { continue: true };+ });+ const pickup = jest.spyOn(bulk as unknown as { processBatch(): Promise<void> }, 'processBatch');+ const batch = await bulk.createBatch('s1', {+ messages: [+ { chatId: '[email protected]', type: 'text', content: { text: 'x' } },+ { chatId: '[email protected]', type: 'image', content: { image: { base64: 'MEDIA-PAYLOAD', mimetype: 'image/png' } } },+ ],+ options: { delayBetweenMessages: 0, randomizeDelay: false },+ } as never);+ await entered.promise;+ const run = pickup.mock.results[0].value as Promise<void>;+ return {+ id: batch.id,+ finish: async () => {+ finish.resolve();+ await run.catch(() => undefined);+ },+ };+ };++ beforeEach(async () => {+ process.env.SIMULATE_TYPING = 'false';+ engine.sendTextMessage.mockReset().mockResolvedValue({ id: 'sent-1', timestamp: 1 });+ engine.sendImageMessage.mockReset().mockResolvedValue({ id: 'sent-2', timestamp: 1 });+ ds = new DataSource({+ type: 'better-sqlite3',+ database: ':memory:',+ entities: [+ Session,+ Webhook,+ Message,+ MessageBatch,+ Template,+ BaileysStoredMessage,+ LidMapping,+ ChatState,+ PluginInstance,+ ConversationMapping,+ IngressEvent,+ WebhookDeliveryFailure,+ WebhookOutboxEvent,+ IntegrationDeliveryFailure,+ StatusUpdate,+ AutomationRule,+ SendIdempotencyKey,+ ],+ synchronize: true,+ });+ await ds.initialize();+ const sessions = ds.getRepository(Session);+ for (const id of ['s1', 's2']) {+ await sessions.save(sessions.create({ id, name: `session-${id}`, status: SessionStatus.READY, config: {} }));+ }+ engines = new EngineRegistry();+ engines.set('s1', engine as unknown as IWhatsAppEngine);+ hooks = new HookManager();+ const messages = ds.getRepository(Message);+ const config = new ConfigService({});+ const pacing = new SendPacingService(messages, sessions, config);+ sender = new MessageSendService(+ messages,+ { findOne: (id: string) => sessions.findOneByOrFail({ id }) } as unknown as SessionService,+ engines,+ hooks,+ {} as never,+ pacing,+ config,+ );+ bulk = new BulkMessageService(+ ds.getRepository(MessageBatch),+ engines,+ sender as unknown as MessageService,+ hooks,+ pacing,+ );+ const dump = await newService().exportData();+ onlyS2 = { ...dump.tables, sessions: dump.tables.sessions?.filter(row => row.id === 's2') };+ });++ afterEach(async () => {+ delete process.env.SIMULATE_TYPING;+ jest.restoreAllMocks();+ await ds.destroy();+ });++ it('waits for a message insert in flight before replacing the session that owns it', async () => {+ const entered = deferred();+ const finish = deferred();+ const messages = ds.getRepository(Message);+ const save = messages.save.bind(messages);+ jest.spyOn(messages, 'save').mockImplementationOnce(async message => {+ entered.resolve();+ await finish.promise;+ return save(message);+ });+ const sending = sender.sendText('s1', { chatId: '[email protected]', text: 'hello' }).catch((error: unknown) => error);+ await entered.promise;+ let imported = false;+ const importing = importNow({ stopOrphans: true }).then(result => {+ imported = true;+ return result;+ });+ await new Promise<void>(setImmediate);+ const importedBeforeInsert = imported;+ finish.resolve();++ expect(await importing).toMatchObject({ imported: true });+ expect(importedBeforeInsert).toBe(false);+ expect(await sending).toMatchObject({ status: 404 });+ expect(engine.sendTextMessage).not.toHaveBeenCalled();+ expect(await ds.getRepository(Message).count({ where: { sessionId: 's1' } })).toBe(0);+ });++ it('keeps the writes of a session the archive holds open throughout', async () => {+ const kept = captureSessionWrites(ds, 's2');++ await expect(importNow({ stopOrphans: true })).resolves.toMatchObject({ imported: true });++ expect(() => kept.assertCurrent()).not.toThrow();+ kept.close();+ });++ it('writes again for a removed session id once the import has committed', async () => {+ await expect(importNow({ stopOrphans: true })).resolves.toMatchObject({ imported: true });++ expect(current('s1')).toBe(true);+ });++ it('restores the batches of a removed session that the archive carries as archived', async () => {+ const batches = ds.getRepository(MessageBatch);+ await batches.save({+ batchId: 'archived',+ sessionId: 's1',+ status: BatchStatus.PENDING,+ messages: [{ chatId: '[email protected]', type: 'text', content: { text: 'x' } }],+ options: { delayBetweenMessages: 0, randomizeDelay: false, stopOnError: false },+ progress: { total: 1, sent: 0, failed: 0, pending: 1, cancelled: 0 },+ results: [],+ });+ const dump = await newService().exportData();++ await expect(+ importNow({ stopOrphans: true }, { ...dump.tables, sessions: dump.tables.sessions?.filter(r => r.id === 's2') }),+ ).resolves.toMatchObject({ imported: true });++ expect(await batches.findOneByOrFail({ batchId: 'archived' })).toMatchObject({ status: BatchStatus.PENDING });+ });++ // The removed sessions survive an import that does not commit, but a run that captured their writes before+ // the import closed them stays stopped. Their engines are gone, so the batch could not send anyway.+ describe.each<[string, () => Promise<unknown>]>([+ ['a row the restore cannot insert', () => importNow({ stopOrphans: true }, withBadRow())],+ [+ 'an archive that restores no rows',+ () => {+ let opened = false;+ onRunner(runner => {+ const begin = runner.startTransaction.bind(runner);+ runner.startTransaction = (...args) => {+ opened = true;+ return begin(...args);+ };+ });+ // The pre-flight refuses an empty archive before it closes anything; this one empties once BEGIN ran.+ const tables = new Proxy(onlyS2, {+ get: (target, key): unknown => (opened ? undefined : target[key as keyof MigrationTables]),+ });+ return importNow({ stopOrphans: true }, tables);+ },+ ],+ [+ 'a statement that fails',+ () => {+ onRunner(runner => {+ const query = runner.query.bind(runner);+ runner.query = ((...args: Parameters<typeof query>) =>+ args[0] === 'DELETE FROM webhooks'+ ? Promise.reject(new Error('disk I/O error'))+ : query(...args)) as typeof runner.query;+ });+ return importNow({ stopOrphans: true });+ },+ ],+ [+ 'a transaction that cannot open',+ () => {+ onRunner(runner => {+ runner.startTransaction = () => Promise.reject(new Error('BEGIN failed'));+ });+ return importNow({ stopOrphans: true });+ },+ ],+ [+ 'another transaction opened while the writes drained',+ () => {+ onRunner(runner => {+ let read = false;+ const query = runner.query.bind(runner);+ runner.query = ((...args: Parameters<typeof query>) => {+ if (/^SELECT id FROM sessions/.test(args[0])) read = true;+ return query(...args);+ }) as typeof runner.query;+ Object.defineProperty(runner, 'isTransactionActive', {+ get: () => read,+ set: () => undefined,+ configurable: true,+ });+ });+ return importNow({ stopOrphans: true });+ },+ ],+ ])('an import that ends without committing: %s', (_exit, run) => {+ it('fails the unfinished batch of the session it meant to remove and reopens its writes', async () => {+ const batch = await parkBatch();++ const outcome = await run().catch((error: unknown) => error);+ await batch.finish();++ expect(outcome).not.toMatchObject({ imported: true });+ const row = await batchRow(batch.id);+ expect(row.status).toBe(BatchStatus.FAILED);+ expect(JSON.stringify(row.messages)).not.toContain('MEDIA-PAYLOAD');+ expect(engine.sendTextMessage).not.toHaveBeenCalled();+ expect(await ds.getRepository(Message).count({ where: { sessionId: 's1' } })).toBe(0);+ expect(await ds.getRepository(Session).count()).toBe(2);+ expect(current('s1')).toBe(true);+ });+ });++ describe.each<[string, () => Promise<unknown>]>([+ ['would orphan a running engine', () => importNow({})],+ [+ 'finds another transaction holding the connection',+ async () => {+ const outer = ds.createQueryRunner();+ await outer.connect();+ await outer.startTransaction();+ try {+ return await importNow({ stopOrphans: true });+ } finally {+ await outer.rollbackTransaction();+ }+ },+ ],+ ])('an import refused before it starts because it %s', (_reason, run) => {+ it('leaves the writes of every session alone', async () => {+ const before = captureSessionWrites(ds, 's1');++ await expect(run()).rejects.toMatchObject({ status: 409 });++ expect(() => before.assertCurrent()).not.toThrow();+ before.close();+ });+ });++ it('leaves a removed session whose engine force keeps running writable, and its batch running', async () => {+ const batch = await parkBatch();++ const result = await importNow({ force: true }, withBadRow());+ await batch.finish();++ expect(result.imported).toBe(false);+ expect(engine.sendTextMessage).toHaveBeenCalledTimes(1);+ expect((await batchRow(batch.id)).status).toBe(BatchStatus.COMPLETED);+ });++ it('leaves the writes and batches of a removed session another node holds alone', async () => {+ engines.delete('s1');+ const batches = ds.getRepository(MessageBatch);+ const running = await batches.save({+ batchId: 'elsewhere',+ sessionId: 's1',+ status: BatchStatus.PROCESSING,+ messages: [{ chatId: '[email protected]', type: 'text', content: { text: 'x' } }],+ options: { delayBetweenMessages: 0, randomizeDelay: false, stopOnError: false },+ progress: { total: 1, sent: 0, failed: 0, pending: 1, cancelled: 0 },+ results: [],+ });+ const before = captureSessionWrites(ds, 's1');+ const ownership = {+ heldByOtherNodes: () => Promise.resolve(['s1']),+ claimableWhere: () => [{}],+ suspendLossDetection: () => () => undefined,+ };++ await expect(importNow({}, withBadRow(), ownership)).resolves.toMatchObject({ imported: false });++ expect((await batchRow(running.id)).status).toBe(BatchStatus.PROCESSING);+ expect(() => before.assertCurrent()).not.toThrow();+ before.close();+ });+});diff --git a/src/modules/message/batch-media.util.ts b/src/modules/message/batch-media.util.tsindex 8fd29f3e1..671d72779 100644--- a/src/modules/message/batch-media.util.ts+++ b/src/modules/message/batch-media.util.ts@@ -1,4 +1,5 @@-import { MessageBatch } from './entities/message-batch.entity';+import { DataSource, In, QueryDeepPartialEntity } from 'typeorm';+import { BatchStatus, MessageBatch } from './entities/message-batch.entity';/*** Drop base64 payloads from a finished batch's stored message list. A completed/cancelled batch is@@ -15,3 +16,24 @@ export function stripBatchMediaPayloads(messages: MessageBatch['messages']): voi}}}++/**+ * Fail a session's unfinished (PENDING or PROCESSING) batches, with their media payloads stripped as on every+ * terminal path. For a session whose write scopes an operation closed and then did not remove: a run that+ * captured a scope before the close stops at its next item without recording anything, and nothing else moves+ * its row on while this process lives. The boot reaper gets any row this misses on the next start.+ */+export async function failUnfinishedSessionBatches(dataSource: DataSource, sessionId: string): Promise<void> {+ const batches = dataSource.getRepository(MessageBatch);+ const unfinished = await batches.find({+ where: { sessionId, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },+ });+ for (const batch of unfinished) {+ stripBatchMediaPayloads(batch.messages);+ await batches.update({ id: batch.id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) }, {+ status: BatchStatus.FAILED,+ completedAt: new Date(),+ messages: batch.messages,+ } as QueryDeepPartialEntity<MessageBatch>);+ }+}diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.tsindex 47c1a12ab..5cb6db61b 100644--- a/src/modules/session/session-engine-controls.ts+++ b/src/modules/session/session-engine-controls.ts@@ -7,11 +7,11 @@ import {ServiceUnavailableException,} from '@nestjs/common';import { ConfigService } from '@nestjs/config';-import { In, QueryDeepPartialEntity, Repository, DataSource } from 'typeorm';+import { In, Repository, DataSource } from 'typeorm';import { Session, SessionStatus } from './entities/session.entity';import { Message } from '../message/entities/message.entity';-import { BatchStatus, MessageBatch } from '../message/entities/message-batch.entity';-import { stripBatchMediaPayloads } from '../message/batch-media.util';+import { MessageBatch } from '../message/entities/message-batch.entity';+import { failUnfinishedSessionBatches } from '../message/batch-media.util';import { SendIdempotencyKey } from '../message/entities/send-idempotency-key.entity';import { Webhook } from '../webhook/entities/webhook.entity';import { WebhookOutboxEvent } from '../webhook/entities/webhook-outbox-event.entity';@@ -915,25 +915,13 @@ export class SessionEngineControls {/*** Fail the session's unfinished bulk batches after a delete that closed its write scopes but did not- * remove it. Written here rather than through the batch service, which this module cannot reach without- * a cycle; the rows are the same ones its reapers fail, with the media payloads stripped as on every- * terminal path. Best effort: the database that just failed the delete may fail this too, and the+ * remove it. Written against the table rather than through the batch service, which this module cannot+ * reach without a cycle. Best effort: the database that just failed the delete may fail this too, and the* boot reaper still gets the row on the next start.*/private async failUnfinishedBatches(sessionId: string): Promise<void> {try {- const batches = this.host.dataSource().getRepository(MessageBatch);- const unfinished = await batches.find({- where: { sessionId, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },- });- for (const batch of unfinished) {- stripBatchMediaPayloads(batch.messages);- await batches.update({ id: batch.id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) }, {- status: BatchStatus.FAILED,- completedAt: new Date(),- messages: batch.messages,- } as QueryDeepPartialEntity<MessageBatch>);- }+ await failUnfinishedSessionBatches(this.host.dataSource(), sessionId);} catch (error) {this.logger.warn(`Could not fail the unfinished batches of session ${sessionId} after a failed delete`, {sessionId,