Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): refuse a start during a delete

Yudhi Armyndharisaca08b47cb4b
diff --git a/src/modules/session/session-delete-race.spec.ts b/src/modules/session/session-delete-race.spec.ts
new file mode 100644
index 000000000..d53534051
--- /dev/null
+++ b/src/modules/session/session-delete-race.spec.ts
@@ -0,0 +1,183 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { getRepositoryToken, getDataSourceToken } from '@nestjs/typeorm';
+import { ConfigService } from '@nestjs/config';
+import { DataSource } from 'typeorm';
+import { SessionService } from './session.service';
+import { SessionEngineLifecycle } from './session-engine-lifecycle.service';
+import { SessionStoppedException } from './session-engine-controls';
+import { SessionErrorStore } from './session-error-store.service';
+import { SessionRestrictionStore } from './session-restriction-store.service';
+import { PresenceStore } from './presence-store.service';
+import { SessionOwnershipService } from './session-ownership.service';
+import { Session, SessionStatus } from './entities/session.entity';
+import { Message } from '../message/entities/message.entity';
+import { MessageBatch } from '../message/entities/message-batch.entity';
+import { SendIdempotencyKey } from '../message/entities/send-idempotency-key.entity';
+import { Webhook } from '../webhook/entities/webhook.entity';
+import { WebhookOutboxEvent } from '../webhook/entities/webhook-outbox-event.entity';
+import { WebhookDeliveryFailure } from '../webhook/entities/webhook-delivery-failure.entity';
+import { IntegrationDeliveryFailure } from '../integration/entities/integration-delivery-failure.entity';
+import { Template } from '../template/entities/template.entity';
+import { BaileysStoredMessage } from '../../engine/adapters/baileys-stored-message.entity';
+import { ChatState } from '../../engine/adapters/baileys-chat-state.entity';
+import { StatusUpdate } from '../status-store/entities/status-update.entity';
+import { EngineFactory } from '../../engine/engine.factory';
+import { EngineRegistry } from '../../engine/engine-registry.service';
+import { SessionLidResolver } from './session-lid-resolver.service';
+import { SessionLivenessWatchdog } from './session-liveness-watchdog.service';
+import { MessageProjector } from './message-projector.service';
+import { LidMappingStoreService } from '../../engine/identity/lid-mapping-store.service';
+import { EventsGateway } from '../events/events.gateway';
+import { WebhookService } from '../webhook/webhook.service';
+import { HookManager } from '../../core/hooks';
+import { StatusStoreService } from '../status-store/status-store.service';
+
+const entities = [
+ Session,
+ Message,
+ MessageBatch,
+ SendIdempotencyKey,
+ Webhook,
+ WebhookOutboxEvent,
+ WebhookDeliveryFailure,
+ IntegrationDeliveryFailure,
+ Template,
+ BaileysStoredMessage,
+ ChatState,
+ StatusUpdate,
+];
+
+const deferred = () => {
+ let resolve!: () => void;
+ const promise = new Promise<void>(res => (resolve = res));
+ return { promise, resolve };
+};
+
+const makeEngine = () => ({
+ initialize: jest.fn().mockResolvedValue(undefined),
+ destroy: jest.fn().mockResolvedValue(undefined),
+ forceDestroy: jest.fn().mockResolvedValue(undefined),
+ disconnect: jest.fn().mockResolvedValue(undefined),
+ getQRCode: jest.fn().mockReturnValue(null),
+});
+
+/**
+ * The real SessionService, lifecycle and ownership service on a real SQLite database; the engine, the
+ * gateway and the webhook sender are fakes. The delete is held where a session with a long history spends
+ * its time, in the batches that remove its messages, after its engine is gone.
+ */
+describe('A session being deleted', () => {
+ let db: DataSource;
+ let module: TestingModule;
+ let service: SessionService;
+ let ownership: SessionOwnershipService;
+ let engineFactory: { create: jest.Mock; purgeSessionData: jest.Mock };
+ let sessionId: string;
+ let reachedBatch: ReturnType<typeof deferred>;
+ let finishBatch: ReturnType<typeof deferred>;
+ let deletion: Promise<void>;
+
+ beforeEach(async () => {
+ db = new DataSource({ type: 'better-sqlite3', database: ':memory:', entities, synchronize: true });
+ await db.initialize();
+ engineFactory = {
+ create: jest.fn().mockImplementation(makeEngine),
+ purgeSessionData: jest.fn().mockResolvedValue(undefined),
+ };
+ const values: Record<string, unknown> = {
+ 'session.nodeId': 'node-a',
+ 'session.leaseTtlMs': 60_000,
+ 'session.leaseHeartbeatMs': 20_000,
+ };
+ module = await Test.createTestingModule({
+ providers: [
+ SessionService,
+ SessionEngineLifecycle,
+ SessionErrorStore,
+ SessionRestrictionStore,
+ PresenceStore,
+ SessionOwnershipService,
+ { provide: getRepositoryToken(Session, 'data'), useValue: db.getRepository(Session) },
+ { provide: getRepositoryToken(Message, 'data'), useValue: db.getRepository(Message) },
+ { provide: getDataSourceToken('data'), useValue: db },
+ { provide: EngineFactory, useValue: engineFactory },
+ EngineRegistry,
+ SessionLidResolver,
+ SessionLivenessWatchdog,
+ MessageProjector,
+ {
+ provide: EventsGateway,
+ useValue: {
+ emitSessionStatus: jest.fn(),
+ emitSessionDisconnected: jest.fn(),
+ emitQRCode: jest.fn(),
+ emitSessionAuthenticated: jest.fn(),
+ },
+ },
+ { provide: WebhookService, useValue: { dispatch: jest.fn().mockResolvedValue(undefined) } },
+ { provide: HookManager, useValue: { execute: jest.fn().mockResolvedValue({ continue: true, data: {} }) } },
+ {
+ provide: ConfigService,
+ useValue: { get: jest.fn(<T>(key: string, def?: T): T => (key in values ? (values[key] as T) : (def as T))) },
+ },
+ { provide: LidMappingStoreService, useValue: { remember: jest.fn() } },
+ { provide: StatusStoreService, useValue: { ingest: jest.fn(), captureWrites: jest.fn() } },
+ ],
+ }).compile();
+ service = module.get(SessionService);
+ ownership = module.get(SessionOwnershipService);
+
+ const row = await db.getRepository(Session).save({
+ name: 'big-session',
+ config: {},
+ phone: '628111111111',
+ status: SessionStatus.READY,
+ });
+ sessionId = row.id;
+ await service.start(sessionId, { explicit: true });
+ expect(service.getEngine(sessionId)).toBeDefined();
+
+ // One stored message is enough: the delete removes it in a batch, and that batch is the one held.
+ const messages = db.getRepository(Message);
+ await messages.insert({ sessionId, chatId: '[email protected]', from: 'me', to: '[email protected]', body: 'm' });
+ const removeBatch = messages.delete.bind(messages);
+ reachedBatch = deferred();
+ finishBatch = deferred();
+ jest.spyOn(messages, 'delete').mockImplementation(async criteria => {
+ reachedBatch.resolve();
+ await finishBatch.promise;
+ return removeBatch(criteria);
+ });
+
+ deletion = service.delete(sessionId);
+ await reachedBatch.promise;
+ });
+
+ afterEach(async () => {
+ finishBatch.resolve();
+ await deletion.catch(() => undefined);
+ jest.restoreAllMocks();
+ await service.onModuleDestroy();
+ await db.destroy();
+ });
+
+ it('refuses a start and leaves no engine running once the rows are gone', async () => {
+ // The engine is gone and the row is still there: the window a start used to slip through.
+ expect(service.getEngine(sessionId)).toBeUndefined();
+ expect(await db.getRepository(Session).countBy({ id: sessionId })).toBe(1);
+
+ await expect(service.start(sessionId, { explicit: true })).rejects.toBeInstanceOf(SessionStoppedException);
+ expect(service.getEngine(sessionId)).toBeUndefined();
+ expect(engineFactory.create).toHaveBeenCalledTimes(1);
+
+ finishBatch.resolve();
+ await deletion;
+
+ expect(await db.getRepository(Session).countBy({ id: sessionId })).toBe(0);
+ expect(service.getEngine(sessionId)).toBeUndefined();
+ expect(service.getActiveSessionIds()).toEqual([]);
+ expect(ownership.ownedIds()).toEqual([]);
+ // The stop of a session that no longer exists answers 404, which it could not if an engine lingered.
+ await expect(service.stop(sessionId)).rejects.toThrow('not found');
+ });
+});
diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.ts
index 3af7d6bed..555d1f037 100644
--- a/src/modules/session/session-engine-controls.ts
+++ b/src/modules/session/session-engine-controls.ts
@@ -173,6 +173,12 @@ export class SessionEngineControls {
private readonly reconnectStates: Map<string, ReconnectState>;
private readonly stuckAuthRecoveryUsed: Set<string>;
private readonly initializingSessions: Set<string>;
+ /**
+ * Deletes in flight per id, counted so that two overlapping deletes of one id do not end each other's hold.
+ * The stop mark of a delete cannot refuse a start for as long as the delete runs: start() takes a mark that
+ * was already set when it began for a stale one and clears it.
+ */
+ private readonly deleting = new Map<string, number>();
constructor(private readonly host: SessionEngineControlsHost) {
this.sessionRepository = host.sessionRepository;
@@ -193,6 +199,11 @@ export class SessionEngineControls {
this.initializingSessions = host.initializingSessions;
}
+ /** True while a delete of the session runs, from its engine teardown until its rows are gone or it failed. */
+ isDeleting(id: string): boolean {
+ return this.deleting.has(id);
+ }
+
/** findOne-or-404 with the runtime-state projection, mirroring SessionService.findOne. */
private async requireSession(id: string): Promise<Session> {
const session = await this.sessionRepository.findOne({ where: { id } });
@@ -282,6 +293,13 @@ export class SessionEngineControls {
// again, so a refused start does not leave a false "already starting" mark behind (briefly held).
await this.fences.awaitPendingTeardown(session.name);
+ // A delete still running outranks every start, however long it has run: the mark it set is not a
+ // stale one, and clearing it below would let an engine start under the delete that is removing the
+ // session (the engine would outlive the row). Checked here, with no await before the clear.
+ if (this.deleting.has(id)) {
+ throw new SessionStoppedException(`Session ${id} is being deleted`);
+ }
+
// A fresh start intentionally (re-)creates the engine — clear any stale stop/delete mark, but
// yield to one set after this start began, leaving it in place.
if (this.stoppingSessions.has(id) && this.stoppingSessions.stamp(id) !== markAtEntry) {
@@ -711,6 +729,7 @@ export class SessionEngineControls {
// rejected 409 from fence #2 leaves them intact (the session still exists).
let parentDeleted = false;
const writes = closeSessionWrites(this.host.dataSource(), id);
+ this.deleting.set(id, (this.deleting.get(id) ?? 0) + 1);
try {
// Stop engine if running — time-bounded + isolated so a stuck Chromium can't wedge the delete;
@@ -829,6 +848,9 @@ export class SessionEngineControls {
throw error;
} finally {
writes.release();
+ const running = (this.deleting.get(id) ?? 1) - 1;
+ if (running > 0) this.deleting.set(id, running);
+ else this.deleting.delete(id);
// Always clear the teardown mark so a later recreate/start with this id isn't suppressed. This
// stop mark was set after fence #1, so clearing it on a rejected 409 only undoes what THIS
// delete added — it does NOT touch reconnect timer / engine / last status / error / recovery
diff --git a/src/modules/session/session-engine-lifecycle-races.spec.ts b/src/modules/session/session-engine-lifecycle-races.spec.ts
index 70551c75d..e9bc83d72 100644
--- a/src/modules/session/session-engine-lifecycle-races.spec.ts
+++ b/src/modules/session/session-engine-lifecycle-races.spec.ts
@@ -76,6 +76,7 @@ const flush = async (): Promise<void> => {
interface Internals {
engines: EngineRegistry;
+ stoppingSessions: Set<string>;
reconnectStates: Map<string, ReconnectState>;
logger: { warn: (...args: unknown[]) => void };
executeReconnect(id: string, session: Session, state: ReconnectState): Promise<void>;
@@ -91,9 +92,11 @@ describe('SessionEngineLifecycle races', () => {
let engineFactory: { create: jest.Mock; purgeSessionData: jest.Mock };
let webhookService: { dispatch: jest.Mock };
let config: Record<string, unknown>;
+ let dataSource: Record<string, unknown>;
beforeEach(async () => {
config = {};
+ dataSource = {};
repository = {
findOne: jest.fn().mockResolvedValue(session()),
update: jest.fn().mockResolvedValue({ affected: 1 }),
@@ -112,7 +115,7 @@ describe('SessionEngineLifecycle races', () => {
PresenceStore,
{ provide: getRepositoryToken(Session, 'data'), useValue: repository },
{ provide: getRepositoryToken(Message, 'data'), useValue: { find: jest.fn().mockResolvedValue([]) } },
- { provide: getDataSourceToken('data'), useValue: {} },
+ { provide: getDataSourceToken('data'), useValue: dataSource },
{ provide: EngineFactory, useValue: engineFactory },
EngineRegistry,
SessionLidResolver,
@@ -222,6 +225,127 @@ describe('SessionEngineLifecycle races', () => {
});
});
+ // A delete removes the rows of a session with a long history in many batches, long after its engine is gone.
+ // The stop mark it set cannot refuse a start for that long: start() takes a mark that was already there when
+ // it began for a stale one and clears it, which for the mark of a delete left an engine running on the
+ // deleted session.
+ describe('start against a delete in flight', () => {
+ type Gate = { promise: Promise<unknown[]>; resolve: (rows: unknown[]) => void };
+ let gates: Gate[];
+ let transaction: jest.Mock;
+
+ // The next delete to look up rows waits here, which is where a delete spends its time on a long history.
+ const hold = (): Gate => {
+ const gate = deferred<unknown[]>();
+ gates.push(gate);
+ return gate;
+ };
+
+ // What the delete ended with: nothing, or its error.
+ const settle = (deleting: Promise<void>): Promise<unknown> =>
+ deleting.then(
+ () => undefined,
+ (e: unknown) => e,
+ );
+
+ // Starts a delete and leaves it waiting on its first batch, after it evicted the engine of a running session.
+ const deleteInBatchPhase = async (): Promise<{ gate: Gate; settled: Promise<unknown> }> => {
+ await lifecycle.start(ID);
+ const gate = hold();
+ const settled = settle(lifecycle.delete(ID));
+ await flush();
+ expect(internals.engines.has(ID)).toBe(false);
+ return { gate, settled };
+ };
+
+ beforeEach(() => {
+ gates = [];
+ const rows = { find: jest.fn(() => gates.shift()?.promise ?? Promise.resolve([])), delete: jest.fn() };
+ const manager = { delete: jest.fn().mockResolvedValue({ affected: 0 }), remove: jest.fn() };
+ transaction = jest.fn((work: (m: unknown) => Promise<void>) => work(manager));
+ Object.assign(dataSource, { getRepository: jest.fn(() => rows), transaction });
+ });
+
+ it.each([
+ ['an operator start', { explicit: true }],
+ ['a boot or takeover start', { explicit: false }],
+ ])('refuses %s and registers no engine', async (_who, options) => {
+ const { gate, settled } = await deleteInBatchPhase();
+
+ await expect(lifecycle.start(ID, options)).rejects.toBeInstanceOf(SessionStoppedException);
+
+ expect(engineFactory.create).toHaveBeenCalledTimes(1);
+ expect(internals.engines.has(ID)).toBe(false);
+ gate.resolve([]);
+ expect(await settled).toBeUndefined();
+ expect(internals.engines.has(ID)).toBe(false);
+ });
+
+ it('leaves the stop mark of the delete in place for the whole delete', async () => {
+ const { gate, settled } = await deleteInBatchPhase();
+
+ await expect(lifecycle.start(ID)).rejects.toBeInstanceOf(SessionStoppedException);
+
+ expect(internals.stoppingSessions.has(ID)).toBe(true);
+ gate.resolve([]);
+ await settled;
+ expect(internals.stoppingSessions.has(ID)).toBe(false);
+ });
+
+ it('lets the session start again when the delete failed', async () => {
+ transaction.mockRejectedValueOnce(new Error('canceling statement due to statement timeout'));
+ const { gate, settled } = await deleteInBatchPhase();
+
+ gate.resolve([]);
+ expect(((await settled) as Error).message).toContain('statement timeout');
+ expect(lifecycle.isEngineActive(ID)).toBe(false);
+
+ await lifecycle.start(ID, { explicit: true });
+
+ expect(internals.engines.has(ID)).toBe(true);
+ });
+
+ it('holds back a start until the last of two overlapping deletes ends', async () => {
+ transaction.mockRejectedValueOnce(new Error('canceling statement due to statement timeout'));
+ const first = await deleteInBatchPhase();
+ const secondGate = hold();
+ const second = settle(lifecycle.delete(ID));
+ await flush();
+
+ // The first delete fails and drops the stop mark the second one is still running under.
+ first.gate.resolve([]);
+ expect(((await first.settled) as Error).message).toContain('statement timeout');
+ await expect(lifecycle.start(ID)).rejects.toBeInstanceOf(SessionStoppedException);
+ expect(internals.engines.has(ID)).toBe(false);
+
+ secondGate.resolve([]);
+ expect(await second).toBeUndefined();
+ await lifecycle.start(ID);
+ expect(internals.engines.has(ID)).toBe(true);
+ });
+
+ it('retires a start whose row read was still pending when the delete began', async () => {
+ const read = deferred<Session>();
+ repository.findOne.mockReturnValueOnce(read.promise);
+ const starting = lifecycle.start(ID, { explicit: true }).then(
+ () => undefined,
+ (e: unknown) => e,
+ );
+ await flush();
+ const gate = hold();
+ const settled = settle(lifecycle.delete(ID));
+ await flush();
+
+ read.resolve(session());
+
+ expect(await starting).toBeInstanceOf(SessionStoppedException);
+ expect(engineFactory.create).not.toHaveBeenCalled();
+ gate.resolve([]);
+ await settled;
+ expect(internals.engines.has(ID)).toBe(false);
+ });
+ });
+
describe('stop against an explicit start still reading its row', () => {
it('yields to a stop that lands while it waits, after an earlier stop', async () => {
repository.findOne.mockResolvedValue(session({ desiredState: 'stopped' }));