Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): keep a session down when its delete stops part-way

Yudhi Armyndharise354a45dc93e
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.md
index 6aa64a6c5..66e227fd6 100644
--- a/docs/06-api-specification.md
+++ b/docs/06-api-specification.md
@@ -1286,7 +1286,7 @@ Delete a session.
**Response** `204` — empty body (`@HttpCode(204)`, returns void). A `findOne` lookup runs first, so a missing id yields `404`.
-The delete first removes the session's messages, webhook outbox rows and webhook delivery-failure records in batches (on SQLite up to 2000 rows, each sized to about 150 ms; 500 rows on PostgreSQL) that are not atomic with the rest: a failure part-way (a database error, a timeout, a restart) leaves the session in place with fewer rows, and repeating the delete finishes it. It then removes, in one transaction, the session's message batches, chat states, received statuses, webhooks, templates, stored Baileys messages, automation rules, integration dead-letter rows and send idempotency claims together with the session itself, then purges its on-disk auth data. It keeps the audit log, the LID mappings, and the integration conversation mappings (so a re-paired session keeps its provider conversations); ingress dedup rows age out on their own after `INGRESS_DEDUP_RETENTION_DAYS` (default 7).
+The delete first removes the session's messages, webhook outbox rows and webhook delivery-failure records in batches (on SQLite up to 2000 rows, each sized to about 150 ms; 500 rows on PostgreSQL) that are not atomic with the rest: a failure part-way (a database error, a timeout, a restart) leaves the session in place with fewer rows and stopped, as `POST /stop` leaves it, and repeating the delete finishes it. It then removes, in one transaction, the session's message batches, chat states, received statuses, webhooks, templates, stored Baileys messages, automation rules, integration dead-letter rows and send idempotency claims together with the session itself, then purges its on-disk auth data. It keeps the audit log, the LID mappings, and the integration conversation mappings (so a re-paired session keeps its provider conversations); ingress dedup rows age out on their own after `INGRESS_DEDUP_RETENTION_DAYS` (default 7).
**Errors:** `401` missing/invalid key · `403` key not scoped to this session, or key role below OPERATOR · `404` session not found · `409` credential teardown for the same session name still in flight (retryable; body carries `code: 'SESSION_NAME_TEARDOWN_PENDING'`; on a `409` the row is **not** deleted and no hook/auth-purge runs — retry after cleanup settles), or, on a multi-node deployment, another node currently holds this session's live engine (no `code`; nothing is deleted; send the delete to the owning node) · `503` the session could not be claimed for the delete because the database did not answer (nothing is deleted; retry)
diff --git a/src/modules/session/entities/session.entity.ts b/src/modules/session/entities/session.entity.ts
index 93b4cfab7..ca082aeb9 100644
--- a/src/modules/session/entities/session.entity.ts
+++ b/src/modules/session/entities/session.entity.ts
@@ -83,10 +83,10 @@ export class Session {
/**
* Operator intent, server-owned and not part of any session response (only the export-data
- * backup carries it): 'stopped' once POST /stop or POST /force-kill took the session down,
- * cleared again by an explicit POST /start. Boot auto-start and the takeover sweep skip a stopped
- * row, so a deliberate stop survives a restart. NULL (every row that predates the column) means
- * eligible, exactly as before.
+ * backup carries it): 'stopped' once POST /stop or POST /force-kill took the session down, or
+ * while a DELETE runs (kept if it fails part-way), cleared again by an explicit POST /start. Boot
+ * auto-start and the takeover sweep skip a stopped row, so a deliberate stop survives a restart.
+ * NULL (every row that predates the column) means eligible, exactly as before.
*/
@Column({ type: 'varchar', length: 20, nullable: true })
desiredState!: 'stopped' | null;
diff --git a/src/modules/session/session-delete-claim.spec.ts b/src/modules/session/session-delete-claim.spec.ts
index 6007b2a61..968d85d58 100644
--- a/src/modules/session/session-delete-claim.spec.ts
+++ b/src/modules/session/session-delete-claim.spec.ts
@@ -13,6 +13,7 @@ import { SessionErrorStore } from './session-error-store.service';
import { SessionRestrictionStore } from './session-restriction-store.service';
import { PresenceStore } from './presence-store.service';
import { SessionOwnershipService } from './session-ownership.service';
+import { SessionLifecycleFences } from './session-lifecycle-fences';
import { Session, SessionStatus } from './entities/session.entity';
import { Message } from '../message/entities/message.entity';
import { MessageBatch } from '../message/entities/message-batch.entity';
@@ -356,6 +357,171 @@ describe('A session deleted on one node while a peer shares the database', () =>
expect(b.ownership.ownedIds()).toEqual([]);
});
+ // A process that dies while its delete runs never hands the claim back, and the claim lapses. The session it
+ // was removing must stay down, wherever it is picked up next.
+ it('leaves a session whose deleting node died part-way for no peer to adopt', async () => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ await startHeldDelete(id);
+ // Node A dies here: nothing resumes its delete or renews its claim.
+ await a.db.getRepository(Session).update(id, { leaseExpiresAt: lapsed() });
+
+ const takeover = new SessionTakeoverService(
+ b.service,
+ b.ownership,
+ new ConfigService({ features: { autoStartSessions: true } }),
+ );
+ await takeover.sweep();
+
+ expect(b.engines).toHaveLength(0);
+ expect(await a.db.getRepository(Session).findOneByOrFail({ id })).toMatchObject({
+ nodeId: 'node-a',
+ desiredState: 'stopped',
+ });
+ });
+
+ // A delete records itself as a stop on the row, so the crash above leaves the session down; one that fails
+ // before it removes anything takes that record back.
+ it.each([
+ ['its first teardown fence refuses', 'fence-1'],
+ ['its second teardown fence refuses', 'fence-2'],
+ ['its row read fails', 'read'],
+ ])('leaves the session as it found it when %s', async (_case, failure) => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ await a.service.start(id, { explicit: true });
+ const fence = jest.spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown');
+ if (failure === 'fence-1') fence.mockRejectedValueOnce(new ConflictException('a logout teardown is running'));
+ if (failure === 'fence-2') {
+ fence.mockResolvedValueOnce(undefined).mockRejectedValueOnce(new ConflictException('a logout is running'));
+ }
+ if (failure === 'read') {
+ jest.spyOn(a.db.getRepository(Session), 'findOne').mockRejectedValueOnce(new Error('connection reset'));
+ }
+
+ await expect(a.service.delete(id)).rejects.toThrow();
+
+ const row = await a.db.getRepository(Session).findOneByOrFail({ id });
+ expect(row.desiredState).toBeNull();
+ // The second fence refuses after the engine is gone, and the claim goes with it.
+ expect(row.nodeId).toBe(failure === 'fence-2' ? null : 'node-a');
+ expect(await rowsOf(id)).toEqual({ sessions: 1, messages: 1, outbox: 0, deliveryFailures: 0 });
+ });
+
+ it('leaves the session as it found it when a retried delete is refused at the first fence again', async () => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ jest
+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')
+ .mockRejectedValueOnce(new ConflictException('a logout teardown is running'))
+ .mockRejectedValueOnce(new ConflictException('a logout teardown is running'));
+
+ await expect(a.service.delete(id)).rejects.toBeInstanceOf(ConflictException);
+ await expect(a.service.delete(id)).rejects.toBeInstanceOf(ConflictException);
+
+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBeNull();
+ });
+
+ it('refuses the delete and takes nothing down when it cannot record the stop', async () => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ await a.service.start(id, { explicit: true });
+ jest.spyOn(a.db.getRepository(Session), 'update').mockRejectedValueOnce(new Error('connection reset'));
+
+ await expect(a.service.delete(id)).rejects.toThrow('connection reset');
+
+ expect(a.service.getEngine(id)).toBe(a.engines[0]);
+ expect(a.engines[0].forceDestroy).not.toHaveBeenCalled();
+ expect(leftovers(id)).toEqual({ stopMark: false, stopCount: false });
+ expect(await a.db.getRepository(Session).findOneByOrFail({ id })).toMatchObject({
+ nodeId: 'node-a',
+ desiredState: null,
+ });
+ });
+
+ it('keeps the stop and answers the delete error when it cannot take the stop back', async () => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ jest
+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')
+ .mockRejectedValueOnce(new ConflictException('a logout teardown is running'));
+ const sessions = a.db.getRepository(Session);
+ const update = sessions.update.bind(sessions);
+ // The stop is recorded; taking it back fails.
+ jest.spyOn(sessions, 'update').mockImplementationOnce(update).mockRejectedValueOnce(new Error('connection reset'));
+
+ await expect(a.service.delete(id)).rejects.toBeInstanceOf(ConflictException);
+
+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');
+ });
+
+ it('keeps a stop the session already had when its delete fails before removing anything', async () => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED, desiredState: 'stopped' });
+ jest
+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')
+ .mockRejectedValueOnce(new ConflictException('a logout teardown is running'));
+
+ await expect(a.service.delete(id)).rejects.toBeInstanceOf(ConflictException);
+
+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('stopped');
+ });
+
+ it.each([['stop'], ['force-kill']])(
+ 'keeps a %s that began before its delete and is recorded after it, when the delete fails before removing',
+ async how => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ await a.service.start(id, { explicit: true });
+ const sessions = a.db.getRepository(Session);
+ const findOne = sessions.findOne.bind(sessions);
+ const update = sessions.update.bind(sessions);
+ const recording = deferred();
+ const recorded = deferred();
+ const reading = deferred();
+ const read = deferred();
+ // The stop's record is held until the delete has written its own; the delete's row read then fails.
+ jest.spyOn(sessions, 'update').mockImplementationOnce(async (...args: Parameters<typeof update>) => {
+ recording.resolve();
+ await recorded.promise;
+ return update(...args);
+ });
+ jest
+ .spyOn(sessions, 'findOne')
+ .mockImplementationOnce(findOne)
+ .mockImplementationOnce(async () => {
+ reading.resolve();
+ await read.promise;
+ throw new Error('connection reset');
+ });
+ const stopping = how === 'stop' ? a.service.stop(id) : a.service.forceKill(id);
+ await recording.promise;
+ const deleting = a.service.delete(id);
+ await reading.promise;
+ recorded.resolve();
+ await stopping;
+ read.resolve();
+
+ await expect(deleting).rejects.toThrow('connection reset');
+
+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');
+ },
+ );
+
+ it('keeps a stop that lands while its delete runs when the delete then fails before removing anything', async () => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ const sessions = a.db.getRepository(Session);
+ const reading = deferred();
+ const read = deferred();
+ // The delete's first row read, which comes after its claim, fails once the stop has finished.
+ jest.spyOn(sessions, 'findOne').mockImplementationOnce(async () => {
+ reading.resolve();
+ await read.promise;
+ throw new Error('connection reset');
+ });
+ const deleting = a.service.delete(id);
+ await reading.promise;
+ await a.service.stop(id);
+ read.resolve();
+
+ await expect(deleting).rejects.toThrow('connection reset');
+
+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');
+ });
+
it('refuses the delete and removes nothing when the claim cannot be written', async () => {
const id = await seed({ status: SessionStatus.DISCONNECTED });
await a.service.start(id, { explicit: true });
@@ -369,6 +535,7 @@ describe('A session deleted on one node while a peer shares the database', () =>
expect(a.hooks.execute).not.toHaveBeenCalledWith('session:deleted', expect.anything(), expect.anything());
expect(a.purge).not.toHaveBeenCalled();
expect(leftovers(id)).toEqual({ stopMark: false, stopCount: false });
+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBeNull();
});
it('answers 409 and removes nothing when a peer claims the session just before the delete does', async () => {
@@ -395,7 +562,11 @@ describe('A session deleted on one node while a peer shares the database', () =>
await expect(a.service.delete(id)).rejects.toThrow('transaction timed out');
- expect((await a.db.getRepository(Session).findOneByOrFail({ id })).nodeId).toBeNull();
+ // The batches ran before the closing transaction failed, so the session stays down like a crashed delete.
+ expect(await a.db.getRepository(Session).findOneByOrFail({ id })).toMatchObject({
+ nodeId: null,
+ desiredState: 'stopped',
+ });
expect(a.ownership.ownedIds()).toEqual([]);
});
diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.ts
index 2563b235e..977ac83dd 100644
--- a/src/modules/session/session-engine-controls.ts
+++ b/src/modules/session/session-engine-controls.ts
@@ -147,6 +147,8 @@ export interface StopHooks {
onReadFailed?: () => void;
/** forceKill() only: runs synchronously once there is an engine to kill; records the stop request. */
onEngineFound?: () => void;
+ /** delete() only: runs once the delete starts removing the session's data, the session:deleted hook first. */
+ onRemoving?: () => void;
}
/**
@@ -850,6 +852,7 @@ export class SessionEngineControls {
}
await writes.drain();
+ hooks.onRemoving?.();
// Execute hook BEFORE delete so plugins can access session data
await this.hookManager.execute(
'session:deleted',
diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.ts
index 4f6c27365..d8c63f45a 100644
--- a/src/modules/session/session.service.spec.ts
+++ b/src/modules/session/session.service.spec.ts
@@ -160,7 +160,7 @@ describe('SessionService', () => {
create: jest.fn(),
save: jest.fn(),
remove: jest.fn(),
- update: jest.fn(),
+ update: jest.fn().mockResolvedValue({ affected: 1 }),
exists: jest.fn().mockResolvedValue(false),
};
diff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.ts
index d8155bda6..9d2b73bd4 100644
--- a/src/modules/session/session.service.ts
+++ b/src/modules/session/session.service.ts
@@ -144,6 +144,8 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
* after it began from a mark left over by an earlier stop, which start() clears by design.
*/
private readonly stopRequests = new Map<string, number>();
+ /** stop() and forceKill() calls in flight per session, whose stop record may still be written; see delete(). */
+ private readonly stopsInFlight = new Map<string, number>();
/**
* Starts past start()'s cap check, held until they return. That check counts them with the
* engine's slot holders, so two starts at the last free slot cannot both pass it while their claims
@@ -609,12 +611,20 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
// post-INITIALIZING check by the time that write settles; awaiting anything first — the fence's
// COUNT, or delete()'s own requireSession — would let the mark land after that window. A mark
// left behind when the fence refuses (409) is harmless and is cleared by the next start().
+ // A stop or kill already running may record its stop after this delete records its own.
+ const stopInFlight = this.stopsInFlight.has(id);
this.markStopping(id);
// Running from here, before the claim: a claim with nothing running behind it would read as idle to the
// heartbeat, and a stop or a failed start here would hand it back while the rows are removed. It ends
// before the claim is handed back.
const endDelete = this.engineLifecycle.holdDelete(id);
+ // This delete's own stop count, and every other stop counted before it: a stop counted while it runs is
+ // the operator's, and outlives a failed delete.
+ let counted = 1;
+ const otherStopsBefore = (this.stopRequests.get(id) ?? 0) - counted;
let claimed = false;
+ let keptDown = false;
+ let removing = false;
try {
try {
if (this.ownership) {
@@ -622,14 +632,32 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
await this.claimForDelete(id, this.ownership);
claimed = true;
}
- // A failed row read deleted nothing: the lifecycle drops the mark, and the count goes with it.
- await this.engineLifecycle.delete(id, { onReadFailed: () => this.uncountStopRequest(id) });
+ // A failed write took nothing down, so the mark and count are undone as for a failed claim.
+ keptDown = await this.keepDownForDelete(id).catch((error: unknown) => {
+ this.engineLifecycle.clearStopping(id);
+ this.uncountStopRequest(id);
+ throw error;
+ });
+ await this.engineLifecycle.delete(id, {
+ // A failed row read deleted nothing: the lifecycle drops the mark, and the count goes with it.
+ onReadFailed: () => {
+ counted = 0;
+ this.uncountStopRequest(id);
+ },
+ onRemoving: () => {
+ removing = true;
+ },
+ });
} finally {
endDelete();
}
await this.releaseClaimBestEffort(id);
this.stopRequests.delete(id);
} catch (error) {
+ // A delete that removed nothing leaves the session as it found it. One that began to remove it leaves
+ // it stopped, as a process that dies part-way does.
+ const stoppedMeanwhile = stopInFlight || (this.stopRequests.get(id) ?? 0) - counted !== otherStopsBefore;
+ if (keptDown && !removing && !stoppedMeanwhile) await this.clearDeleteStop(id);
// The row survived. A claim left on it lapses with nothing to renew it, and a peer would then
// adopt and restart the session the operator was deleting.
if (claimed) await this.releaseAfterTeardown(id);
@@ -667,6 +695,31 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
throw new ConflictException(`Session ${id} is running on another node`);
}
+ /**
+ * Record the delete as a stop on a row that has none, so a session whose delete ends part-way, with the
+ * process that ran it, stays down: no peer adopts it and no boot starts it. Answers whether it wrote one.
+ */
+ private async keepDownForDelete(id: string): Promise<boolean> {
+ const { affected } = await this.sessionRepository.update(
+ { id, desiredState: IsNull() },
+ { desiredState: 'stopped' },
+ );
+ return affected === 1;
+ }
+
+ /** Undo keepDownForDelete. Best effort: the delete's own error is the answer, and a stop left on is safe. */
+ private async clearDeleteStop(id: string): Promise<void> {
+ try {
+ await this.sessionRepository.update({ id, desiredState: 'stopped' }, { desiredState: null });
+ } catch (error) {
+ this.logger.warn(`Could not clear the stop a failed delete recorded for session ${id}`, {
+ sessionId: id,
+ action: 'session_delete_stop_clear_failed',
+ error: error instanceof Error ? error.message : String(error),
+ });
+ }
+ }
+
/**
* Reclaim the entry-time stop mark when the id turns out to have no session row.
*
@@ -872,6 +925,7 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
async stop(id: string): Promise<Session> {
// Synchronous stop-mark before the awaited fence — see delete() for why.
this.markStopping(id);
+ const endStop = this.countStopInFlight(id);
let session: Session;
try {
if (this.ownership) await this.assertNotHeldElsewhere(id);
@@ -897,6 +951,8 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
if (error instanceof BadGatewayException) await this.releaseAfterTeardown(id);
this.discardStopMarkForMissingSession(id, error);
throw error;
+ } finally {
+ endStop();
}
// Handed back on the way out so a peer can pick it up immediately rather than waiting for the
// lease to lapse. Stop is the deliberate end of this process's ownership — but a start() that
@@ -940,6 +996,7 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
this.uncountStopRequest(id);
};
if (this.engines.get(id)) count();
+ const endStop = this.countStopInFlight(id);
try {
// The engine kill records the stop itself, once it has an engine to kill.
const session = await this.engineLifecycle.forceKill(id, { onReadFailed: uncount, onEngineFound: count });
@@ -951,9 +1008,20 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
if (error instanceof BadRequestException) uncount();
if (!(error instanceof BadRequestException)) await this.releaseAfterTeardown(id);
throw error;
+ } finally {
+ endStop();
}
}
+ private countStopInFlight(id: string): () => void {
+ this.stopsInFlight.set(id, (this.stopsInFlight.get(id) ?? 0) + 1);
+ return () => {
+ const left = (this.stopsInFlight.get(id) ?? 1) - 1;
+ if (left > 0) this.stopsInFlight.set(id, left);
+ else this.stopsInFlight.delete(id);
+ };
+ }
+
/** Persist the operator's stop so boot auto-start and the takeover sweep leave the session down. */
private async keepDown(id: string): Promise<void> {
await this.sessionRepository.update(id, { desiredState: 'stopped' });