Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): keep a stop a peer records during a failed delete
Yudhi Armyndharisf5be4a597188
diff --git a/docs/05-database-design.md b/docs/05-database-design.mdindex 153d798aa..0b1587905 100644--- a/docs/05-database-design.md+++ b/docs/05-database-design.md@@ -346,8 +346,9 @@ CREATE TABLE sessions ("leaseExpiresAt" TIMESTAMP,-- 'stopped' after POST /stop or /force-kill, NULL again after an explicit POST /start. Boot-- auto-start and the takeover sweep skip a stopped row. Server-owned: not in any session- -- response, only in the /api/infra/export-data backup. A DELETE writes it too: kept when the- -- delete fails part-way or its node dies, taken back when it fails before removing anything.+ -- response, only in the /api/infra/export-data backup. A DELETE writes 'deleting', read the+ -- same way: kept when the delete fails part-way or its node dies, taken back when it fails+ -- before removing anything, unless a stop was recorded over it meanwhile."desiredState" VARCHAR(20),"createdAt" TIMESTAMP NOT NULL DEFAULT NOW(),"updatedAt" TIMESTAMP NOT NULL DEFAULT NOW()diff --git a/docs/13-horizontal-scaling.md b/docs/13-horizontal-scaling.mdindex 17602dbf7..e88ea66b2 100644--- a/docs/13-horizontal-scaling.md+++ b/docs/13-horizontal-scaling.md@@ -50,6 +50,8 @@> Nothing else revisits such a row, since the boot reset skips a foreign claim that is still live.> A session an operator stopped (`POST /stop` or `/force-kill`), or whose `DELETE` failed part-way or> died with its node, is never adopted, even when its claim did lapse, until an explicit `POST /start`.+> While versions are mixed during an upgrade, a node still on 0.25.0 does not read a `DELETE`'s record:+> it can adopt such a session, and a `POST /start` it serves leaves the record in place.>> Known limitation: a holder that is alive but cannot reach the database for more than three lease> TTLs minus one heartbeat (160s at defaults) is marked disconnected by a peer too, and does not write its status back when it reconnects. Itsdiff --git a/src/modules/infra/table-importers.spec.ts b/src/modules/infra/table-importers.spec.tsindex 881778ca0..0e5ffa2f3 100644--- a/src/modules/infra/table-importers.spec.ts+++ b/src/modules/infra/table-importers.spec.ts@@ -37,6 +37,8 @@ describe('sessions table importer: desiredState', () => {expect(sessions?.sql).toContain('"desiredState"');expect(sessions?.sql).toContain('$13');expect(mapped({ desiredState: 'stopped' })[12]).toBe('stopped');+ // No delete runs for a restored row, so the record of one that failed part-way is a stop.+ expect(mapped({ desiredState: 'deleting' })[12]).toBe('stopped');});it('restores a row without the field, or with an unknown value, as eligible (NULL)', () => {diff --git a/src/modules/infra/table-importers.ts b/src/modules/infra/table-importers.tsindex 7b6db5b49..bfff08a27 100644--- a/src/modules/infra/table-importers.ts+++ b/src/modules/infra/table-importers.ts@@ -165,9 +165,10 @@ export const TABLE_IMPORTERS: AnyTableImporter[] = [session.lastActiveAt,session.createdAt,session.updatedAt,- // 'stopped' is the only value the column holds. A backup written before the column existed,- // or an unknown value, restores as NULL: eligible for auto-start and takeover, as before.- session.desiredState === 'stopped' ? 'stopped' : null,+ // A failed delete's 'deleting' restores as 'stopped': no delete runs for the restored row. A+ // backup written before the column existed, or an unknown value, restores as NULL: eligible for+ // auto-start and takeover, as before.+ session.desiredState === 'stopped' || session.desiredState === 'deleting' ? 'stopped' : null,],}),diff --git a/src/modules/session/entities/session.entity.ts b/src/modules/session/entities/session.entity.tsindex ca082aeb9..c970999f1 100644--- a/src/modules/session/entities/session.entity.ts+++ b/src/modules/session/entities/session.entity.ts@@ -83,13 +83,13 @@ export class Session {/*** Operator intent, server-owned and not part of any session response (only the export-data- * backup carries it): 'stopped' once POST /stop or POST /force-kill took the session down, or- * while a DELETE runs (kept if it fails part-way), cleared again by an explicit POST /start. Boot- * auto-start and the takeover sweep skip a stopped row, so a deliberate stop survives a restart.- * NULL (every row that predates the column) means eligible, exactly as before.+ * backup carries it): 'stopped' once POST /stop or POST /force-kill took the session down, and+ * 'deleting' while a DELETE runs (kept if it fails part-way), both cleared again by an explicit+ * POST /start. Boot auto-start and takeover skip a row holding either, so a deliberate stop+ * survives a restart. NULL (every row that predates the column) means eligible, as before.*/@Column({ type: 'varchar', length: 20, nullable: true })- desiredState!: 'stopped' | null;+ desiredState!: 'stopped' | 'deleting' | null;@CreateDateColumn()createdAt!: Date;diff --git a/src/modules/session/session-delete-claim.spec.ts b/src/modules/session/session-delete-claim.spec.tsindex 68c9fcfbd..2e7a4a4ec 100644--- a/src/modules/session/session-delete-claim.spec.ts+++ b/src/modules/session/session-delete-claim.spec.ts@@ -14,6 +14,7 @@ import { SessionRestrictionStore } from './session-restriction-store.service';import { PresenceStore } from './presence-store.service';import { SessionOwnershipService } from './session-ownership.service';import { SessionLifecycleFences } from './session-lifecycle-fences';+import { SessionStoppedException } from './session-engine-controls';import { Session, SessionStatus } from './entities/session.entity';import { Message } from '../message/entities/message.entity';import { MessageBatch } from '../message/entities/message-batch.entity';@@ -384,10 +385,66 @@ describe('A session deleted on one node while a peer shares the database', () =>expect(b.engines).toHaveLength(0);expect(await a.db.getRepository(Session).findOneByOrFail({ id })).toMatchObject({nodeId: 'node-a',- desiredState: 'stopped',+ desiredState: 'deleting',});});+ it('keeps a session a failed delete left down until an explicit start', async () => {+ const id = await seed({+ status: SessionStatus.DISCONNECTED,+ desiredState: 'deleting',+ nodeId: 'node-c',+ leaseExpiresAt: lapsed(),+ });+ const takeover = new SessionTakeoverService(+ b.service,+ b.ownership,+ new ConfigService({ features: { autoStartSessions: true } }),+ );++ await takeover.sweep();+ expect(b.engines).toHaveLength(0);+ // Refused on the record, before the teardown fence would refuse it for another reason.+ const fence = jest+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')+ .mockRejectedValue(new ConflictException('a logout teardown is running'));+ await expect(b.service.start(id)).rejects.toBeInstanceOf(SessionStoppedException);+ fence.mockRestore();+ expect(b.engines).toHaveLength(0);++ await b.service.start(id, { explicit: true });+ expect(b.service.getEngine(id)).toBe(b.engines[0]);+ expect((await b.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBeNull();+ });++ it('keeps the record of a delete that lands while an explicit start reads the session again', async () => {+ const id = await seed({ status: SessionStatus.DISCONNECTED });+ const sessions = a.db.getRepository(Session);+ const findOne = sessions.findOne.bind(sessions);+ const reading = deferred();+ const read = deferred();+ // The start's second row read, past its teardown fence, is held while the delete records itself.+ jest+ .spyOn(sessions, 'findOne')+ .mockImplementationOnce(findOne)+ .mockImplementationOnce(async (...args: Parameters<typeof findOne>) => {+ reading.resolve();+ await read.promise;+ return findOne(...args);+ });+ const starting = a.service.start(id, { explicit: true });+ await reading.promise;+ await startHeldDelete(id);+ read.resolve();++ await expect(starting).rejects.toBeInstanceOf(SessionStoppedException);++ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('deleting');+ finish!();+ await deletion;+ expect(await rowsOf(id)).toEqual(gone);+ });+// A delete records itself as a stop on the row, so the crash above leaves the session down; one that fails// before it removes anything takes that record back.it.each([@@ -456,7 +513,7 @@ describe('A session deleted on one node while a peer shares the database', () =>await expect(a.service.delete(id)).rejects.toBeInstanceOf(ConflictException);- expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('deleting');});it('keeps a stop the session already had when its delete fails before removing anything', async () => {@@ -698,12 +755,12 @@ describe('A session deleted on one node while a peer shares the database', () =>.mockRejectedValueOnce(new ConflictException('a logout teardown is running'));await expect(a.service.delete(id)).rejects.toBeInstanceOf(ConflictException);- expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('deleting');read.resolve();await reached.promise;// The rows are being removed: a process that died here would leave the session down.- expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('deleting');release.resolve();await deletion;@@ -734,7 +791,7 @@ describe('A session deleted on one node while a peer shares the database', () =>if (takeBackFails) throw new Error('connection reset');}const result = await update(...args);- if (args[1].desiredState === 'stopped') stopsWritten++;+ if (args[1].desiredState) stopsWritten++;return result;});// The failed delete is refused at its first fence, and so are the next `refusals - 1` deletes.@@ -772,7 +829,7 @@ describe('A session deleted on one node while a peer shares the database', () =>});// The rows are being removed: a process that died here would leave the session down.- expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('stopped');+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('deleting');release.resolve();await deletion;expect(await rowsOf(id)).toEqual(gone);@@ -805,6 +862,39 @@ describe('A session deleted on one node while a peer shares the database', () =>expect(bookkeeping(id)).toEqual({ deletes: false, stopWrites: false });});+ // The peer's stop passes its own ownership check while nobody holds the session, before this node claims it.+ it('keeps a stop another node records while its delete fails before removing anything', async () => {+ const id = await seed({ status: SessionStatus.DISCONNECTED });+ const peerSessions = b.db.getRepository(Session);+ const findOne = peerSessions.findOne.bind(peerSessions);+ const reading = deferred();+ const read = deferred();+ jest.spyOn(peerSessions, 'findOne').mockImplementationOnce(async (...args: Parameters<typeof findOne>) => {+ reading.resolve();+ await read.promise;+ return findOne(...args);+ });+ const stopping = b.service.stop(id);+ await reading.promise;+ // The delete records its stop and waits at its first fence until the peer's stop has answered.+ const fenced = deferred();+ const refuse = deferred();+ jest.spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown').mockImplementationOnce(async () => {+ fenced.resolve();+ await refuse.promise;+ throw new ConflictException('a logout teardown is running');+ });+ const deleting = a.service.delete(id);+ await fenced.promise;+ read.resolve();+ await stopping;+ refuse.resolve();++ await expect(deleting).rejects.toBeInstanceOf(ConflictException);++ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('stopped');+ });+it('refuses the delete and removes nothing when the claim cannot be written', async () => {const id = await seed({ status: SessionStatus.DISCONNECTED });await a.service.start(id, { explicit: true });@@ -848,7 +938,7 @@ describe('A session deleted on one node while a peer shares the database', () =>// The batches ran before the closing transaction failed, so the session stays down like a crashed delete.expect(await a.db.getRepository(Session).findOneByOrFail({ id })).toMatchObject({nodeId: null,- desiredState: 'stopped',+ desiredState: 'deleting',});expect(a.ownership.ownedIds()).toEqual([]);});diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.tsindex 8e32e2281..0576a27eb 100644--- a/src/modules/session/session-engine-controls.ts+++ b/src/modules/session/session-engine-controls.ts@@ -326,8 +326,8 @@ export class SessionEngineControls {// An operator's stop outranks any start but POST /start, so this refuses boot auto-start and// the takeover sweep, which pick their rows once and launch them seconds apart, when a stop- // landed after that read.- if (session.desiredState === 'stopped' && !explicit) {+ // landed after that read. A delete's record ('deleting') counts as one.+ if (session.desiredState && !explicit) {throw new SessionStoppedException(`Session ${id} was stopped`);}@@ -390,7 +390,7 @@ export class SessionEngineControls {// mark again and retires this start after its init. A launch that fails after the clear still// counts as the start the operator asked for.const { desiredState } = await this.requireSession(id);- if (desiredState === 'stopped' && (!explicit || session.desiredState !== 'stopped')) {+ if (desiredState && (!explicit || !session.desiredState)) {this.stoppingSessions.add(id);throw new SessionStoppedException(`Session ${id} was stopped`);}@@ -406,8 +406,8 @@ export class SessionEngineControls {if (this.host.isClosed()) {throw closedRefusal();}- if (desiredState === 'stopped') {- await this.sessionRepository.update({ id, desiredState: 'stopped' }, { desiredState: null });+ if (desiredState) {+ await this.sessionRepository.update({ id, desiredState }, { desiredState: null });// The clear stands (this start was the operator's), but the hook must not fire for a start// shutdown overtook during the write.if (this.host.isClosed()) {diff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.tsindex 40210bb3f..06b23f91f 100644--- a/src/modules/session/session.service.ts+++ b/src/modules/session/session.service.ts@@ -748,24 +748,25 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat/*** Record the delete as a stop on a row that has none, so a session whose delete ends part-way, with the* process that ran it, stays down: no peer adopts it and no boot starts it. Written after a take-back in- * progress, which would otherwise clear it.+ * progress, which would otherwise clear it. 'deleting' rather than 'stopped', so the take-back can tell it+ * from a stop that replaced it, also one another node recorded.*/private async keepDownForDelete(id: string, stops: DeleteStops): Promise<void> {if (stops.givingBack) await stops.givingBack;const { affected } = await this.sessionRepository.update({ id, desiredState: IsNull() },- { desiredState: 'stopped' },+ { desiredState: 'deleting' },);if (affected === 1) stops.recorded = true;}/*** Undo keepDownForDelete; answers whether the record is off the row. Best effort: the delete's own error is- * the answer, and a stop left on is safe.+ * the answer, and a stop left on is safe. A stop recorded over it since is left alone.*/private async clearDeleteStop(id: string): Promise<boolean> {try {- await this.sessionRepository.update({ id, desiredState: 'stopped' }, { desiredState: null });+ await this.sessionRepository.update({ id, desiredState: 'deleting' }, { desiredState: null });return true;} catch (error) {this.logger.warn(`Could not clear the stop a failed delete recorded for session ${id}`, {@@ -1095,7 +1096,7 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat/*** Re-record a stop an explicit start may have erased. The start clears the stop record with a- * write conditional on the row still reading 'stopped', and a stop's own write leaves that value+ * write conditional on the row still reading what it read, and a stop's own write leaves 'stopped'* unchanged, so a stop landing just before the clear was wiped from the row while its in-memory* mark retired the start. The session was down, but the next boot would relaunch it. Any stop* counted while the start ran and leaving no engine behind is the operator's last word. A faileddiff --git a/src/modules/takeover/session-takeover.service.ts b/src/modules/takeover/session-takeover.service.tsindex 4e9dc9583..1be077556 100644--- a/src/modules/takeover/session-takeover.service.ts+++ b/src/modules/takeover/session-takeover.service.ts@@ -208,7 +208,8 @@ export class SessionTakeoverService implements OnApplicationBootstrap, OnModuleDprivate isEligible(session: Session): boolean {// Only authenticated sessions (phone set): an engine is worth relaunching exactly when the// saved credentials can restore the link without a human scanning anything. A session an- // operator stopped stays down until an explicit start, wherever its claim lapsed.- return Boolean(session.phone) && TAKEOVER_STATUSES.has(session.status) && session.desiredState !== 'stopped';+ // operator stopped, or whose delete failed part-way, stays down until an explicit start, wherever+ // its claim lapsed.+ return Boolean(session.phone) && TAKEOVER_STATUSES.has(session.status) && !session.desiredState;}}