Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): take a failed delete's stop back after overlaps
Yudhi Armyndharisae92309de279
diff --git a/src/modules/session/session-delete-claim.spec.ts b/src/modules/session/session-delete-claim.spec.tsindex 968d85d58..68c9fcfbd 100644--- a/src/modules/session/session-delete-claim.spec.ts+++ b/src/modules/session/session-delete-claim.spec.ts@@ -269,6 +269,15 @@ describe('A session deleted on one node while a peer shares the database', () =>};}+ /** What a settled delete or stop must not leave behind: its share of a stop record, a stop write counted. */+ function bookkeeping(id: string): { deletes: boolean; stopWrites: boolean } {+ const service = a.service as unknown as {+ deleteStops: Map<string, unknown>;+ stopWritesInFlight: Map<string, number>;+ };+ return { deletes: service.deleteStops.has(id), stopWrites: service.stopWritesInFlight.has(id) };+ }+const gone = { sessions: 0, messages: 0, outbox: 0, deliveryFailures: 0 };it.each([@@ -522,6 +531,280 @@ describe('A session deleted on one node while a peer shares the database', () =>expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');});+ it('keeps a stop whose record has landed but not yet returned when its delete fails', async () => {+ const id = await seed({ status: SessionStatus.DISCONNECTED });+ const sessions = a.db.getRepository(Session);+ const update = sessions.update.bind(sessions);+ const reading = deferred();+ const read = deferred();+ jest.spyOn(sessions, 'findOne').mockImplementationOnce(async () => {+ reading.resolve();+ await read.promise;+ throw new Error('connection reset');+ });+ const deleting = a.service.delete(id);+ await reading.promise;+ const landed = deferred();+ const answer = deferred();+ // The stop's record is on the row; its answer is held until the delete has failed.+ jest.spyOn(sessions, 'update').mockImplementationOnce(async (...args: Parameters<typeof update>) => {+ const result = await update(...args);+ landed.resolve();+ await answer.promise;+ return result;+ });+ const stopping = a.service.stop(id);+ await landed.promise;+ read.resolve();++ await expect(deleting).rejects.toThrow('connection reset');+ answer.resolve();+ await stopping;++ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');+ });++ it('takes its stop back when another delete of it is refused at the first fence too', async () => {+ const id = await seed({ status: SessionStatus.DISCONNECTED });+ await a.service.start(id, { explicit: true });+ jest+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')+ .mockRejectedValueOnce(new ConflictException('a logout teardown is running'))+ .mockRejectedValueOnce(new ConflictException('a logout teardown is running'));++ const outcomes = await Promise.allSettled([a.service.delete(id), a.service.delete(id)]);++ expect(outcomes.map(outcome => outcome.status)).toEqual(['rejected', 'rejected']);+ expect(a.service.getEngine(id)).toBe(a.engines[0]);+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBeNull();+ expect(bookkeeping(id)).toEqual({ deletes: false, stopWrites: false });+ });++ // Each of these begins before the delete and ends, having recorded no stop, before the delete fails.+ it.each([+ ['a force-kill with no engine to kill', 'kill-refused', 'BadRequestException'],+ ['a force-kill that cannot record its stop', 'kill-write', 'ok'],+ ['a stop whose row read fails', 'stop-read', 'Error'],+ ['a stop that cannot record itself', 'stop-write', 'Error'],+ ])('takes its stop back when %s overlaps it', async (_case, other, answer) => {+ const id = await seed({ status: SessionStatus.DISCONNECTED });+ if (other === 'kill-write') await a.service.start(id, { explicit: true });+ const sessions = a.db.getRepository(Session);+ const findOne = sessions.findOne.bind(sessions);+ const reached = deferred();+ const go = deferred();+ // The other request is held at its row read or at its stop record until the delete has begun.+ const hold = async (): Promise<void> => {+ reached.resolve();+ await go.promise;+ };+ if (other === 'kill-refused') {+ jest.spyOn(sessions, 'findOne').mockImplementationOnce(async (...args: Parameters<typeof findOne>) => {+ await hold();+ return findOne(...args);+ });+ } else if (other === 'stop-read') {+ jest.spyOn(sessions, 'findOne').mockImplementationOnce(async () => {+ await hold();+ throw new Error('connection reset');+ });+ } else {+ jest.spyOn(sessions, 'update').mockImplementationOnce(async () => {+ await hold();+ throw new Error('connection reset');+ });+ }+ // The delete waits at its first fence until the other request has answered, and is refused there.+ const fenced = deferred();+ const refuse = deferred();+ jest.spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown').mockImplementationOnce(async () => {+ fenced.resolve();+ await refuse.promise;+ throw new ConflictException('a logout teardown is running');+ });+ const request = other.startsWith('kill') ? a.service.forceKill(id) : a.service.stop(id);+ const answered = request.then(+ () => 'ok',+ (error: unknown) => (error as Error).constructor.name,+ );+ await reached.promise;+ const deleting = a.service.delete(id);+ await fenced.promise;+ go.resolve();+ expect(await answered).toBe(answer);+ refuse.resolve();++ await expect(deleting).rejects.toBeInstanceOf(ConflictException);++ expect((await sessions.findOneByOrFail({ id })).desiredState).toBeNull();+ expect(bookkeeping(id)).toEqual({ deletes: false, stopWrites: false });+ });++ it.each([+ ['records it', 'stopped'],+ ['fails to read the session', null],+ ])('leaves the session to a stop still reading it when its delete fails, which then %s', async (_case, after) => {+ const id = await seed({ status: SessionStatus.DISCONNECTED });+ const sessions = a.db.getRepository(Session);+ const findOne = sessions.findOne.bind(sessions);+ const reading = deferred();+ const read = deferred();+ jest.spyOn(sessions, 'findOne').mockImplementationOnce(async (...args: Parameters<typeof findOne>) => {+ reading.resolve();+ await read.promise;+ if (after === null) throw new Error('connection reset');+ return findOne(...args);+ });+ jest+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')+ .mockRejectedValueOnce(new ConflictException('a logout teardown is running'));+ const stopping = a.service.stop(id).catch((error: unknown) => error);+ await reading.promise;++ await expect(a.service.delete(id)).rejects.toBeInstanceOf(ConflictException);+ read.resolve();+ await stopping;++ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe(after);+ });++ // Two deletes of one session share the stop the first of them recorded; the last one out decides.+ it('keeps the session down for a delete still running when another delete of it is refused', async () => {+ const id = await seed({ status: SessionStatus.DISCONNECTED });+ const sessions = a.db.getRepository(Session);+ const findOne = sessions.findOne.bind(sessions);+ const reading = deferred();+ const read = deferred();+ // The first delete's row read, which comes after its stop record, is held.+ jest.spyOn(sessions, 'findOne').mockImplementationOnce(async (...args: Parameters<typeof findOne>) => {+ reading.resolve();+ await read.promise;+ return findOne(...args);+ });+ const reached = deferred();+ const release = deferred();+ a.hooks.execute.mockImplementation(async (event: string, data: unknown) => {+ if (event === 'session:deleted') {+ reached.resolve();+ await release.promise;+ }+ return { continue: true, data };+ });+ finish = release.resolve;+ deletion = a.service.delete(id);+ await reading.promise;+ jest+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')+ .mockRejectedValueOnce(new ConflictException('a logout teardown is running'));++ await expect(a.service.delete(id)).rejects.toBeInstanceOf(ConflictException);+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');++ read.resolve();+ await reached.promise;+ // The rows are being removed: a process that died here would leave the session down.+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');+ release.resolve();+ await deletion;++ expect(await rowsOf(id)).toEqual(gone);+ });++ /**+ * A failed delete's take-back is held while `next` runs: until `next` has written a stop record or 200 ms+ * have passed, then let go.+ */+ async function duringTakeBack(+ next: (id: string) => Promise<unknown>,+ { takeBackFails = false, refusals = 1 } = {},+ ): Promise<string> {+ const id = await seed({ status: SessionStatus.DISCONNECTED });+ const sessions = a.db.getRepository(Session);+ const update = sessions.update.bind(sessions);+ const clearing = deferred();+ const cleared = deferred();+ let held = false;+ let stopsWritten = 0;+ // Only the first take-back is held, and fails with `takeBackFails`; any later one runs at once.+ jest.spyOn(sessions, 'update').mockImplementation(async (...args: Parameters<typeof update>) => {+ if (args[1].desiredState === null && !held) {+ held = true;+ clearing.resolve();+ await cleared.promise;+ if (takeBackFails) throw new Error('connection reset');+ }+ const result = await update(...args);+ if (args[1].desiredState === 'stopped') stopsWritten++;+ return result;+ });+ // The failed delete is refused at its first fence, and so are the next `refusals - 1` deletes.+ const fence = jest.spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown');+ for (let refused = 0; refused < refusals; refused++) {+ fence.mockRejectedValueOnce(new ConflictException('a logout teardown is running'));+ }+ const failing = a.service.delete(id).catch((error: unknown) => error);+ await clearing.promise;+ const written = stopsWritten;+ const running = next(id);+ for (let waited = 0; waited < 200 && stopsWritten === written; waited += 5) {+ await new Promise(resolve => setTimeout(resolve, 5));+ }+ cleared.resolve();+ expect(await failing).toBeInstanceOf(ConflictException);+ await running;+ return id;+ }++ it('records its own stop when it begins while a failed delete takes its stop back', async () => {+ const reached = deferred();+ const release = deferred();+ a.hooks.execute.mockImplementation(async (event: string, data: unknown) => {+ if (event === 'session:deleted') {+ reached.resolve();+ await release.promise;+ }+ return { continue: true, data };+ });+ finish = release.resolve;+ const id = await duringTakeBack(async next => {+ deletion = a.service.delete(next);+ await reached.promise;+ });++ // The rows are being removed: a process that died here would leave the session down.+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('stopped');+ release.resolve();+ await deletion;+ expect(await rowsOf(id)).toEqual(gone);+ });++ it('keeps a stop sent while a failed delete takes its stop back', async () => {+ const id = await duringTakeBack(next => a.service.stop(next));++ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('stopped');+ expect(bookkeeping(id)).toEqual({ deletes: false, stopWrites: false });+ });++ it('leaves the take-back to the failed delete when one begun during it ends before recording its own', async () => {+ const id = await duringTakeBack(async next => {+ jest.spyOn(a.ownership, 'claim').mockRejectedValueOnce(new Error('Connection terminated unexpectedly'));+ await expect(a.service.delete(next)).rejects.toBeInstanceOf(ServiceUnavailableException);+ await a.service.stop(next);+ });++ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('stopped');+ });++ it('takes the stop back after a failed take-back when a delete begun during it fails too', async () => {+ const id = await duringTakeBack(next => expect(a.service.delete(next)).rejects.toBeInstanceOf(ConflictException), {+ takeBackFails: true,+ refusals: 2,+ });++ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBeNull();+ expect(bookkeeping(id)).toEqual({ deletes: false, stopWrites: false });+ });+it('refuses the delete and removes nothing when the claim cannot be written', async () => {const id = await seed({ status: SessionStatus.DISCONNECTED });await a.service.start(id, { explicit: true });diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.tsindex 1296bfce2..8e32e2281 100644--- a/src/modules/session/session-engine-controls.ts+++ b/src/modules/session/session-engine-controls.ts@@ -149,6 +149,8 @@ export interface StopHooks {onReadFailed?: () => void;/** forceKill() only: runs synchronously once there is an engine to kill; records the stop request. */onEngineFound?: () => void;+ /** forceKill() only: runs the write of the kill's stop record, which it is handed; without it, the write runs alone. */+ recordStop?: (write: () => Promise<unknown>) => Promise<void>;/** delete() only: runs once the delete starts removing the session's data, the session:deleted hook first. */onRemoving?: () => void;}@@ -747,15 +749,14 @@ export class SessionEngineControls {// explicit start. Recorded only once there is an engine to kill, and before the teardown, so a// POST /start that lands after the eviction clears it and stands. A failed write must not keep a// wedged engine alive, so it is logged and the kill goes on.- await this.sessionRepository- .update(id, { desiredState: 'stopped' })- .catch((error: unknown) =>- this.logger.error(- 'Failed to record the force-kill as a stop',- error instanceof Error ? error.message : String(error),- { sessionId: id, action: 'force_kill_keep_down_failed' },- ),- );+ const write = (): Promise<unknown> => this.sessionRepository.update(id, { desiredState: 'stopped' });+ await (hooks.recordStop ? hooks.recordStop(write) : write()).catch((error: unknown) =>+ this.logger.error(+ 'Failed to record the force-kill as a stop',+ error instanceof Error ? error.message : String(error),+ { sessionId: id, action: 'force_kill_keep_down_failed' },+ ),+ );// Announced by the write below rather than from the engine callback, as in stop().this.operatorTeardowns.set(id, engine);diff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.tsindex db960f5ad..40210bb3f 100644--- a/src/modules/session/session.service.ts+++ b/src/modules/session/session.service.ts@@ -103,6 +103,19 @@ export interface SessionListOptions extends ListOptions {name?: string;}+/** The deletes of one session running on this node, and the stop record they share; see SessionService.delete. */+interface DeleteStops {+ running: number;+ /** One of them wrote the record on a row that had none. */+ recorded: boolean;+ /** One of them began removing the session's data. */+ removing: boolean;+ /** An operator stop or a kill recorded a stop while they ran. */+ overtaken: boolean;+ /** The last one out taking the record back. */+ givingBack?: Promise<void>;+}+/*** Statuses that assert an engine is running somewhere. The boot reset clears them for every row this* node may claim; markLapsedDisconnected clears them for a row whose holder never came back, a@@ -145,8 +158,10 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat* after it began from a mark left over by an earlier stop, which start() clears by design.*/private readonly stopRequests = new Map<string, number>();- /** stop() and forceKill() calls in flight per session, whose stop record may still be written; see delete(). */- private readonly stopsInFlight = new Map<string, number>();+ /** Stop records of stop() and forceKill() being written per session, which may land before a take-back; see delete(). */+ private readonly stopWritesInFlight = new Map<string, number>();+ /** Per session, the deletes running here; see delete(). */+ private readonly deleteStops = new Map<string, DeleteStops>();/*** Starts past start()'s cap check, held until they return. That check counts them with the* engine's slot holders, so two starts at the last free slot cannot both pass it while their claims@@ -619,20 +634,14 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat// post-INITIALIZING check by the time that write settles; awaiting anything first — the fence's// COUNT, or delete()'s own requireSession — would let the mark land after that window. A mark// left behind when the fence refuses (409) is harmless and is cleared by the next start().- // A stop or kill already running may record its stop after this delete records its own.- const stopInFlight = this.stopsInFlight.has(id);this.markStopping(id);// Running from here, before the claim: a claim with nothing running behind it would read as idle to the// heartbeat, and a stop or a failed start here would hand it back while the rows are removed. It ends// before the claim is handed back.const endDelete = this.engineLifecycle.holdDelete(id);- // This delete's own stop count, and every other stop counted before it: a stop counted while it runs is- // the operator's, and outlives a failed delete.- let counted = 1;- const otherStopsBefore = (this.stopRequests.get(id) ?? 0) - counted;+ // From here an operator stop or kill that records a stop is seen, also one whose record is already on its way.+ const stops = this.joinDeleteStops(id);let claimed = false;- let keptDown = false;- let removing = false;try {try {if (this.ownership) {@@ -641,19 +650,16 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicatclaimed = true;}// A failed write took nothing down, so the mark and count are undone as for a failed claim.- keptDown = await this.keepDownForDelete(id).catch((error: unknown) => {+ await this.keepDownForDelete(id, stops).catch((error: unknown) => {this.engineLifecycle.clearStopping(id);this.uncountStopRequest(id);throw error;});await this.engineLifecycle.delete(id, {// A failed row read deleted nothing: the lifecycle drops the mark, and the count goes with it.- onReadFailed: () => {- counted = 0;- this.uncountStopRequest(id);- },+ onReadFailed: () => this.uncountStopRequest(id),onRemoving: () => {- removing = true;+ stops.removing = true;},});} finally {@@ -664,14 +670,50 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat} catch (error) {// A delete that removed nothing leaves the session as it found it. One that began to remove it leaves// it stopped, as a process that dies part-way does.- const stoppedMeanwhile = stopInFlight || (this.stopRequests.get(id) ?? 0) - counted !== otherStopsBefore;- if (keptDown && !removing && !stoppedMeanwhile) await this.clearDeleteStop(id);+ await this.leaveDelete(id, stops);// The row survived. A claim left on it lapses with nothing to renew it, and a peer would then// adopt and restart the session the operator was deleting.if (claimed) await this.releaseAfterTeardown(id);this.discardStopMarkForMissingSession(id, error);throw error;}+ await this.leaveDelete(id, stops);+ }++ private joinDeleteStops(id: string): DeleteStops {+ let stops = this.deleteStops.get(id);+ if (!stops) {+ stops = { running: 0, recorded: false, removing: false, overtaken: false };+ this.deleteStops.set(id, stops);+ }+ stops.running += 1;+ return stops;+ }++ /**+ * End a delete's share of the stop record. Another delete of the session still running here relies on it, so+ * only the last one out takes it back, and only when they wrote it, none began removing the session, and no+ * operator stop or kill recorded one meanwhile or is still writing one (it may land before the take-back).+ */+ private async leaveDelete(id: string, stops: DeleteStops): Promise<void> {+ stops.running -= 1;+ if (+ stops.running === 0 &&+ stops.recorded &&+ !stops.removing &&+ !stops.overtaken &&+ !this.stopWritesInFlight.has(id)+ ) {+ // Not theirs while it is cleared, so a delete that begins and ends meanwhile leaves it alone.+ stops.recorded = false;+ stops.givingBack = this.clearDeleteStop(id).then(cleared => {+ // Still on the row: a delete that began meanwhile shares it.+ if (!cleared) stops.recorded = true;+ stops.givingBack = undefined;+ });+ await stops.givingBack;+ }+ if (stops.running === 0 && !stops.givingBack) this.deleteStops.delete(id);}/**@@ -705,26 +747,33 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat/*** Record the delete as a stop on a row that has none, so a session whose delete ends part-way, with the- * process that ran it, stays down: no peer adopts it and no boot starts it. Answers whether it wrote one.+ * process that ran it, stays down: no peer adopts it and no boot starts it. Written after a take-back in+ * progress, which would otherwise clear it.*/- private async keepDownForDelete(id: string): Promise<boolean> {+ private async keepDownForDelete(id: string, stops: DeleteStops): Promise<void> {+ if (stops.givingBack) await stops.givingBack;const { affected } = await this.sessionRepository.update({ id, desiredState: IsNull() },{ desiredState: 'stopped' },);- return affected === 1;+ if (affected === 1) stops.recorded = true;}- /** Undo keepDownForDelete. Best effort: the delete's own error is the answer, and a stop left on is safe. */- private async clearDeleteStop(id: string): Promise<void> {+ /**+ * Undo keepDownForDelete; answers whether the record is off the row. Best effort: the delete's own error is+ * the answer, and a stop left on is safe.+ */+ private async clearDeleteStop(id: string): Promise<boolean> {try {await this.sessionRepository.update({ id, desiredState: 'stopped' }, { desiredState: null });+ return true;} catch (error) {this.logger.warn(`Could not clear the stop a failed delete recorded for session ${id}`, {sessionId: id,action: 'session_delete_stop_clear_failed',error: error instanceof Error ? error.message : String(error),});+ return false;}}@@ -933,7 +982,6 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicatasync stop(id: string): Promise<Session> {// Synchronous stop-mark before the awaited fence — see delete() for why.this.markStopping(id);- const endStop = this.countStopInFlight(id);let session: Session;try {if (this.ownership) await this.assertNotHeldElsewhere(id);@@ -959,8 +1007,6 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicatif (error instanceof BadGatewayException) await this.releaseAfterTeardown(id);this.discardStopMarkForMissingSession(id, error);throw error;- } finally {- endStop();}// Handed back on the way out so a peer can pick it up immediately rather than waiting for the// lease to lapse. Stop is the deliberate end of this process's ownership — but a start() that@@ -1004,10 +1050,13 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicatthis.uncountStopRequest(id);};if (this.engines.get(id)) count();- const endStop = this.countStopInFlight(id);try {// The engine kill records the stop itself, once it has an engine to kill.- const session = await this.engineLifecycle.forceKill(id, { onReadFailed: uncount, onEngineFound: count });+ const session = await this.engineLifecycle.forceKill(id, {+ onReadFailed: uncount,+ onEngineFound: count,+ recordStop: write => this.recordStop(id, write),+ });await this.releaseAfterTeardown(id);return session;} catch (error) {@@ -1016,23 +1065,32 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicatif (error instanceof BadRequestException) uncount();if (!(error instanceof BadRequestException)) await this.releaseAfterTeardown(id);throw error;- } finally {- endStop();}}- private countStopInFlight(id: string): () => void {- this.stopsInFlight.set(id, (this.stopsInFlight.get(id) ?? 0) + 1);- return () => {- const left = (this.stopsInFlight.get(id) ?? 1) - 1;- if (left > 0) this.stopsInFlight.set(id, left);- else this.stopsInFlight.delete(id);- };- }-/** Persist the operator's stop so boot auto-start and the takeover sweep leave the session down. */private async keepDown(id: string): Promise<void> {- await this.sessionRepository.update(id, { desiredState: 'stopped' });+ await this.recordStop(id, () => this.sessionRepository.update(id, { desiredState: 'stopped' }));+ }++ /**+ * Write an operator stop's or a kill's record of a stop. The deletes of the session running here keep their+ * own record once it lands, and leave it alone while it is being written; it is written after a take-back+ * in progress, which would otherwise clear it.+ */+ private async recordStop(id: string, write: () => Promise<unknown>): Promise<void> {+ this.stopWritesInFlight.set(id, (this.stopWritesInFlight.get(id) ?? 0) + 1);+ try {+ const givingBack = this.deleteStops.get(id)?.givingBack;+ if (givingBack) await givingBack;+ await write();+ const stops = this.deleteStops.get(id);+ if (stops) stops.overtaken = true;+ } finally {+ const left = (this.stopWritesInFlight.get(id) ?? 1) - 1;+ if (left > 0) this.stopWritesInFlight.set(id, left);+ else this.stopWritesInFlight.delete(id);+ }}/**