Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): keep a stop a peer records during a failed delete

Yudhi Armyndharisf5be4a597188
diff --git a/docs/05-database-design.md b/docs/05-database-design.md
index 153d798aa..0b1587905 100644
--- a/docs/05-database-design.md
+++ b/docs/05-database-design.md
@@ -346,8 +346,9 @@ CREATE TABLE sessions (
"leaseExpiresAt" TIMESTAMP,
-- 'stopped' after POST /stop or /force-kill, NULL again after an explicit POST /start. Boot
-- auto-start and the takeover sweep skip a stopped row. Server-owned: not in any session
- -- response, only in the /api/infra/export-data backup. A DELETE writes it too: kept when the
- -- delete fails part-way or its node dies, taken back when it fails before removing anything.
+ -- response, only in the /api/infra/export-data backup. A DELETE writes 'deleting', read the
+ -- same way: kept when the delete fails part-way or its node dies, taken back when it fails
+ -- before removing anything, unless a stop was recorded over it meanwhile.
"desiredState" VARCHAR(20),
"createdAt" TIMESTAMP NOT NULL DEFAULT NOW(),
"updatedAt" TIMESTAMP NOT NULL DEFAULT NOW()
diff --git a/docs/13-horizontal-scaling.md b/docs/13-horizontal-scaling.md
index 17602dbf7..e88ea66b2 100644
--- a/docs/13-horizontal-scaling.md
+++ b/docs/13-horizontal-scaling.md
@@ -50,6 +50,8 @@
> Nothing else revisits such a row, since the boot reset skips a foreign claim that is still live.
> A session an operator stopped (`POST /stop` or `/force-kill`), or whose `DELETE` failed part-way or
> died with its node, is never adopted, even when its claim did lapse, until an explicit `POST /start`.
+> While versions are mixed during an upgrade, a node still on 0.25.0 does not read a `DELETE`'s record:
+> it can adopt such a session, and a `POST /start` it serves leaves the record in place.
>
> Known limitation: a holder that is alive but cannot reach the database for more than three lease
> TTLs minus one heartbeat (160s at defaults) is marked disconnected by a peer too, and does not write its status back when it reconnects. Its
diff --git a/src/modules/infra/table-importers.spec.ts b/src/modules/infra/table-importers.spec.ts
index 881778ca0..0e5ffa2f3 100644
--- a/src/modules/infra/table-importers.spec.ts
+++ b/src/modules/infra/table-importers.spec.ts
@@ -37,6 +37,8 @@ describe('sessions table importer: desiredState', () => {
expect(sessions?.sql).toContain('"desiredState"');
expect(sessions?.sql).toContain('$13');
expect(mapped({ desiredState: 'stopped' })[12]).toBe('stopped');
+ // No delete runs for a restored row, so the record of one that failed part-way is a stop.
+ expect(mapped({ desiredState: 'deleting' })[12]).toBe('stopped');
});
it('restores a row without the field, or with an unknown value, as eligible (NULL)', () => {
diff --git a/src/modules/infra/table-importers.ts b/src/modules/infra/table-importers.ts
index 7b6db5b49..bfff08a27 100644
--- a/src/modules/infra/table-importers.ts
+++ b/src/modules/infra/table-importers.ts
@@ -165,9 +165,10 @@ export const TABLE_IMPORTERS: AnyTableImporter[] = [
session.lastActiveAt,
session.createdAt,
session.updatedAt,
- // 'stopped' is the only value the column holds. A backup written before the column existed,
- // or an unknown value, restores as NULL: eligible for auto-start and takeover, as before.
- session.desiredState === 'stopped' ? 'stopped' : null,
+ // A failed delete's 'deleting' restores as 'stopped': no delete runs for the restored row. A
+ // backup written before the column existed, or an unknown value, restores as NULL: eligible for
+ // auto-start and takeover, as before.
+ session.desiredState === 'stopped' || session.desiredState === 'deleting' ? 'stopped' : null,
],
}),
diff --git a/src/modules/session/entities/session.entity.ts b/src/modules/session/entities/session.entity.ts
index ca082aeb9..c970999f1 100644
--- a/src/modules/session/entities/session.entity.ts
+++ b/src/modules/session/entities/session.entity.ts
@@ -83,13 +83,13 @@ export class Session {
/**
* Operator intent, server-owned and not part of any session response (only the export-data
- * backup carries it): 'stopped' once POST /stop or POST /force-kill took the session down, or
- * while a DELETE runs (kept if it fails part-way), cleared again by an explicit POST /start. Boot
- * auto-start and the takeover sweep skip a stopped row, so a deliberate stop survives a restart.
- * NULL (every row that predates the column) means eligible, exactly as before.
+ * backup carries it): 'stopped' once POST /stop or POST /force-kill took the session down, and
+ * 'deleting' while a DELETE runs (kept if it fails part-way), both cleared again by an explicit
+ * POST /start. Boot auto-start and takeover skip a row holding either, so a deliberate stop
+ * survives a restart. NULL (every row that predates the column) means eligible, as before.
*/
@Column({ type: 'varchar', length: 20, nullable: true })
- desiredState!: 'stopped' | null;
+ desiredState!: 'stopped' | 'deleting' | null;
@CreateDateColumn()
createdAt!: Date;
diff --git a/src/modules/session/session-delete-claim.spec.ts b/src/modules/session/session-delete-claim.spec.ts
index 68c9fcfbd..2e7a4a4ec 100644
--- a/src/modules/session/session-delete-claim.spec.ts
+++ b/src/modules/session/session-delete-claim.spec.ts
@@ -14,6 +14,7 @@ import { SessionRestrictionStore } from './session-restriction-store.service';
import { PresenceStore } from './presence-store.service';
import { SessionOwnershipService } from './session-ownership.service';
import { SessionLifecycleFences } from './session-lifecycle-fences';
+import { SessionStoppedException } from './session-engine-controls';
import { Session, SessionStatus } from './entities/session.entity';
import { Message } from '../message/entities/message.entity';
import { MessageBatch } from '../message/entities/message-batch.entity';
@@ -384,10 +385,66 @@ describe('A session deleted on one node while a peer shares the database', () =>
expect(b.engines).toHaveLength(0);
expect(await a.db.getRepository(Session).findOneByOrFail({ id })).toMatchObject({
nodeId: 'node-a',
- desiredState: 'stopped',
+ desiredState: 'deleting',
});
});
+ it('keeps a session a failed delete left down until an explicit start', async () => {
+ const id = await seed({
+ status: SessionStatus.DISCONNECTED,
+ desiredState: 'deleting',
+ nodeId: 'node-c',
+ leaseExpiresAt: lapsed(),
+ });
+ const takeover = new SessionTakeoverService(
+ b.service,
+ b.ownership,
+ new ConfigService({ features: { autoStartSessions: true } }),
+ );
+
+ await takeover.sweep();
+ expect(b.engines).toHaveLength(0);
+ // Refused on the record, before the teardown fence would refuse it for another reason.
+ const fence = jest
+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')
+ .mockRejectedValue(new ConflictException('a logout teardown is running'));
+ await expect(b.service.start(id)).rejects.toBeInstanceOf(SessionStoppedException);
+ fence.mockRestore();
+ expect(b.engines).toHaveLength(0);
+
+ await b.service.start(id, { explicit: true });
+ expect(b.service.getEngine(id)).toBe(b.engines[0]);
+ expect((await b.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBeNull();
+ });
+
+ it('keeps the record of a delete that lands while an explicit start reads the session again', async () => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ const sessions = a.db.getRepository(Session);
+ const findOne = sessions.findOne.bind(sessions);
+ const reading = deferred();
+ const read = deferred();
+ // The start's second row read, past its teardown fence, is held while the delete records itself.
+ jest
+ .spyOn(sessions, 'findOne')
+ .mockImplementationOnce(findOne)
+ .mockImplementationOnce(async (...args: Parameters<typeof findOne>) => {
+ reading.resolve();
+ await read.promise;
+ return findOne(...args);
+ });
+ const starting = a.service.start(id, { explicit: true });
+ await reading.promise;
+ await startHeldDelete(id);
+ read.resolve();
+
+ await expect(starting).rejects.toBeInstanceOf(SessionStoppedException);
+
+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('deleting');
+ finish!();
+ await deletion;
+ expect(await rowsOf(id)).toEqual(gone);
+ });
+
// A delete records itself as a stop on the row, so the crash above leaves the session down; one that fails
// before it removes anything takes that record back.
it.each([
@@ -456,7 +513,7 @@ describe('A session deleted on one node while a peer shares the database', () =>
await expect(a.service.delete(id)).rejects.toBeInstanceOf(ConflictException);
- expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');
+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('deleting');
});
it('keeps a stop the session already had when its delete fails before removing anything', async () => {
@@ -698,12 +755,12 @@ describe('A session deleted on one node while a peer shares the database', () =>
.mockRejectedValueOnce(new ConflictException('a logout teardown is running'));
await expect(a.service.delete(id)).rejects.toBeInstanceOf(ConflictException);
- expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');
+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('deleting');
read.resolve();
await reached.promise;
// The rows are being removed: a process that died here would leave the session down.
- expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');
+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('deleting');
release.resolve();
await deletion;
@@ -734,7 +791,7 @@ describe('A session deleted on one node while a peer shares the database', () =>
if (takeBackFails) throw new Error('connection reset');
}
const result = await update(...args);
- if (args[1].desiredState === 'stopped') stopsWritten++;
+ if (args[1].desiredState) stopsWritten++;
return result;
});
// The failed delete is refused at its first fence, and so are the next `refusals - 1` deletes.
@@ -772,7 +829,7 @@ describe('A session deleted on one node while a peer shares the database', () =>
});
// The rows are being removed: a process that died here would leave the session down.
- expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('stopped');
+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('deleting');
release.resolve();
await deletion;
expect(await rowsOf(id)).toEqual(gone);
@@ -805,6 +862,39 @@ describe('A session deleted on one node while a peer shares the database', () =>
expect(bookkeeping(id)).toEqual({ deletes: false, stopWrites: false });
});
+ // The peer's stop passes its own ownership check while nobody holds the session, before this node claims it.
+ it('keeps a stop another node records while its delete fails before removing anything', async () => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ const peerSessions = b.db.getRepository(Session);
+ const findOne = peerSessions.findOne.bind(peerSessions);
+ const reading = deferred();
+ const read = deferred();
+ jest.spyOn(peerSessions, 'findOne').mockImplementationOnce(async (...args: Parameters<typeof findOne>) => {
+ reading.resolve();
+ await read.promise;
+ return findOne(...args);
+ });
+ const stopping = b.service.stop(id);
+ await reading.promise;
+ // The delete records its stop and waits at its first fence until the peer's stop has answered.
+ const fenced = deferred();
+ const refuse = deferred();
+ jest.spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown').mockImplementationOnce(async () => {
+ fenced.resolve();
+ await refuse.promise;
+ throw new ConflictException('a logout teardown is running');
+ });
+ const deleting = a.service.delete(id);
+ await fenced.promise;
+ read.resolve();
+ await stopping;
+ refuse.resolve();
+
+ await expect(deleting).rejects.toBeInstanceOf(ConflictException);
+
+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('stopped');
+ });
+
it('refuses the delete and removes nothing when the claim cannot be written', async () => {
const id = await seed({ status: SessionStatus.DISCONNECTED });
await a.service.start(id, { explicit: true });
@@ -848,7 +938,7 @@ describe('A session deleted on one node while a peer shares the database', () =>
// The batches ran before the closing transaction failed, so the session stays down like a crashed delete.
expect(await a.db.getRepository(Session).findOneByOrFail({ id })).toMatchObject({
nodeId: null,
- desiredState: 'stopped',
+ desiredState: 'deleting',
});
expect(a.ownership.ownedIds()).toEqual([]);
});
diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.ts
index 8e32e2281..0576a27eb 100644
--- a/src/modules/session/session-engine-controls.ts
+++ b/src/modules/session/session-engine-controls.ts
@@ -326,8 +326,8 @@ export class SessionEngineControls {
// An operator's stop outranks any start but POST /start, so this refuses boot auto-start and
// the takeover sweep, which pick their rows once and launch them seconds apart, when a stop
- // landed after that read.
- if (session.desiredState === 'stopped' && !explicit) {
+ // landed after that read. A delete's record ('deleting') counts as one.
+ if (session.desiredState && !explicit) {
throw new SessionStoppedException(`Session ${id} was stopped`);
}
@@ -390,7 +390,7 @@ export class SessionEngineControls {
// mark again and retires this start after its init. A launch that fails after the clear still
// counts as the start the operator asked for.
const { desiredState } = await this.requireSession(id);
- if (desiredState === 'stopped' && (!explicit || session.desiredState !== 'stopped')) {
+ if (desiredState && (!explicit || !session.desiredState)) {
this.stoppingSessions.add(id);
throw new SessionStoppedException(`Session ${id} was stopped`);
}
@@ -406,8 +406,8 @@ export class SessionEngineControls {
if (this.host.isClosed()) {
throw closedRefusal();
}
- if (desiredState === 'stopped') {
- await this.sessionRepository.update({ id, desiredState: 'stopped' }, { desiredState: null });
+ if (desiredState) {
+ await this.sessionRepository.update({ id, desiredState }, { desiredState: null });
// The clear stands (this start was the operator's), but the hook must not fire for a start
// shutdown overtook during the write.
if (this.host.isClosed()) {
diff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.ts
index 40210bb3f..06b23f91f 100644
--- a/src/modules/session/session.service.ts
+++ b/src/modules/session/session.service.ts
@@ -748,24 +748,25 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
/**
* Record the delete as a stop on a row that has none, so a session whose delete ends part-way, with the
* process that ran it, stays down: no peer adopts it and no boot starts it. Written after a take-back in
- * progress, which would otherwise clear it.
+ * progress, which would otherwise clear it. 'deleting' rather than 'stopped', so the take-back can tell it
+ * from a stop that replaced it, also one another node recorded.
*/
private async keepDownForDelete(id: string, stops: DeleteStops): Promise<void> {
if (stops.givingBack) await stops.givingBack;
const { affected } = await this.sessionRepository.update(
{ id, desiredState: IsNull() },
- { desiredState: 'stopped' },
+ { desiredState: 'deleting' },
);
if (affected === 1) stops.recorded = true;
}
/**
* Undo keepDownForDelete; answers whether the record is off the row. Best effort: the delete's own error is
- * the answer, and a stop left on is safe.
+ * the answer, and a stop left on is safe. A stop recorded over it since is left alone.
*/
private async clearDeleteStop(id: string): Promise<boolean> {
try {
- await this.sessionRepository.update({ id, desiredState: 'stopped' }, { desiredState: null });
+ await this.sessionRepository.update({ id, desiredState: 'deleting' }, { desiredState: null });
return true;
} catch (error) {
this.logger.warn(`Could not clear the stop a failed delete recorded for session ${id}`, {
@@ -1095,7 +1096,7 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
/**
* Re-record a stop an explicit start may have erased. The start clears the stop record with a
- * write conditional on the row still reading 'stopped', and a stop's own write leaves that value
+ * write conditional on the row still reading what it read, and a stop's own write leaves 'stopped'
* unchanged, so a stop landing just before the clear was wiped from the row while its in-memory
* mark retired the start. The session was down, but the next boot would relaunch it. Any stop
* counted while the start ran and leaving no engine behind is the operator's last word. A failed
diff --git a/src/modules/takeover/session-takeover.service.ts b/src/modules/takeover/session-takeover.service.ts
index 4e9dc9583..1be077556 100644
--- a/src/modules/takeover/session-takeover.service.ts
+++ b/src/modules/takeover/session-takeover.service.ts
@@ -208,7 +208,8 @@ export class SessionTakeoverService implements OnApplicationBootstrap, OnModuleD
private isEligible(session: Session): boolean {
// Only authenticated sessions (phone set): an engine is worth relaunching exactly when the
// saved credentials can restore the link without a human scanning anything. A session an
- // operator stopped stays down until an explicit start, wherever its claim lapsed.
- return Boolean(session.phone) && TAKEOVER_STATUSES.has(session.status) && session.desiredState !== 'stopped';
+ // operator stopped, or whose delete failed part-way, stays down until an explicit start, wherever
+ // its claim lapsed.
+ return Boolean(session.phone) && TAKEOVER_STATUSES.has(session.status) && !session.desiredState;
}
}