Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): detect two processes sharing one NODE_ID
Yudhi Armyndharisae56df92cb1f
diff --git a/.env.example b/.env.exampleindex 6c67d7126..99a6a7f60 100644--- a/.env.example+++ b/.env.example@@ -547,6 +547,9 @@ WEBHOOK_SSRF_PROTECT=true# A single-process deployment needs none of this: unclaimed rows read as free, which is what every# existing row is.# NODE_ID= # defaults to the hostname; must be STABLE across restarts+# # and UNIQUE per process: two processes on one host (host+# # networking, pm2 cluster) must each set it; a shared one+# # is logged as duplicate_node_id# SESSION_LEASE_TTL_MS=60000 # how long a claim survives unrenewed (default 60s) —# # also the worst-case failover delay after an unclean exit# SESSION_LEASE_HEARTBEAT_MS=20000 # renewal cadence (default 20s) — MUST be strictly lessdiff --git a/docs/13-horizontal-scaling.md b/docs/13-horizontal-scaling.mdindex b887665d1..6ac996ca2 100644--- a/docs/13-horizontal-scaling.md+++ b/docs/13-horizontal-scaling.md@@ -12,7 +12,9 @@> resets only the sessions it may claim, instead of reporting a live peer's sessions as> disconnected. Claims are released on a clean shutdown and expire otherwise, so failover> does not depend on one. `NODE_ID` names the process; it defaults to the hostname and must-> be stable across restarts.+> be stable across restarts and unique per running process: two processes sharing a hostname+> (host networking, pm2 cluster mode) must each set it, and a node that sees another process+> renewing leases under its `NODE_ID` logs `duplicate_node_id`.>> A node that loses its claim gives up the engine. A lease can lapse while the process is> perfectly healthy — a slow query is enough — after which a peer may legitimately take thediff --git a/src/modules/session/session-ownership.service.spec.ts b/src/modules/session/session-ownership.service.spec.tsindex fe236eac6..40ab6d5b2 100644--- a/src/modules/session/session-ownership.service.spec.ts+++ b/src/modules/session/session-ownership.service.spec.ts@@ -536,4 +536,126 @@ describe('SessionOwnershipService', () => {expect(bare.nodeId).not.toContain(String(process.pid));});});++ /**+ * Two processes sharing a nodeId (two instances on one host with no NODE_ID) each treat the other's+ * rows as their own and both keep renewing them, so no loss is ever seen. Only the lease values+ * changing under this process give it away.+ */+ describe('duplicate NODE_ID detection', () => {+ const errorSpy = (svc: SessionOwnershipService): jest.SpyInstance =>+ jest+ .spyOn((svc as unknown as { logger: { error: (...a: unknown[]) => void } }).logger, 'error')+ .mockImplementation(() => undefined);+ const duplicateLogs = (spy: jest.SpyInstance): unknown[][] =>+ (spy.mock.calls as unknown[][]).filter(+ c => (c[2] as { action?: string } | undefined)?.action === 'duplicate_node_id',+ );+ // Each tick lands a second later, so every renewal writes a distinct expiry.+ const tick = async (svc: SessionOwnershipService): Promise<void> => {+ jest.setSystemTime(Date.now() + 1_000);+ await svc.renew();+ };++ beforeEach(() => {+ jest.useFakeTimers({ doNotFake: ['nextTick', 'setImmediate', 'queueMicrotask'] });+ });+ afterEach(() => {+ jest.useRealTimers();+ jest.restoreAllMocks();+ });++ it('reports twins that both claimed and renew the same row, once each', async () => {+ const session = await seed();+ const twinA = service('node-a');+ const twinB = service('node-a');+ const [logA, logB] = [errorSpy(twinA), errorSpy(twinB)];+ await expect(twinA.claim(session.id)).resolves.toBe(true);+ await expect(twinB.claim(session.id)).resolves.toBe(true);++ for (let k = 0; k < 4; k++) {+ await tick(twinA);+ await tick(twinB);+ }++ expect(duplicateLogs(logA)).toHaveLength(1);+ expect(duplicateLogs(logB)).toHaveLength(1);+ expect(duplicateLogs(logA)[0][2]).toEqual({+ action: 'duplicate_node_id',+ nodeId: 'node-a',+ sessionIds: [session.id],+ });+ });++ it('reports a twin that holds nothing but sees its nodeId renewed by someone else', async () => {+ const session = await seed();+ const owner = service('node-a');+ const idle = service('node-a');+ const log = errorSpy(idle);+ await owner.claim(session.id);++ for (let k = 0; k < 4; k++) {+ await tick(owner);+ await tick(idle);+ }++ expect(duplicateLogs(log)).toHaveLength(1);+ });++ it('stays quiet for a single process renewing its own rows', async () => {+ const session = await seed();+ const node = service('node-a');+ const log = errorSpy(node);+ await node.claim(session.id);++ for (let k = 0; k < 5; k++) await tick(node);++ expect(duplicateLogs(log)).toHaveLength(0);+ });++ it('stays quiet after a one-off rewrite, like a data import carrying a lease forward', async () => {+ const session = await seed();+ const node = service('node-a');+ const log = errorSpy(node);+ await node.claim(session.id);+ await tick(node);+ await sessions.update({ id: session.id }, { leaseExpiresAt: new Date(Date.now() + 45_000) });++ for (let k = 0; k < 4; k++) await tick(node);++ expect(duplicateLogs(log)).toHaveLength(0);+ });++ it("stays quiet for a previous incarnation's lease that nobody renews", async () => {+ await seed({ nodeId: 'node-a', leaseExpiresAt: new Date(Date.now() + 60_000) });+ const fresh = service('node-a');+ const log = errorSpy(fresh);++ for (let k = 0; k < 4; k++) await tick(fresh);++ expect(duplicateLogs(log)).toHaveLength(0);+ });++ it('does not count renewals seen while an import holds the suspension token', async () => {+ const session = await seed();+ const owner = service('node-a');+ const idle = service('node-a');+ const log = errorSpy(idle);+ await owner.claim(session.id);+ const release = idle.suspendLossDetection();++ for (let k = 0; k < 4; k++) {+ await tick(owner);+ await tick(idle);+ }+ expect(duplicateLogs(log)).toHaveLength(0);++ release();+ await tick(owner);+ await tick(idle); // baseline after the suspension+ await tick(owner);+ await tick(idle);+ expect(duplicateLogs(log)).toHaveLength(0);+ });+ });});diff --git a/src/modules/session/session-ownership.service.ts b/src/modules/session/session-ownership.service.tsindex 1b10619d8..f9e70e372 100644--- a/src/modules/session/session-ownership.service.ts+++ b/src/modules/session/session-ownership.service.ts@@ -51,6 +51,14 @@ export class SessionOwnershipService {private lossDetectionSuspended = 0;/** Answers "does anything still run for this id here?" — consulted by renew(). See setEngineLiveness. */private engineLiveness?: (sessionId: string) => boolean;+ /**+ * Duplicate-NODE_ID detection (see noteForeignRenewals): the lease expiry this process last wrote+ * or saw per row under its nodeId, how many consecutive ticks it changed without this process+ * writing it, and whether the duplicate has been reported.+ */+ private readonly lastSeenLease = new Map<string, number>();+ private readonly foreignStreak = new Map<string, number>();+ private duplicateReported = false;constructor(@InjectRepository(Session, 'data')@@ -65,6 +73,9 @@ export class SessionOwnershipService {* Deliberately not tied to the pid: a restarted process must recognise its own previous rows in* order to reset them, and a pid never matches after a restart. The hostname is stable for the* lifetime of a container or a host; where that is not the right boundary, `NODE_ID` overrides it.+ * It must also be unique per running process: two processes sharing a hostname (host networking,+ * pm2 cluster mode, two instances on one host) must each set `NODE_ID`, or each treats the other's+ * sessions as its own. renew() detects that and logs `duplicate_node_id`.*/get nodeId(): string {return this.configService?.get<string>('session.nodeId') || process.env.NODE_ID || hostname();@@ -113,13 +124,14 @@ export class SessionOwnershipService {*/async claim(sessionId: string): Promise<boolean> {const now = new Date();+ const leaseExpiresAt = new Date(now.getTime() + this.leaseTtlMs);const result = await this.sessions.createQueryBuilder().update(Session).set({nodeId: this.nodeId,claimedAt: now,- leaseExpiresAt: new Date(now.getTime() + this.leaseTtlMs),+ leaseExpiresAt,nodeUrl: this.nodeUrl || null,}).where('id = :id', { id: sessionId })@@ -132,8 +144,10 @@ export class SessionOwnershipService {.execute();const claimed = (result.affected ?? 0) > 0;- if (claimed) this.owned.add(sessionId);- else this.logger.warn('Session is held by another node', { sessionId, nodeId: this.nodeId });+ if (claimed) {+ this.owned.add(sessionId);+ this.lastSeenLease.set(sessionId, leaseExpiresAt.getTime());+ } else this.logger.warn('Session is held by another node', { sessionId, nodeId: this.nodeId });return claimed;}@@ -239,7 +253,6 @@ export class SessionOwnershipService {*/async renew(): Promise<void> {const held = [...this.owned];- if (held.length === 0) return;// Only claims that still cover something alive on this process are pushed out. A claim whose// engine is gone (a failed start, an exhausted reconnect) must be allowed to lapse — renewing@@ -250,13 +263,24 @@ export class SessionOwnershipService {let kept: Set<string>;try {+ // Read BEFORE this tick's own write, so a lease another process renewed under this nodeId is+ // still visible. Runs even when nothing is held: a twin that never claimed must notice too.+ const mine = await this.sessions+ .createQueryBuilder('s')+ .select(['s.id', 's.leaseExpiresAt'])+ .where('s.nodeId = :me', { me: this.nodeId })+ .getMany();+ this.noteForeignRenewals(mine);+ if (held.length === 0) return;if (live.length > 0) {+ const leaseExpiresAt = new Date(Date.now() + this.leaseTtlMs);await this.sessions.createQueryBuilder().update(Session)- .set({ leaseExpiresAt: new Date(Date.now() + this.leaseTtlMs) })+ .set({ leaseExpiresAt }).where({ id: In(live), nodeId: this.nodeId }).execute();+ for (const id of live) this.lastSeenLease.set(id, leaseExpiresAt.getTime());}const rows = await this.sessions.find({ where: { id: In(held), nodeId: this.nodeId }, select: { id: true } });kept = new Set(rows.map(row => row.id));@@ -299,6 +323,47 @@ export class SessionOwnershipService {}}+ /**+ * Notice another process renewing leases under this nodeId. A lease expiry that changed without+ * this process writing it, on two consecutive ticks, can only be a live twin: it renews every+ * heartbeat. A one-off change (a data import carrying leases forward, a claim racing a renewal)+ * resets on the next tick, and a previous incarnation's lapsed or static lease never changes.+ * Reported once per process. Detection only: nothing is refused on it.+ */+ private noteForeignRenewals(rows: Array<Pick<Session, 'id' | 'leaseExpiresAt'>>): void {+ if (this.lossDetectionSuspended > 0) {+ // An import is rewriting the table under this tick; start over once it is done.+ this.lastSeenLease.clear();+ this.foreignStreak.clear();+ return;+ }+ const now = Date.now();+ const seen = new Set<string>();+ const flagged: string[] = [];+ for (const row of rows) {+ const at = row.leaseExpiresAt?.getTime();+ if (at === undefined || at <= now) continue;+ seen.add(row.id);+ const prev = this.lastSeenLease.get(row.id);+ const streak = prev !== undefined && prev !== at ? (this.foreignStreak.get(row.id) ?? 0) + 1 : 0;+ this.foreignStreak.set(row.id, streak);+ this.lastSeenLease.set(row.id, at);+ if (streak >= 2) flagged.push(row.id);+ }+ for (const id of [...this.lastSeenLease.keys()]) {+ if (seen.has(id)) continue;+ this.lastSeenLease.delete(id);+ this.foreignStreak.delete(id);+ }+ if (flagged.length === 0 || this.duplicateReported) return;+ this.duplicateReported = true;+ this.logger.error(+ 'Another process is renewing session leases under this NODE_ID; set a unique NODE_ID per process',+ undefined,+ { action: 'duplicate_node_id', nodeId: this.nodeId, sessionIds: flagged },+ );+ }+/** For the boot reset, which must run before anything is claimed. */ownedByOtherLiveNode(session: Pick<Session, 'nodeId' | 'leaseExpiresAt'>, now = new Date()): boolean {if (!session.nodeId || session.nodeId === this.nodeId) return false;