Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): detect two processes sharing one NODE_ID

Yudhi Armyndharisae56df92cb1f
diff --git a/.env.example b/.env.example
index 6c67d7126..99a6a7f60 100644
--- a/.env.example
+++ b/.env.example
@@ -547,6 +547,9 @@ WEBHOOK_SSRF_PROTECT=true
# A single-process deployment needs none of this: unclaimed rows read as free, which is what every
# existing row is.
# NODE_ID= # defaults to the hostname; must be STABLE across restarts
+# # and UNIQUE per process: two processes on one host (host
+# # networking, pm2 cluster) must each set it; a shared one
+# # is logged as duplicate_node_id
# SESSION_LEASE_TTL_MS=60000 # how long a claim survives unrenewed (default 60s) —
# # also the worst-case failover delay after an unclean exit
# SESSION_LEASE_HEARTBEAT_MS=20000 # renewal cadence (default 20s) — MUST be strictly less
diff --git a/docs/13-horizontal-scaling.md b/docs/13-horizontal-scaling.md
index b887665d1..6ac996ca2 100644
--- a/docs/13-horizontal-scaling.md
+++ b/docs/13-horizontal-scaling.md
@@ -12,7 +12,9 @@
> resets only the sessions it may claim, instead of reporting a live peer's sessions as
> disconnected. Claims are released on a clean shutdown and expire otherwise, so failover
> does not depend on one. `NODE_ID` names the process; it defaults to the hostname and must
-> be stable across restarts.
+> be stable across restarts and unique per running process: two processes sharing a hostname
+> (host networking, pm2 cluster mode) must each set it, and a node that sees another process
+> renewing leases under its `NODE_ID` logs `duplicate_node_id`.
>
> A node that loses its claim gives up the engine. A lease can lapse while the process is
> perfectly healthy — a slow query is enough — after which a peer may legitimately take the
diff --git a/src/modules/session/session-ownership.service.spec.ts b/src/modules/session/session-ownership.service.spec.ts
index fe236eac6..40ab6d5b2 100644
--- a/src/modules/session/session-ownership.service.spec.ts
+++ b/src/modules/session/session-ownership.service.spec.ts
@@ -536,4 +536,126 @@ describe('SessionOwnershipService', () => {
expect(bare.nodeId).not.toContain(String(process.pid));
});
});
+
+ /**
+ * Two processes sharing a nodeId (two instances on one host with no NODE_ID) each treat the other's
+ * rows as their own and both keep renewing them, so no loss is ever seen. Only the lease values
+ * changing under this process give it away.
+ */
+ describe('duplicate NODE_ID detection', () => {
+ const errorSpy = (svc: SessionOwnershipService): jest.SpyInstance =>
+ jest
+ .spyOn((svc as unknown as { logger: { error: (...a: unknown[]) => void } }).logger, 'error')
+ .mockImplementation(() => undefined);
+ const duplicateLogs = (spy: jest.SpyInstance): unknown[][] =>
+ (spy.mock.calls as unknown[][]).filter(
+ c => (c[2] as { action?: string } | undefined)?.action === 'duplicate_node_id',
+ );
+ // Each tick lands a second later, so every renewal writes a distinct expiry.
+ const tick = async (svc: SessionOwnershipService): Promise<void> => {
+ jest.setSystemTime(Date.now() + 1_000);
+ await svc.renew();
+ };
+
+ beforeEach(() => {
+ jest.useFakeTimers({ doNotFake: ['nextTick', 'setImmediate', 'queueMicrotask'] });
+ });
+ afterEach(() => {
+ jest.useRealTimers();
+ jest.restoreAllMocks();
+ });
+
+ it('reports twins that both claimed and renew the same row, once each', async () => {
+ const session = await seed();
+ const twinA = service('node-a');
+ const twinB = service('node-a');
+ const [logA, logB] = [errorSpy(twinA), errorSpy(twinB)];
+ await expect(twinA.claim(session.id)).resolves.toBe(true);
+ await expect(twinB.claim(session.id)).resolves.toBe(true);
+
+ for (let k = 0; k < 4; k++) {
+ await tick(twinA);
+ await tick(twinB);
+ }
+
+ expect(duplicateLogs(logA)).toHaveLength(1);
+ expect(duplicateLogs(logB)).toHaveLength(1);
+ expect(duplicateLogs(logA)[0][2]).toEqual({
+ action: 'duplicate_node_id',
+ nodeId: 'node-a',
+ sessionIds: [session.id],
+ });
+ });
+
+ it('reports a twin that holds nothing but sees its nodeId renewed by someone else', async () => {
+ const session = await seed();
+ const owner = service('node-a');
+ const idle = service('node-a');
+ const log = errorSpy(idle);
+ await owner.claim(session.id);
+
+ for (let k = 0; k < 4; k++) {
+ await tick(owner);
+ await tick(idle);
+ }
+
+ expect(duplicateLogs(log)).toHaveLength(1);
+ });
+
+ it('stays quiet for a single process renewing its own rows', async () => {
+ const session = await seed();
+ const node = service('node-a');
+ const log = errorSpy(node);
+ await node.claim(session.id);
+
+ for (let k = 0; k < 5; k++) await tick(node);
+
+ expect(duplicateLogs(log)).toHaveLength(0);
+ });
+
+ it('stays quiet after a one-off rewrite, like a data import carrying a lease forward', async () => {
+ const session = await seed();
+ const node = service('node-a');
+ const log = errorSpy(node);
+ await node.claim(session.id);
+ await tick(node);
+ await sessions.update({ id: session.id }, { leaseExpiresAt: new Date(Date.now() + 45_000) });
+
+ for (let k = 0; k < 4; k++) await tick(node);
+
+ expect(duplicateLogs(log)).toHaveLength(0);
+ });
+
+ it("stays quiet for a previous incarnation's lease that nobody renews", async () => {
+ await seed({ nodeId: 'node-a', leaseExpiresAt: new Date(Date.now() + 60_000) });
+ const fresh = service('node-a');
+ const log = errorSpy(fresh);
+
+ for (let k = 0; k < 4; k++) await tick(fresh);
+
+ expect(duplicateLogs(log)).toHaveLength(0);
+ });
+
+ it('does not count renewals seen while an import holds the suspension token', async () => {
+ const session = await seed();
+ const owner = service('node-a');
+ const idle = service('node-a');
+ const log = errorSpy(idle);
+ await owner.claim(session.id);
+ const release = idle.suspendLossDetection();
+
+ for (let k = 0; k < 4; k++) {
+ await tick(owner);
+ await tick(idle);
+ }
+ expect(duplicateLogs(log)).toHaveLength(0);
+
+ release();
+ await tick(owner);
+ await tick(idle); // baseline after the suspension
+ await tick(owner);
+ await tick(idle);
+ expect(duplicateLogs(log)).toHaveLength(0);
+ });
+ });
});
diff --git a/src/modules/session/session-ownership.service.ts b/src/modules/session/session-ownership.service.ts
index 1b10619d8..f9e70e372 100644
--- a/src/modules/session/session-ownership.service.ts
+++ b/src/modules/session/session-ownership.service.ts
@@ -51,6 +51,14 @@ export class SessionOwnershipService {
private lossDetectionSuspended = 0;
/** Answers "does anything still run for this id here?" — consulted by renew(). See setEngineLiveness. */
private engineLiveness?: (sessionId: string) => boolean;
+ /**
+ * Duplicate-NODE_ID detection (see noteForeignRenewals): the lease expiry this process last wrote
+ * or saw per row under its nodeId, how many consecutive ticks it changed without this process
+ * writing it, and whether the duplicate has been reported.
+ */
+ private readonly lastSeenLease = new Map<string, number>();
+ private readonly foreignStreak = new Map<string, number>();
+ private duplicateReported = false;
constructor(
@InjectRepository(Session, 'data')
@@ -65,6 +73,9 @@ export class SessionOwnershipService {
* Deliberately not tied to the pid: a restarted process must recognise its own previous rows in
* order to reset them, and a pid never matches after a restart. The hostname is stable for the
* lifetime of a container or a host; where that is not the right boundary, `NODE_ID` overrides it.
+ * It must also be unique per running process: two processes sharing a hostname (host networking,
+ * pm2 cluster mode, two instances on one host) must each set `NODE_ID`, or each treats the other's
+ * sessions as its own. renew() detects that and logs `duplicate_node_id`.
*/
get nodeId(): string {
return this.configService?.get<string>('session.nodeId') || process.env.NODE_ID || hostname();
@@ -113,13 +124,14 @@ export class SessionOwnershipService {
*/
async claim(sessionId: string): Promise<boolean> {
const now = new Date();
+ const leaseExpiresAt = new Date(now.getTime() + this.leaseTtlMs);
const result = await this.sessions
.createQueryBuilder()
.update(Session)
.set({
nodeId: this.nodeId,
claimedAt: now,
- leaseExpiresAt: new Date(now.getTime() + this.leaseTtlMs),
+ leaseExpiresAt,
nodeUrl: this.nodeUrl || null,
})
.where('id = :id', { id: sessionId })
@@ -132,8 +144,10 @@ export class SessionOwnershipService {
.execute();
const claimed = (result.affected ?? 0) > 0;
- if (claimed) this.owned.add(sessionId);
- else this.logger.warn('Session is held by another node', { sessionId, nodeId: this.nodeId });
+ if (claimed) {
+ this.owned.add(sessionId);
+ this.lastSeenLease.set(sessionId, leaseExpiresAt.getTime());
+ } else this.logger.warn('Session is held by another node', { sessionId, nodeId: this.nodeId });
return claimed;
}
@@ -239,7 +253,6 @@ export class SessionOwnershipService {
*/
async renew(): Promise<void> {
const held = [...this.owned];
- if (held.length === 0) return;
// Only claims that still cover something alive on this process are pushed out. A claim whose
// engine is gone (a failed start, an exhausted reconnect) must be allowed to lapse — renewing
@@ -250,13 +263,24 @@ export class SessionOwnershipService {
let kept: Set<string>;
try {
+ // Read BEFORE this tick's own write, so a lease another process renewed under this nodeId is
+ // still visible. Runs even when nothing is held: a twin that never claimed must notice too.
+ const mine = await this.sessions
+ .createQueryBuilder('s')
+ .select(['s.id', 's.leaseExpiresAt'])
+ .where('s.nodeId = :me', { me: this.nodeId })
+ .getMany();
+ this.noteForeignRenewals(mine);
+ if (held.length === 0) return;
if (live.length > 0) {
+ const leaseExpiresAt = new Date(Date.now() + this.leaseTtlMs);
await this.sessions
.createQueryBuilder()
.update(Session)
- .set({ leaseExpiresAt: new Date(Date.now() + this.leaseTtlMs) })
+ .set({ leaseExpiresAt })
.where({ id: In(live), nodeId: this.nodeId })
.execute();
+ for (const id of live) this.lastSeenLease.set(id, leaseExpiresAt.getTime());
}
const rows = await this.sessions.find({ where: { id: In(held), nodeId: this.nodeId }, select: { id: true } });
kept = new Set(rows.map(row => row.id));
@@ -299,6 +323,47 @@ export class SessionOwnershipService {
}
}
+ /**
+ * Notice another process renewing leases under this nodeId. A lease expiry that changed without
+ * this process writing it, on two consecutive ticks, can only be a live twin: it renews every
+ * heartbeat. A one-off change (a data import carrying leases forward, a claim racing a renewal)
+ * resets on the next tick, and a previous incarnation's lapsed or static lease never changes.
+ * Reported once per process. Detection only: nothing is refused on it.
+ */
+ private noteForeignRenewals(rows: Array<Pick<Session, 'id' | 'leaseExpiresAt'>>): void {
+ if (this.lossDetectionSuspended > 0) {
+ // An import is rewriting the table under this tick; start over once it is done.
+ this.lastSeenLease.clear();
+ this.foreignStreak.clear();
+ return;
+ }
+ const now = Date.now();
+ const seen = new Set<string>();
+ const flagged: string[] = [];
+ for (const row of rows) {
+ const at = row.leaseExpiresAt?.getTime();
+ if (at === undefined || at <= now) continue;
+ seen.add(row.id);
+ const prev = this.lastSeenLease.get(row.id);
+ const streak = prev !== undefined && prev !== at ? (this.foreignStreak.get(row.id) ?? 0) + 1 : 0;
+ this.foreignStreak.set(row.id, streak);
+ this.lastSeenLease.set(row.id, at);
+ if (streak >= 2) flagged.push(row.id);
+ }
+ for (const id of [...this.lastSeenLease.keys()]) {
+ if (seen.has(id)) continue;
+ this.lastSeenLease.delete(id);
+ this.foreignStreak.delete(id);
+ }
+ if (flagged.length === 0 || this.duplicateReported) return;
+ this.duplicateReported = true;
+ this.logger.error(
+ 'Another process is renewing session leases under this NODE_ID; set a unique NODE_ID per process',
+ undefined,
+ { action: 'duplicate_node_id', nodeId: this.nodeId, sessionIds: flagged },
+ );
+ }
+
/** For the boot reset, which must run before anything is claimed. */
ownedByOtherLiveNode(session: Pick<Session, 'nodeId' | 'leaseExpiresAt'>, now = new Date()): boolean {
if (!session.nodeId || session.nodeId === this.nodeId) return false;