Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(events): cap the commands the publisher keeps unanswered
Yudhi Armyndharisad842da7b53a
diff --git a/docs/13-horizontal-scaling.md b/docs/13-horizontal-scaling.mdindex f1f004104..a15e65004 100644--- a/docs/13-horizontal-scaling.md+++ b/docs/13-horizontal-scaling.md@@ -156,9 +156,13 @@> publishes on, and what was written to it may be lost with it. The events that node raises while it is> disconnected, plus its reconnect backoff of up to 5 s once Redis is back, are dropped for the clients> connected to the other replicas: they are not delivered late and not twice. Clients connected to the node-> that raised an event are served directly, and webhooks do not use this path, so neither is affected. The-> node holds at most 10 s of unanswered events, which bounds what a stalled Redis can cost in memory. A-> replica whose own Redis connection is reconnecting also misses what the others publish meanwhile.+> that raised an event are served directly, and webhooks do not use this path, so neither is affected. Memory+> is bounded in two ways. A Redis that sends nothing for 10 s has the connection destroyed, which releases+> what the node held for it: at most 10 s of events. A Redis that keeps answering, but slower than events are+> raised, is never silent for 10 s, so the node holds at most 12,000 unanswered events (10 s at 1,000 events a+> second, with a fifth to spare) and drops further frames for the other replicas, with one warning per burst,+> until Redis catches up. A replica whose own Redis connection is reconnecting also misses what the others+> publish meanwhile.>> **Mid-connection key eviction converges on a timer, not a broadcast, and only on one node.** The> node that processes a revoke, delete, or narrowing tears down that key's sockets synchronously, indiff --git a/src/modules/events/redis-io.adapter.spec.ts b/src/modules/events/redis-io.adapter.spec.tsindex 1e7c168c7..f8ca1ebca 100644--- a/src/modules/events/redis-io.adapter.spec.ts+++ b/src/modules/events/redis-io.adapter.spec.ts@@ -39,6 +39,7 @@ import {isWsRedisEnabled,wsRedisOptions,WS_REDIS_COMMAND_TIMEOUT_MS,+ WS_REDIS_MAX_UNANSWERED,WS_REDIS_QUIT_TIMEOUT_MS,WS_REDIS_SOCKET_TIMEOUT_MS,} from './redis-io.adapter';@@ -245,6 +246,46 @@ describe('RedisIoAdapter', () => {}});+ // The cap reads the length of the client's command queue, which ioredis keeps on an internal property.+ describe('the cap on unanswered commands', () => {+ const withPublisher = async (+ queue: { length: number } | undefined,+ check: (pub: FakeRedis) => Promise<void>,+ ): Promise<void> => {+ process.env.REDIS_ENABLED = 'true';+ const spy = withBaseServer(fakeServer().server);+ try {+ new RedisIoAdapter({} as never).createIOServer(2785);+ const [pub] = redisInstances;+ if (queue) (pub as unknown as { commandQueue: { length: number } }).commandQueue = queue;+ await check(pub);+ } finally {+ spy.mockRestore();+ }+ };++ it('does not publish a frame while the queue is at the cap, and publishes again below it', async () => {+ // The figure docs/13 quotes. A 10 s silence at 1000 events a second must still fit, with room to spare.+ expect(WS_REDIS_MAX_UNANSWERED).toBe(12_000);+ expect(WS_REDIS_MAX_UNANSWERED).toBeGreaterThanOrEqual((WS_REDIS_SOCKET_TIMEOUT_MS / 1000) * 1000 * 1.2);+ const queue = { length: WS_REDIS_MAX_UNANSWERED };+ await withPublisher(queue, async pub => {+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(0);+ expect(pub.sent).not.toHaveBeenCalled();+ queue.length = WS_REDIS_MAX_UNANSWERED - 1;+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);+ expect(pub.sent).toHaveBeenCalledTimes(1);+ });+ });++ it('applies no cap, and does not fail, when the client exposes no command queue', async () => {+ await withPublisher(undefined, async pub => {+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);+ expect(pub.sent).toHaveBeenCalledTimes(1);+ });+ });+ });+it('falls back to the base server (no crash) if wiring the adapter throws', () => {process.env.REDIS_ENABLED = 'true';const { server, adapterFn } = fakeServer();diff --git a/src/modules/events/redis-io.adapter.stall.spec.ts b/src/modules/events/redis-io.adapter.stall.spec.tsindex fddb4e604..7926b08e4 100644--- a/src/modules/events/redis-io.adapter.stall.spec.ts+++ b/src/modules/events/redis-io.adapter.stall.spec.ts@@ -3,7 +3,12 @@ import * as net from 'node:net';import type { AddressInfo } from 'node:net';import { IoAdapter } from '@nestjs/platform-socket.io';import { Server } from 'socket.io';-import { RedisIoAdapter, WS_REDIS_COMMAND_TIMEOUT_MS, WS_REDIS_SOCKET_TIMEOUT_MS } from './redis-io.adapter';+import {+ RedisIoAdapter,+ WS_REDIS_COMMAND_TIMEOUT_MS,+ WS_REDIS_MAX_UNANSWERED,+ WS_REDIS_SOCKET_TIMEOUT_MS,+} from './redis-io.adapter';/*** A Redis that keeps the connection open and stops answering never errors the socket, so the publish@@ -16,11 +21,24 @@ class FakeRedis {/** Connections accepted so far: a client that destroys its socket and reconnects raises it. */connections = 0;readonly sockets = new Set<net.Socket>();+ /** When set, a PUBLISH is answered one at a time through answerOne(): a Redis that is slow, not silent. */+ trickle = false;private stalled = false;+ private readonly slowReplies: Array<() => void> = [];/** What a stalled peer owes its clients. Sent when it comes back, as a Redis that was paused does. */private readonly owed: Array<() => void> = [];private readonly server: net.Server;+ /** Answer the oldest PUBLISH a slow peer still owes. */+ answerOne(): void {+ this.slowReplies.shift()?.();+ }++ /** Answer every PUBLISH a slow peer still owes. */+ answerAll(): void {+ for (const send of this.slowReplies.splice(0)) send();+ }+get silent(): boolean {return this.stalled;}@@ -46,7 +64,9 @@ class FakeRedis {// that no reply ever settles is rejected as an unhandled rejection when the client is closed, which ends// a Jest worker.const reply = FakeRedis.reply(command);- if (!this.silent || /^p?subscribe$/i.test(command[0])) socket.write(reply);+ if (this.trickle && !this.silent && command[0].toUpperCase() === 'PUBLISH') {+ this.slowReplies.push(() => !socket.destroyed && socket.write(reply));+ } else if (!this.silent || /^p?subscribe$/i.test(command[0])) socket.write(reply);else this.owed.push(() => !socket.destroyed && socket.write(reply));}});@@ -230,6 +250,38 @@ describe('RedisIoAdapter publish client against a Redis that stops answering', (expect(pub.status).toBe('ready');}, 15_000);+ // A Redis that answers, but slower than events are raised, resets the socket timer with every reply, so the socket+ // deadline never fires and the client would hold the whole backlog. Only the cap on unanswered commands bounds it.+ it('holds no more unanswered events than the cap when the peer answers too slowly to ever look silent', async () => {+ const connections = redis.connections;+ redis.trickle = true;+ const answering = setInterval(() => redis.answerOne(), 1000);+ try {+ broadcast(WS_REDIS_MAX_UNANSWERED + 3000);+ await sleep(2500);++ expect(held()).toBeLessThanOrEqual(WS_REDIS_MAX_UNANSWERED);+ expect(held()).toBeGreaterThan(WS_REDIS_MAX_UNANSWERED - 10);+ expect(redis.published.length).toBeLessThanOrEqual(WS_REDIS_MAX_UNANSWERED + 3);+ expect(redis.connections).toBe(connections);+ expect(pub.status).toBe('ready');+ // Once per burst, not once per frame.+ const capLines = warn.mock.calls.flat().filter(line => String(line).includes('fan-out frames are dropped'));+ expect(capLines).toHaveLength(1);+ } finally {+ clearInterval(answering);+ }++ // The peer catches up: everything it owed is answered, and events are published again.+ redis.trickle = false;+ redis.answerAll();+ expect(await waitUntil(() => held() === 0, 3000)).toBe(true);+ const published = redis.published.length;+ broadcast(1);+ expect(await waitUntil(() => redis.published.length === published + 1, 3000)).toBe(true);+ expect(warn.mock.calls.flat().some(line => String(line).includes('were dropped'))).toBe(true);+ }, 30_000);+it('settles a publish within its deadline, then drops the connection after the socket deadline, holds nothing more, and recovers', async () => {redis.silent = true;diff --git a/src/modules/events/redis-io.adapter.ts b/src/modules/events/redis-io.adapter.tsindex b41c16e15..0259629e3 100644--- a/src/modules/events/redis-io.adapter.ts+++ b/src/modules/events/redis-io.adapter.ts@@ -22,10 +22,31 @@ export const WS_REDIS_COMMAND_TIMEOUT_MS = 2000;* releases what a stalled Redis left in the client. The timer runs on this process's own event loop, so a healthy* Redis looks silent whenever the loop stays blocked for that long right after a write, and the connection would* be dropped for nothing. That is why it sits far above any block this process runs, and above the command- * timeout. What the client holds during a stall is bounded by this many seconds of traffic.+ * timeout. It bounds a Redis that sends nothing: after this long the socket is destroyed and what the client+ * holds is released. A Redis that answers, however slowly, resets it with every reply, and is bounded by+ * WS_REDIS_MAX_UNANSWERED instead.*/export const WS_REDIS_SOCKET_TIMEOUT_MS = 10_000;+/**+ * Most commands the publisher keeps written and unanswered. A Redis that answers slower than events are raised+ * never trips the socket deadline, so without a cap the backlog grows with the load for as long as the slowness+ * lasts. Frames raised above the cap are dropped for the other replicas, once warned per burst, until the queue+ * drains. 12,000 holds a full socket deadline of traffic at 1,000 events a second with a fifth to spare, which+ * measured about 25 MB of heap with 1 KiB frames. It counts commands, not bytes: events with inline media are larger.+ */+export const WS_REDIS_MAX_UNANSWERED = 12_000;++/**+ * Commands the client has written and not had answered. ioredis keeps them on an internal queue; if this version+ * does not expose it, the answer is 0 and the cap does nothing, so a change of ioredis loses the bound rather than+ * the fan-out (the socket deadline still applies).+ */+function unansweredCommands(client: Redis): number {+ const queue = (client as unknown as { commandQueue?: { length?: unknown } }).commandQueue;+ return typeof queue?.length === 'number' ? queue.length : 0;+}+/*** Whether cross-replica WebSocket fan-out is switched on. Same `REDIS_ENABLED === 'true'` gate the* throttler and cache read at boot, so a deployment already using Redis for those gets event@@ -76,14 +97,16 @@ export function rejectRootNamespace(server: Server): void {* adapter — the local node keeps working, only cross-node fan-out is lost — rather than refusing to* boot. A Redis outage, at boot or after it, is ioredis's problem to retry; the adapter recovers on reconnect.*- * Delivery while Redis is away: the fan-out to the other replicas is best effort and keeps no replay queue.- * A Redis that pauses for less than WS_REDIS_SOCKET_TIMEOUT_MS answers afterwards, and every event raised- * meanwhile reaches them late. A longer silence, a restart, a failover or an outage drops the publisher's- * connection, and what was written to it may be lost with it. The events this node raises while it is- * disconnected, plus its reconnect backoff of up to 5 s once Redis is back (wsRedisOptions), are dropped for- * the other replicas, never delivered late and never twice. Clients on this node are served by the local- * broadcast and webhooks do not use this path, so neither is affected, and what the publisher holds is- * bounded by one socket deadline of traffic.+ * Delivery while Redis is away or slow: the fan-out to the other replicas is best effort and keeps no replay+ * queue. A Redis that pauses for less than WS_REDIS_SOCKET_TIMEOUT_MS answers afterwards, and every event raised+ * meanwhile reaches them late. Silence for that long destroys the publisher's socket, and so does a restart, a+ * failover or an outage: what was written to the dropped connection may be lost with it. The events this node+ * raises while it is disconnected, plus its reconnect backoff of up to 5 s once Redis is back (wsRedisOptions),+ * are dropped for the other replicas, never delivered late and never twice. A Redis that keeps answering, but+ * slower than events are raised, never looks silent: it is bounded by WS_REDIS_MAX_UNANSWERED, above which+ * frames are dropped for the other replicas until it catches up. Clients on this node are served by the local+ * broadcast and webhooks do not use this path, so neither is affected. What the publisher holds is bounded by one+ * socket deadline of traffic for a silent Redis and by that cap for a slow one.*/export class RedisIoAdapter extends IoAdapter {private pubClient?: Redis;@@ -105,8 +128,9 @@ export class RedisIoAdapter extends IoAdapter {// answer in time is rejected (the command stays on the wire, so a slow Redis still delivers it), a socket// that stays silent for the socket deadline is destroyed and the client reconnects, nothing is queued for// a connection it does not have, and nothing the old socket may already have delivered is sent again. An- // event raised while disconnected is therefore dropped for the other replicas rather than replayed late,- // and a stall holds at most one socket deadline of traffic.+ // event raised while disconnected is therefore dropped for the other replicas rather than replayed late.+ // A silent Redis holds at most one socket deadline of traffic; a slow one that keeps answering is held to+ // WS_REDIS_MAX_UNANSWERED by the wrapper below.const pubClient = new Redis({...wsRedisOptions(),commandTimeout: WS_REDIS_COMMAND_TIMEOUT_MS,@@ -132,11 +156,27 @@ export class RedisIoAdapter extends IoAdapter {// confirmed delivery is only worth a debug line. A timed-out command was still written, and a slow// Redis delivers it late; one rejected for lack of a connection was never sent.const publish = pubClient.publish.bind(pubClient);- pubClient.publish = (channel: string, message: string | Buffer) =>- publish(channel, message).catch((err: Error) => {+ let capped = 0;+ pubClient.publish = (channel: string, message: string | Buffer) => {+ const unanswered = unansweredCommands(pubClient);+ if (unanswered >= WS_REDIS_MAX_UNANSWERED) {+ // Nothing else reports a Redis that answers slowly, so the first frame of a burst is warned about.+ if (capped++ === 0) {+ logger.warn(+ `Redis has ${unanswered} fan-out frames unanswered; fan-out frames are dropped for the other replicas until it catches up`,+ );+ }+ return Promise.resolve(0);+ }+ if (capped > 0) {+ logger.warn(`Redis caught up; ${capped} fan-out frames were dropped for the other replicas`);+ capped = 0;+ }+ return publish(channel, message).catch((err: Error) => {logger.debug(`Redis publish not confirmed: ${err.message}`);return 0;});+ };this.pubClient = pubClient;this.subClient = subClient;server.adapter(createAdapter(pubClient, subClient));