Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(events): cap the commands the publisher keeps unanswered

Yudhi Armyndharisad842da7b53a
diff --git a/docs/13-horizontal-scaling.md b/docs/13-horizontal-scaling.md
index f1f004104..a15e65004 100644
--- a/docs/13-horizontal-scaling.md
+++ b/docs/13-horizontal-scaling.md
@@ -156,9 +156,13 @@
> publishes on, and what was written to it may be lost with it. The events that node raises while it is
> disconnected, plus its reconnect backoff of up to 5 s once Redis is back, are dropped for the clients
> connected to the other replicas: they are not delivered late and not twice. Clients connected to the node
-> that raised an event are served directly, and webhooks do not use this path, so neither is affected. The
-> node holds at most 10 s of unanswered events, which bounds what a stalled Redis can cost in memory. A
-> replica whose own Redis connection is reconnecting also misses what the others publish meanwhile.
+> that raised an event are served directly, and webhooks do not use this path, so neither is affected. Memory
+> is bounded in two ways. A Redis that sends nothing for 10 s has the connection destroyed, which releases
+> what the node held for it: at most 10 s of events. A Redis that keeps answering, but slower than events are
+> raised, is never silent for 10 s, so the node holds at most 12,000 unanswered events (10 s at 1,000 events a
+> second, with a fifth to spare) and drops further frames for the other replicas, with one warning per burst,
+> until Redis catches up. A replica whose own Redis connection is reconnecting also misses what the others
+> publish meanwhile.
>
> **Mid-connection key eviction converges on a timer, not a broadcast, and only on one node.** The
> node that processes a revoke, delete, or narrowing tears down that key's sockets synchronously, in
diff --git a/src/modules/events/redis-io.adapter.spec.ts b/src/modules/events/redis-io.adapter.spec.ts
index 1e7c168c7..f8ca1ebca 100644
--- a/src/modules/events/redis-io.adapter.spec.ts
+++ b/src/modules/events/redis-io.adapter.spec.ts
@@ -39,6 +39,7 @@ import {
isWsRedisEnabled,
wsRedisOptions,
WS_REDIS_COMMAND_TIMEOUT_MS,
+ WS_REDIS_MAX_UNANSWERED,
WS_REDIS_QUIT_TIMEOUT_MS,
WS_REDIS_SOCKET_TIMEOUT_MS,
} from './redis-io.adapter';
@@ -245,6 +246,46 @@ describe('RedisIoAdapter', () => {
}
});
+ // The cap reads the length of the client's command queue, which ioredis keeps on an internal property.
+ describe('the cap on unanswered commands', () => {
+ const withPublisher = async (
+ queue: { length: number } | undefined,
+ check: (pub: FakeRedis) => Promise<void>,
+ ): Promise<void> => {
+ process.env.REDIS_ENABLED = 'true';
+ const spy = withBaseServer(fakeServer().server);
+ try {
+ new RedisIoAdapter({} as never).createIOServer(2785);
+ const [pub] = redisInstances;
+ if (queue) (pub as unknown as { commandQueue: { length: number } }).commandQueue = queue;
+ await check(pub);
+ } finally {
+ spy.mockRestore();
+ }
+ };
+
+ it('does not publish a frame while the queue is at the cap, and publishes again below it', async () => {
+ // The figure docs/13 quotes. A 10 s silence at 1000 events a second must still fit, with room to spare.
+ expect(WS_REDIS_MAX_UNANSWERED).toBe(12_000);
+ expect(WS_REDIS_MAX_UNANSWERED).toBeGreaterThanOrEqual((WS_REDIS_SOCKET_TIMEOUT_MS / 1000) * 1000 * 1.2);
+ const queue = { length: WS_REDIS_MAX_UNANSWERED };
+ await withPublisher(queue, async pub => {
+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(0);
+ expect(pub.sent).not.toHaveBeenCalled();
+ queue.length = WS_REDIS_MAX_UNANSWERED - 1;
+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);
+ expect(pub.sent).toHaveBeenCalledTimes(1);
+ });
+ });
+
+ it('applies no cap, and does not fail, when the client exposes no command queue', async () => {
+ await withPublisher(undefined, async pub => {
+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);
+ expect(pub.sent).toHaveBeenCalledTimes(1);
+ });
+ });
+ });
+
it('falls back to the base server (no crash) if wiring the adapter throws', () => {
process.env.REDIS_ENABLED = 'true';
const { server, adapterFn } = fakeServer();
diff --git a/src/modules/events/redis-io.adapter.stall.spec.ts b/src/modules/events/redis-io.adapter.stall.spec.ts
index fddb4e604..7926b08e4 100644
--- a/src/modules/events/redis-io.adapter.stall.spec.ts
+++ b/src/modules/events/redis-io.adapter.stall.spec.ts
@@ -3,7 +3,12 @@ import * as net from 'node:net';
import type { AddressInfo } from 'node:net';
import { IoAdapter } from '@nestjs/platform-socket.io';
import { Server } from 'socket.io';
-import { RedisIoAdapter, WS_REDIS_COMMAND_TIMEOUT_MS, WS_REDIS_SOCKET_TIMEOUT_MS } from './redis-io.adapter';
+import {
+ RedisIoAdapter,
+ WS_REDIS_COMMAND_TIMEOUT_MS,
+ WS_REDIS_MAX_UNANSWERED,
+ WS_REDIS_SOCKET_TIMEOUT_MS,
+} from './redis-io.adapter';
/**
* A Redis that keeps the connection open and stops answering never errors the socket, so the publish
@@ -16,11 +21,24 @@ class FakeRedis {
/** Connections accepted so far: a client that destroys its socket and reconnects raises it. */
connections = 0;
readonly sockets = new Set<net.Socket>();
+ /** When set, a PUBLISH is answered one at a time through answerOne(): a Redis that is slow, not silent. */
+ trickle = false;
private stalled = false;
+ private readonly slowReplies: Array<() => void> = [];
/** What a stalled peer owes its clients. Sent when it comes back, as a Redis that was paused does. */
private readonly owed: Array<() => void> = [];
private readonly server: net.Server;
+ /** Answer the oldest PUBLISH a slow peer still owes. */
+ answerOne(): void {
+ this.slowReplies.shift()?.();
+ }
+
+ /** Answer every PUBLISH a slow peer still owes. */
+ answerAll(): void {
+ for (const send of this.slowReplies.splice(0)) send();
+ }
+
get silent(): boolean {
return this.stalled;
}
@@ -46,7 +64,9 @@ class FakeRedis {
// that no reply ever settles is rejected as an unhandled rejection when the client is closed, which ends
// a Jest worker.
const reply = FakeRedis.reply(command);
- if (!this.silent || /^p?subscribe$/i.test(command[0])) socket.write(reply);
+ if (this.trickle && !this.silent && command[0].toUpperCase() === 'PUBLISH') {
+ this.slowReplies.push(() => !socket.destroyed && socket.write(reply));
+ } else if (!this.silent || /^p?subscribe$/i.test(command[0])) socket.write(reply);
else this.owed.push(() => !socket.destroyed && socket.write(reply));
}
});
@@ -230,6 +250,38 @@ describe('RedisIoAdapter publish client against a Redis that stops answering', (
expect(pub.status).toBe('ready');
}, 15_000);
+ // A Redis that answers, but slower than events are raised, resets the socket timer with every reply, so the socket
+ // deadline never fires and the client would hold the whole backlog. Only the cap on unanswered commands bounds it.
+ it('holds no more unanswered events than the cap when the peer answers too slowly to ever look silent', async () => {
+ const connections = redis.connections;
+ redis.trickle = true;
+ const answering = setInterval(() => redis.answerOne(), 1000);
+ try {
+ broadcast(WS_REDIS_MAX_UNANSWERED + 3000);
+ await sleep(2500);
+
+ expect(held()).toBeLessThanOrEqual(WS_REDIS_MAX_UNANSWERED);
+ expect(held()).toBeGreaterThan(WS_REDIS_MAX_UNANSWERED - 10);
+ expect(redis.published.length).toBeLessThanOrEqual(WS_REDIS_MAX_UNANSWERED + 3);
+ expect(redis.connections).toBe(connections);
+ expect(pub.status).toBe('ready');
+ // Once per burst, not once per frame.
+ const capLines = warn.mock.calls.flat().filter(line => String(line).includes('fan-out frames are dropped'));
+ expect(capLines).toHaveLength(1);
+ } finally {
+ clearInterval(answering);
+ }
+
+ // The peer catches up: everything it owed is answered, and events are published again.
+ redis.trickle = false;
+ redis.answerAll();
+ expect(await waitUntil(() => held() === 0, 3000)).toBe(true);
+ const published = redis.published.length;
+ broadcast(1);
+ expect(await waitUntil(() => redis.published.length === published + 1, 3000)).toBe(true);
+ expect(warn.mock.calls.flat().some(line => String(line).includes('were dropped'))).toBe(true);
+ }, 30_000);
+
it('settles a publish within its deadline, then drops the connection after the socket deadline, holds nothing more, and recovers', async () => {
redis.silent = true;
diff --git a/src/modules/events/redis-io.adapter.ts b/src/modules/events/redis-io.adapter.ts
index b41c16e15..0259629e3 100644
--- a/src/modules/events/redis-io.adapter.ts
+++ b/src/modules/events/redis-io.adapter.ts
@@ -22,10 +22,31 @@ export const WS_REDIS_COMMAND_TIMEOUT_MS = 2000;
* releases what a stalled Redis left in the client. The timer runs on this process's own event loop, so a healthy
* Redis looks silent whenever the loop stays blocked for that long right after a write, and the connection would
* be dropped for nothing. That is why it sits far above any block this process runs, and above the command
- * timeout. What the client holds during a stall is bounded by this many seconds of traffic.
+ * timeout. It bounds a Redis that sends nothing: after this long the socket is destroyed and what the client
+ * holds is released. A Redis that answers, however slowly, resets it with every reply, and is bounded by
+ * WS_REDIS_MAX_UNANSWERED instead.
*/
export const WS_REDIS_SOCKET_TIMEOUT_MS = 10_000;
+/**
+ * Most commands the publisher keeps written and unanswered. A Redis that answers slower than events are raised
+ * never trips the socket deadline, so without a cap the backlog grows with the load for as long as the slowness
+ * lasts. Frames raised above the cap are dropped for the other replicas, once warned per burst, until the queue
+ * drains. 12,000 holds a full socket deadline of traffic at 1,000 events a second with a fifth to spare, which
+ * measured about 25 MB of heap with 1 KiB frames. It counts commands, not bytes: events with inline media are larger.
+ */
+export const WS_REDIS_MAX_UNANSWERED = 12_000;
+
+/**
+ * Commands the client has written and not had answered. ioredis keeps them on an internal queue; if this version
+ * does not expose it, the answer is 0 and the cap does nothing, so a change of ioredis loses the bound rather than
+ * the fan-out (the socket deadline still applies).
+ */
+function unansweredCommands(client: Redis): number {
+ const queue = (client as unknown as { commandQueue?: { length?: unknown } }).commandQueue;
+ return typeof queue?.length === 'number' ? queue.length : 0;
+}
+
/**
* Whether cross-replica WebSocket fan-out is switched on. Same `REDIS_ENABLED === 'true'` gate the
* throttler and cache read at boot, so a deployment already using Redis for those gets event
@@ -76,14 +97,16 @@ export function rejectRootNamespace(server: Server): void {
* adapter — the local node keeps working, only cross-node fan-out is lost — rather than refusing to
* boot. A Redis outage, at boot or after it, is ioredis's problem to retry; the adapter recovers on reconnect.
*
- * Delivery while Redis is away: the fan-out to the other replicas is best effort and keeps no replay queue.
- * A Redis that pauses for less than WS_REDIS_SOCKET_TIMEOUT_MS answers afterwards, and every event raised
- * meanwhile reaches them late. A longer silence, a restart, a failover or an outage drops the publisher's
- * connection, and what was written to it may be lost with it. The events this node raises while it is
- * disconnected, plus its reconnect backoff of up to 5 s once Redis is back (wsRedisOptions), are dropped for
- * the other replicas, never delivered late and never twice. Clients on this node are served by the local
- * broadcast and webhooks do not use this path, so neither is affected, and what the publisher holds is
- * bounded by one socket deadline of traffic.
+ * Delivery while Redis is away or slow: the fan-out to the other replicas is best effort and keeps no replay
+ * queue. A Redis that pauses for less than WS_REDIS_SOCKET_TIMEOUT_MS answers afterwards, and every event raised
+ * meanwhile reaches them late. Silence for that long destroys the publisher's socket, and so does a restart, a
+ * failover or an outage: what was written to the dropped connection may be lost with it. The events this node
+ * raises while it is disconnected, plus its reconnect backoff of up to 5 s once Redis is back (wsRedisOptions),
+ * are dropped for the other replicas, never delivered late and never twice. A Redis that keeps answering, but
+ * slower than events are raised, never looks silent: it is bounded by WS_REDIS_MAX_UNANSWERED, above which
+ * frames are dropped for the other replicas until it catches up. Clients on this node are served by the local
+ * broadcast and webhooks do not use this path, so neither is affected. What the publisher holds is bounded by one
+ * socket deadline of traffic for a silent Redis and by that cap for a slow one.
*/
export class RedisIoAdapter extends IoAdapter {
private pubClient?: Redis;
@@ -105,8 +128,9 @@ export class RedisIoAdapter extends IoAdapter {
// answer in time is rejected (the command stays on the wire, so a slow Redis still delivers it), a socket
// that stays silent for the socket deadline is destroyed and the client reconnects, nothing is queued for
// a connection it does not have, and nothing the old socket may already have delivered is sent again. An
- // event raised while disconnected is therefore dropped for the other replicas rather than replayed late,
- // and a stall holds at most one socket deadline of traffic.
+ // event raised while disconnected is therefore dropped for the other replicas rather than replayed late.
+ // A silent Redis holds at most one socket deadline of traffic; a slow one that keeps answering is held to
+ // WS_REDIS_MAX_UNANSWERED by the wrapper below.
const pubClient = new Redis({
...wsRedisOptions(),
commandTimeout: WS_REDIS_COMMAND_TIMEOUT_MS,
@@ -132,11 +156,27 @@ export class RedisIoAdapter extends IoAdapter {
// confirmed delivery is only worth a debug line. A timed-out command was still written, and a slow
// Redis delivers it late; one rejected for lack of a connection was never sent.
const publish = pubClient.publish.bind(pubClient);
- pubClient.publish = (channel: string, message: string | Buffer) =>
- publish(channel, message).catch((err: Error) => {
+ let capped = 0;
+ pubClient.publish = (channel: string, message: string | Buffer) => {
+ const unanswered = unansweredCommands(pubClient);
+ if (unanswered >= WS_REDIS_MAX_UNANSWERED) {
+ // Nothing else reports a Redis that answers slowly, so the first frame of a burst is warned about.
+ if (capped++ === 0) {
+ logger.warn(
+ `Redis has ${unanswered} fan-out frames unanswered; fan-out frames are dropped for the other replicas until it catches up`,
+ );
+ }
+ return Promise.resolve(0);
+ }
+ if (capped > 0) {
+ logger.warn(`Redis caught up; ${capped} fan-out frames were dropped for the other replicas`);
+ capped = 0;
+ }
+ return publish(channel, message).catch((err: Error) => {
logger.debug(`Redis publish not confirmed: ${err.message}`);
return 0;
});
+ };
this.pubClient = pubClient;
this.subClient = subClient;
server.adapter(createAdapter(pubClient, subClient));