Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(events): judge a slow Redis by its backlog's age, not its depth
Yudhi Armyndharise24c803cc019
diff --git a/docs/13-horizontal-scaling.md b/docs/13-horizontal-scaling.mdindex 609e3d401..ef28ee264 100644--- a/docs/13-horizontal-scaling.md+++ b/docs/13-horizontal-scaling.md@@ -160,11 +160,13 @@> that raised an event are served directly, and webhooks do not use this path, so neither is affected. Memory> is bounded in two ways. A Redis that sends nothing for 10 s has the connection destroyed, and the node holds> nothing more for it; what it had already held, at most 10 s of events, is freed once the node has reconnected.-> A Redis that keeps answering, but slower than events are raised, is never silent for 10 s, so the node holds-> at most 12,000 unanswered events (10 s at 1,000 events a second, with a fifth to spare) and drops further-> frames for the other replicas until Redis catches up, with a warning when dropping starts and another, with-> the count, when it ends. A replica whose own Redis connection is reconnecting also misses what the others-> publish meanwhile.+> A Redis that keeps answering, but slower than events are raised, is never silent for 10 s, so a cap bounds+> it instead: once the node holds 12,000 unanswered events (10 s at 1,000 events a second, with a fifth to+> spare) and the backlog has outlived 2 s, meaning the count has not been below 6,000 for that long, it drops+> further frames for the other replicas until the count is back below 6,000, with a warning when dropping+> starts and another, with the count, when it ends. A burst raised in a single tick is never cut, however+> large: a healthy Redis answers it right after, and the node holds it until then. A replica whose own Redis+> connection is reconnecting also misses what the others publish meanwhile.>> **Mid-connection key eviction converges on a timer, not a broadcast, and only on one node.** The> node that processes a revoke, delete, or narrowing tears down that key's sockets synchronously, indiff --git a/src/modules/events/redis-io.adapter.spec.ts b/src/modules/events/redis-io.adapter.spec.tsindex fca04d1da..b1a401dde 100644--- a/src/modules/events/redis-io.adapter.spec.ts+++ b/src/modules/events/redis-io.adapter.spec.ts@@ -267,20 +267,88 @@ describe('RedisIoAdapter', () => {}};- it('does not publish a frame while the queue is at the cap, and publishes again below it', async () => {- // The figure docs/13 quotes. A 10 s silence at 1000 events a second must still fit, with room to spare.+ it('pins the figures docs/13 quotes, and a 10 s silence at 1000 events a second fits under them', () => {expect(WS_REDIS_MAX_UNANSWERED).toBe(12_000);expect(WS_REDIS_MAX_UNANSWERED).toBeGreaterThanOrEqual((WS_REDIS_SOCKET_TIMEOUT_MS / 1000) * 1000 * 1.2);- const queue = { length: WS_REDIS_MAX_UNANSWERED };+ });++ // Slowness is judged by how long the backlog has lasted, not by its depth: a burst raised in one tick is all on the+ // queue before any reply can be read.+ it('does not cut a deep queue that is new, however large', async () => {+ const queue = { length: 0 };await withPublisher(queue, async pub => {- await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(0);- expect(pub.sent).not.toHaveBeenCalled();- queue.length = WS_REDIS_MAX_UNANSWERED - 1;+ await pub.publish('socket.io#/events#', 'frame');+ queue.length = WS_REDIS_MAX_UNANSWERED * 2;await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);- expect(pub.sent).toHaveBeenCalledTimes(1);+ expect(pub.sent).toHaveBeenCalledTimes(2);});});+ it('drops frames once the queue is at the cap and has been above half of it for longer than a command is given', async () => {+ let clock = 1_000_000;+ const now = jest.spyOn(Date, 'now').mockImplementation(() => clock);+ const queue = { length: 0 };+ try {+ await withPublisher(queue, async pub => {+ await pub.publish('socket.io#/events#', 'frame');+ queue.length = WS_REDIS_MAX_UNANSWERED / 2;+ clock += WS_REDIS_COMMAND_TIMEOUT_MS;+ // At the cap, but only exactly as long as a command is given: not yet.+ queue.length = WS_REDIS_MAX_UNANSWERED;+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);+ clock += 1;+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(0);+ expect(pub.sent).toHaveBeenCalledTimes(2);+ // Dropping goes on until the queue is back below half the cap, not until it is below the cap.+ queue.length = WS_REDIS_MAX_UNANSWERED - 1;+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(0);+ queue.length = WS_REDIS_MAX_UNANSWERED / 2;+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(0);+ queue.length = WS_REDIS_MAX_UNANSWERED / 2 - 1;+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);+ expect(pub.sent).toHaveBeenCalledTimes(3);+ });+ } finally {+ now.mockRestore();+ }+ });++ it('warns once when an episode starts and once when it ends, each with its own count', async () => {+ let clock = 1_000_000;+ const now = jest.spyOn(Date, 'now').mockImplementation(() => clock);+ const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined);+ const lines = (text: string): string[] =>+ warn.mock.calls.map(([line]) => String(line)).filter(line => line.includes(text));+ const queue = { length: 0 };+ try {+ await withPublisher(queue, async pub => {+ const episode = async (frames: number): Promise<void> => {+ queue.length = 0;+ await pub.publish('socket.io#/events#', 'frame');+ queue.length = WS_REDIS_MAX_UNANSWERED / 2;+ clock += WS_REDIS_COMMAND_TIMEOUT_MS + 1;+ queue.length = WS_REDIS_MAX_UNANSWERED;+ for (let frame = 0; frame < frames; frame++) await pub.publish('socket.io#/events#', 'frame');+ queue.length = 0;+ await pub.publish('socket.io#/events#', 'frame');+ await pub.publish('socket.io#/events#', 'frame');+ };++ await episode(5);+ await episode(2);++ expect(lines('fan-out frames are dropped')).toHaveLength(2);+ const ended = lines('were dropped');+ expect(ended).toHaveLength(2);+ expect(ended[0]).toContain('5 fan-out frames');+ expect(ended[1]).toContain('2 fan-out frames');+ });+ } finally {+ now.mockRestore();+ warn.mockRestore();+ }+ });+it('applies no cap, and does not fail, when the client exposes no command queue', async () => {await withPublisher(undefined, async pub => {await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);diff --git a/src/modules/events/redis-io.adapter.stall.spec.ts b/src/modules/events/redis-io.adapter.stall.spec.tsindex 7926b08e4..8bbfda6c8 100644--- a/src/modules/events/redis-io.adapter.stall.spec.ts+++ b/src/modules/events/redis-io.adapter.stall.spec.ts@@ -250,36 +250,69 @@ describe('RedisIoAdapter publish client against a Redis that stops answering', (expect(pub.status).toBe('ready');}, 15_000);+ const warned = (text: string): string[] =>+ warn.mock.calls+ .flat()+ .map(line => String(line))+ .filter(line => line.includes(text));++ // Every frame of a burst raised in one tick is on the queue before the first reply can be read, however healthy the+ // peer is, so the depth of the queue alone says nothing about a Redis that is behind.+ it('publishes every frame of a burst larger than the cap raised in one tick to a peer that answers', async () => {+ const burst = WS_REDIS_MAX_UNANSWERED + 3000;++ broadcast(burst);++ expect(await waitUntil(() => redis.published.length === burst && held() === 0, 15_000)).toBe(true);+ expect(warned('fan-out frames are dropped')).toHaveLength(0);+ expect(warned('were dropped')).toHaveLength(0);+ }, 30_000);+// A Redis that answers, but slower than events are raised, resets the socket timer with every reply, so the socket- // deadline never fires and the client would hold the whole backlog. Only the cap on unanswered commands bounds it.- it('holds no more unanswered events than the cap when the peer answers too slowly to ever look silent', async () => {+ // deadline never fires and the client would hold the whole backlog. Only the cap on unanswered commands bounds it,+ // once the backlog has outlived a command's own deadline. The clock is moved by hand: the rule is about elapsed time.+ it('holds a slow peer to the cap, warning once when dropping starts and once when it ends, however often it answers', async () => {+ let clock = 1_000_000;+ jest.spyOn(Date, 'now').mockImplementation(() => clock);const connections = redis.connections;redis.trickle = true;- const answering = setInterval(() => redis.answerOne(), 1000);- try {- broadcast(WS_REDIS_MAX_UNANSWERED + 3000);- await sleep(2500);-- expect(held()).toBeLessThanOrEqual(WS_REDIS_MAX_UNANSWERED);- expect(held()).toBeGreaterThan(WS_REDIS_MAX_UNANSWERED - 10);- expect(redis.published.length).toBeLessThanOrEqual(WS_REDIS_MAX_UNANSWERED + 3);- expect(redis.connections).toBe(connections);- expect(pub.status).toBe('ready');- // Once per burst, not once per frame.- const capLines = warn.mock.calls.flat().filter(line => String(line).includes('fan-out frames are dropped'));- expect(capLines).toHaveLength(1);- } finally {- clearInterval(answering);++ // The queue is half full, which is the last moment it counted as being on top of its work...+ broadcast(WS_REDIS_MAX_UNANSWERED / 2);+ // ...and it stays above that for longer than a command is given to be answered.+ clock += WS_REDIS_COMMAND_TIMEOUT_MS + 100;+ broadcast(WS_REDIS_MAX_UNANSWERED);++ expect(held()).toBe(WS_REDIS_MAX_UNANSWERED);+ expect(await waitUntil(() => redis.published.length >= WS_REDIS_MAX_UNANSWERED, 10_000)).toBe(true);+ expect(redis.published.length).toBe(WS_REDIS_MAX_UNANSWERED);+ expect(redis.connections).toBe(connections);+ expect(pub.status).toBe('ready');+ expect(warned('fan-out frames are dropped')).toHaveLength(1);++ // Each reply frees one slot and the next frame arrives: that must not end the episode, or every reply would+ // log two lines and let one frame through. It ends when the queue is back below half the cap.+ for (let reply = 0; reply < 30; reply++) {+ const before = held();+ redis.answerOne();+ expect(await waitUntil(() => held() < before, 3000)).toBe(true);+ broadcast(1);}+ expect(held()).toBe(WS_REDIS_MAX_UNANSWERED - 30);+ expect(warned('fan-out frames are dropped')).toHaveLength(1);+ expect(warned('were dropped')).toHaveLength(0);// The peer catches up: everything it owed is answered, and events are published again.redis.trickle = false;redis.answerAll();- expect(await waitUntil(() => held() === 0, 3000)).toBe(true);+ expect(await waitUntil(() => held() === 0, 5000)).toBe(true);const published = redis.published.length;- broadcast(1);- expect(await waitUntil(() => redis.published.length === published + 1, 3000)).toBe(true);- expect(warn.mock.calls.flat().some(line => String(line).includes('were dropped'))).toBe(true);+ broadcast(2);+ expect(await waitUntil(() => redis.published.length === published + 2, 5000)).toBe(true);+ // One line for the whole episode, with its count: 6,000 dropped by the burst and 30 while it was answering.+ const ended = warned('were dropped');+ expect(ended).toHaveLength(1);+ expect(ended[0]).toContain(`${WS_REDIS_MAX_UNANSWERED / 2 + 30} fan-out frames`);}, 30_000);it('settles a publish within its deadline, then drops the connection after the socket deadline, holds nothing more, and recovers', async () => {diff --git a/src/modules/events/redis-io.adapter.ts b/src/modules/events/redis-io.adapter.tsindex d6ccc8110..925d54f18 100644--- a/src/modules/events/redis-io.adapter.ts+++ b/src/modules/events/redis-io.adapter.ts@@ -30,15 +30,21 @@ export const WS_REDIS_COMMAND_TIMEOUT_MS = 2000;export const WS_REDIS_SOCKET_TIMEOUT_MS = 10_000;/**- * Most commands the publisher keeps written and unanswered. A Redis that answers slower than events are raised- * never trips the socket deadline, so without a cap the backlog grows with the load for as long as the slowness- * lasts. Frames raised above the cap are dropped for the other replicas until the queue drains, with a warning- * when dropping starts and another, with the count, when Redis catches up. 12,000 holds a full socket deadline of- * traffic at 1,000 events a second with a fifth to spare, which measured about 25 MB of heap with 1 KiB frames.- * It counts commands, not bytes: events with inline media are larger.+ * Most commands the publisher keeps written and unanswered once the backlog has lasted. A Redis that answers slower+ * than events are raised never trips the socket deadline, so without a cap the backlog grows with the load for as+ * long as the slowness lasts. The rule: frames are dropped for the other replicas while the queue holds this many+ * commands and has not been below half of it for longer than a command is given to be answered+ * (WS_REDIS_COMMAND_TIMEOUT_MS), and they go on being dropped until the queue is back below half of it. A burst+ * raised in one tick is never cut, however large, because depth alone says nothing: it is all on the queue before a+ * reply can be read. Dropping starts with a warning and ends with another, with the count. 12,000 holds a full+ * socket deadline of traffic at 1,000 events a second with a fifth to spare, which measured about 25 MB of heap with+ * 1 KiB frames. It counts commands, not bytes: events with inline media are larger.*/export const WS_REDIS_MAX_UNANSWERED = 12_000;+/** The queue length below which a backlog counts as over, and dropped frames are published again. */+const WS_REDIS_RESUME_UNANSWERED = WS_REDIS_MAX_UNANSWERED / 2;+/*** Commands the client has written and not had answered. ioredis keeps them on an internal queue; if this version* does not expose it, the answer is 0 and the cap does nothing, so a change of ioredis loses the bound rather than@@ -106,10 +112,11 @@ export function rejectRootNamespace(server: Server): void {* other replicas late and after newer events. The events this node raises while it is disconnected, plus its* reconnect backoff of up to 5 s once Redis is back (wsRedisOptions), are dropped for the other replicas, never* delivered late and never twice. A Redis that keeps answering, but slower than events are raised, never looks- * silent: it is bounded by WS_REDIS_MAX_UNANSWERED, above which frames are dropped for the other replicas until- * it catches up. Clients on this node are served by the local broadcast and webhooks do not use this path, so- * neither is affected. What the publisher holds is bounded by one socket deadline of traffic for a silent Redis- * and by that cap for a slow one.+ * silent: it is bounded by WS_REDIS_MAX_UNANSWERED, which drops frames for the other replicas once the backlog has+ * outlived a command's deadline and until it is back below half of it (a burst raised in one tick is not cut).+ * Clients on this node are served by the local broadcast and webhooks do not use this path, so neither is+ * affected. What the publisher holds is bounded by one socket deadline of traffic for a silent Redis and by that+ * cap, plus the traffic of one command deadline, for a slow one.*/export class RedisIoAdapter extends IoAdapter {private pubClient?: Redis;@@ -133,7 +140,7 @@ export class RedisIoAdapter extends IoAdapter {// a connection it does not have, and nothing the old socket may already have delivered is sent again. An// event raised while disconnected is therefore dropped for the other replicas rather than replayed late.// A silent Redis holds at most one socket deadline of traffic; a slow one that keeps answering is held to- // WS_REDIS_MAX_UNANSWERED by the wrapper below.+ // WS_REDIS_MAX_UNANSWERED (see there for the rule) by the wrapper below.const pubClient = new Redis({...wsRedisOptions(),commandTimeout: WS_REDIS_COMMAND_TIMEOUT_MS,@@ -159,21 +166,34 @@ export class RedisIoAdapter extends IoAdapter {// confirmed delivery is only worth a debug line. A timed-out command was still written, and a slow// Redis delivers it late; one rejected for lack of a connection was never sent.const publish = pubClient.publish.bind(pubClient);- let capped = 0;+ let dropping = false;+ let dropped = 0;+ // When the queue was last seen below half the cap: the backlog is as old as the time since.+ let lastBelowResume = 0;pubClient.publish = (channel: string, message: string | Buffer) => {const unanswered = unansweredCommands(pubClient);- if (unanswered >= WS_REDIS_MAX_UNANSWERED) {- // Nothing else reports a Redis that answers slowly, so the first frame of a burst is warned about.- if (capped++ === 0) {- logger.warn(- `Redis has ${unanswered} fan-out frames unanswered; fan-out frames are dropped for the other replicas until it catches up`,- );+ const now = Date.now();+ if (unanswered < WS_REDIS_RESUME_UNANSWERED) {+ lastBelowResume = now;+ if (dropping) {+ logger.warn(`Redis caught up; ${dropped} fan-out frames were dropped for the other replicas`);+ dropping = false;+ dropped = 0;}- return Promise.resolve(0);+ } else if (+ !dropping &&+ unanswered >= WS_REDIS_MAX_UNANSWERED &&+ now - lastBelowResume > WS_REDIS_COMMAND_TIMEOUT_MS+ ) {+ // Nothing else reports a Redis that answers slowly, so the start of an episode is warned about.+ dropping = true;+ logger.warn(+ `Redis has ${unanswered} fan-out frames unanswered; fan-out frames are dropped for the other replicas until it catches up`,+ );}- if (capped > 0) {- logger.warn(`Redis caught up; ${capped} fan-out frames were dropped for the other replicas`);- capped = 0;+ if (dropping) {+ dropped++;+ return Promise.resolve(0);}return publish(channel, message).catch((err: Error) => {logger.debug(`Redis publish not confirmed: ${err.message}`);