Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(events): judge a slow Redis by its backlog's age, not its depth

Yudhi Armyndharise24c803cc019
diff --git a/docs/13-horizontal-scaling.md b/docs/13-horizontal-scaling.md
index 609e3d401..ef28ee264 100644
--- a/docs/13-horizontal-scaling.md
+++ b/docs/13-horizontal-scaling.md
@@ -160,11 +160,13 @@
> that raised an event are served directly, and webhooks do not use this path, so neither is affected. Memory
> is bounded in two ways. A Redis that sends nothing for 10 s has the connection destroyed, and the node holds
> nothing more for it; what it had already held, at most 10 s of events, is freed once the node has reconnected.
-> A Redis that keeps answering, but slower than events are raised, is never silent for 10 s, so the node holds
-> at most 12,000 unanswered events (10 s at 1,000 events a second, with a fifth to spare) and drops further
-> frames for the other replicas until Redis catches up, with a warning when dropping starts and another, with
-> the count, when it ends. A replica whose own Redis connection is reconnecting also misses what the others
-> publish meanwhile.
+> A Redis that keeps answering, but slower than events are raised, is never silent for 10 s, so a cap bounds
+> it instead: once the node holds 12,000 unanswered events (10 s at 1,000 events a second, with a fifth to
+> spare) and the backlog has outlived 2 s, meaning the count has not been below 6,000 for that long, it drops
+> further frames for the other replicas until the count is back below 6,000, with a warning when dropping
+> starts and another, with the count, when it ends. A burst raised in a single tick is never cut, however
+> large: a healthy Redis answers it right after, and the node holds it until then. A replica whose own Redis
+> connection is reconnecting also misses what the others publish meanwhile.
>
> **Mid-connection key eviction converges on a timer, not a broadcast, and only on one node.** The
> node that processes a revoke, delete, or narrowing tears down that key's sockets synchronously, in
diff --git a/src/modules/events/redis-io.adapter.spec.ts b/src/modules/events/redis-io.adapter.spec.ts
index fca04d1da..b1a401dde 100644
--- a/src/modules/events/redis-io.adapter.spec.ts
+++ b/src/modules/events/redis-io.adapter.spec.ts
@@ -267,20 +267,88 @@ describe('RedisIoAdapter', () => {
}
};
- it('does not publish a frame while the queue is at the cap, and publishes again below it', async () => {
- // The figure docs/13 quotes. A 10 s silence at 1000 events a second must still fit, with room to spare.
+ it('pins the figures docs/13 quotes, and a 10 s silence at 1000 events a second fits under them', () => {
expect(WS_REDIS_MAX_UNANSWERED).toBe(12_000);
expect(WS_REDIS_MAX_UNANSWERED).toBeGreaterThanOrEqual((WS_REDIS_SOCKET_TIMEOUT_MS / 1000) * 1000 * 1.2);
- const queue = { length: WS_REDIS_MAX_UNANSWERED };
+ });
+
+ // Slowness is judged by how long the backlog has lasted, not by its depth: a burst raised in one tick is all on the
+ // queue before any reply can be read.
+ it('does not cut a deep queue that is new, however large', async () => {
+ const queue = { length: 0 };
await withPublisher(queue, async pub => {
- await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(0);
- expect(pub.sent).not.toHaveBeenCalled();
- queue.length = WS_REDIS_MAX_UNANSWERED - 1;
+ await pub.publish('socket.io#/events#', 'frame');
+ queue.length = WS_REDIS_MAX_UNANSWERED * 2;
await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);
- expect(pub.sent).toHaveBeenCalledTimes(1);
+ expect(pub.sent).toHaveBeenCalledTimes(2);
});
});
+ it('drops frames once the queue is at the cap and has been above half of it for longer than a command is given', async () => {
+ let clock = 1_000_000;
+ const now = jest.spyOn(Date, 'now').mockImplementation(() => clock);
+ const queue = { length: 0 };
+ try {
+ await withPublisher(queue, async pub => {
+ await pub.publish('socket.io#/events#', 'frame');
+ queue.length = WS_REDIS_MAX_UNANSWERED / 2;
+ clock += WS_REDIS_COMMAND_TIMEOUT_MS;
+ // At the cap, but only exactly as long as a command is given: not yet.
+ queue.length = WS_REDIS_MAX_UNANSWERED;
+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);
+ clock += 1;
+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(0);
+ expect(pub.sent).toHaveBeenCalledTimes(2);
+ // Dropping goes on until the queue is back below half the cap, not until it is below the cap.
+ queue.length = WS_REDIS_MAX_UNANSWERED - 1;
+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(0);
+ queue.length = WS_REDIS_MAX_UNANSWERED / 2;
+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(0);
+ queue.length = WS_REDIS_MAX_UNANSWERED / 2 - 1;
+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);
+ expect(pub.sent).toHaveBeenCalledTimes(3);
+ });
+ } finally {
+ now.mockRestore();
+ }
+ });
+
+ it('warns once when an episode starts and once when it ends, each with its own count', async () => {
+ let clock = 1_000_000;
+ const now = jest.spyOn(Date, 'now').mockImplementation(() => clock);
+ const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined);
+ const lines = (text: string): string[] =>
+ warn.mock.calls.map(([line]) => String(line)).filter(line => line.includes(text));
+ const queue = { length: 0 };
+ try {
+ await withPublisher(queue, async pub => {
+ const episode = async (frames: number): Promise<void> => {
+ queue.length = 0;
+ await pub.publish('socket.io#/events#', 'frame');
+ queue.length = WS_REDIS_MAX_UNANSWERED / 2;
+ clock += WS_REDIS_COMMAND_TIMEOUT_MS + 1;
+ queue.length = WS_REDIS_MAX_UNANSWERED;
+ for (let frame = 0; frame < frames; frame++) await pub.publish('socket.io#/events#', 'frame');
+ queue.length = 0;
+ await pub.publish('socket.io#/events#', 'frame');
+ await pub.publish('socket.io#/events#', 'frame');
+ };
+
+ await episode(5);
+ await episode(2);
+
+ expect(lines('fan-out frames are dropped')).toHaveLength(2);
+ const ended = lines('were dropped');
+ expect(ended).toHaveLength(2);
+ expect(ended[0]).toContain('5 fan-out frames');
+ expect(ended[1]).toContain('2 fan-out frames');
+ });
+ } finally {
+ now.mockRestore();
+ warn.mockRestore();
+ }
+ });
+
it('applies no cap, and does not fail, when the client exposes no command queue', async () => {
await withPublisher(undefined, async pub => {
await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);
diff --git a/src/modules/events/redis-io.adapter.stall.spec.ts b/src/modules/events/redis-io.adapter.stall.spec.ts
index 7926b08e4..8bbfda6c8 100644
--- a/src/modules/events/redis-io.adapter.stall.spec.ts
+++ b/src/modules/events/redis-io.adapter.stall.spec.ts
@@ -250,36 +250,69 @@ describe('RedisIoAdapter publish client against a Redis that stops answering', (
expect(pub.status).toBe('ready');
}, 15_000);
+ const warned = (text: string): string[] =>
+ warn.mock.calls
+ .flat()
+ .map(line => String(line))
+ .filter(line => line.includes(text));
+
+ // Every frame of a burst raised in one tick is on the queue before the first reply can be read, however healthy the
+ // peer is, so the depth of the queue alone says nothing about a Redis that is behind.
+ it('publishes every frame of a burst larger than the cap raised in one tick to a peer that answers', async () => {
+ const burst = WS_REDIS_MAX_UNANSWERED + 3000;
+
+ broadcast(burst);
+
+ expect(await waitUntil(() => redis.published.length === burst && held() === 0, 15_000)).toBe(true);
+ expect(warned('fan-out frames are dropped')).toHaveLength(0);
+ expect(warned('were dropped')).toHaveLength(0);
+ }, 30_000);
+
// A Redis that answers, but slower than events are raised, resets the socket timer with every reply, so the socket
- // deadline never fires and the client would hold the whole backlog. Only the cap on unanswered commands bounds it.
- it('holds no more unanswered events than the cap when the peer answers too slowly to ever look silent', async () => {
+ // deadline never fires and the client would hold the whole backlog. Only the cap on unanswered commands bounds it,
+ // once the backlog has outlived a command's own deadline. The clock is moved by hand: the rule is about elapsed time.
+ it('holds a slow peer to the cap, warning once when dropping starts and once when it ends, however often it answers', async () => {
+ let clock = 1_000_000;
+ jest.spyOn(Date, 'now').mockImplementation(() => clock);
const connections = redis.connections;
redis.trickle = true;
- const answering = setInterval(() => redis.answerOne(), 1000);
- try {
- broadcast(WS_REDIS_MAX_UNANSWERED + 3000);
- await sleep(2500);
-
- expect(held()).toBeLessThanOrEqual(WS_REDIS_MAX_UNANSWERED);
- expect(held()).toBeGreaterThan(WS_REDIS_MAX_UNANSWERED - 10);
- expect(redis.published.length).toBeLessThanOrEqual(WS_REDIS_MAX_UNANSWERED + 3);
- expect(redis.connections).toBe(connections);
- expect(pub.status).toBe('ready');
- // Once per burst, not once per frame.
- const capLines = warn.mock.calls.flat().filter(line => String(line).includes('fan-out frames are dropped'));
- expect(capLines).toHaveLength(1);
- } finally {
- clearInterval(answering);
+
+ // The queue is half full, which is the last moment it counted as being on top of its work...
+ broadcast(WS_REDIS_MAX_UNANSWERED / 2);
+ // ...and it stays above that for longer than a command is given to be answered.
+ clock += WS_REDIS_COMMAND_TIMEOUT_MS + 100;
+ broadcast(WS_REDIS_MAX_UNANSWERED);
+
+ expect(held()).toBe(WS_REDIS_MAX_UNANSWERED);
+ expect(await waitUntil(() => redis.published.length >= WS_REDIS_MAX_UNANSWERED, 10_000)).toBe(true);
+ expect(redis.published.length).toBe(WS_REDIS_MAX_UNANSWERED);
+ expect(redis.connections).toBe(connections);
+ expect(pub.status).toBe('ready');
+ expect(warned('fan-out frames are dropped')).toHaveLength(1);
+
+ // Each reply frees one slot and the next frame arrives: that must not end the episode, or every reply would
+ // log two lines and let one frame through. It ends when the queue is back below half the cap.
+ for (let reply = 0; reply < 30; reply++) {
+ const before = held();
+ redis.answerOne();
+ expect(await waitUntil(() => held() < before, 3000)).toBe(true);
+ broadcast(1);
}
+ expect(held()).toBe(WS_REDIS_MAX_UNANSWERED - 30);
+ expect(warned('fan-out frames are dropped')).toHaveLength(1);
+ expect(warned('were dropped')).toHaveLength(0);
// The peer catches up: everything it owed is answered, and events are published again.
redis.trickle = false;
redis.answerAll();
- expect(await waitUntil(() => held() === 0, 3000)).toBe(true);
+ expect(await waitUntil(() => held() === 0, 5000)).toBe(true);
const published = redis.published.length;
- broadcast(1);
- expect(await waitUntil(() => redis.published.length === published + 1, 3000)).toBe(true);
- expect(warn.mock.calls.flat().some(line => String(line).includes('were dropped'))).toBe(true);
+ broadcast(2);
+ expect(await waitUntil(() => redis.published.length === published + 2, 5000)).toBe(true);
+ // One line for the whole episode, with its count: 6,000 dropped by the burst and 30 while it was answering.
+ const ended = warned('were dropped');
+ expect(ended).toHaveLength(1);
+ expect(ended[0]).toContain(`${WS_REDIS_MAX_UNANSWERED / 2 + 30} fan-out frames`);
}, 30_000);
it('settles a publish within its deadline, then drops the connection after the socket deadline, holds nothing more, and recovers', async () => {
diff --git a/src/modules/events/redis-io.adapter.ts b/src/modules/events/redis-io.adapter.ts
index d6ccc8110..925d54f18 100644
--- a/src/modules/events/redis-io.adapter.ts
+++ b/src/modules/events/redis-io.adapter.ts
@@ -30,15 +30,21 @@ export const WS_REDIS_COMMAND_TIMEOUT_MS = 2000;
export const WS_REDIS_SOCKET_TIMEOUT_MS = 10_000;
/**
- * Most commands the publisher keeps written and unanswered. A Redis that answers slower than events are raised
- * never trips the socket deadline, so without a cap the backlog grows with the load for as long as the slowness
- * lasts. Frames raised above the cap are dropped for the other replicas until the queue drains, with a warning
- * when dropping starts and another, with the count, when Redis catches up. 12,000 holds a full socket deadline of
- * traffic at 1,000 events a second with a fifth to spare, which measured about 25 MB of heap with 1 KiB frames.
- * It counts commands, not bytes: events with inline media are larger.
+ * Most commands the publisher keeps written and unanswered once the backlog has lasted. A Redis that answers slower
+ * than events are raised never trips the socket deadline, so without a cap the backlog grows with the load for as
+ * long as the slowness lasts. The rule: frames are dropped for the other replicas while the queue holds this many
+ * commands and has not been below half of it for longer than a command is given to be answered
+ * (WS_REDIS_COMMAND_TIMEOUT_MS), and they go on being dropped until the queue is back below half of it. A burst
+ * raised in one tick is never cut, however large, because depth alone says nothing: it is all on the queue before a
+ * reply can be read. Dropping starts with a warning and ends with another, with the count. 12,000 holds a full
+ * socket deadline of traffic at 1,000 events a second with a fifth to spare, which measured about 25 MB of heap with
+ * 1 KiB frames. It counts commands, not bytes: events with inline media are larger.
*/
export const WS_REDIS_MAX_UNANSWERED = 12_000;
+/** The queue length below which a backlog counts as over, and dropped frames are published again. */
+const WS_REDIS_RESUME_UNANSWERED = WS_REDIS_MAX_UNANSWERED / 2;
+
/**
* Commands the client has written and not had answered. ioredis keeps them on an internal queue; if this version
* does not expose it, the answer is 0 and the cap does nothing, so a change of ioredis loses the bound rather than
@@ -106,10 +112,11 @@ export function rejectRootNamespace(server: Server): void {
* other replicas late and after newer events. The events this node raises while it is disconnected, plus its
* reconnect backoff of up to 5 s once Redis is back (wsRedisOptions), are dropped for the other replicas, never
* delivered late and never twice. A Redis that keeps answering, but slower than events are raised, never looks
- * silent: it is bounded by WS_REDIS_MAX_UNANSWERED, above which frames are dropped for the other replicas until
- * it catches up. Clients on this node are served by the local broadcast and webhooks do not use this path, so
- * neither is affected. What the publisher holds is bounded by one socket deadline of traffic for a silent Redis
- * and by that cap for a slow one.
+ * silent: it is bounded by WS_REDIS_MAX_UNANSWERED, which drops frames for the other replicas once the backlog has
+ * outlived a command's deadline and until it is back below half of it (a burst raised in one tick is not cut).
+ * Clients on this node are served by the local broadcast and webhooks do not use this path, so neither is
+ * affected. What the publisher holds is bounded by one socket deadline of traffic for a silent Redis and by that
+ * cap, plus the traffic of one command deadline, for a slow one.
*/
export class RedisIoAdapter extends IoAdapter {
private pubClient?: Redis;
@@ -133,7 +140,7 @@ export class RedisIoAdapter extends IoAdapter {
// a connection it does not have, and nothing the old socket may already have delivered is sent again. An
// event raised while disconnected is therefore dropped for the other replicas rather than replayed late.
// A silent Redis holds at most one socket deadline of traffic; a slow one that keeps answering is held to
- // WS_REDIS_MAX_UNANSWERED by the wrapper below.
+ // WS_REDIS_MAX_UNANSWERED (see there for the rule) by the wrapper below.
const pubClient = new Redis({
...wsRedisOptions(),
commandTimeout: WS_REDIS_COMMAND_TIMEOUT_MS,
@@ -159,21 +166,34 @@ export class RedisIoAdapter extends IoAdapter {
// confirmed delivery is only worth a debug line. A timed-out command was still written, and a slow
// Redis delivers it late; one rejected for lack of a connection was never sent.
const publish = pubClient.publish.bind(pubClient);
- let capped = 0;
+ let dropping = false;
+ let dropped = 0;
+ // When the queue was last seen below half the cap: the backlog is as old as the time since.
+ let lastBelowResume = 0;
pubClient.publish = (channel: string, message: string | Buffer) => {
const unanswered = unansweredCommands(pubClient);
- if (unanswered >= WS_REDIS_MAX_UNANSWERED) {
- // Nothing else reports a Redis that answers slowly, so the first frame of a burst is warned about.
- if (capped++ === 0) {
- logger.warn(
- `Redis has ${unanswered} fan-out frames unanswered; fan-out frames are dropped for the other replicas until it catches up`,
- );
+ const now = Date.now();
+ if (unanswered < WS_REDIS_RESUME_UNANSWERED) {
+ lastBelowResume = now;
+ if (dropping) {
+ logger.warn(`Redis caught up; ${dropped} fan-out frames were dropped for the other replicas`);
+ dropping = false;
+ dropped = 0;
}
- return Promise.resolve(0);
+ } else if (
+ !dropping &&
+ unanswered >= WS_REDIS_MAX_UNANSWERED &&
+ now - lastBelowResume > WS_REDIS_COMMAND_TIMEOUT_MS
+ ) {
+ // Nothing else reports a Redis that answers slowly, so the start of an episode is warned about.
+ dropping = true;
+ logger.warn(
+ `Redis has ${unanswered} fan-out frames unanswered; fan-out frames are dropped for the other replicas until it catches up`,
+ );
}
- if (capped > 0) {
- logger.warn(`Redis caught up; ${capped} fan-out frames were dropped for the other replicas`);
- capped = 0;
+ if (dropping) {
+ dropped++;
+ return Promise.resolve(0);
}
return publish(channel, message).catch((err: Error) => {
logger.debug(`Redis publish not confirmed: ${err.message}`);