Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(events): keep the publish socket through a busy event loop
Yudhi Armyndharisbe92482a0520
diff --git a/src/modules/events/redis-io.adapter.spec.ts b/src/modules/events/redis-io.adapter.spec.tsindex 5730fb266..1e7c168c7 100644--- a/src/modules/events/redis-io.adapter.spec.ts+++ b/src/modules/events/redis-io.adapter.spec.ts@@ -209,7 +209,11 @@ describe('RedisIoAdapter', () => {enableOfflineQueue: false,autoResendUnfulfilledCommands: false,});- expect(WS_REDIS_SOCKET_TIMEOUT_MS).toBeLessThan(WS_REDIS_COMMAND_TIMEOUT_MS);+ // Both are real deadlines, and the socket may not be cut off before a command has had its full time: the+ // socket timer runs on this process's own event loop, so a short one drops healthy connections.+ expect(Number.isFinite(WS_REDIS_COMMAND_TIMEOUT_MS)).toBe(true);+ expect(Number.isFinite(WS_REDIS_SOCKET_TIMEOUT_MS)).toBe(true);+ expect(WS_REDIS_SOCKET_TIMEOUT_MS).toBeGreaterThanOrEqual(WS_REDIS_COMMAND_TIMEOUT_MS);for (const option of ['commandTimeout','socketTimeout',diff --git a/src/modules/events/redis-io.adapter.stall.spec.ts b/src/modules/events/redis-io.adapter.stall.spec.tsindex 455b48cbe..fddb4e604 100644--- a/src/modules/events/redis-io.adapter.stall.spec.ts+++ b/src/modules/events/redis-io.adapter.stall.spec.ts@@ -1,8 +1,9 @@+import * as fs from 'node:fs';import * as net from 'node:net';import type { AddressInfo } from 'node:net';import { IoAdapter } from '@nestjs/platform-socket.io';import { Server } from 'socket.io';-import { RedisIoAdapter, WS_REDIS_COMMAND_TIMEOUT_MS } from './redis-io.adapter';+import { RedisIoAdapter, WS_REDIS_COMMAND_TIMEOUT_MS, WS_REDIS_SOCKET_TIMEOUT_MS } from './redis-io.adapter';/*** A Redis that keeps the connection open and stops answering never errors the socket, so the publish@@ -10,14 +11,28 @@ import { RedisIoAdapter, WS_REDIS_COMMAND_TIMEOUT_MS } from './redis-io.adapter'* client against a peer that speaks just enough RESP to bring a client up, and can then go silent.*/class FakeRedis {- silent = false;/** Every PUBLISH payload that reached the peer, answered or not. */readonly published: string[] = [];+ /** Connections accepted so far: a client that destroys its socket and reconnects raises it. */+ connections = 0;readonly sockets = new Set<net.Socket>();+ private stalled = false;+ /** What a stalled peer owes its clients. Sent when it comes back, as a Redis that was paused does. */+ private readonly owed: Array<() => void> = [];private readonly server: net.Server;+ get silent(): boolean {+ return this.stalled;+ }++ set silent(value: boolean) {+ this.stalled = value;+ if (!value) for (const send of this.owed.splice(0)) send();+ }+constructor() {this.server = net.createServer(socket => {+ this.connections++;this.sockets.add(socket);socket.on('error', () => undefined);socket.on('close', () => this.sockets.delete(socket));@@ -26,10 +41,13 @@ class FakeRedis {pending = Buffer.concat([pending, chunk]);for (const command of FakeRedis.takeCommands(pending, rest => (pending = rest))) {if (command[0].toUpperCase() === 'PUBLISH') this.published.push(command[2]);- // A stalled peer answers nothing the publisher or its reconnect asks. Subscriptions are exempt: the adapter- // issues them without a handler, when a namespace is first used, and a SUBSCRIBE that no reply ever- // settles is rejected as an unhandled rejection when the client is closed, which ends a Jest worker.- if (!this.silent || /^p?subscribe$/i.test(command[0])) socket.write(FakeRedis.reply(command));+ // A stalled peer answers nothing the publisher or its reconnect asks until it comes back. Subscriptions+ // are exempt: the adapter issues them without a handler, when a namespace is first used, and a SUBSCRIBE+ // that no reply ever settles is rejected as an unhandled rejection when the client is closed, which ends+ // a Jest worker.+ const reply = FakeRedis.reply(command);+ if (!this.silent || /^p?subscribe$/i.test(command[0])) socket.write(reply);+ else this.owed.push(() => !socket.destroyed && socket.write(reply));}});});@@ -107,10 +125,18 @@ describe('RedisIoAdapter publish client against a Redis that stops answering', (let adapter: RedisIoAdapter;let io: Server;let pub: PublishClient;+ let warn: jest.SpyInstance;const sleep = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms));- /** Everything the client holds for a reply or for a connection: the three places ioredis keeps a command. */- const held = (): number => pub.commandQueue.length + pub.offlineQueue.length + (pub.prevCommandQueue?.length ?? 0);+ /**+ * Everything the client holds for a reply or for a connection: the three places ioredis keeps a command. Between+ * a destroyed socket and the next connect the command queue and the previous one are the same object.+ */+ const held = (): number => {+ const queues = new Set<Queue>([pub.commandQueue, pub.offlineQueue]);+ if (pub.prevCommandQueue) queues.add(pub.prevCommandQueue);+ return [...queues].reduce((total, queue) => total + queue.length, 0);+ };const broadcast = (count: number): void => {for (let i = 0; i < count; i++)io.of('/events')@@ -134,7 +160,7 @@ describe('RedisIoAdapter publish client against a Redis that stops answering', (io = new Server();jest.spyOn(IoAdapter.prototype, 'createIOServer').mockReturnValue(io);jest.spyOn(IoAdapter.prototype, 'close').mockResolvedValue(undefined);- jest.spyOn(console, 'warn').mockImplementation(() => undefined);+ warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined);jest.spyOn(console, 'log').mockImplementation(() => undefined);adapter = new RedisIoAdapter({} as never);adapter.createIOServer(0);@@ -157,7 +183,54 @@ describe('RedisIoAdapter publish client against a Redis that stops answering', (expect(held()).toBe(0);});- it('settles a publish within its deadline, holds no more events while the peer is silent, and recovers', async () => {+ // ioredis arms the socket timer on the write, on this process's own event loop. A loop that stays busy for longer+ // than the deadline right after a write expires that timer before the reply already waiting in the socket is+ // read, so a healthy peer would look stalled and the client would drop its connection. The block runs in an I/O+ // callback, as the engine and database callbacks do; inside a timer callback the loop clock is stale and the same+ // block would not show it.+ it('does not treat a healthy peer as stalled when this process blocks its loop right after a publish', async () => {+ const connections = redis.connections;++ await new Promise<void>(resolve =>+ fs.readFile(__filename, () => {+ broadcast(1);+ for (const until = Date.now() + 2000; Date.now() < until;) {+ // hold the loop+ }+ resolve();+ }),+ );+ // Long enough for a destroyed socket to have been replaced.+ await sleep(1500);++ expect(redis.published.length).toBe(1);+ expect(redis.connections).toBe(connections);+ expect(pub.status).toBe('ready');+ expect(held()).toBe(0);+ expect(warn.mock.calls.flat().some(line => String(line).includes('Socket timeout'))).toBe(false);+ }, 15_000);++ // The pause of a busy Redis (a snapshot fork, an fsync) ends. Everything raised meanwhile is on the wire, so it+ // arrives once the peer is back, provided the client has not given up on the connection first.+ it('loses no event when the peer stays silent for longer than the command deadline but less than the socket deadline', async () => {+ const connections = redis.connections;+ let sent = 0;++ redis.silent = true;+ for (let batch = 0; batch < 25; batch++) {+ broadcast(4);+ sent += 4;+ await sleep(100);+ }+ redis.silent = false;++ expect(await waitUntil(() => held() === 0, 3000)).toBe(true);+ expect(redis.published.length).toBe(sent);+ expect(redis.connections).toBe(connections);+ expect(pub.status).toBe('ready');+ }, 15_000);++ it('settles a publish within its deadline, then drops the connection after the socket deadline, holds nothing more, and recovers', async () => {redis.silent = true;// The peer is silent but the connection stays open: nothing errors, so the deadline is the only thing@@ -170,20 +243,33 @@ describe('RedisIoAdapter publish client against a Redis that stops answering', (expect(outcome).toBe(0);expect(Date.now() - started).toBeLessThan(WS_REDIS_COMMAND_TIMEOUT_MS + 1000);- // Past the first deadline, events keep arriving at a steady rate. None of them may pile up on the- // client, whichever queue ioredis would park them in.- const before = held();- for (let batch = 0; batch < 5; batch++) {- broadcast(100);- await sleep(100);+ // Events keep arriving at a steady rate. The client holds them until its socket deadline destroys the+ // connection, and that is all it holds.+ const connections = redis.connections;+ let heldAtDestroy = 0;+ const destroyed = await waitUntil(() => {+ broadcast(10);+ heldAtDestroy = held();+ return pub.status !== 'ready';+ }, WS_REDIS_SOCKET_TIMEOUT_MS + 3000);+ expect(destroyed).toBe(true);+ expect(heldAtDestroy).toBeLessThan((WS_REDIS_SOCKET_TIMEOUT_MS / 1000 + 2) * 200);++ // From here the client is reconnecting: nothing is queued for a connection it does not have, however many+ // events are raised. (Each batch of the wait above was 10 events per 50 ms.) The window outlasts the command+ // deadline, so the timers of what the abandoned socket still owes are gone before the test ends.+ for (let waited = 0; waited < WS_REDIS_COMMAND_TIMEOUT_MS + 500; waited += 50) {+ broadcast(10);+ await sleep(50);}- expect(held() - before).toBeLessThan(20);+ expect(held() - heldAtDestroy).toBeLessThan(20);// The peer answers again: the client comes back without a restart, and does not send again what the// abandoned socket had already put on the wire, which the peer may have delivered.redis.silent = false;expect(await waitUntil(async () => (await pub.publish('socket.io#/events#', 'probe')) === 1, 10_000)).toBe(true);expect(held()).toBe(0);+ expect(redis.connections).toBeGreaterThan(connections);expect(redis.published.filter(payload => payload === 'stalled-frame')).toHaveLength(1);- }, 30_000);+ }, 60_000);});diff --git a/src/modules/events/redis-io.adapter.ts b/src/modules/events/redis-io.adapter.tsindex 8cf551b6c..732258164 100644--- a/src/modules/events/redis-io.adapter.ts+++ b/src/modules/events/redis-io.adapter.ts@@ -11,14 +11,20 @@ const logger = createLogger('RedisIoAdapter');/** Max time to await a graceful `quit()` per client on shutdown before force-disconnecting. */export const WS_REDIS_QUIT_TIMEOUT_MS = 2000;-/** Upper bound for one fan-out publish, the same bound as the other Redis clients that serve a request or an event. */+/**+ * How long the promise of one fan-out publish waits for its reply. Only the promise: the command stays on the+ * wire, and a Redis that was merely slow still delivers it, so this bounds nothing the client holds.+ */export const WS_REDIS_COMMAND_TIMEOUT_MS = 2000;/**- * How long the publish client's socket may stay silent after a write before it is destroyed. Kept below the- * command timeout, as the throttler's is, so the socket is already being replaced when the first command gives up.+ * How long the publish client's socket may stay silent after a write before it is destroyed, which is what+ * releases what a stalled Redis left in the client. The timer runs on this process's own event loop, so a healthy+ * Redis looks silent whenever the loop stays blocked for that long right after a write, and the connection would+ * be dropped for nothing. That is why it sits far above any block this process runs, and above the command+ * timeout. What the client holds during a stall is bounded by this many seconds of traffic.*/-export const WS_REDIS_SOCKET_TIMEOUT_MS = 1500;+export const WS_REDIS_SOCKET_TIMEOUT_MS = 10_000;/*** Whether cross-replica WebSocket fan-out is switched on. Same `REDIS_ENABLED === 'true'` gate the@@ -87,10 +93,11 @@ export class RedisIoAdapter extends IoAdapter {try {// The publisher is the only client that waits for replies, and a Redis that keeps the connection open but// stops answering (a frozen process, a stalled disk) never errors it: every broadcast would wait in the- // client, payload and all, for as long as the stall lasts. So it gets the throttler's posture: a command- // the peer does not answer in time is rejected, a socket that stays silent is destroyed and the client- // reconnects, nothing is queued while it is disconnected, and nothing the old socket may already have- // delivered is sent again. A stall costs fan-out frames, never memory or a second delivery.+ // client, payload and all, for as long as the stall lasts. So the promise of a command the peer does not+ // answer in time is rejected (the command stays on the wire), a socket that stays silent for the socket+ // deadline is destroyed and the client reconnects, nothing is queued while it is disconnected, and nothing+ // the old socket may already have delivered is sent again. A stall holds at most that many seconds of+ // traffic and never delivers an event twice.const pubClient = new Redis({...wsRedisOptions(),commandTimeout: WS_REDIS_COMMAND_TIMEOUT_MS,