Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(events): bound the WebSocket Redis publish client
Yudhi Armyndharis8f8fd69eb818
diff --git a/src/modules/events/redis-io.adapter.spec.ts b/src/modules/events/redis-io.adapter.spec.tsindex 169be78d7..0341b5851 100644--- a/src/modules/events/redis-io.adapter.spec.ts+++ b/src/modules/events/redis-io.adapter.spec.ts@@ -38,7 +38,14 @@ jest.mock('@socket.io/redis-adapter', () => ({createAdapter: (pub: unknown, sub: unknown) => createAdapterMock(pub, sub),}));-import { RedisIoAdapter, isWsRedisEnabled, wsRedisOptions, WS_REDIS_QUIT_TIMEOUT_MS } from './redis-io.adapter';+import {+ RedisIoAdapter,+ isWsRedisEnabled,+ wsRedisOptions,+ WS_REDIS_COMMAND_TIMEOUT_MS,+ WS_REDIS_QUIT_TIMEOUT_MS,+ WS_REDIS_SOCKET_TIMEOUT_MS,+} from './redis-io.adapter';import { IoAdapter } from '@nestjs/platform-socket.io';import type { Server } from 'socket.io';@@ -191,6 +198,36 @@ describe('RedisIoAdapter', () => {}});+ // A Redis that keeps the connection open and stops answering never errors the publisher's socket, so only+ // its own deadlines release what it holds. The subscriber must not inherit them: it waits for no reply, and+ // a destroyed socket or a dropped queue would strand its one-time SUBSCRIBE.+ it('puts the deadlines on the publisher only', () => {+ process.env.REDIS_ENABLED = 'true';+ const spy = withBaseServer(fakeServer().server);+ try {+ new RedisIoAdapter({} as never).createIOServer(2785);+ const [pub, sub] = redisInstances as [FakeRedis, FakeRedis];+ expect(pub.opts).toMatchObject({+ commandTimeout: WS_REDIS_COMMAND_TIMEOUT_MS,+ socketTimeout: WS_REDIS_SOCKET_TIMEOUT_MS,+ enableOfflineQueue: false,+ autoResendUnfulfilledCommands: false,+ });+ expect(WS_REDIS_SOCKET_TIMEOUT_MS).toBeLessThan(WS_REDIS_COMMAND_TIMEOUT_MS);+ for (const option of [+ 'commandTimeout',+ 'socketTimeout',+ 'enableOfflineQueue',+ 'autoResendUnfulfilledCommands',+ ]) {+ expect(sub.opts).not.toHaveProperty(option);+ }+ expect(sub.opts).toMatchObject({ maxRetriesPerRequest: null });+ } finally {+ spy.mockRestore();+ }+ });+it('gives a failed publish an owner, since the adapter drops the promise it returns', async () => {process.env.REDIS_ENABLED = 'true';const { server } = fakeServer();diff --git a/src/modules/events/redis-io.adapter.stall.spec.ts b/src/modules/events/redis-io.adapter.stall.spec.tsnew file mode 100644index 000000000..422d1be18--- /dev/null+++ b/src/modules/events/redis-io.adapter.stall.spec.ts@@ -0,0 +1,186 @@+import * as net from 'node:net';+import type { AddressInfo } from 'node:net';+import { IoAdapter } from '@nestjs/platform-socket.io';+import { Server } from 'socket.io';+import { RedisIoAdapter, WS_REDIS_COMMAND_TIMEOUT_MS } from './redis-io.adapter';++/**+ * A Redis that keeps the connection open and stops answering never errors the socket, so the publish+ * client of the fan-out pair has to bound it itself. This runs the real adapter and the real ioredis+ * client against a peer that speaks just enough RESP to bring a client up, and can then go silent.+ */+class FakeRedis {+ silent = false;+ /** Every PUBLISH payload that reached the peer, answered or not. */+ readonly published: string[] = [];+ readonly sockets = new Set<net.Socket>();+ private readonly server: net.Server;++ constructor() {+ this.server = net.createServer(socket => {+ this.sockets.add(socket);+ socket.on('error', () => undefined);+ socket.on('close', () => this.sockets.delete(socket));+ let pending: Buffer = Buffer.alloc(0);+ socket.on('data', chunk => {+ pending = Buffer.concat([pending, chunk]);+ for (const command of FakeRedis.takeCommands(pending, rest => (pending = rest))) {+ if (command[0].toUpperCase() === 'PUBLISH') this.published.push(command[2]);+ if (!this.silent) socket.write(FakeRedis.reply(command));+ }+ });+ });+ }++ listen(): Promise<number> {+ return new Promise(resolve =>+ this.server.listen(0, '127.0.0.1', () => resolve((this.server.address() as AddressInfo).port)),+ );+ }++ close(): Promise<void> {+ for (const socket of this.sockets) socket.destroy();+ return new Promise(resolve => this.server.close(() => resolve()));+ }++ /** Parse every complete `*N` array of bulk strings off the front of `buf`; the remainder goes to `keep`. */+ private static takeCommands(buf: Buffer, keep: (rest: Buffer) => void): string[][] {+ const commands: string[][] = [];+ let offset = 0;+ for (;;) {+ if (buf[offset] !== 0x2a) break;+ const eol = buf.indexOf('\r\n', offset);+ if (eol < 0) break;+ const count = parseInt(buf.toString('utf8', offset + 1, eol), 10);+ let cursor = eol + 2;+ const parts: string[] = [];+ for (let i = 0; i < count; i++) {+ const lineEnd = buf.indexOf('\r\n', cursor);+ if (lineEnd < 0) break;+ const length = parseInt(buf.toString('utf8', cursor + 1, lineEnd), 10);+ if (buf.length < lineEnd + 2 + length + 2) break;+ parts.push(buf.toString('utf8', lineEnd + 2, lineEnd + 2 + length));+ cursor = lineEnd + 2 + length + 2;+ }+ if (parts.length < count) break;+ commands.push(parts);+ offset = cursor;+ }+ keep(buf.subarray(offset));+ return commands;+ }++ private static reply([name, ...args]: string[]): string {+ switch (name.toUpperCase()) {+ case 'INFO': {+ const info = '# Server\r\nredis_version:7.2.0\r\nloading:0\r\n';+ return `$${info.length}\r\n${info}\r\n`;+ }+ case 'PUBLISH':+ return ':1\r\n';+ case 'SUBSCRIBE':+ case 'PSUBSCRIBE':+ return args+ .map((channel, i) => `*3\r\n$9\r\nsubscribe\r\n$${channel.length}\r\n${channel}\r\n:${i + 1}\r\n`)+ .join('');+ default:+ return '+OK\r\n';+ }+ }+}++type Queue = { length: number };+type PublishClient = {+ status: string;+ commandQueue: Queue;+ offlineQueue: Queue;+ prevCommandQueue?: Queue | null;+ publish: (channel: string, message: string) => Promise<number>;+};++describe('RedisIoAdapter publish client against a Redis that stops answering', () => {+ const prevEnv = { ...process.env };+ let redis: FakeRedis;+ let adapter: RedisIoAdapter;+ let io: Server;+ let pub: PublishClient;++ const sleep = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms));+ /** Everything the client holds for a reply or for a connection: the three places ioredis keeps a command. */+ const held = (): number => pub.commandQueue.length + pub.offlineQueue.length + (pub.prevCommandQueue?.length ?? 0);+ const broadcast = (count: number): void => {+ for (let i = 0; i < count; i++)+ io.of('/events')+ .to('room')+ .emit('message', { body: 'x'.repeat(1024) });+ };+ const waitUntil = async (condition: () => boolean | Promise<boolean>, timeoutMs: number): Promise<boolean> => {+ for (let waited = 0; waited < timeoutMs; waited += 50) {+ if (await condition()) return true;+ await sleep(50);+ }+ return false;+ };++ beforeEach(async () => {+ redis = new FakeRedis();+ const port = await redis.listen();+ process.env.REDIS_ENABLED = 'true';+ process.env.REDIS_HOST = '127.0.0.1';+ process.env.REDIS_PORT = String(port);+ io = new Server();+ jest.spyOn(IoAdapter.prototype, 'createIOServer').mockReturnValue(io);+ jest.spyOn(IoAdapter.prototype, 'close').mockResolvedValue(undefined);+ jest.spyOn(console, 'warn').mockImplementation(() => undefined);+ jest.spyOn(console, 'log').mockImplementation(() => undefined);+ adapter = new RedisIoAdapter({} as never);+ adapter.createIOServer(0);+ pub = (adapter as unknown as { pubClient: PublishClient }).pubClient;+ expect(await waitUntil(() => pub.status === 'ready', 3000)).toBe(true);+ });++ afterEach(async () => {+ redis.silent = false;+ await adapter.close(io);+ await redis.close();+ jest.restoreAllMocks();+ process.env = { ...prevEnv };+ });++ it('still publishes to a healthy peer and holds nothing afterwards', async () => {+ broadcast(20);++ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);+ expect(held()).toBe(0);+ });++ it('settles a publish within its deadline, holds no more events while the peer is silent, and recovers', async () => {+ redis.silent = true;++ // The peer is silent but the connection stays open: nothing errors, so the deadline is the only thing+ // that settles the command.+ const started = Date.now();+ const outcome = await Promise.race([+ pub.publish('socket.io#/events#', 'stalled-frame'),+ sleep(WS_REDIS_COMMAND_TIMEOUT_MS * 2).then(() => 'still pending'),+ ]);+ expect(outcome).toBe(0);+ expect(Date.now() - started).toBeLessThan(WS_REDIS_COMMAND_TIMEOUT_MS + 1000);++ // Past the first deadline, events keep arriving at a steady rate. None of them may pile up on the+ // client, whichever queue ioredis would park them in.+ const before = held();+ for (let batch = 0; batch < 5; batch++) {+ broadcast(100);+ await sleep(100);+ }+ expect(held() - before).toBeLessThan(20);++ // The peer answers again: the client comes back without a restart, and does not send again what the+ // abandoned socket had already put on the wire, which the peer may have delivered.+ redis.silent = false;+ expect(await waitUntil(async () => (await pub.publish('socket.io#/events#', 'probe')) === 1, 10_000)).toBe(true);+ expect(held()).toBe(0);+ expect(redis.published.filter(payload => payload === 'stalled-frame')).toHaveLength(1);+ }, 30_000);+});diff --git a/src/modules/events/redis-io.adapter.ts b/src/modules/events/redis-io.adapter.tsindex fe828d5b0..8cf551b6c 100644--- a/src/modules/events/redis-io.adapter.ts+++ b/src/modules/events/redis-io.adapter.ts@@ -11,6 +11,15 @@ const logger = createLogger('RedisIoAdapter');/** Max time to await a graceful `quit()` per client on shutdown before force-disconnecting. */export const WS_REDIS_QUIT_TIMEOUT_MS = 2000;+/** Upper bound for one fan-out publish, the same bound as the other Redis clients that serve a request or an event. */+export const WS_REDIS_COMMAND_TIMEOUT_MS = 2000;++/**+ * How long the publish client's socket may stay silent after a write before it is destroyed. Kept below the+ * command timeout, as the throttler's is, so the socket is already being replaced when the first command gives up.+ */+export const WS_REDIS_SOCKET_TIMEOUT_MS = 1500;+/*** Whether cross-replica WebSocket fan-out is switched on. Same `REDIS_ENABLED === 'true'` gate the* throttler and cache read at boot, so a deployment already using Redis for those gets event@@ -60,6 +69,7 @@ export function rejectRootNamespace(server: Server): void {* Failure posture: if the pub/sub clients cannot be created the server falls back to the in-memory* adapter — the local node keeps working, only cross-node fan-out is lost — rather than refusing to* boot. A Redis outage, at boot or after it, is ioredis's problem to retry; the adapter recovers on reconnect.+ * Events raised on this node while Redis is unreachable or silent are not replayed to the other replicas.*/export class RedisIoAdapter extends IoAdapter {private pubClient?: Redis;@@ -75,22 +85,35 @@ export class RedisIoAdapter extends IoAdapter {if (!isWsRedisEnabled()) return server;try {- const pubClient = new Redis(wsRedisOptions());+ // The publisher is the only client that waits for replies, and a Redis that keeps the connection open but+ // stops answering (a frozen process, a stalled disk) never errors it: every broadcast would wait in the+ // client, payload and all, for as long as the stall lasts. So it gets the throttler's posture: a command+ // the peer does not answer in time is rejected, a socket that stays silent is destroyed and the client+ // reconnects, nothing is queued while it is disconnected, and nothing the old socket may already have+ // delivered is sent again. A stall costs fan-out frames, never memory or a second delivery.+ const pubClient = new Redis({+ ...wsRedisOptions(),+ commandTimeout: WS_REDIS_COMMAND_TIMEOUT_MS,+ socketTimeout: WS_REDIS_SOCKET_TIMEOUT_MS,+ enableOfflineQueue: false,+ autoResendUnfulfilledCommands: false,+ });// The adapter issues its SUBSCRIBE/PSUBSCRIBE once, at construction. With a bounded per-request// retry, an outage at boot flushes them before Redis is ever reached, and ioredis only replays// subscriptions the server confirmed, so the replica would never subscribe. This connection- // carries nothing else, so keep its commands queued until the first connect.- const subClient = pubClient.duplicate({ maxRetriesPerRequest: null });+ // carries nothing else, so keep its commands queued until the first connect. It waits for no+ // reply either, which is why none of the publisher's deadlines apply to it.+ const subClient = new Redis({ ...wsRedisOptions(), maxRetriesPerRequest: null });for (const [name, client] of [['pub', pubClient],['sub', subClient],] as const) {client.on('error', err => logger.warn(`Redis ${name} client error: ${err.message}`));}- // The adapter fires publish() on every broadcast and drops the promise. During an outage ioredis- // queues those commands and, each time its retries run out, rejects the whole queue, so every- // event emitted meanwhile surfaced as an unhandled rejection. The 'error' listener above already- // reports the outage once; a lost fan-out frame is only worth a debug line.+ // The adapter fires publish() on every broadcast and drops the promise. During an outage each of+ // those commands is rejected, so every event emitted meanwhile surfaced as an unhandled+ // rejection. The 'error' listener above already reports the outage once; a lost fan-out frame is+ // only worth a debug line.const publish = pubClient.publish.bind(pubClient);pubClient.publish = (channel: string, message: string | Buffer) =>publish(channel, message).catch((err: Error) => {