Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(events): bound the WebSocket Redis publish client

Yudhi Armyndharis8f8fd69eb818
diff --git a/src/modules/events/redis-io.adapter.spec.ts b/src/modules/events/redis-io.adapter.spec.ts
index 169be78d7..0341b5851 100644
--- a/src/modules/events/redis-io.adapter.spec.ts
+++ b/src/modules/events/redis-io.adapter.spec.ts
@@ -38,7 +38,14 @@ jest.mock('@socket.io/redis-adapter', () => ({
createAdapter: (pub: unknown, sub: unknown) => createAdapterMock(pub, sub),
}));
-import { RedisIoAdapter, isWsRedisEnabled, wsRedisOptions, WS_REDIS_QUIT_TIMEOUT_MS } from './redis-io.adapter';
+import {
+ RedisIoAdapter,
+ isWsRedisEnabled,
+ wsRedisOptions,
+ WS_REDIS_COMMAND_TIMEOUT_MS,
+ WS_REDIS_QUIT_TIMEOUT_MS,
+ WS_REDIS_SOCKET_TIMEOUT_MS,
+} from './redis-io.adapter';
import { IoAdapter } from '@nestjs/platform-socket.io';
import type { Server } from 'socket.io';
@@ -191,6 +198,36 @@ describe('RedisIoAdapter', () => {
}
});
+ // A Redis that keeps the connection open and stops answering never errors the publisher's socket, so only
+ // its own deadlines release what it holds. The subscriber must not inherit them: it waits for no reply, and
+ // a destroyed socket or a dropped queue would strand its one-time SUBSCRIBE.
+ it('puts the deadlines on the publisher only', () => {
+ process.env.REDIS_ENABLED = 'true';
+ const spy = withBaseServer(fakeServer().server);
+ try {
+ new RedisIoAdapter({} as never).createIOServer(2785);
+ const [pub, sub] = redisInstances as [FakeRedis, FakeRedis];
+ expect(pub.opts).toMatchObject({
+ commandTimeout: WS_REDIS_COMMAND_TIMEOUT_MS,
+ socketTimeout: WS_REDIS_SOCKET_TIMEOUT_MS,
+ enableOfflineQueue: false,
+ autoResendUnfulfilledCommands: false,
+ });
+ expect(WS_REDIS_SOCKET_TIMEOUT_MS).toBeLessThan(WS_REDIS_COMMAND_TIMEOUT_MS);
+ for (const option of [
+ 'commandTimeout',
+ 'socketTimeout',
+ 'enableOfflineQueue',
+ 'autoResendUnfulfilledCommands',
+ ]) {
+ expect(sub.opts).not.toHaveProperty(option);
+ }
+ expect(sub.opts).toMatchObject({ maxRetriesPerRequest: null });
+ } finally {
+ spy.mockRestore();
+ }
+ });
+
it('gives a failed publish an owner, since the adapter drops the promise it returns', async () => {
process.env.REDIS_ENABLED = 'true';
const { server } = fakeServer();
diff --git a/src/modules/events/redis-io.adapter.stall.spec.ts b/src/modules/events/redis-io.adapter.stall.spec.ts
new file mode 100644
index 000000000..422d1be18
--- /dev/null
+++ b/src/modules/events/redis-io.adapter.stall.spec.ts
@@ -0,0 +1,186 @@
+import * as net from 'node:net';
+import type { AddressInfo } from 'node:net';
+import { IoAdapter } from '@nestjs/platform-socket.io';
+import { Server } from 'socket.io';
+import { RedisIoAdapter, WS_REDIS_COMMAND_TIMEOUT_MS } from './redis-io.adapter';
+
+/**
+ * A Redis that keeps the connection open and stops answering never errors the socket, so the publish
+ * client of the fan-out pair has to bound it itself. This runs the real adapter and the real ioredis
+ * client against a peer that speaks just enough RESP to bring a client up, and can then go silent.
+ */
+class FakeRedis {
+ silent = false;
+ /** Every PUBLISH payload that reached the peer, answered or not. */
+ readonly published: string[] = [];
+ readonly sockets = new Set<net.Socket>();
+ private readonly server: net.Server;
+
+ constructor() {
+ this.server = net.createServer(socket => {
+ this.sockets.add(socket);
+ socket.on('error', () => undefined);
+ socket.on('close', () => this.sockets.delete(socket));
+ let pending: Buffer = Buffer.alloc(0);
+ socket.on('data', chunk => {
+ pending = Buffer.concat([pending, chunk]);
+ for (const command of FakeRedis.takeCommands(pending, rest => (pending = rest))) {
+ if (command[0].toUpperCase() === 'PUBLISH') this.published.push(command[2]);
+ if (!this.silent) socket.write(FakeRedis.reply(command));
+ }
+ });
+ });
+ }
+
+ listen(): Promise<number> {
+ return new Promise(resolve =>
+ this.server.listen(0, '127.0.0.1', () => resolve((this.server.address() as AddressInfo).port)),
+ );
+ }
+
+ close(): Promise<void> {
+ for (const socket of this.sockets) socket.destroy();
+ return new Promise(resolve => this.server.close(() => resolve()));
+ }
+
+ /** Parse every complete `*N` array of bulk strings off the front of `buf`; the remainder goes to `keep`. */
+ private static takeCommands(buf: Buffer, keep: (rest: Buffer) => void): string[][] {
+ const commands: string[][] = [];
+ let offset = 0;
+ for (;;) {
+ if (buf[offset] !== 0x2a) break;
+ const eol = buf.indexOf('\r\n', offset);
+ if (eol < 0) break;
+ const count = parseInt(buf.toString('utf8', offset + 1, eol), 10);
+ let cursor = eol + 2;
+ const parts: string[] = [];
+ for (let i = 0; i < count; i++) {
+ const lineEnd = buf.indexOf('\r\n', cursor);
+ if (lineEnd < 0) break;
+ const length = parseInt(buf.toString('utf8', cursor + 1, lineEnd), 10);
+ if (buf.length < lineEnd + 2 + length + 2) break;
+ parts.push(buf.toString('utf8', lineEnd + 2, lineEnd + 2 + length));
+ cursor = lineEnd + 2 + length + 2;
+ }
+ if (parts.length < count) break;
+ commands.push(parts);
+ offset = cursor;
+ }
+ keep(buf.subarray(offset));
+ return commands;
+ }
+
+ private static reply([name, ...args]: string[]): string {
+ switch (name.toUpperCase()) {
+ case 'INFO': {
+ const info = '# Server\r\nredis_version:7.2.0\r\nloading:0\r\n';
+ return `$${info.length}\r\n${info}\r\n`;
+ }
+ case 'PUBLISH':
+ return ':1\r\n';
+ case 'SUBSCRIBE':
+ case 'PSUBSCRIBE':
+ return args
+ .map((channel, i) => `*3\r\n$9\r\nsubscribe\r\n$${channel.length}\r\n${channel}\r\n:${i + 1}\r\n`)
+ .join('');
+ default:
+ return '+OK\r\n';
+ }
+ }
+}
+
+type Queue = { length: number };
+type PublishClient = {
+ status: string;
+ commandQueue: Queue;
+ offlineQueue: Queue;
+ prevCommandQueue?: Queue | null;
+ publish: (channel: string, message: string) => Promise<number>;
+};
+
+describe('RedisIoAdapter publish client against a Redis that stops answering', () => {
+ const prevEnv = { ...process.env };
+ let redis: FakeRedis;
+ let adapter: RedisIoAdapter;
+ let io: Server;
+ let pub: PublishClient;
+
+ const sleep = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms));
+ /** Everything the client holds for a reply or for a connection: the three places ioredis keeps a command. */
+ const held = (): number => pub.commandQueue.length + pub.offlineQueue.length + (pub.prevCommandQueue?.length ?? 0);
+ const broadcast = (count: number): void => {
+ for (let i = 0; i < count; i++)
+ io.of('/events')
+ .to('room')
+ .emit('message', { body: 'x'.repeat(1024) });
+ };
+ const waitUntil = async (condition: () => boolean | Promise<boolean>, timeoutMs: number): Promise<boolean> => {
+ for (let waited = 0; waited < timeoutMs; waited += 50) {
+ if (await condition()) return true;
+ await sleep(50);
+ }
+ return false;
+ };
+
+ beforeEach(async () => {
+ redis = new FakeRedis();
+ const port = await redis.listen();
+ process.env.REDIS_ENABLED = 'true';
+ process.env.REDIS_HOST = '127.0.0.1';
+ process.env.REDIS_PORT = String(port);
+ io = new Server();
+ jest.spyOn(IoAdapter.prototype, 'createIOServer').mockReturnValue(io);
+ jest.spyOn(IoAdapter.prototype, 'close').mockResolvedValue(undefined);
+ jest.spyOn(console, 'warn').mockImplementation(() => undefined);
+ jest.spyOn(console, 'log').mockImplementation(() => undefined);
+ adapter = new RedisIoAdapter({} as never);
+ adapter.createIOServer(0);
+ pub = (adapter as unknown as { pubClient: PublishClient }).pubClient;
+ expect(await waitUntil(() => pub.status === 'ready', 3000)).toBe(true);
+ });
+
+ afterEach(async () => {
+ redis.silent = false;
+ await adapter.close(io);
+ await redis.close();
+ jest.restoreAllMocks();
+ process.env = { ...prevEnv };
+ });
+
+ it('still publishes to a healthy peer and holds nothing afterwards', async () => {
+ broadcast(20);
+
+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(1);
+ expect(held()).toBe(0);
+ });
+
+ it('settles a publish within its deadline, holds no more events while the peer is silent, and recovers', async () => {
+ redis.silent = true;
+
+ // The peer is silent but the connection stays open: nothing errors, so the deadline is the only thing
+ // that settles the command.
+ const started = Date.now();
+ const outcome = await Promise.race([
+ pub.publish('socket.io#/events#', 'stalled-frame'),
+ sleep(WS_REDIS_COMMAND_TIMEOUT_MS * 2).then(() => 'still pending'),
+ ]);
+ expect(outcome).toBe(0);
+ expect(Date.now() - started).toBeLessThan(WS_REDIS_COMMAND_TIMEOUT_MS + 1000);
+
+ // Past the first deadline, events keep arriving at a steady rate. None of them may pile up on the
+ // client, whichever queue ioredis would park them in.
+ const before = held();
+ for (let batch = 0; batch < 5; batch++) {
+ broadcast(100);
+ await sleep(100);
+ }
+ expect(held() - before).toBeLessThan(20);
+
+ // The peer answers again: the client comes back without a restart, and does not send again what the
+ // abandoned socket had already put on the wire, which the peer may have delivered.
+ redis.silent = false;
+ expect(await waitUntil(async () => (await pub.publish('socket.io#/events#', 'probe')) === 1, 10_000)).toBe(true);
+ expect(held()).toBe(0);
+ expect(redis.published.filter(payload => payload === 'stalled-frame')).toHaveLength(1);
+ }, 30_000);
+});
diff --git a/src/modules/events/redis-io.adapter.ts b/src/modules/events/redis-io.adapter.ts
index fe828d5b0..8cf551b6c 100644
--- a/src/modules/events/redis-io.adapter.ts
+++ b/src/modules/events/redis-io.adapter.ts
@@ -11,6 +11,15 @@ const logger = createLogger('RedisIoAdapter');
/** Max time to await a graceful `quit()` per client on shutdown before force-disconnecting. */
export const WS_REDIS_QUIT_TIMEOUT_MS = 2000;
+/** Upper bound for one fan-out publish, the same bound as the other Redis clients that serve a request or an event. */
+export const WS_REDIS_COMMAND_TIMEOUT_MS = 2000;
+
+/**
+ * How long the publish client's socket may stay silent after a write before it is destroyed. Kept below the
+ * command timeout, as the throttler's is, so the socket is already being replaced when the first command gives up.
+ */
+export const WS_REDIS_SOCKET_TIMEOUT_MS = 1500;
+
/**
* Whether cross-replica WebSocket fan-out is switched on. Same `REDIS_ENABLED === 'true'` gate the
* throttler and cache read at boot, so a deployment already using Redis for those gets event
@@ -60,6 +69,7 @@ export function rejectRootNamespace(server: Server): void {
* Failure posture: if the pub/sub clients cannot be created the server falls back to the in-memory
* adapter — the local node keeps working, only cross-node fan-out is lost — rather than refusing to
* boot. A Redis outage, at boot or after it, is ioredis's problem to retry; the adapter recovers on reconnect.
+ * Events raised on this node while Redis is unreachable or silent are not replayed to the other replicas.
*/
export class RedisIoAdapter extends IoAdapter {
private pubClient?: Redis;
@@ -75,22 +85,35 @@ export class RedisIoAdapter extends IoAdapter {
if (!isWsRedisEnabled()) return server;
try {
- const pubClient = new Redis(wsRedisOptions());
+ // The publisher is the only client that waits for replies, and a Redis that keeps the connection open but
+ // stops answering (a frozen process, a stalled disk) never errors it: every broadcast would wait in the
+ // client, payload and all, for as long as the stall lasts. So it gets the throttler's posture: a command
+ // the peer does not answer in time is rejected, a socket that stays silent is destroyed and the client
+ // reconnects, nothing is queued while it is disconnected, and nothing the old socket may already have
+ // delivered is sent again. A stall costs fan-out frames, never memory or a second delivery.
+ const pubClient = new Redis({
+ ...wsRedisOptions(),
+ commandTimeout: WS_REDIS_COMMAND_TIMEOUT_MS,
+ socketTimeout: WS_REDIS_SOCKET_TIMEOUT_MS,
+ enableOfflineQueue: false,
+ autoResendUnfulfilledCommands: false,
+ });
// The adapter issues its SUBSCRIBE/PSUBSCRIBE once, at construction. With a bounded per-request
// retry, an outage at boot flushes them before Redis is ever reached, and ioredis only replays
// subscriptions the server confirmed, so the replica would never subscribe. This connection
- // carries nothing else, so keep its commands queued until the first connect.
- const subClient = pubClient.duplicate({ maxRetriesPerRequest: null });
+ // carries nothing else, so keep its commands queued until the first connect. It waits for no
+ // reply either, which is why none of the publisher's deadlines apply to it.
+ const subClient = new Redis({ ...wsRedisOptions(), maxRetriesPerRequest: null });
for (const [name, client] of [
['pub', pubClient],
['sub', subClient],
] as const) {
client.on('error', err => logger.warn(`Redis ${name} client error: ${err.message}`));
}
- // The adapter fires publish() on every broadcast and drops the promise. During an outage ioredis
- // queues those commands and, each time its retries run out, rejects the whole queue, so every
- // event emitted meanwhile surfaced as an unhandled rejection. The 'error' listener above already
- // reports the outage once; a lost fan-out frame is only worth a debug line.
+ // The adapter fires publish() on every broadcast and drops the promise. During an outage each of
+ // those commands is rejected, so every event emitted meanwhile surfaced as an unhandled
+ // rejection. The 'error' listener above already reports the outage once; a lost fan-out frame is
+ // only worth a debug line.
const publish = pubClient.publish.bind(pubClient);
pubClient.publish = (channel: string, message: string | Buffer) =>
publish(channel, message).catch((err: Error) => {