Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(events): ping the WebSocket Redis subscriber connection

Yudhi Armyndharisdc5ea512fc3c
diff --git a/docs/13-horizontal-scaling.md b/docs/13-horizontal-scaling.md
index 0f0749a97..e2ef0b30c 100644
--- a/docs/13-horizontal-scaling.md
+++ b/docs/13-horizontal-scaling.md
@@ -170,7 +170,13 @@
> as raising it past its 6,000th frame takes under 2 s: a healthy Redis answers it right after, and the node
> holds it until then. A synchronous burst that takes longer than that to raise past 6,000 frames is cut at
> 12,000 even against a healthy Redis, and the warning names the unanswered count. A replica whose own Redis
-> connection is reconnecting also misses what the others publish meanwhile.
+> connection is reconnecting also misses what the others publish meanwhile. The connection a replica receives
+> on only listens, so it is pinged every 15 s and replaced when an answer takes over 30 s: one that stops
+> answering without being closed (a proxy in the path that keeps its side open, a NAT or a firewall that drops
+> the flow) costs that replica at most about 45 s of the others' events before it subscribes again. A Redis
+> that pauses for up to 30 s keeps that connection, so the replica still receives late what the others had
+> written before their own 10 s deadline; a longer silence while a ping waits replaces it, and what the others
+> had published into that silence is then lost for that replica.
>
> **Mid-connection key eviction converges on a timer, not a broadcast, and only on one node.** The
> node that processes a revoke, delete, or narrowing tears down that key's sockets synchronously, in
diff --git a/src/modules/events/redis-io.adapter.spec.ts b/src/modules/events/redis-io.adapter.spec.ts
index 026083ce9..2f7f5ae58 100644
--- a/src/modules/events/redis-io.adapter.spec.ts
+++ b/src/modules/events/redis-io.adapter.spec.ts
@@ -11,6 +11,8 @@ class FakeRedis {
readonly handlers: Record<string, (err: Error) => void> = {};
quit = jest.fn().mockResolvedValue('OK');
disconnect = jest.fn();
+ status = 'ready';
+ ping = jest.fn().mockResolvedValue('PONG');
readonly sent = jest.fn().mockResolvedValue(1);
publish(channel: string, message: string): Promise<number> {
return this.sent(channel, message) as Promise<number>;
@@ -42,6 +44,8 @@ import {
WS_REDIS_MAX_UNANSWERED,
WS_REDIS_QUIT_TIMEOUT_MS,
WS_REDIS_SOCKET_TIMEOUT_MS,
+ WS_REDIS_SUBSCRIBER_PING_MS,
+ WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS,
} from './redis-io.adapter';
import { IoAdapter } from '@nestjs/platform-socket.io';
import type { Server } from 'socket.io';
@@ -196,9 +200,10 @@ describe('RedisIoAdapter', () => {
});
// A Redis that keeps the connection open and stops answering never errors the publisher's socket, so only
- // its own deadlines release what it holds. The subscriber must not inherit them: it waits for no reply, and
- // a destroyed socket or a dropped queue would strand its one-time SUBSCRIBE.
- it('puts the deadlines on the publisher only', () => {
+ // its own deadlines release what it holds. The subscriber gets a socket deadline of its own, which its pings
+ // arm, and nothing else: a rejected or dropped command would strand its one-time SUBSCRIBE, while a destroyed
+ // socket does not, since ioredis subscribes again on the next connection.
+ it('puts the command deadlines on the publisher only, and a socket deadline on each', () => {
process.env.REDIS_ENABLED = 'true';
const spy = withBaseServer(fakeServer().server);
try {
@@ -218,15 +223,13 @@ describe('RedisIoAdapter', () => {
// The figure docs/13 and the adapter comment quote ("a pause under 10 s is delivered late", "at most 10 s of
// events"): the other tests derive their waits from the constant and would follow it to any value.
expect(WS_REDIS_SOCKET_TIMEOUT_MS).toBe(10_000);
- for (const option of [
- 'commandTimeout',
- 'socketTimeout',
- 'enableOfflineQueue',
- 'autoResendUnfulfilledCommands',
- ]) {
+ for (const option of ['commandTimeout', 'enableOfflineQueue', 'autoResendUnfulfilledCommands']) {
expect(sub.opts).not.toHaveProperty(option);
}
- expect(sub.opts).toMatchObject({ maxRetriesPerRequest: null });
+ expect(sub.opts).toMatchObject({
+ maxRetriesPerRequest: null,
+ socketTimeout: WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS,
+ });
} finally {
spy.mockRestore();
}
@@ -400,6 +403,97 @@ describe('RedisIoAdapter', () => {
});
});
+ // The subscriber only receives, so a connection that stops answering without being closed shows nothing until TCP
+ // keepalive gives up, and never behind a proxy that keeps its side open. A ping arms the socket deadline that
+ // replaces it.
+ describe('the subscriber ping', () => {
+ const withSubscriber = async (
+ check: (pub: FakeRedis, sub: FakeRedis, adapter: RedisIoAdapter, server: Server) => Promise<void>,
+ ): Promise<void> => {
+ jest.useFakeTimers({ doNotFake: ['setImmediate'] });
+ process.env.REDIS_ENABLED = 'true';
+ const { server } = fakeServer();
+ const createSpy = withBaseServer(server);
+ const closeSpy = jest.spyOn(IoAdapter.prototype, 'close').mockResolvedValue(undefined);
+ try {
+ const adapter = new RedisIoAdapter({} as never);
+ adapter.createIOServer(2785);
+ const [pub, sub] = redisInstances as [FakeRedis, FakeRedis];
+ await check(pub, sub, adapter, server);
+ } finally {
+ jest.useRealTimers();
+ createSpy.mockRestore();
+ closeSpy.mockRestore();
+ }
+ };
+
+ it('pins the figures docs/13 quotes, and keeps the subscriber through a pause the publisher gives up on', () => {
+ expect(WS_REDIS_SUBSCRIBER_PING_MS).toBe(15_000);
+ expect(WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS).toBe(30_000);
+ expect(WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS).toBeGreaterThan(WS_REDIS_SOCKET_TIMEOUT_MS);
+ // "At most about 45 s": a flow dropped right after an answered ping, the next ping, then the deadline.
+ expect(WS_REDIS_SUBSCRIBER_PING_MS + WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS).toBe(45_000);
+ });
+
+ it('pings the subscriber every interval while it is ready, and never while it is not', async () => {
+ await withSubscriber(async (pub, sub) => {
+ await jest.advanceTimersByTimeAsync(WS_REDIS_SUBSCRIBER_PING_MS - 1);
+ expect(sub.ping).not.toHaveBeenCalled();
+ await jest.advanceTimersByTimeAsync(1);
+ expect(sub.ping).toHaveBeenCalledTimes(1);
+
+ // A ping issued while reconnecting would wait in the offline queue for as long as the outage lasts.
+ sub.status = 'reconnecting';
+ await jest.advanceTimersByTimeAsync(WS_REDIS_SUBSCRIBER_PING_MS * 4);
+ expect(sub.ping).toHaveBeenCalledTimes(1);
+
+ sub.status = 'ready';
+ await jest.advanceTimersByTimeAsync(WS_REDIS_SUBSCRIBER_PING_MS);
+ expect(sub.ping).toHaveBeenCalledTimes(2);
+ expect(pub.ping).not.toHaveBeenCalled();
+ });
+ });
+
+ it('gives a failed ping an owner', async () => {
+ const unhandled = jest.fn();
+ process.on('unhandledRejection', unhandled);
+ try {
+ await withSubscriber(async (_pub, sub) => {
+ sub.ping.mockRejectedValue(new Error('Connection is closed.'));
+ await jest.advanceTimersByTimeAsync(WS_REDIS_SUBSCRIBER_PING_MS);
+ await new Promise(resolve => setImmediate(resolve));
+ expect(sub.ping).toHaveBeenCalledTimes(1);
+ });
+ expect(unhandled).not.toHaveBeenCalled();
+ } finally {
+ process.off('unhandledRejection', unhandled);
+ }
+ });
+
+ it('does not keep the process alive with its timer', () => {
+ process.env.REDIS_ENABLED = 'true';
+ const spy = withBaseServer(fakeServer().server);
+ try {
+ const adapter = new RedisIoAdapter({} as never);
+ adapter.createIOServer(2785);
+ const timer = (adapter as unknown as { subPing: NodeJS.Timeout }).subPing;
+ expect(timer.hasRef()).toBe(false);
+ clearInterval(timer);
+ } finally {
+ spy.mockRestore();
+ }
+ });
+
+ it('stops pinging once the adapter is closed', async () => {
+ await withSubscriber(async (_pub, sub, adapter, server) => {
+ await adapter.close(server);
+ await jest.advanceTimersByTimeAsync(WS_REDIS_SUBSCRIBER_PING_MS * 3);
+ expect(sub.ping).not.toHaveBeenCalled();
+ expect(jest.getTimerCount()).toBe(0);
+ });
+ });
+ });
+
describe('close', () => {
it('quits both pub and sub clients', async () => {
process.env.REDIS_ENABLED = 'true';
diff --git a/src/modules/events/redis-io.adapter.stall.spec.ts b/src/modules/events/redis-io.adapter.stall.spec.ts
index 11a31f2e1..f0156dec4 100644
--- a/src/modules/events/redis-io.adapter.stall.spec.ts
+++ b/src/modules/events/redis-io.adapter.stall.spec.ts
@@ -7,7 +7,10 @@ import {
RedisIoAdapter,
WS_REDIS_COMMAND_TIMEOUT_MS,
WS_REDIS_MAX_UNANSWERED,
+ WS_REDIS_QUIT_TIMEOUT_MS,
WS_REDIS_SOCKET_TIMEOUT_MS,
+ WS_REDIS_SUBSCRIBER_PING_MS,
+ WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS,
} from './redis-io.adapter';
/**
@@ -23,6 +26,10 @@ class FakeRedis {
readonly sockets = new Set<net.Socket>();
/** When set, a PUBLISH is answered one at a time through answerOne(): a Redis that is slow, not silent. */
trickle = false;
+ /** PINGs answered so far. */
+ pings = 0;
+ /** The subscriptions each connection asked for, in the order the connections asked. */
+ readonly subscriptions = new Map<net.Socket, string[]>();
private stalled = false;
private readonly slowReplies: Array<() => void> = [];
/** What a stalled peer owes its clients. Sent when it comes back, as a Redis that was paused does. */
@@ -39,6 +46,19 @@ class FakeRedis {
for (const send of this.slowReplies.splice(0)) send();
}
+ /**
+ * Stop reading every connection that has subscribed, without closing it, as behind a proxy that lost its upstream
+ * or a paused Redis. New connections are served. What arrives meanwhile waits in the socket and is answered once
+ * they are resumed.
+ */
+ pauseSubscribers(): void {
+ for (const socket of this.subscriptions.keys()) socket.pause();
+ }
+
+ resumeSubscribers(): void {
+ for (const socket of this.subscriptions.keys()) socket.resume();
+ }
+
get silent(): boolean {
return this.stalled;
}
@@ -59,6 +79,11 @@ class FakeRedis {
pending = Buffer.concat([pending, chunk]);
for (const command of FakeRedis.takeCommands(pending, rest => (pending = rest))) {
if (command[0].toUpperCase() === 'PUBLISH') this.published.push(command[2]);
+ if (command[0].toUpperCase() === 'PING') this.pings++;
+ if (/^p?subscribe$/i.test(command[0])) {
+ const asked = this.subscriptions.get(socket) ?? [];
+ this.subscriptions.set(socket, [...asked, ...command.slice(1).map(channel => `${command[0]} ${channel}`)]);
+ }
// A stalled peer answers nothing the publisher or its reconnect asks until it comes back. Subscriptions
// are exempt: the adapter issues them without a handler, when a namespace is first used, and a SUBSCRIBE
// that no reply ever settles is rejected as an unhandled rejection when the client is closed, which ends
@@ -119,11 +144,17 @@ class FakeRedis {
}
case 'PUBLISH':
return ':1\r\n';
+ case 'PING':
+ return '+PONG\r\n';
+ // The client said HELLO 3, so each subscription is confirmed as a push of its own kind, as Redis 6 and
+ // later do. ioredis records what a push confirms and subscribes to exactly that again after a reconnect.
case 'SUBSCRIBE':
- case 'PSUBSCRIBE':
+ case 'PSUBSCRIBE': {
+ const kind = name.toLowerCase();
return args
- .map((channel, i) => `*3\r\n$9\r\nsubscribe\r\n$${channel.length}\r\n${channel}\r\n:${i + 1}\r\n`)
+ .map((channel, i) => `>3\r\n$${kind.length}\r\n${kind}\r\n$${channel.length}\r\n${channel}\r\n:${i + 1}\r\n`)
.join('');
+ }
default:
return '+OK\r\n';
}
@@ -377,3 +408,116 @@ describe('RedisIoAdapter publish client against a Redis that stops answering', (
expect(redis.published.filter(payload => payload === 'stalled-frame')).toHaveLength(1);
}, 60_000);
});
+
+/**
+ * The subscriber only receives, so a connection that stops answering while the far side keeps it open, as a proxy
+ * does, errors nothing: the client stays ready and the replica misses what the others publish. Real adapter and real
+ * ioredis against the same peer, on fake timers so the ping interval and the socket deadline cost no wall time;
+ * socket I/O stays real and is read between steps.
+ */
+describe('RedisIoAdapter subscriber against a connection that goes quiet while idle', () => {
+ const prevEnv = { ...process.env };
+ let redis: FakeRedis;
+ let adapter: RedisIoAdapter;
+ let io: Server;
+ let sub: { status: string; commandQueue: Queue };
+ let warn: jest.SpyInstance;
+
+ /** Let real socket I/O run until the condition holds, for up to `ms` of wall time. */
+ const settle = async (condition: () => boolean, ms = 3000): Promise<boolean> => {
+ const until = process.hrtime.bigint() + BigInt(ms) * 1_000_000n;
+ while (!condition() && process.hrtime.bigint() < until) await new Promise(resolve => setImmediate(resolve));
+ return condition();
+ };
+ const idle = (): boolean => sub.status === 'ready' && sub.commandQueue.length === 0;
+ const warned = (text: string): number =>
+ warn.mock.calls
+ .flat()
+ .map(line => String(line))
+ .filter(line => line.includes(text)).length;
+
+ beforeEach(async () => {
+ jest.useFakeTimers({ doNotFake: ['nextTick', 'setImmediate', 'clearImmediate', 'queueMicrotask', 'hrtime'] });
+ redis = new FakeRedis();
+ const port = await redis.listen();
+ process.env.REDIS_ENABLED = 'true';
+ process.env.REDIS_HOST = '127.0.0.1';
+ process.env.REDIS_PORT = String(port);
+ io = new Server();
+ jest.spyOn(IoAdapter.prototype, 'createIOServer').mockReturnValue(io);
+ jest.spyOn(IoAdapter.prototype, 'close').mockResolvedValue(undefined);
+ warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined);
+ jest.spyOn(console, 'log').mockImplementation(() => undefined);
+ adapter = new RedisIoAdapter({} as never);
+ adapter.createIOServer(0);
+ io.of('/events');
+ sub = (adapter as unknown as { subClient: typeof sub }).subClient;
+ expect(await settle(() => idle() && redis.subscriptions.size === 1)).toBe(true);
+ });
+
+ afterEach(async () => {
+ const closed = adapter.close(io);
+ await jest.advanceTimersByTimeAsync(WS_REDIS_QUIT_TIMEOUT_MS);
+ await closed;
+ jest.useRealTimers();
+ await redis.close();
+ jest.restoreAllMocks();
+ process.env = { ...prevEnv };
+ });
+
+ it('replaces a subscriber connection that stops answering, and subscribes again on the new one', async () => {
+ const [first] = redis.subscriptions.keys();
+ const connections = redis.connections;
+ redis.pauseSubscribers();
+
+ // The next ping goes unanswered. Until the deadline the client still looks ready.
+ await jest.advanceTimersByTimeAsync(WS_REDIS_SUBSCRIBER_PING_MS);
+ expect(await settle(() => sub.commandQueue.length === 1)).toBe(true);
+ await jest.advanceTimersByTimeAsync(WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS - 1);
+ expect(await settle(() => sub.status !== 'ready', 500)).toBe(false);
+
+ // At the deadline the socket is destroyed, and after the first reconnect delay a new connection subscribes.
+ await jest.advanceTimersByTimeAsync(1);
+ expect(await settle(() => sub.status === 'reconnecting')).toBe(true);
+ expect(warned('Redis sub client error: Socket timeout')).toBe(1);
+ await jest.advanceTimersByTimeAsync(500);
+ expect(await settle(() => idle() && redis.subscriptions.size === 2)).toBe(true);
+ expect(redis.connections).toBe(connections + 1);
+ const [, second] = redis.subscriptions.keys();
+ expect([...redis.subscriptions.get(second)!].sort()).toEqual([...redis.subscriptions.get(first)!].sort());
+ }, 20_000);
+
+ it('keeps an idle subscriber connection that answers its pings', async () => {
+ const connections = redis.connections;
+
+ // Each step lets the answer be read before the clock moves on, as it is in real time. The six intervals span the
+ // socket deadline several times over, so a deadline the answers did not clear would have fired.
+ for (let ping = 1; ping <= 6; ping++) {
+ await jest.advanceTimersByTimeAsync(WS_REDIS_SUBSCRIBER_PING_MS);
+ expect(await settle(() => redis.pings === ping && idle())).toBe(true);
+ }
+ expect(WS_REDIS_SUBSCRIBER_PING_MS * 6).toBeGreaterThan(WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS * 2);
+
+ expect(redis.connections).toBe(connections);
+ expect(sub.status).toBe('ready');
+ expect(warned('Socket timeout')).toBe(0);
+ }, 20_000);
+
+ // A Redis that pauses (a snapshot fork, a slow disk) for longer than the publisher's deadline still delivers, once
+ // it resumes, what the others had written before giving up. The subscriber keeps its connection to receive it.
+ it('keeps a subscriber connection whose ping is answered after a pause longer than the publish deadline', async () => {
+ const connections = redis.connections;
+ redis.pauseSubscribers();
+
+ await jest.advanceTimersByTimeAsync(WS_REDIS_SUBSCRIBER_PING_MS);
+ expect(await settle(() => sub.commandQueue.length === 1)).toBe(true);
+ await jest.advanceTimersByTimeAsync(WS_REDIS_SOCKET_TIMEOUT_MS + 3000);
+ redis.resumeSubscribers();
+ expect(await settle(() => idle() && redis.pings === 1)).toBe(true);
+ await jest.advanceTimersByTimeAsync(WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS);
+
+ expect(await settle(() => sub.status !== 'ready', 500)).toBe(false);
+ expect(redis.connections).toBe(connections);
+ expect(warned('Socket timeout')).toBe(0);
+ }, 20_000);
+});
diff --git a/src/modules/events/redis-io.adapter.ts b/src/modules/events/redis-io.adapter.ts
index fbcdbcc36..a439e5efe 100644
--- a/src/modules/events/redis-io.adapter.ts
+++ b/src/modules/events/redis-io.adapter.ts
@@ -29,6 +29,23 @@ export const WS_REDIS_COMMAND_TIMEOUT_MS = 2000;
*/
export const WS_REDIS_SOCKET_TIMEOUT_MS = 10_000;
+/**
+ * How often the subscriber connection is pinged. It only receives, so a connection that stops answering without
+ * being closed (a NAT or a firewall that drops the flow, a proxy that keeps its own side open) would otherwise go
+ * unnoticed until TCP keepalive gives up, or for good behind such a proxy, and the node would miss what the other
+ * replicas publish meanwhile. A ping arms the subscriber's socket deadline, so a connection that does not answer is
+ * replaced, and its subscriptions restored, within this interval plus WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS.
+ */
+export const WS_REDIS_SUBSCRIBER_PING_MS = 15_000;
+
+/**
+ * How long the subscriber's socket may stay silent after a ping before it is destroyed. Above the publisher's
+ * deadline on purpose: a replaced subscriber misses what Redis delivers before it has subscribed again, so through
+ * a pause longer than WS_REDIS_SOCKET_TIMEOUT_MS this node still receives, late, what the others had written before
+ * their own deadline, rather than losing it. Up to this long a pause costs nothing more than it did without pings.
+ */
+export const WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS = 30_000;
+
/**
* Most commands the publisher keeps written and unanswered once the backlog has lasted. A Redis that answers slower
* than events are raised never trips the socket deadline, so without a cap the backlog grows with the load for as
@@ -107,18 +124,23 @@ export function rejectRootNamespace(server: Server): void {
* Failure posture: if the pub/sub clients cannot be created the server falls back to the in-memory
* adapter — the local node keeps working, only cross-node fan-out is lost — rather than refusing to
* boot. A Redis outage, at boot or after it, is ioredis's problem to retry; the adapter recovers on reconnect.
+ * The subscriber only listens, so a connection that stops answering without being closed errors nothing until TCP
+ * keepalive gives up, and never when a proxy in the path keeps its side open: it is pinged every
+ * WS_REDIS_SUBSCRIBER_PING_MS and replaced, and subscribed again, when an answer takes longer than
+ * WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS.
*
* Delivery while Redis is away or slow: the fan-out to the other replicas is best effort and keeps no replay
* queue. A Redis that pauses for less than WS_REDIS_SOCKET_TIMEOUT_MS answers afterwards, and every event raised
* meanwhile reaches them late, at up to about 1,200 events a second; above that the cap below applies first, and
* drops the frames beyond it. Silence for that long destroys the publisher's socket, and so does a restart, a
* failover or an outage: what was written to the dropped connection may be lost with it, or may still reach the
- * other replicas late and after newer events. The events this node raises while it is disconnected, plus its
- * reconnect backoff of up to 5 s once Redis is back (wsRedisOptions), are dropped for the other replicas, never
- * delivered late and never twice. A Redis that keeps answering, but slower than events are raised, never looks
- * silent: it is bounded by WS_REDIS_MAX_UNANSWERED, which drops frames for the other replicas once the backlog has
- * outlived a command's deadline and until it is back below half of it (a burst raised in one tick is not cut
- * unless raising it past half the cap takes over 2 s).
+ * other replicas late and after newer events. A subscriber whose ping waits through a silence longer than
+ * WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS is replaced too, and this node then misses what the others had published
+ * into it. The events this node raises while it is disconnected, plus its reconnect backoff of up to 5 s once Redis
+ * is back (wsRedisOptions), are dropped for the other replicas, never delivered late and never twice. A Redis that
+ * keeps answering, but slower than events are raised, never looks silent: it is bounded by WS_REDIS_MAX_UNANSWERED,
+ * which drops frames for the other replicas once the backlog has outlived a command's deadline and until it is back
+ * below half of it (a burst raised in one tick is not cut unless raising it past half the cap takes over 2 s).
* Clients on this node are served by the local broadcast and webhooks do not use this path, so neither is
* affected. What the publisher holds is bounded by one socket deadline of traffic for a silent Redis and by that
* cap, plus the traffic of one command deadline, for a slow one.
@@ -126,6 +148,7 @@ export function rejectRootNamespace(server: Server): void {
export class RedisIoAdapter extends IoAdapter {
private pubClient?: Redis;
private subClient?: Redis;
+ private subPing?: NodeJS.Timeout;
constructor(app: INestApplicationContext) {
super(app);
@@ -156,9 +179,16 @@ export class RedisIoAdapter extends IoAdapter {
// The adapter issues its SUBSCRIBE/PSUBSCRIBE once, at construction. With a bounded per-request
// retry, an outage at boot flushes them before Redis is ever reached, and ioredis only replays
// subscriptions the server confirmed, so the replica would never subscribe. This connection
- // carries nothing else, so keep its commands queued until the first connect. It waits for no
- // reply either, which is why none of the publisher's deadlines apply to it.
- const subClient = new Redis({ ...wsRedisOptions(), maxRetriesPerRequest: null });
+ // carries nothing else, so keep its commands queued until the first connect. Nothing waits on
+ // its replies, so the publisher's command deadline and dropped queues do not apply to it. A
+ // socket deadline of its own does, armed by the pings below: a destroyed socket loses no
+ // subscription, since on the next ready ioredis subscribes again to what the server had
+ // confirmed and resends the commands it had not answered.
+ const subClient = new Redis({
+ ...wsRedisOptions(),
+ maxRetriesPerRequest: null,
+ socketTimeout: WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS,
+ });
for (const [name, client] of [
['pub', pubClient],
['sub', subClient],
@@ -210,6 +240,12 @@ export class RedisIoAdapter extends IoAdapter {
this.pubClient = pubClient;
this.subClient = subClient;
server.adapter(createAdapter(pubClient, subClient));
+ // Only a ready connection is pinged: one issued while reconnecting would wait in the offline queue
+ // for as long as the outage lasts. A failed ping needs nothing more, the deadline or the next ping
+ // covers the connection. The timer alone does not keep the process alive.
+ this.subPing = setInterval(() => {
+ if (subClient.status === 'ready') subClient.ping().catch(() => undefined);
+ }, WS_REDIS_SUBSCRIBER_PING_MS).unref();
logger.log('WebSocket events fan out across replicas via the Redis adapter');
} catch (error) {
logger.error(
@@ -229,6 +265,7 @@ export class RedisIoAdapter extends IoAdapter {
// a shutdown that had in fact gone fine. Close the server first, then release the pair. Under
// forceCloseConnections Nest skips the server close when the socket.io server shares the HTTP
// server, so only the pub/sub pair is released here and the HTTP close tears the sockets down.
+ clearInterval(this.subPing);
await super.close(server);
await Promise.allSettled([this.quitClient(this.pubClient), this.quitClient(this.subClient)]);
this.pubClient = undefined;