Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(throttler): bound a silent Redis connection with a timeout

Yudhi Armyndharisa145753b8eb0
diff --git a/src/common/throttler/throttler-redis.client.spec.ts b/src/common/throttler/throttler-redis.client.spec.ts
index 992353557..ba8de33aa 100644
--- a/src/common/throttler/throttler-redis.client.spec.ts
+++ b/src/common/throttler/throttler-redis.client.spec.ts
@@ -2,6 +2,7 @@ import {
buildThrottlerRedisOptions,
createThrottlerRedisClient,
THROTTLER_REDIS_COMMAND_TIMEOUT_MS,
+ THROTTLER_REDIS_SOCKET_TIMEOUT_MS,
} from './throttler-redis.client';
const KEYS = ['REDIS_HOST', 'REDIS_PORT', 'REDIS_USERNAME', 'REDIS_PASSWORD', 'REDIS_CONNECT_TIMEOUT_MS', 'REDIS_TLS'];
@@ -53,6 +54,12 @@ describe('buildThrottlerRedisOptions', () => {
expect(options.autoResendUnfulfilledCommands).toBe(false);
// Every command is deadline-bounded, so a dropped in-flight call cannot hang the request.
expect(options.commandTimeout).toBe(THROTTLER_REDIS_COMMAND_TIMEOUT_MS);
+ // A peer that stays connected but stops answering never errors its socket, so the commandTimeout
+ // above alone costs every guarded request one full timeout per throttler tier. The socket timeout
+ // destroys that socket so the offline-queue-off rejection makes the following requests fail open at once,
+ // and it has to fire before the first command's own deadline or the next tier is written first.
+ expect(options.socketTimeout).toBe(THROTTLER_REDIS_SOCKET_TIMEOUT_MS);
+ expect(THROTTLER_REDIS_SOCKET_TIMEOUT_MS).toBeLessThan(THROTTLER_REDIS_COMMAND_TIMEOUT_MS);
// No queue-based retry flushing — per-command latency is bounded by commandTimeout alone.
expect(options.maxRetriesPerRequest).toBeNull();
});
@@ -66,6 +73,7 @@ describe('createThrottlerRedisClient', () => {
expect(client.options.enableOfflineQueue).toBe(false);
expect(client.options.autoResendUnfulfilledCommands).toBe(false);
expect(client.options.commandTimeout).toBe(THROTTLER_REDIS_COMMAND_TIMEOUT_MS);
+ expect(client.options.socketTimeout).toBe(THROTTLER_REDIS_SOCKET_TIMEOUT_MS);
expect(client.options.maxRetriesPerRequest).toBeNull();
expect(client.options.host).toBe('127.0.0.1');
} finally {
diff --git a/src/common/throttler/throttler-redis.client.ts b/src/common/throttler/throttler-redis.client.ts
index c7a1fe73b..d8389b72c 100644
--- a/src/common/throttler/throttler-redis.client.ts
+++ b/src/common/throttler/throttler-redis.client.ts
@@ -9,6 +9,13 @@ import { redisConnectionOptions } from '../../config/redis-options';
*/
export const THROTTLER_REDIS_COMMAND_TIMEOUT_MS = 2000;
+/**
+ * How long the throttler's socket may stay silent after a write before it is destroyed. Kept below the command
+ * timeout: with the two equal, the socket is torn down in the same millisecond the first command gives up, and
+ * the next tier's command is written to the still-open socket first, so a second request waits out its own timeout.
+ */
+export const THROTTLER_REDIS_SOCKET_TIMEOUT_MS = 1500;
+
/**
* Client options for the throttler's Redis backend, tuned so the fail-open posture of
* RedisThrottlerStorage engages IMMEDIATELY instead of after a stall:
@@ -25,6 +32,11 @@ export const THROTTLER_REDIS_COMMAND_TIMEOUT_MS = 2000;
* - maxRetriesPerRequest: null — disables ioredis's periodic pending-queue flush; with the
* offline queue disabled there is nothing to flush, and per-command latency is bounded by
* commandTimeout instead of a retry counter.
+ * - socketTimeout: a peer that stays connected but stops answering never errors the socket, so
+ * commandTimeout alone makes every guarded request wait one full timeout per throttler tier
+ * (three tiers, 6 s in total) until TCP gives up. socketTimeout destroys the stalled socket before the
+ * first command's own deadline, so the first request waits that one timeout and the tiers behind it, and
+ * every later request, fail open at once while the client reconnects (the offline queue is off).
*/
export function buildThrottlerRedisOptions(): RedisOptions {
return {
@@ -32,6 +44,7 @@ export function buildThrottlerRedisOptions(): RedisOptions {
enableOfflineQueue: false,
autoResendUnfulfilledCommands: false,
commandTimeout: THROTTLER_REDIS_COMMAND_TIMEOUT_MS,
+ socketTimeout: THROTTLER_REDIS_SOCKET_TIMEOUT_MS,
maxRetriesPerRequest: null,
};
}
diff --git a/test/throttler-redis.e2e-spec.ts b/test/throttler-redis.e2e-spec.ts
index 228815176..170158788 100644
--- a/test/throttler-redis.e2e-spec.ts
+++ b/test/throttler-redis.e2e-spec.ts
@@ -1,7 +1,10 @@
import { spawnSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
+import * as net from 'node:net';
+import type { AddressInfo } from 'node:net';
import Redis from 'ioredis';
import { RedisThrottlerStorage } from '../src/common/throttler/redis-throttler.storage';
+import { buildThrottlerRedisOptions } from '../src/common/throttler/throttler-redis.client';
// Runs the throttler's Lua script against a real Redis: the unit spec mocks eval, so a wrong ARGV
// index or a repair branch that is never taken would pass there, and increment() fails open on a
@@ -83,3 +86,75 @@ describeWithRedis('RedisThrottlerStorage against a real Redis', () => {
expect(pttl).toBeGreaterThan(0);
});
});
+
+// A Redis that stays connected and stops answering never errors its socket. The client the app builds has to
+// turn that into one command timeout for the first guarded request and an immediate allow for everything after
+// it, instead of one timeout per throttler tier on every request.
+describeWithRedis('the throttler client against a Redis that stops answering', () => {
+ const throttler = `e2e-stall-${randomUUID()}`;
+ const sockets = new Set<net.Socket>();
+ let blackhole = false;
+ let proxy: net.Server;
+ let storage: RedisThrottlerStorage;
+ let cleaner: Redis;
+
+ beforeAll(async () => {
+ proxy = net.createServer(client => {
+ const upstream = net.connect(REDIS_PORT, REDIS_HOST);
+ sockets.add(client).add(upstream);
+ client.on('data', data => {
+ if (!blackhole) upstream.write(data);
+ });
+ upstream.on('data', data => {
+ if (!blackhole) client.write(data);
+ });
+ const end = (): void => {
+ client.destroy();
+ upstream.destroy();
+ };
+ client.on('close', end);
+ upstream.on('close', end);
+ client.on('error', () => undefined);
+ upstream.on('error', () => undefined);
+ });
+ await new Promise<void>(resolve => proxy.listen(0, '127.0.0.1', resolve));
+ const client = new Redis({
+ ...buildThrottlerRedisOptions(),
+ host: '127.0.0.1',
+ port: (proxy.address() as AddressInfo).port,
+ });
+ client.on('error', () => undefined);
+ await new Promise<void>(resolve => client.once('ready', () => resolve()));
+ storage = new RedisThrottlerStorage(client);
+ jest.spyOn((storage as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});
+ cleaner = new Redis({ host: REDIS_HOST, port: REDIS_PORT });
+ });
+
+ afterAll(async () => {
+ blackhole = false;
+ const keys = await cleaner.keys(`openwa:throttle:*:${throttler}*`);
+ if (keys.length) await cleaner.del(...keys);
+ cleaner.disconnect();
+ await storage.onModuleDestroy();
+ for (const socket of sockets) socket.destroy();
+ await new Promise<void>(resolve => proxy.close(() => resolve()));
+ });
+
+ const timed = async (tier: string): Promise<number> => {
+ const started = Date.now();
+ await storage.increment(throttler, 10_000, 5, 0, tier);
+ return Date.now() - started;
+ };
+
+ it('costs the first request one command timeout and every request after it nothing', async () => {
+ expect((await storage.increment(throttler, 10_000, 5, 0, 'warmup')).totalHits).toBe(1);
+
+ blackhole = true;
+ const firstRequest = [await timed('short'), await timed('medium'), await timed('long')];
+
+ // The tiers behind the first command do not each wait out a timeout of their own.
+ expect(firstRequest[1] + firstRequest[2]).toBeLessThan(1000);
+ expect(firstRequest[0] + firstRequest[1] + firstRequest[2]).toBeLessThan(3000);
+ expect(await timed('short')).toBeLessThan(500);
+ }, 20_000);
+});