Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(throttler): bound a silent Redis connection with a timeout
Yudhi Armyndharisa145753b8eb0
diff --git a/src/common/throttler/throttler-redis.client.spec.ts b/src/common/throttler/throttler-redis.client.spec.tsindex 992353557..ba8de33aa 100644--- a/src/common/throttler/throttler-redis.client.spec.ts+++ b/src/common/throttler/throttler-redis.client.spec.ts@@ -2,6 +2,7 @@ import {buildThrottlerRedisOptions,createThrottlerRedisClient,THROTTLER_REDIS_COMMAND_TIMEOUT_MS,+ THROTTLER_REDIS_SOCKET_TIMEOUT_MS,} from './throttler-redis.client';const KEYS = ['REDIS_HOST', 'REDIS_PORT', 'REDIS_USERNAME', 'REDIS_PASSWORD', 'REDIS_CONNECT_TIMEOUT_MS', 'REDIS_TLS'];@@ -53,6 +54,12 @@ describe('buildThrottlerRedisOptions', () => {expect(options.autoResendUnfulfilledCommands).toBe(false);// Every command is deadline-bounded, so a dropped in-flight call cannot hang the request.expect(options.commandTimeout).toBe(THROTTLER_REDIS_COMMAND_TIMEOUT_MS);+ // A peer that stays connected but stops answering never errors its socket, so the commandTimeout+ // above alone costs every guarded request one full timeout per throttler tier. The socket timeout+ // destroys that socket so the offline-queue-off rejection makes the following requests fail open at once,+ // and it has to fire before the first command's own deadline or the next tier is written first.+ expect(options.socketTimeout).toBe(THROTTLER_REDIS_SOCKET_TIMEOUT_MS);+ expect(THROTTLER_REDIS_SOCKET_TIMEOUT_MS).toBeLessThan(THROTTLER_REDIS_COMMAND_TIMEOUT_MS);// No queue-based retry flushing — per-command latency is bounded by commandTimeout alone.expect(options.maxRetriesPerRequest).toBeNull();});@@ -66,6 +73,7 @@ describe('createThrottlerRedisClient', () => {expect(client.options.enableOfflineQueue).toBe(false);expect(client.options.autoResendUnfulfilledCommands).toBe(false);expect(client.options.commandTimeout).toBe(THROTTLER_REDIS_COMMAND_TIMEOUT_MS);+ expect(client.options.socketTimeout).toBe(THROTTLER_REDIS_SOCKET_TIMEOUT_MS);expect(client.options.maxRetriesPerRequest).toBeNull();expect(client.options.host).toBe('127.0.0.1');} finally {diff --git a/src/common/throttler/throttler-redis.client.ts b/src/common/throttler/throttler-redis.client.tsindex c7a1fe73b..d8389b72c 100644--- a/src/common/throttler/throttler-redis.client.ts+++ b/src/common/throttler/throttler-redis.client.ts@@ -9,6 +9,13 @@ import { redisConnectionOptions } from '../../config/redis-options';*/export const THROTTLER_REDIS_COMMAND_TIMEOUT_MS = 2000;+/**+ * How long the throttler's socket may stay silent after a write before it is destroyed. Kept below the command+ * timeout: with the two equal, the socket is torn down in the same millisecond the first command gives up, and+ * the next tier's command is written to the still-open socket first, so a second request waits out its own timeout.+ */+export const THROTTLER_REDIS_SOCKET_TIMEOUT_MS = 1500;+/*** Client options for the throttler's Redis backend, tuned so the fail-open posture of* RedisThrottlerStorage engages IMMEDIATELY instead of after a stall:@@ -25,6 +32,11 @@ export const THROTTLER_REDIS_COMMAND_TIMEOUT_MS = 2000;* - maxRetriesPerRequest: null — disables ioredis's periodic pending-queue flush; with the* offline queue disabled there is nothing to flush, and per-command latency is bounded by* commandTimeout instead of a retry counter.+ * - socketTimeout: a peer that stays connected but stops answering never errors the socket, so+ * commandTimeout alone makes every guarded request wait one full timeout per throttler tier+ * (three tiers, 6 s in total) until TCP gives up. socketTimeout destroys the stalled socket before the+ * first command's own deadline, so the first request waits that one timeout and the tiers behind it, and+ * every later request, fail open at once while the client reconnects (the offline queue is off).*/export function buildThrottlerRedisOptions(): RedisOptions {return {@@ -32,6 +44,7 @@ export function buildThrottlerRedisOptions(): RedisOptions {enableOfflineQueue: false,autoResendUnfulfilledCommands: false,commandTimeout: THROTTLER_REDIS_COMMAND_TIMEOUT_MS,+ socketTimeout: THROTTLER_REDIS_SOCKET_TIMEOUT_MS,maxRetriesPerRequest: null,};}diff --git a/test/throttler-redis.e2e-spec.ts b/test/throttler-redis.e2e-spec.tsindex 228815176..170158788 100644--- a/test/throttler-redis.e2e-spec.ts+++ b/test/throttler-redis.e2e-spec.ts@@ -1,7 +1,10 @@import { spawnSync } from 'node:child_process';import { randomUUID } from 'node:crypto';+import * as net from 'node:net';+import type { AddressInfo } from 'node:net';import Redis from 'ioredis';import { RedisThrottlerStorage } from '../src/common/throttler/redis-throttler.storage';+import { buildThrottlerRedisOptions } from '../src/common/throttler/throttler-redis.client';// Runs the throttler's Lua script against a real Redis: the unit spec mocks eval, so a wrong ARGV// index or a repair branch that is never taken would pass there, and increment() fails open on a@@ -83,3 +86,75 @@ describeWithRedis('RedisThrottlerStorage against a real Redis', () => {expect(pttl).toBeGreaterThan(0);});});++// A Redis that stays connected and stops answering never errors its socket. The client the app builds has to+// turn that into one command timeout for the first guarded request and an immediate allow for everything after+// it, instead of one timeout per throttler tier on every request.+describeWithRedis('the throttler client against a Redis that stops answering', () => {+ const throttler = `e2e-stall-${randomUUID()}`;+ const sockets = new Set<net.Socket>();+ let blackhole = false;+ let proxy: net.Server;+ let storage: RedisThrottlerStorage;+ let cleaner: Redis;++ beforeAll(async () => {+ proxy = net.createServer(client => {+ const upstream = net.connect(REDIS_PORT, REDIS_HOST);+ sockets.add(client).add(upstream);+ client.on('data', data => {+ if (!blackhole) upstream.write(data);+ });+ upstream.on('data', data => {+ if (!blackhole) client.write(data);+ });+ const end = (): void => {+ client.destroy();+ upstream.destroy();+ };+ client.on('close', end);+ upstream.on('close', end);+ client.on('error', () => undefined);+ upstream.on('error', () => undefined);+ });+ await new Promise<void>(resolve => proxy.listen(0, '127.0.0.1', resolve));+ const client = new Redis({+ ...buildThrottlerRedisOptions(),+ host: '127.0.0.1',+ port: (proxy.address() as AddressInfo).port,+ });+ client.on('error', () => undefined);+ await new Promise<void>(resolve => client.once('ready', () => resolve()));+ storage = new RedisThrottlerStorage(client);+ jest.spyOn((storage as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});+ cleaner = new Redis({ host: REDIS_HOST, port: REDIS_PORT });+ });++ afterAll(async () => {+ blackhole = false;+ const keys = await cleaner.keys(`openwa:throttle:*:${throttler}*`);+ if (keys.length) await cleaner.del(...keys);+ cleaner.disconnect();+ await storage.onModuleDestroy();+ for (const socket of sockets) socket.destroy();+ await new Promise<void>(resolve => proxy.close(() => resolve()));+ });++ const timed = async (tier: string): Promise<number> => {+ const started = Date.now();+ await storage.increment(throttler, 10_000, 5, 0, tier);+ return Date.now() - started;+ };++ it('costs the first request one command timeout and every request after it nothing', async () => {+ expect((await storage.increment(throttler, 10_000, 5, 0, 'warmup')).totalHits).toBe(1);++ blackhole = true;+ const firstRequest = [await timed('short'), await timed('medium'), await timed('long')];++ // The tiers behind the first command do not each wait out a timeout of their own.+ expect(firstRequest[1] + firstRequest[2]).toBeLessThan(1000);+ expect(firstRequest[0] + firstRequest[1] + firstRequest[2]).toBeLessThan(3000);+ expect(await timed('short')).toBeLessThan(500);+ }, 20_000);+});