Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(queue): bound producer commands once Redis has connected
Yudhi Armyndharisde133a3dad5c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e0b966c22..9a74765bf 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -43,6 +43,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Bound each webhook and ingress queue command at 2 s once Redis has connected, so a Redis that stops answering falls back to direct delivery and inline dispatch instead of holding every dispatch and ingress request.- Answer `400` for a send route's `chatId` or a bulk `batchId` over 255 characters, where PostgreSQL answered `500` once the value passed its index entry limit.- Index ingress dead letters by delivery id, so the reconciler's per-delivery checks stop scanning every dead letter of an instance; the first boot builds the index.- Replay stranded ingress events past one whose stored payload is not valid JSON, instead of failing every sweep on it.diff --git a/src/modules/queue/redis-connection.spec.ts b/src/modules/queue/redis-connection.spec.tsindex 35de02889..70a9e0671 100644--- a/src/modules/queue/redis-connection.spec.ts+++ b/src/modules/queue/redis-connection.spec.ts@@ -1,3 +1,5 @@+import { createServer, type AddressInfo, type Server, type Socket } from 'node:net';+import Redis from 'ioredis';import {ingressWorkerConcurrency,producerConnectWaitMs,@@ -22,6 +24,10 @@ describe('workerConnectionOptions (webhook Worker connection)', () => {expect(opts.enableOfflineQueue).toBeUndefined();});+ it('sets no command timeout: the Worker blocks on Redis by design', () => {+ expect((workerConnectionOptions() as unknown as Record<string, unknown>).commandTimeout).toBeUndefined();+ });+it('reads host/port/username/password/connectTimeout from env with safe defaults', () => {process.env = { ...ORIGINAL_ENV };delete process.env.REDIS_HOST;@@ -58,6 +64,83 @@ describe('workerConnectionOptions (webhook Worker connection)', () => {});});+/** The command names in a chunk of RESP arrays, lower-cased: `*2\r\n$3\r\nGET\r\n$1\r\na\r\n` is ['get']. */+function commandNames(chunk: string): string[] {+ const lines = chunk.split('\r\n');+ const names: string[] = [];+ for (let i = 0; i < lines.length;) {+ const header = /^\*(\d+)$/.exec(lines[i]);+ if (!header) {+ i += 1;+ continue;+ }+ names.push((lines[i + 2] ?? '').toLowerCase());+ i += 1 + Number(header[1]) * 2;+ }+ return names;+}++describe('queueConnectionOptions on a Redis that stops answering', () => {+ // The socket stays open and nothing comes back, as in a network partition or a frozen Redis process.+ // Without a deadline every producer call (the queue add of a delivery, a job lookup) then never settles,+ // so the dispatch slot or the ingress request holding it never frees and the fallback never runs.+ let server: Server;+ let sockets: Socket[];+ let answering: boolean;+ let client: Redis;++ beforeEach(async () => {+ sockets = [];+ answering = true;+ server = createServer(socket => {+ sockets.push(socket);+ socket.on('error', () => undefined);+ socket.on('data', chunk => {+ if (!answering) return;+ // One reply per command in the chunk, enough RESP for the handshake and a nil GET.+ for (const name of commandNames(chunk.toString())) {+ socket.write(+ name === 'hello' ? "-ERR unknown command 'hello'\r\n" : name === 'client' ? '+OK\r\n' : '$-1\r\n',+ );+ }+ });+ });+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));+ client = new Redis({+ ...queueConnectionOptions(),+ host: '127.0.0.1',+ port: (server.address() as AddressInfo).port,+ enableReadyCheck: false,+ retryStrategy: () => null,+ });+ // With no offline queue, a command issued before the connect rejects at once.+ await new Promise(resolve => client.once('ready', resolve));+ });++ afterEach(async () => {+ client.disconnect();+ sockets.forEach(socket => socket.destroy());+ await new Promise(resolve => server.close(resolve));+ });++ it('rejects a command that gets no reply, instead of holding its caller for good', async () => {+ await expect(client.get('first')).resolves.toBeNull();+ answering = false;++ // Read off the options, with the 2 s the producer is held to as the fallback of a build without one.+ const budget = (queueConnectionOptions() as { commandTimeout?: number }).commandTimeout ?? 2000;+ const outcome = await Promise.race([+ client.get('second').then(+ () => 'answered',+ (error: unknown) => (error instanceof Error ? error.message : String(error)),+ ),+ new Promise<string>(resolve => setTimeout(resolve, budget + 1500, 'still waiting')),+ ]);++ expect(outcome).toMatch(/timed out/i);+ });+});+describe('webhookWorkerConcurrency', () => {const ORIGINAL_ENV = process.env;afterEach(() => {diff --git a/src/modules/queue/redis-connection.ts b/src/modules/queue/redis-connection.tsindex 13dc1ceda..5842ae940 100644--- a/src/modules/queue/redis-connection.ts+++ b/src/modules/queue/redis-connection.ts@@ -6,14 +6,26 @@ import {type RedisConnectionOptions,} from '../../config/redis-options';+/** Upper bound for one producer command once connected. Same bound as the throttler's Redis client. */+export const QUEUE_COMMAND_TIMEOUT_MS = 2000;+/*** BullMQ connection for the shared producer (`BullModule.forRootAsync` in queue.module.ts): the common* Redis fields plus `enableOfflineQueue: false`, so once connected, `queue.add()` fails fast when Redis* drops and the dispatch path can fall back to direct delivery instead of buffering forever. Before the* first connect that option never applies: see producerReady.+ *+ * `commandTimeout` covers the other failure: a Redis that stays connected and stops answering (a+ * network partition, a frozen process). Nothing is rejected then, so without a deadline the add of a+ * delivery or a job lookup never settles, the dispatch slot or the ingress request holding it never+ * frees, and the direct-delivery fallback never runs. The Worker connection below must not inherit it:+ * its blocking commands wait on Redis by design.*/-export function queueConnectionOptions(): RedisConnectionOptions & { enableOfflineQueue: false } {- return { ...redisConnectionOptions(), enableOfflineQueue: false };+export function queueConnectionOptions(): RedisConnectionOptions & {+ enableOfflineQueue: false;+ commandTimeout: number;+} {+ return { ...redisConnectionOptions(), enableOfflineQueue: false, commandTimeout: QUEUE_COMMAND_TIMEOUT_MS };}/**