Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(queue): bound producer commands once Redis has connected

Yudhi Armyndharisde133a3dad5c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0b966c22..9a74765bf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -43,6 +43,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Bound each webhook and ingress queue command at 2 s once Redis has connected, so a Redis that stops answering falls back to direct delivery and inline dispatch instead of holding every dispatch and ingress request.
- Answer `400` for a send route's `chatId` or a bulk `batchId` over 255 characters, where PostgreSQL answered `500` once the value passed its index entry limit.
- Index ingress dead letters by delivery id, so the reconciler's per-delivery checks stop scanning every dead letter of an instance; the first boot builds the index.
- Replay stranded ingress events past one whose stored payload is not valid JSON, instead of failing every sweep on it.
diff --git a/src/modules/queue/redis-connection.spec.ts b/src/modules/queue/redis-connection.spec.ts
index 35de02889..70a9e0671 100644
--- a/src/modules/queue/redis-connection.spec.ts
+++ b/src/modules/queue/redis-connection.spec.ts
@@ -1,3 +1,5 @@
+import { createServer, type AddressInfo, type Server, type Socket } from 'node:net';
+import Redis from 'ioredis';
import {
ingressWorkerConcurrency,
producerConnectWaitMs,
@@ -22,6 +24,10 @@ describe('workerConnectionOptions (webhook Worker connection)', () => {
expect(opts.enableOfflineQueue).toBeUndefined();
});
+ it('sets no command timeout: the Worker blocks on Redis by design', () => {
+ expect((workerConnectionOptions() as unknown as Record<string, unknown>).commandTimeout).toBeUndefined();
+ });
+
it('reads host/port/username/password/connectTimeout from env with safe defaults', () => {
process.env = { ...ORIGINAL_ENV };
delete process.env.REDIS_HOST;
@@ -58,6 +64,83 @@ describe('workerConnectionOptions (webhook Worker connection)', () => {
});
});
+/** The command names in a chunk of RESP arrays, lower-cased: `*2\r\n$3\r\nGET\r\n$1\r\na\r\n` is ['get']. */
+function commandNames(chunk: string): string[] {
+ const lines = chunk.split('\r\n');
+ const names: string[] = [];
+ for (let i = 0; i < lines.length;) {
+ const header = /^\*(\d+)$/.exec(lines[i]);
+ if (!header) {
+ i += 1;
+ continue;
+ }
+ names.push((lines[i + 2] ?? '').toLowerCase());
+ i += 1 + Number(header[1]) * 2;
+ }
+ return names;
+}
+
+describe('queueConnectionOptions on a Redis that stops answering', () => {
+ // The socket stays open and nothing comes back, as in a network partition or a frozen Redis process.
+ // Without a deadline every producer call (the queue add of a delivery, a job lookup) then never settles,
+ // so the dispatch slot or the ingress request holding it never frees and the fallback never runs.
+ let server: Server;
+ let sockets: Socket[];
+ let answering: boolean;
+ let client: Redis;
+
+ beforeEach(async () => {
+ sockets = [];
+ answering = true;
+ server = createServer(socket => {
+ sockets.push(socket);
+ socket.on('error', () => undefined);
+ socket.on('data', chunk => {
+ if (!answering) return;
+ // One reply per command in the chunk, enough RESP for the handshake and a nil GET.
+ for (const name of commandNames(chunk.toString())) {
+ socket.write(
+ name === 'hello' ? "-ERR unknown command 'hello'\r\n" : name === 'client' ? '+OK\r\n' : '$-1\r\n',
+ );
+ }
+ });
+ });
+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));
+ client = new Redis({
+ ...queueConnectionOptions(),
+ host: '127.0.0.1',
+ port: (server.address() as AddressInfo).port,
+ enableReadyCheck: false,
+ retryStrategy: () => null,
+ });
+ // With no offline queue, a command issued before the connect rejects at once.
+ await new Promise(resolve => client.once('ready', resolve));
+ });
+
+ afterEach(async () => {
+ client.disconnect();
+ sockets.forEach(socket => socket.destroy());
+ await new Promise(resolve => server.close(resolve));
+ });
+
+ it('rejects a command that gets no reply, instead of holding its caller for good', async () => {
+ await expect(client.get('first')).resolves.toBeNull();
+ answering = false;
+
+ // Read off the options, with the 2 s the producer is held to as the fallback of a build without one.
+ const budget = (queueConnectionOptions() as { commandTimeout?: number }).commandTimeout ?? 2000;
+ const outcome = await Promise.race([
+ client.get('second').then(
+ () => 'answered',
+ (error: unknown) => (error instanceof Error ? error.message : String(error)),
+ ),
+ new Promise<string>(resolve => setTimeout(resolve, budget + 1500, 'still waiting')),
+ ]);
+
+ expect(outcome).toMatch(/timed out/i);
+ });
+});
+
describe('webhookWorkerConcurrency', () => {
const ORIGINAL_ENV = process.env;
afterEach(() => {
diff --git a/src/modules/queue/redis-connection.ts b/src/modules/queue/redis-connection.ts
index 13dc1ceda..5842ae940 100644
--- a/src/modules/queue/redis-connection.ts
+++ b/src/modules/queue/redis-connection.ts
@@ -6,14 +6,26 @@ import {
type RedisConnectionOptions,
} from '../../config/redis-options';
+/** Upper bound for one producer command once connected. Same bound as the throttler's Redis client. */
+export const QUEUE_COMMAND_TIMEOUT_MS = 2000;
+
/**
* BullMQ connection for the shared producer (`BullModule.forRootAsync` in queue.module.ts): the common
* Redis fields plus `enableOfflineQueue: false`, so once connected, `queue.add()` fails fast when Redis
* drops and the dispatch path can fall back to direct delivery instead of buffering forever. Before the
* first connect that option never applies: see producerReady.
+ *
+ * `commandTimeout` covers the other failure: a Redis that stays connected and stops answering (a
+ * network partition, a frozen process). Nothing is rejected then, so without a deadline the add of a
+ * delivery or a job lookup never settles, the dispatch slot or the ingress request holding it never
+ * frees, and the direct-delivery fallback never runs. The Worker connection below must not inherit it:
+ * its blocking commands wait on Redis by design.
*/
-export function queueConnectionOptions(): RedisConnectionOptions & { enableOfflineQueue: false } {
- return { ...redisConnectionOptions(), enableOfflineQueue: false };
+export function queueConnectionOptions(): RedisConnectionOptions & {
+ enableOfflineQueue: false;
+ commandTimeout: number;
+} {
+ return { ...redisConnectionOptions(), enableOfflineQueue: false, commandTimeout: QUEUE_COMMAND_TIMEOUT_MS };
}
/**