Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(engine): answer 503 for a throttled Baileys send
Yudhi Armyndharis8a20def7b462
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.mdindex c0a039f82..105101968 100644--- a/docs/06-api-specification.md+++ b/docs/06-api-specification.md@@ -1303,7 +1303,7 @@ The key is 1-255 visible ASCII characters (a UUID works), unique per session, an| first request still running | `409` `IDEMPOTENCY_KEY_IN_PROGRESS`; retry after it finishes || first request failed with an uncertain delivery outcome | `409` `IDEMPOTENCY_OUTCOME_UNKNOWN`: the message may have gone out; check the chat, then use a **new** key || first request was refused with `4xx` or `501` before calling the engine | the key was freed, so the retry runs as a new request |-| transport explicitly reported that nothing was sent, such as a Baileys recipient lookup timeout | the key was freed, so the retry runs as a new request |+| transport explicitly reported that nothing was sent, or WhatsApp rate-limited the request first | the key was freed, so the retry runs as a new request || different route or body (key order in the JSON does not matter) | `422` `IDEMPOTENCY_KEY_REUSED` |A malformed key, or the header sent twice, gets `400` `IDEMPOTENCY_KEY_INVALID`. A request still pending ten minutes after it was claimed (its node most likely stopped) also reads as `IDEMPOTENCY_OUTCOME_UNKNOWN`. HTTP status alone does not prove that nothing was sent: an engine can report `409` after attempting delivery, so engine-stage failures retain the key unless transport explicitly proves that nothing was sent. Keys are stored in the data database (`send_idempotency_keys`); concurrent processes using that database share the claim, and the session-owner proxy forwards the header. This does not change the deployment support limits in [Horizontal scaling](./13-horizontal-scaling.md). `send-bulk` does not take the header: retrying its request can create another batch. A call that outlives the 24-hour key window is outside the retry guarantee.diff --git a/src/engine/adapters/baileys-messaging.ts b/src/engine/adapters/baileys-messaging.tsindex 167bafb39..0e3ae8c34 100644--- a/src/engine/adapters/baileys-messaging.ts+++ b/src/engine/adapters/baileys-messaging.ts@@ -22,7 +22,7 @@ import {Product,Quotable,} from '../interfaces/whatsapp-engine.interface';-import { toEngineParticipants } from './baileys-groups';+import { refusedStatusCode, toEngineParticipants } from './baileys-groups';import { findSelfParticipant } from './baileys-group-mapper';import { buildVCard } from './vcard';import {@@ -40,6 +40,7 @@ import { EngineRefusedError } from '../../common/errors/engine-refused.error';import { MessageNotFoundError } from '../../common/errors/message-not-found.error';import { type createLogger } from '../../common/services/logger.service';import { EngineTransportError } from '../../common/errors/engine-transport.error';+import { EngineThrottledError } from '../../common/errors/engine-throttled.error';import { EngineNotSentError } from '../../common/errors/engine-not-sent.error';import { parseWaId, userPart } from '../identity/wa-id';import { BAILEYS_QUERY_BUDGET_MS, withQueryDeadline } from './baileys-query-deadline';@@ -271,6 +272,19 @@ export async function resolveMediaBuffer(return { data: Buffer.from(media.data, 'base64'), mimetype: media.mimetype };}+/**+ * Give a WhatsApp throttle or timeout answer (WA code 429 or 408) its retryable 503 instead of letting+ * the raw Boom surface as a 500. `sendMessage` awaits several IQs before it relays the message (group+ * roster, devices, sessions) and nothing after the relay awaits a result, so a numeric code here is a+ * query WhatsApp turned away: a 429 provably sent nothing. Every other failure is returned untouched.+ */+export function mapSendRefusal(error: unknown): unknown {+ const code = refusedStatusCode(error);+ if (code === 429) return new EngineThrottledError('WhatsApp rate-limited the send (code 429)');+ if (code === 408) return new EngineTransportError('WhatsApp timed out a query the send depends on (code 408)');+ return error;+}+export class BaileysMessaging {constructor(private readonly host: BaileysMessagingHost,@@ -929,7 +943,7 @@ export class BaileysMessaging {sent = options ? await sock.sendMessage(jid, content, options) : await sock.sendMessage(jid, content);} catch (error) {if (this.host.getSocketOrNull() !== sock) throw new EngineNotReadyError();- throw error;+ throw mapSendRefusal(error);}// Mutations are announced by their library echo; no local callback replaces it.if (!('react' in content || 'delete' in content || 'edit' in content)) this.host.rememberOwnSend(sent?.key?.id);diff --git a/src/engine/adapters/baileys-status.ts b/src/engine/adapters/baileys-status.tsindex adc55f736..5158ffa76 100644--- a/src/engine/adapters/baileys-status.ts+++ b/src/engine/adapters/baileys-status.ts@@ -1,7 +1,7 @@import type { AnyMessageContent, WAMessage, WASocket } from '@whiskeysockets/baileys';import { MediaInput, StatusPostOptions, StatusResult } from '../interfaces/whatsapp-engine.interface';import { BadRequestException } from '@nestjs/common';-import { resolveMediaBuffer } from './baileys-messaging';+import { mapSendRefusal, resolveMediaBuffer } from './baileys-messaging';import { describeAudio } from './baileys-audio-probe';import { EngineRefusedError } from '../../common/errors/engine-refused.error';import { EngineNotReadyError } from '../../common/errors/engine-not-ready.error';@@ -54,7 +54,7 @@ export class BaileysStatus {return await sock.sendMessage('status@broadcast', content, options);} catch (error) {if (this.host.getSocketOrNull() !== sock) throw new EngineNotReadyError();- throw error;+ throw mapSendRefusal(error);}}diff --git a/src/engine/adapters/baileys.adapter.spec.ts b/src/engine/adapters/baileys.adapter.spec.tsindex a2c7ad6f6..8f6d7340e 100644--- a/src/engine/adapters/baileys.adapter.spec.ts+++ b/src/engine/adapters/baileys.adapter.spec.ts@@ -183,6 +183,7 @@ import { ChannelNotFoundError } from '../../common/errors/channel-not-found.erroimport { ChatLabelsUnsupportedError } from '../../common/errors/chat-labels-unsupported.error';import { Boom } from '@hapi/boom';import { EngineTransportError } from '../../common/errors/engine-transport.error';+import { EngineThrottledError } from '../../common/errors/engine-throttled.error';import { LidNotMappedError } from '../../common/errors/lid-not-mapped.error';import { countsTowardSendBreaker, sentNothing } from '../../modules/message/send-pacing.service';import { loadRemoteMediaBuffer } from '../../common/media/load-remote-media';@@ -1902,6 +1903,46 @@ describe('BaileysAdapter messaging', () => {await expect(adapter.sendTextMessage('[email protected]', 'hello')).rejects.toBe(failure);});+ // sendMessage awaits IQs (group roster, devices, sessions) before it relays the message, and WhatsApp+ // answers a throttled one with a numeric code on the Boom. Every other path maps these to a retryable 503.+ it.each([+ [429, EngineThrottledError],+ [408, EngineTransportError],+ ])('a WhatsApp %s answer to a query the send awaits is a retryable 503, not a bare 500', async (code, expected) => {+ const adapter = await readyAdapter();+ fakeSock.sendMessage.mockRejectedValue(new Boom('rate-overlimit', { data: code }));+ const rejection = adapter.sendTextMessage('[email protected]', 'hello');+ await expect(rejection).rejects.toBeInstanceOf(expected);+ await expect(rejection).rejects.toBeInstanceOf(EngineTransportError);+ });++ // The throttle is WhatsApp refusing this account's traffic, which is what the breaker watches for; the send+ // itself provably went nowhere, so its admission is given back.+ it('a throttled send is known to have sent nothing and still feeds the send breaker', async () => {+ const adapter = await readyAdapter();+ fakeSock.sendMessage.mockRejectedValue(new Boom('rate-overlimit', { data: 429 }));+ const error: unknown = await adapter.sendTextMessage('[email protected]', 'hello').catch((e: unknown) => e);+ expect(sentNothing(error)).toBe(true);+ expect(countsTowardSendBreaker(error)).toBe(true);+ });++ // A query that timed out may have been answered after the caller gave up, so nothing is claimed about the+ // send: its admission stays held, and the timeout is a transport fault the breaker leaves to reconnects.+ it('a send whose awaited query timed out is not known to have sent nothing and stays off the breaker', async () => {+ const adapter = await readyAdapter();+ fakeSock.sendMessage.mockRejectedValue(new Boom('timed out', { data: 408 }));+ const error: unknown = await adapter.sendTextMessage('[email protected]', 'hello').catch((e: unknown) => e);+ expect(sentNothing(error)).toBe(false);+ expect(countsTowardSendBreaker(error)).toBe(false);+ });++ it('leaves any other WhatsApp refusal code to propagate untouched', async () => {+ const adapter = await readyAdapter();+ const refusal = new Boom('forbidden', { data: 403 });+ fakeSock.sendMessage.mockRejectedValue(refusal);+ await expect(adapter.sendTextMessage('[email protected]', 'hello')).rejects.toBe(refusal);+ });+/** An adapter for a session started behind a proxy, which every fetch it makes must leave through. */const proxiedAdapter = async (): Promise<BaileysAdapter> => {const adapter = new BaileysAdapter({@@ -7424,6 +7465,14 @@ describe('BaileysAdapter status posting', () => {fakeSock.sendMessage.mockRejectedValue(failure);await expect(adapter.postTextStatus('hello', { recipients: ['[email protected]'] })).rejects.toBe(failure);});++ it('a status post throttled by WhatsApp is a retryable 503, not a bare 500', async () => {+ const adapter = await ready();+ fakeSock.sendMessage.mockRejectedValue(new Boom('rate-overlimit', { data: 429 }));+ await expect(adapter.postTextStatus('hello', { recipients: ['[email protected]'] })).rejects.toBeInstanceOf(+ EngineThrottledError,+ );+ });});describe('BaileysAdapter proxy support', () => {diff --git a/src/modules/message/idempotency/send-idempotency.interceptor.spec.ts b/src/modules/message/idempotency/send-idempotency.interceptor.spec.tsindex 7008cecc1..8edc0dc39 100644--- a/src/modules/message/idempotency/send-idempotency.interceptor.spec.ts+++ b/src/modules/message/idempotency/send-idempotency.interceptor.spec.ts@@ -12,6 +12,8 @@ import { lastValueFrom, of, throwError } from 'rxjs';import { EngineNotReadyError } from '../../../common/errors/engine-not-ready.error';import { markEngineSendFailure } from '../../../common/errors/engine-send-failure';import { EngineNotSentError } from '../../../common/errors/engine-not-sent.error';+import { EngineThrottledError } from '../../../common/errors/engine-throttled.error';+import { EngineTransportError } from '../../../common/errors/engine-transport.error';import { SendIdempotencyInterceptor, sendFailureProvesNothingSent } from './send-idempotency.interceptor';import { hashSendRequest, SendIdempotencyService } from './send-idempotency.service';@@ -152,6 +154,13 @@ describe('SendIdempotencyInterceptor', () => {expect(store.markFailed).not.toHaveBeenCalled();});+ it('releases the key when WhatsApp rate-limited the request before it ran', async () => {+ const error = markEngineSendFailure(new EngineThrottledError('WhatsApp rate-limited the send (code 429)'));+ await expect(run(contextFor({ 'idempotency-key': 'k-1' }), () => throwError(() => error))).rejects.toBe(error);+ expect(store.release).toHaveBeenCalledWith('row-1');+ expect(store.markFailed).not.toHaveBeenCalled();+ });+it('still answers the send when recording its outcome fails', async () => {store.complete.mockRejectedValue(new Error('db down'));@@ -182,6 +191,11 @@ describe('sendFailureProvesNothingSent', () => {expect(sendFailureProvesNothingSent(error)).toBe(expected);});+ it('frees the key for a WhatsApp rate limit, which turned the request away, but not for a timeout', () => {+ expect(sendFailureProvesNothingSent(markEngineSendFailure(new EngineThrottledError('rate limited')))).toBe(true);+ expect(sendFailureProvesNothingSent(markEngineSendFailure(new EngineTransportError('timed out')))).toBe(false);+ });+it('treats 501 as a refusal', () => {expect(HttpStatus.NOT_IMPLEMENTED).toBe(501);});diff --git a/src/modules/message/idempotency/send-idempotency.interceptor.ts b/src/modules/message/idempotency/send-idempotency.interceptor.tsindex 235068360..b4d465880 100644--- a/src/modules/message/idempotency/send-idempotency.interceptor.ts+++ b/src/modules/message/idempotency/send-idempotency.interceptor.ts@@ -14,6 +14,7 @@ import { from, lastValueFrom, Observable, of } from 'rxjs';import { createLogger } from '../../../common/services/logger.service';import { isEngineSendFailure } from '../../../common/errors/engine-send-failure';import { EngineNotSentError } from '../../../common/errors/engine-not-sent.error';+import { EngineThrottledError } from '../../../common/errors/engine-throttled.error';import { hashSendRequest, isValidIdempotencyKey, SendIdempotencyService } from './send-idempotency.service';/** Request header a client sets to make a send safe to retry. */@@ -27,11 +28,12 @@ const CLAIMABLE_SESSION_ID_MAX_LENGTH = 255;* Whether a failed send proves nothing reached WhatsApp, so the key can be freed for a retry.** A 4xx or 501 before the engine call is a refusal. Errors escaping the shared engine-send failure- * boundary stay uncertain unless transport explicitly reports nothing was sent: a socket can report 409 after- * WhatsApp accepted the message. Other server errors/timeouts also keep the key taken.+ * boundary stay uncertain unless transport explicitly reports nothing was sent, or WhatsApp itself turned the+ * request away with a rate limit (EngineThrottledError): a socket can report 409 after WhatsApp accepted the+ * message. Other server errors/timeouts also keep the key taken.*/export function sendFailureProvesNothingSent(error: unknown): boolean {- if (error instanceof EngineNotSentError) return true;+ if (error instanceof EngineNotSentError || error instanceof EngineThrottledError) return true;if (isEngineSendFailure(error) || !(error instanceof HttpException)) return false;const status: number = error.getStatus();const notImplemented: number = HttpStatus.NOT_IMPLEMENTED;diff --git a/src/modules/message/send-pacing.service.spec.ts b/src/modules/message/send-pacing.service.spec.tsindex c951158e4..8b9a40edb 100644--- a/src/modules/message/send-pacing.service.spec.ts+++ b/src/modules/message/send-pacing.service.spec.ts@@ -10,6 +10,7 @@ import { SendPacingService, SEND_PACING_LIMITED, countsTowardSendBreaker } fromimport { EnginePageError } from '../../common/errors/engine-page.error';import { EngineRefusedError } from '../../common/errors/engine-refused.error';import { EngineNotSupportedError } from '../../common/errors/engine-not-supported.error';+import { EngineThrottledError } from '../../common/errors/engine-throttled.error';import { SsrfBlockedError } from '../../common/security/ssrf-guard';import { computeSendPacingConfig, type SendPacingConfig } from './send-pacing.config';import { getSendPacingRefusals, resetSendPacingRefusals } from '../../common/metrics/send-pacing-metrics';@@ -629,6 +630,7 @@ describe('countsTowardSendBreaker', () => {it.each([['a WhatsApp refusal (403 EngineRefusedError)', new EngineRefusedError('not allowed to send here')],+ ['a WhatsApp rate limit (503 EngineThrottledError)', new EngineThrottledError('WhatsApp rate-limited the send')],['a raw engine error', new Error('ack error 500')],['a server-side fault', new InternalServerErrorException('boom')],[diff --git a/src/modules/message/send-pacing.service.ts b/src/modules/message/send-pacing.service.tsindex a6a447b2f..2dccfff1a 100644--- a/src/modules/message/send-pacing.service.ts+++ b/src/modules/message/send-pacing.service.ts@@ -39,10 +39,12 @@ export function isPacingLimitedError(error: unknown): boolean {* open the breaker on a perfectly healthy session and 429 every send for the cooldown. The rule is* therefore: a client-fault or engine-state HTTP status (4xx/501/503) does NOT count; a refusal* WhatsApp itself returned — EngineRefusedError (403) — and anything unclassified (a raw engine- * error, a timeout, a 5xx) does.+ * error, a timeout, a 5xx) does. A rate limit WhatsApp answered (EngineThrottledError, a 503 on the wire)+ * counts as well: it is WhatsApp refusing this account's traffic, the signal the breaker exists to act on,+ * even though the send provably went nowhere (see sentNothing).*/export function countsTowardSendBreaker(error: unknown): boolean {- if (error instanceof EngineRefusedError) return true;+ if (error instanceof EngineRefusedError || error instanceof EngineThrottledError) return true;if (error instanceof SsrfBlockedError) return false;if (error instanceof HttpException) {const status = error.getStatus();