Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
import { Optional } from '@nestjs/common';
import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
import { DelayedError, Job, Queue, RATE_LIMIT_ERROR } from 'bullmq';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { QUEUE_NAMES } from '../queue-names';
import { workerConnectionOptions, ingressWorkerConcurrency, producerReady } from '../redis-connection';
import { closeWorkerIfStarted, MAX_WORKER_CLOSE_WAIT_MS, startedWorker } from './close-worker';
import { IntegrationDeliveryFailure } from '../../integration/entities/integration-delivery-failure.entity';
import { IngressEvent } from '../../integration/entities/ingress-event.entity';
import { PluginLoaderService } from '../../../core/plugins/plugin-loader.service';
import { HookManager } from '../../../core/hooks';
import { createLogger } from '../../../common/services/logger.service';
import { ShutdownService } from '../../../common/services/shutdown.service';
import { KeyedAsyncLock, orderingKeyFor } from '../../integration/ordering-lock';
import { requeuedJobIds, resolveIngressJobOptions } from '../../integration/ingress-enqueue.service';
// BullMQ's failedReason for a job that stalled more than maxStalledCount (see WebhookProcessor).
const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';
// How long a delivery whose dead-letter row could not be written waits before it runs again.
const REQUEUE_DELAY_MS = 60_000;
// The id suffix deadLetterOrRequeue gives a re-queued delivery (see requeuedJobIds).
const REQUEUED_JOB_ID = /-requeued-[12]$/;
export interface IngressJobData {
pluginId: string;
instanceId: string;
route: string;
// Optional for backward compatibility with jobs and DLQ rows persisted before method forwarding.
method?: string;
deliveryId: string;
sessionId?: string;
// Best-effort provider conversation id, extracted host-side from the manifest's conversationId
// pointer. Undefined when the route declares no pointer — the per-conversation ordering lock then
// serializes per instance instead (see orderingKeyFor in ../../integration/ordering-lock).
providerConversationId?: string;
payload: {
headers: Record<string, string>;
query: Record<string, string>;
body: string;
rawBody: string;
// The host's verdict when it accepted the request; see IngressEvent.payload.
admission?: { verified: boolean };
};
}
// The KeyedAsyncLock wrapping dispatch below guarantees no two dispatches for the SAME conversation
// run concurrently (mutual exclusion + in-order START for events as they reach the worker), so the
// worker no longer needs concurrency 1: raising it parallelizes unrelated conversations. The lock is
// taken inside the job, though, so a job waiting on a busy key still holds a worker slot. A burst on
// one key larger than the concurrency fills every slot with waiters, and other keys queue behind it
// until it drains. The key is per instance when the route declares no conversationId pointer, so
// routes should declare one, and the concurrency should exceed the largest expected per-key burst.
//
// Strict end-to-end order is NOT preserved across a BullMQ retry: a retried job re-enters lock.run()
// after its backoff and chains at the conversation's CURRENT tail, so it can overtake a
// same-conversation successor that dispatched
// during the backoff window. This is a deliberate tradeoff — BullMQ retries release the worker slot
// during backoff (better throughput under transient failure), where retrying inside the lock would
// hold it — and is acceptable because ingress order is best-effort regardless: the provider delivers
// over unordered HTTP. Order-strict plugins must not assume retry-involved events arrive in sequence.
@Processor(QUEUE_NAMES.INGRESS, { connection: workerConnectionOptions(), concurrency: ingressWorkerConcurrency() })
export class IngressProcessor extends WorkerHost {
private readonly logger = createLogger('IngressProcessor');
private readonly lock = new KeyedAsyncLock();
constructor(
private readonly loader: PluginLoaderService,
@InjectRepository(IntegrationDeliveryFailure, 'data')
private readonly failures: Repository<IntegrationDeliveryFailure>,
private readonly hooks: HookManager,
@InjectQueue(QUEUE_NAMES.INGRESS)
private readonly ingressQueue: Queue<IngressJobData>,
@InjectRepository(IngressEvent, 'data')
private readonly events: Repository<IngressEvent>,
@Optional() shutdown?: ShutdownService,
) {
super();
// A plugin's ingress handler sends through the session engines, and SessionModule tears them down
// before QueueModule is destroyed, so a job taken in between fails and spends an attempt. Stop taking
// jobs once shutdown begins: the jobs already running keep the drain window to finish while the
// engines are up, onModuleDestroy below waits for them, and the rest stay in Redis for another node
// or the next start. A close that skips ShutdownService (a signal Nest handles) closes there only.
shutdown?.onShutdown(() => {
const worker = startedWorker(this);
void worker?.close().catch(() => undefined);
});
}
/**
* BullModule closes the worker only in onApplicationShutdown, after every destroy hook, so it would
* keep taking jobs while PluginLoaderService (a global module, destroyed last) disables the plugins:
* each one would fail against a terminated sandbox, spending an attempt or landing in the DLQ.
* Closing here, before the global modules, lets the jobs already running finish against live plugins
* and leaves the rest in Redis for the next start. Running jobs that share an ordering key dispatch
* one after another, so a full worker of them can outlast any wait; this one takes the whole cap.
* A job still running at the cap is cut mid-dispatch and re-run after the next start; BullMQ allows one
* stall per job, so a second one fails it.
*/
async onModuleDestroy(): Promise<void> {
await closeWorkerIfStarted(this, MAX_WORKER_CLOSE_WAIT_MS);
}
async process(job: Job<IngressJobData>, token?: string): Promise<void> {
const d = job.data;
if (this.loader.isRestorePending(d.pluginId)) {
// The plugin's worker does not exist until the boot restore reaches it, and a restore can outlast the
// whole retry budget. Before the try below on purpose: this is not an attempt, so it must never be
// logged as a failure or file a dead-letter row. moveToDelayed does not spend one of the job's attempts.
// Every held job is delayed to the next multiple of the retry delay (at least a second: a job delayed to
// "now" is promoted straight back and would spin), so they all wake together and BullMQ promotes them in the
// order they were delayed. A fixed delay from "now" would give two deliveries of one conversation, a fraction of
// a period apart, polls a fraction of a period apart, and a restore ending between them lets the later through
// first. The delay carries no jitter for the same reason. Jobs of one wake that the worker reaches after
// the restore ends still pass those it held again before, and BullMQ's delayed score has 4096 insertion
// ranks per millisecond: past them the jobs of one wake come back ordered by id.
const delay = Math.max(1000, resolveIngressJobOptions().backoff.delay);
await job.moveToDelayed((Math.floor(Date.now() / delay) + 1) * delay, token);
throw new DelayedError();
}
try {
await this.lock.run(orderingKeyFor(d), () => this.loader.dispatchWebhookForInstance(d));
} catch (err) {
const errorMessage = err instanceof Error ? err.message : String(err);
const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1);
this.logger.error('Ingress dispatch failed', errorMessage, {
pluginId: d.pluginId,
instanceId: d.instanceId,
route: d.route,
deliveryId: d.deliveryId,
attempt: job.attemptsMade + 1,
isFinalAttempt,
action: 'ingress_dispatch_failed',
});
if (isFinalAttempt) await this.deadLetterOrRequeue(job, job.attemptsMade + 1, errorMessage);
// BullMQ reads an error with exactly this text as a rate limit, moves the job back to wait without
// counting the attempt and runs it again: here a hot loop that never reaches the final attempt, or on
// it writes a dead letter on every run. A handler may relay such text from elsewhere.
if (errorMessage === RATE_LIMIT_ERROR) {
throw new Error(`Ingress dispatch failed: ${errorMessage}`, { cause: err });
}
// Re-throw to trigger BullMQ's exponential backoff / retry.
throw err;
}
if (REQUEUED_JOB_ID.test(String(job.id))) await this.settleRequeued(d);
}
/**
* A re-queued copy runs while the original job stays 'failed' under the original id. The reconciler
* takes a live or completed copy for the job, but one it misses (pruned once completed) reads as
* dead-lettered: for an event still 'pending' it writes a DLQ row and marks the event 'failed'.
* Record the delivery on both so neither is left redrivable for an event the plugin already
* received. Never rejects: the dispatch succeeded, and a retry would deliver it again.
*/
private async settleRequeued(d: IngressJobData): Promise<void> {
try {
await this.events.update(
{ pluginId: d.pluginId, instanceId: d.instanceId, providerDeliveryId: d.deliveryId },
{ dispatchState: 'dispatched', payload: null },
);
await this.failures.update(
{
direction: 'inbound',
pluginId: d.pluginId,
instanceId: d.instanceId,
deliveryId: d.deliveryId,
redriven: false,
},
{ redriven: true },
);
} catch (err) {
this.logger.error(
'Could not record a re-queued ingress delivery',
err instanceof Error ? err.message : String(err),
{
pluginId: d.pluginId,
instanceId: d.instanceId,
deliveryId: d.deliveryId,
action: 'ingress_requeue_settle_failed',
},
);
}
}
/**
* A job failed by stall exhaustion never enters process(): the worker fails it internally and only
* emits 'failed'. The ingress_events row already retired its payload when the enqueue returned
* 'queued', so without this the failed BullMQ job (pruned by removeOnFail) is the only copy and no
* DLQ row exists to redrive. Any other failure was already recorded by process() on the final
* attempt, so only the stall sentinel is handled here. `job` is undefined once removeOnFail pruned it.
*/
@OnWorkerEvent('failed')
async onWorkerFailed(job: Job<IngressJobData> | undefined, error: Error): Promise<void> {
// The sentinel text alone is not proof: a failed attempt whose error reads the same is retried and is not
// this. BullMQ marks a job it fails for stalling by storing the reason as its deferred failure.
if (!job || job.deferredFailure !== STALL_EXHAUSTION_MESSAGE || error.message !== STALL_EXHAUSTION_MESSAGE) return;
const d = job.data;
this.logger.error('Ingress job failed after stalling beyond the recovery limit', error.message, {
pluginId: d.pluginId,
instanceId: d.instanceId,
route: d.route,
deliveryId: d.deliveryId,
attemptsMade: job.attemptsMade,
action: 'ingress_stall_exhausted',
});
// Never rejects: an event listener's rejection would surface as an unhandled rejection.
await this.deadLetterOrRequeue(job, job.attemptsMade, error.message);
}
/**
* Dead-letter a job that has spent its attempts. That row is the only durable copy left, so when the
* data database refuses it too (an outage longer than the retry window fails dispatch and the write
* alike), the delivery goes back on the queue, which still works, and runs again once the database
* is back. A fresh job id, because the failed job keeps the original one until removeOnFail prunes it
* and BullMQ would resolve an add under that id to the existing job. A copy that fails the same way
* takes the other of the two copy ids, removing the failed copy before it that still holds it, so the
* reconciler can find the live one by id. Never rejects.
*/
private async deadLetterOrRequeue(job: Job<IngressJobData>, attempts: number, errorMessage: string): Promise<void> {
const d = job.data;
try {
await this.deadLetter(d, attempts, errorMessage, REQUEUED_JOB_ID.test(String(job.id)));
} catch (err) {
const meta = { jobId: job.id, pluginId: d.pluginId, instanceId: d.instanceId, deliveryId: d.deliveryId };
const reason = err instanceof Error ? err.message : String(err);
try {
const [first, second] = requeuedJobIds(String(job.id).replace(REQUEUED_JOB_ID, ''));
const jobId = String(job.id) === first ? second : first;
// The producer queue holds its calls until its first connect, which the worker's own connection does not
// wait for; bounded, so the recovery gives up instead of waiting on a Redis that never answered it.
await producerReady(this.ingressQueue);
await this.ingressQueue.remove(jobId);
await this.ingressQueue.add(job.name, d, {
jobId,
attempts: job.opts.attempts,
backoff: job.opts.backoff,
delay: REQUEUE_DELAY_MS,
});
this.logger.error('Could not dead-letter a failed ingress job; re-queued it', reason, {
...meta,
action: 'ingress_dlq_failed_requeued',
});
} catch (queueErr) {
// Only the failed BullMQ job is left; the job id lets an operator retry it before it is pruned.
this.logger.error('Could not dead-letter or re-queue a failed ingress job', reason, {
...meta,
requeueError: queueErr instanceof Error ? queueErr.message : String(queueErr),
action: 'ingress_dlq_failed',
});
}
}
}
private async deadLetter(
d: IngressJobData,
attempts: number,
errorMessage: string,
requeued: boolean,
): Promise<void> {
await this.hooks.execute(
'ingress:error',
{ ...d, error: errorMessage },
{ sessionId: d.sessionId, source: 'IngressProcessor' },
);
// The reconciler writes this row itself when it finds the original job failed with none, so a
// re-queued copy that fails again must not add a second redrivable one. Only a re-queued copy: a
// redrive job can fail while the row it replays is still open, and that row is retired regardless.
if (
requeued &&
(await this.failures.count({
where: {
direction: 'inbound',
pluginId: d.pluginId,
instanceId: d.instanceId,
deliveryId: d.deliveryId,
redriven: false,
},
})) > 0
) {
return;
}
await this.failures.save({
direction: 'inbound',
pluginId: d.pluginId,
instanceId: d.instanceId,
sessionId: d.sessionId ?? null,
deliveryId: d.deliveryId,
attempts,
lastError: errorMessage,
// Persist the FULL ingress payload (route + headers/rawBody) so P1 redrive is
// self-contained and never has to re-read ingress_events.
payload: {
route: d.route,
method: d.method,
providerConversationId: d.providerConversationId,
ingress: d.payload,
},
redriven: false,
});
}
}