Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Webhook } from './entities/webhook.entity';
import { WebhookOutboxService } from './webhook-outbox.service';
import {
DEFAULT_WEBHOOK_SHUTDOWN_DRAIN_MS,
isAbortedBackoff,
WebhookDeliveryService,
} from './webhook-delivery.service';
import { isDeliverableWebhook } from './utils/deliver-once';
import { createLogger } from '../../common/services/logger.service';
import { MAX_TIMER_MS, resolveNonNegativeIntEnv } from '../../config/configuration';
export interface WebhookReconcilerOptions {
/** 0 disables the sweep entirely. */
intervalMs: number;
/** How long a row may sit 'pending' before it counts as stranded rather than in flight. */
graceMs: number;
batchSize: number;
/**
* Replay budget per row. Past it the row is marked 'failed' and left alone: a stuck delivery must
* not become an infinite replay loop, and the failure row is where recovery continues.
*/
maxAttempts: number;
}
export function resolveWebhookReconcilerOptions(env: NodeJS.ProcessEnv = process.env): WebhookReconcilerOptions {
const batch = Number(env.WEBHOOK_RECONCILE_BATCH_SIZE);
const maxAttempts = Number(env.WEBHOOK_RECONCILE_MAX_ATTEMPTS);
return {
intervalMs: resolveNonNegativeIntEnv(env.WEBHOOK_RECONCILE_INTERVAL_MS, 60_000),
graceMs: resolveNonNegativeIntEnv(env.WEBHOOK_RECONCILE_GRACE_MS, 60_000),
batchSize: Number.isInteger(batch) && batch >= 1 ? batch : 50,
maxAttempts: Number.isInteger(maxAttempts) && maxAttempts >= 1 ? maxAttempts : 5,
};
}
export interface WebhookReconcileStats {
scanned: number;
replayed: number;
failed: number;
skipped: number;
}
/**
* Closes the crash window on outbound webhook delivery.
*
* Fan-out is fire-and-forget from the projector, so before the outbox row existed a hard crash
* between persisting a message and completing its POST lost the delivery with nothing left behind
* in either mode, while the documented contract promises at-least-once. The row makes the intent
* durable; this sweep is what turns durability into delivery.
*
* Mirrors IngressReconcilerService on the inbound side: an unref'd interval started on module init,
* an overlap guard so a slow pass never stacks, a bounded batch, and a per-row replay budget.
*
* The sweep does NOT claim a row, so two nodes running against one database can both replay the
* same delivery. That is deliberate and matches the inbound reconciler: the replay carries the
* stored idempotency key, which is exactly the header a receiver dedups on, so the cost of the
* race is a duplicate the contract already tells consumers to expect. Claiming would trade that
* for a lock whose holder can die mid-flight.
*/
@Injectable()
export class WebhookReconcilerService implements OnModuleInit, OnModuleDestroy {
private readonly logger = createLogger('WebhookReconcilerService');
private timer?: ReturnType<typeof setInterval>;
// The pass in flight, settled when it ends; doubles as the overlap guard.
private inFlight?: Promise<void>;
// Aborted on destroy: stops the pass at the next row and the replay in hand before its next retry.
// A row already past its last check, or a retry POST already started, still sends that one POST.
private readonly stop = new AbortController();
private cursor?: string;
constructor(
@InjectRepository(Webhook, 'data') private readonly webhooks: Repository<Webhook>,
private readonly outbox: WebhookOutboxService,
private readonly delivery: WebhookDeliveryService,
private readonly configService: ConfigService,
) {}
onModuleInit(): void {
const opts = resolveWebhookReconcilerOptions();
if (opts.intervalMs <= 0) {
this.logger.log('Webhook delivery reconciler disabled (WEBHOOK_RECONCILE_INTERVAL_MS=0)');
return;
}
this.timer = setInterval(() => {
this.sweep(opts).catch(err =>
this.logger.error('Webhook reconcile sweep failed', err instanceof Error ? err.stack : String(err)),
);
}, opts.intervalMs);
this.timer.unref?.();
}
/**
* Clearing the interval only stops the NEXT pass. A replay holds no dispatch slot, so the delivery
* drain neither waits for nor stops a pass already running: it would go on POSTing after the drain,
* and after PluginLoaderService has unregistered the `webhook:before` hooks. Stop it at the next row,
* stop the replay in hand without a further retry, and wait for it, at most WEBHOOK_SHUTDOWN_DRAIN_MS
* like the delivery drain beside it: the replay's hooks are not capped, and a queue lookup or add made
* before Redis has ever connected waits up to REDIS_CONNECT_TIMEOUT_MS. A row given up on stays
* 'pending' unless the replay in hand completes before the process exits.
*/
async onModuleDestroy(): Promise<void> {
this.stop.abort();
if (this.timer) clearInterval(this.timer);
if (!this.inFlight) return;
// Above MAX_TIMER_MS a Node timer fires after 1 ms; the delivery drain honours such a value in full.
const waitMs = Math.min(
this.configService.get<number>('webhook.shutdownDrainMs', DEFAULT_WEBHOOK_SHUTDOWN_DRAIN_MS),
MAX_TIMER_MS,
);
let timer: NodeJS.Timeout | undefined;
const settled = await Promise.race([
this.inFlight.then(() => true),
new Promise<false>(resolve => (timer = setTimeout(resolve, waitMs, false))),
]);
clearTimeout(timer);
if (!settled) {
this.logger.warn('Webhook reconcile pass still running at shutdown; continuing without it', {
waitMs,
action: 'webhook_reconcile_shutdown_timeout',
});
}
}
/** One bounded pass over the stranded backlog. Overlap-guarded. */
async sweep(opts: WebhookReconcilerOptions, now: Date = new Date()): Promise<WebhookReconcileStats> {
const stats: WebhookReconcileStats = { scanned: 0, replayed: 0, failed: 0, skipped: 0 };
if (this.inFlight || this.stop.signal.aborted) return stats;
let settle!: () => void;
this.inFlight = new Promise(resolve => (settle = resolve));
try {
const cutoff = new Date(now.getTime() - opts.graceMs);
// The last id the page covered, which can lie beyond its last readable row: a row whose stored
// payload is unreadable is skipped, and the cursor must pass it or a page of them is read forever.
let scannedThrough: string | undefined;
const onScanned = (id: string): void => {
scannedThrough = id;
};
let rows = await this.outbox.findStale(cutoff, opts.batchSize, this.cursor, onScanned);
if (rows.length === 0 && !scannedThrough && this.cursor) {
this.cursor = undefined;
rows = await this.outbox.findStale(cutoff, opts.batchSize, undefined, onScanned);
}
// Live jobs may occupy a whole page. Advance past them without spending their replay budget.
this.cursor = scannedThrough ?? rows.at(-1)?.id;
stats.scanned = rows.length;
// Webhooks whose replay did not deliver in this pass. Direct mode replays one row at a time with
// the full retry schedule, so a receiver that never answers would cost that schedule per stranded
// row and hold every webhook behind it. The first failure has shown it is down: its further rows
// make one attempt each, and the next sweep is their retry.
const failedWebhooks = new Set<string>();
for (const row of rows) {
if (this.stop.signal.aborted) break;
if (this.delivery.isLocallyPending(row.idempotencyKey)) {
// Still owned by a dispatch on this node (parked in the limiter or mid retry loop), so it
// is slow rather than stranded. Replaying it would POST alongside the original and outside
// the dispatch concurrency bound, and would spend its budget while it is still running.
stats.skipped++;
continue;
}
const webhook = await this.webhooks.findOne({ where: { id: row.webhookId } });
if (!isDeliverableWebhook(webhook, row.event, row.sessionId)) {
// The subscription is gone, switched off or no longer lists this event; replaying it would
// deliver an event the operator has already unsubscribed from. Same test as the queue
// processor applies before every attempt.
await this.outbox.close(row.webhookId, row.idempotencyKey, 'failed');
stats.skipped++;
continue;
}
if (row.deliveryId && (await this.delivery.isQueueJobPending(row.deliveryId))) {
stats.skipped++;
continue;
}
if (row.deliveryId && (await this.delivery.hasQueueJobCompleted(row.deliveryId))) {
// A worker settled this job and its outbox row was left behind (a worker that never closes the
// outbox, or a close that failed). It is not stranded: a replay would POST an event the receiver
// already has, up to the budget, and then record a failure for a delivery that succeeded.
await this.outbox.close(row.webhookId, row.idempotencyKey, 'dispatched');
stats.skipped++;
continue;
}
if (row.attempts >= opts.maxAttempts) {
// A database fault may have prevented every terminal failure write. Keep the outbox
// payload until this handoff succeeds, without sending beyond the replay budget.
if (await this.delivery.recordReplayExhaustion(row, webhook.url)) {
await this.outbox.close(row.webhookId, row.idempotencyKey, 'failed');
}
stats.failed++;
continue;
}
// Shutdown may have begun during the lookups above; stop before spending budget or POSTing.
if (this.stop.signal.aborted) break;
if (!(await this.outbox.countAttempt(row.id, row.attempts))) {
// Settled since the batch was read, typically a local dispatch that finished while an
// earlier row in this pass was replaying. The copy in hand is stale; replaying it duplicates.
stats.skipped++;
continue;
}
try {
// The outcome is a RETURN VALUE, not an exception. Every delivery failure is handled in
// place (dead-letter row, hook, log), so redeliver resolves either way and a catch here
// would see nothing: retiring on resolve alone marked dead-lettered events 'dispatched'
// and nulled their payload, spending the whole budget on one sweep.
const outcome = await this.delivery.redeliver(
webhook,
row.sessionId,
row.event,
row.idempotencyKey,
row.payload,
{ signal: this.stop.signal, oneDirectAttempt: failedWebhooks.has(row.webhookId) },
);
if (outcome === 'failed' || outcome === 'unrecorded') {
// Left 'pending' on purpose: the next sweep retries it until the budget is spent.
this.logger.warn(`Replay of ${row.event} to webhook ${row.webhookId} did not deliver`);
failedWebhooks.add(row.webhookId);
stats.failed++;
continue;
}
// Enqueued rows retain their copy until the worker settles them, and redeliver has already
// retired a delivered one. A deliberate cancellation retires here.
if (outcome === 'cancelled') await this.outbox.close(row.webhookId, row.idempotencyKey, 'dispatched');
stats.replayed++;
} catch (error) {
// A replay cut short in its retry backoff by shutdown was interrupted, not failed. Any other
// exception is an unexpected fault rather than a delivery failure. The row stays pending.
if (isAbortedBackoff(error, this.stop.signal)) {
this.logger.log(`Replay of ${row.event} to webhook ${row.webhookId} interrupted by shutdown`);
stats.skipped++;
continue;
}
this.logger.warn(`Replay of ${row.event} to webhook ${row.webhookId} failed: ${String(error)}`);
stats.failed++;
}
}
} finally {
this.inFlight = undefined;
settle();
}
return stats;
}
}