Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(integration): refuse ingress for a plugin still restoring
Yudhi Armyndharis70e3647d185a
diff --git a/src/modules/integration/ingress-preflight.ts b/src/modules/integration/ingress-preflight.tsindex 166004a9a..925b84fb0 100644--- a/src/modules/integration/ingress-preflight.ts+++ b/src/modules/integration/ingress-preflight.ts@@ -13,6 +13,23 @@ export type PreflightRejection = { status: number; body: string; headers?: Recor*/const PREFLIGHT_RETRY_AFTER_SECONDS = 5;+/**+ * A refusal the provider is told to retry. Attached per rejection, not to every one: a future+ * non-retryable check must not inherit a header that tells the provider to come back.+ */+const retryable503 = (body: string): PreflightRejection => ({+ status: 503,+ body,+ headers: { 'Retry-After': String(PREFLIGHT_RETRY_AFTER_SECONDS) },+});++/**+ * The refusal for a delivery whose plugin the boot restore has not finished with. No worker can take it+ * yet, so an ack would only dead-letter it after the retry budget, and the provider would never send it+ * again. IngressService answers it in the same stage as the preflight, so it leaves no dedup row either.+ */+export const pluginRestoringRejection = (): PreflightRejection => retryable503('plugin restoring');+/*** Evaluates a route's host-side preflight checks. Returns null to PASS (proceed to dedup + ack + enqueue),* or a `{status, body}` to REJECT synchronously. Runs AFTER signature verify and BEFORE the dedup persist@@ -35,15 +52,7 @@ export function evaluatePreflight(if (!sessionScope || sessionScope === '*') continue; // wildcard: no single session to probeif (!sessionStatus) continue; // unwired (pure unit mode): skip rather than false-rejectconst status = sessionStatus(sessionScope);- if (status === undefined || status === EngineStatus.FAILED) {- // Attached at the return site, not to every rejection: a future non-retryable check must not- // inherit a header that tells the provider to come back.- return {- status: 503,- body: 'session not ready',- headers: { 'Retry-After': String(PREFLIGHT_RETRY_AFTER_SECONDS) },- };- }+ if (status === undefined || status === EngineStatus.FAILED) return retryable503('session not ready');}}return null;diff --git a/src/modules/integration/ingress.service.spec.ts b/src/modules/integration/ingress.service.spec.tsindex 5daf0eef7..3ea91b2d7 100644--- a/src/modules/integration/ingress.service.spec.ts+++ b/src/modules/integration/ingress.service.spec.ts@@ -729,6 +729,62 @@ describe('IngressService.handle — response contract', () => {expect(d.enqueue).toHaveBeenCalled();});+ // While the boot restore is still bringing a plugin up, no worker can take its delivery: an ack would+ // only dead-letter it once the retry budget is spent, and the provider would never send it again.+ describe('a plugin that is still being restored after boot', () => {+ it('refuses 503 with Retry-After BEFORE dedup or enqueue, so the retry is not swallowed as a duplicate', async () => {+ const restorePending = jest.fn().mockReturnValue(true);+ const d = depsWith({ restorePending });++ const res = await new IngressService(d).handle(baseReq);++ expect(res).toEqual({ status: 503, body: 'plugin restoring', headers: { 'Retry-After': '5' } });+ expect(restorePending).toHaveBeenCalledWith('p');+ expect(d.events.recordOrSkip).not.toHaveBeenCalled();+ expect(d.enqueue).not.toHaveBeenCalled();+ expect(d.log).toHaveBeenCalledWith(+ 'ingress_preflight_rejected',+ expect.objectContaining({ status: 503, reason: 'plugin restoring', route: 'send-sms' }),+ );+ });++ it('accepts the delivery once the restore is done with the plugin', async () => {+ const restorePending = jest.fn().mockReturnValue(false);+ const d = depsWith({ restorePending });++ const res = await new IngressService(d).handle(baseReq);++ expect(res.status).toBe(202);+ expect(d.events.recordOrSkip).toHaveBeenCalledTimes(1);+ expect(d.enqueue).toHaveBeenCalledTimes(1);+ });++ it('still answers a failed signature with 401, so an unauthenticated caller learns nothing', async () => {+ const d = depsWith({+ restorePending: jest.fn().mockReturnValue(true),+ manifestRoute: jest.fn().mockReturnValue({+ route: 'send-sms',+ maxBodyBytes: 1024,+ signature: { scheme: 'hmac-sha256', header: 'x-sig' },+ }),+ });++ const res = await new IngressService(d).handle({ ...baseReq, headers: { 'x-sig': 'sha256=bad' } });++ expect(res.status).toBe(401);+ });++ it('charges the bucket before it refuses, so a signed provider that keeps retrying is still bounded', async () => {+ const admitInstance = jest.fn().mockResolvedValue({ ok: false, headers: { 'Retry-After': '7' } });+ const d = depsWith({ restorePending: jest.fn().mockReturnValue(true), admitInstance });++ const res = await new IngressService(d).handle(baseReq);++ expect(res.status).toBe(429);+ expect(admitInstance).toHaveBeenCalledTimes(1);+ });+ });+it('renders a declared ack status/body/headers', async () => {const d = depsWith({manifestRoute: jest.fn().mockReturnValue({diff --git a/src/modules/integration/ingress.service.ts b/src/modules/integration/ingress.service.tsindex bb28ff1b5..63f423098 100644--- a/src/modules/integration/ingress.service.ts+++ b/src/modules/integration/ingress.service.ts@@ -5,7 +5,7 @@ import { constantTimeEqual } from '../../common/security/constantTimeEqual';import { PluginIngressRoute } from '../../core/plugins/plugin.interfaces';import { IngressJobData } from '../queue/processors/ingress.processor';import type { EngineStatus } from '../../engine/interfaces/whatsapp-engine.interface';-import { evaluatePreflight } from './ingress-preflight';+import { evaluatePreflight, pluginRestoringRejection } from './ingress-preflight';import { renderAck } from './ingress-ack';import type { IngressAdmission } from './ingress-instance-limit';import { parseBodyLimitBytes } from '../../config/inflight-body-budget';@@ -69,6 +69,10 @@ export interface IngressDeps {// field read); MUST NOT call engine.initialize() or any blocking call. Absent (pure unit tests) → the// session-alive check skips (passes) rather than false-rejecting.sessionStatus?: (scope: string) => EngineStatus | undefined;+ // Optional host-side probe: whether the boot restore has yet to finish with this plugin. A delivery for+ // such a plugin is refused with a retryable 503 in the preflight stage, since no worker can take it+ // yet. O(1) (a Set lookup). Absent in pure unit tests: every plugin counts as restored.+ restorePending?: (pluginId: string) => boolean;// Optional structured sink for preflight rejections, so operators can audit deliveries that were// rejected host-side (and therefore leave no dedup/DLQ row). Absent in pure unit tests.log?: (event: string, meta: Record<string, unknown>) => void;@@ -183,13 +187,17 @@ export class IngressService {// cannot probe liveness) and BEFORE the dedup persist (so a 5xx-rejected delivery never writes a// dedup row that would swallow the provider's retry as a 200 'duplicate' — the dedup trap). A// rejection leaves no dedup/DLQ row, so log it for operator audit.- const preflight = evaluatePreflight(route, instance.sessionScope, this.deps.sessionStatus);+ // A plugin the boot restore has not reached yet is refused in the same stage: no worker can take its delivery.+ const preflight = this.deps.restorePending?.(req.pluginId)+ ? pluginRestoringRejection()+ : evaluatePreflight(route, instance.sessionScope, this.deps.sessionStatus);if (preflight) {this.deps.log?.('ingress_preflight_rejected', {pluginId: req.pluginId,instanceId: req.instanceId,route: req.route,status: preflight.status,+ reason: preflight.body,sessionScope: instance.sessionScope,});// Returned whole, so a rejection's headers reach the wire. Re-packing the two fields dropped thediff --git a/src/modules/integration/integration-queue-wiring.spec.ts b/src/modules/integration/integration-queue-wiring.spec.tsindex 95125431e..88bb27884 100644--- a/src/modules/integration/integration-queue-wiring.spec.ts+++ b/src/modules/integration/integration-queue-wiring.spec.ts@@ -190,4 +190,27 @@ describe('IntegrationModule queue wiring (QUEUE_ENABLED conditional)', () => {await moduleRef.close();}});++ // The ingress door refuses a delivery for a plugin the boot restore has not finished with. The+ // refusal is a dependency of the pure pipeline, so only the real graph can show it is connected.+ it('hands IngressService the restore state of the real plugin loader', async () => {+ const { moduleRef } = await compileIntegrationModule('false');+ try {+ /* eslint-disable @typescript-eslint/no-require-imports */+ const { IngressService } = require('./ingress.service') as typeof import('./ingress.service');+ const { PluginLoaderService } =+ require('../../core/plugins/plugin-loader.service') as typeof import('../../core/plugins/plugin-loader.service');+ /* eslint-enable @typescript-eslint/no-require-imports */+ const ingress = moduleRef.get(IngressService, { strict: false });+ const loader = moduleRef.get(PluginLoaderService, { strict: false });+ const { restorePending } = (ingress as unknown as { deps: { restorePending?: (id: string) => boolean } }).deps;++ expect(restorePending?.('adapter')).toBe(false);+ (loader as unknown as { restorePending: Set<string> }).restorePending.add('adapter');+ expect(restorePending?.('adapter')).toBe(true);+ expect(restorePending?.('other')).toBe(false);+ } finally {+ await moduleRef.close();+ }+ });});diff --git a/src/modules/integration/integration.module.ts b/src/modules/integration/integration.module.tsindex 610484692..cc930dfbf 100644--- a/src/modules/integration/integration.module.ts+++ b/src/modules/integration/integration.module.ts@@ -93,6 +93,9 @@ if (process.env.QUEUE_ENABLED === 'true') {// MUST stay cheap — never call engine.initialize() here (that is the slow/blocking path bounded// separately by the #667/#696 init-timeout). Undefined = no live engine (stopped/deleted).sessionStatus: (scope: string) => registry.get(scope)?.getStatus(),+ // A plugin the boot restore has not reached has no worker to take a delivery: the door answers a+ // retryable 503 before persisting it, as the port did by staying closed until the restore was done.+ restorePending: (pluginId: string) => loader.isRestorePending(pluginId),// Audit sink for preflight rejections (they leave no dedup/DLQ row). The Prometheus counter is// a separate follow-up (see plan notes); the structured log is the MVP audit surface.log: (event, meta) => ingressLogger.warn(event, meta),diff --git a/test/integration-fabric.e2e-spec.ts b/test/integration-fabric.e2e-spec.tsindex 7f862cd1f..c69c02009 100644--- a/test/integration-fabric.e2e-spec.ts+++ b/test/integration-fabric.e2e-spec.ts@@ -169,6 +169,39 @@ describe('Integration Fabric ingress (e2e)', () => {expect(dispatchWebhookForInstance).not.toHaveBeenCalled();});+ it('refuses 503 with Retry-After and stores nothing while the plugin restore is pending, then accepts the retry', async () => {+ const eventRepo = app.get<Repository<IngressEvent>>(getRepositoryToken(IngressEvent, 'data'));+ const failureRepo = app.get<Repository<IntegrationDeliveryFailure>>(+ getRepositoryToken(IntegrationDeliveryFailure, 'data'),+ );+ const post = () =>+ request(app.getHttpServer())+ .post(INGRESS_PATH)+ .set('X-Chatwoot-Signature', sig)+ .set('X-Chatwoot-Delivery', 'delivery-restoring-1')+ .set('Content-Type', 'application/json')+ .send(raw);+ const restoring = jest.spyOn(app.get(PluginLoaderService), 'isRestorePending').mockReturnValue(true);++ try {+ const refused = await post();++ expect(refused.status).toBe(503);+ expect(refused.headers['retry-after']).toBe('5');+ expect(dispatchWebhookForInstance).not.toHaveBeenCalled();+ // A stored row would answer the provider's retry with a 'duplicate' ack and lose the delivery.+ expect(await eventRepo.countBy({ providerDeliveryId: 'delivery-restoring-1' })).toBe(0);+ expect(await failureRepo.countBy({ deliveryId: 'delivery-restoring-1' })).toBe(0);+ } finally {+ restoring.mockRestore();+ }++ const retried = await post();++ expect(retried.status).toBe(202);+ expect(dispatchWebhookForInstance).toHaveBeenCalledTimes(1);+ });+it('marks the persisted event dispatched once a successful inline dispatch is recorded', async () => {const eventRepo = app.get<Repository<IngressEvent>>(getRepositoryToken(IngressEvent, 'data'));const res = await request(app.getHttpServer())