Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(integration): refuse ingress for a plugin still restoring

Yudhi Armyndharis70e3647d185a
diff --git a/src/modules/integration/ingress-preflight.ts b/src/modules/integration/ingress-preflight.ts
index 166004a9a..925b84fb0 100644
--- a/src/modules/integration/ingress-preflight.ts
+++ b/src/modules/integration/ingress-preflight.ts
@@ -13,6 +13,23 @@ export type PreflightRejection = { status: number; body: string; headers?: Recor
*/
const PREFLIGHT_RETRY_AFTER_SECONDS = 5;
+/**
+ * A refusal the provider is told to retry. Attached per rejection, not to every one: a future
+ * non-retryable check must not inherit a header that tells the provider to come back.
+ */
+const retryable503 = (body: string): PreflightRejection => ({
+ status: 503,
+ body,
+ headers: { 'Retry-After': String(PREFLIGHT_RETRY_AFTER_SECONDS) },
+});
+
+/**
+ * The refusal for a delivery whose plugin the boot restore has not finished with. No worker can take it
+ * yet, so an ack would only dead-letter it after the retry budget, and the provider would never send it
+ * again. IngressService answers it in the same stage as the preflight, so it leaves no dedup row either.
+ */
+export const pluginRestoringRejection = (): PreflightRejection => retryable503('plugin restoring');
+
/**
* Evaluates a route's host-side preflight checks. Returns null to PASS (proceed to dedup + ack + enqueue),
* or a `{status, body}` to REJECT synchronously. Runs AFTER signature verify and BEFORE the dedup persist
@@ -35,15 +52,7 @@ export function evaluatePreflight(
if (!sessionScope || sessionScope === '*') continue; // wildcard: no single session to probe
if (!sessionStatus) continue; // unwired (pure unit mode): skip rather than false-reject
const status = sessionStatus(sessionScope);
- if (status === undefined || status === EngineStatus.FAILED) {
- // Attached at the return site, not to every rejection: a future non-retryable check must not
- // inherit a header that tells the provider to come back.
- return {
- status: 503,
- body: 'session not ready',
- headers: { 'Retry-After': String(PREFLIGHT_RETRY_AFTER_SECONDS) },
- };
- }
+ if (status === undefined || status === EngineStatus.FAILED) return retryable503('session not ready');
}
}
return null;
diff --git a/src/modules/integration/ingress.service.spec.ts b/src/modules/integration/ingress.service.spec.ts
index 5daf0eef7..3ea91b2d7 100644
--- a/src/modules/integration/ingress.service.spec.ts
+++ b/src/modules/integration/ingress.service.spec.ts
@@ -729,6 +729,62 @@ describe('IngressService.handle — response contract', () => {
expect(d.enqueue).toHaveBeenCalled();
});
+ // While the boot restore is still bringing a plugin up, no worker can take its delivery: an ack would
+ // only dead-letter it once the retry budget is spent, and the provider would never send it again.
+ describe('a plugin that is still being restored after boot', () => {
+ it('refuses 503 with Retry-After BEFORE dedup or enqueue, so the retry is not swallowed as a duplicate', async () => {
+ const restorePending = jest.fn().mockReturnValue(true);
+ const d = depsWith({ restorePending });
+
+ const res = await new IngressService(d).handle(baseReq);
+
+ expect(res).toEqual({ status: 503, body: 'plugin restoring', headers: { 'Retry-After': '5' } });
+ expect(restorePending).toHaveBeenCalledWith('p');
+ expect(d.events.recordOrSkip).not.toHaveBeenCalled();
+ expect(d.enqueue).not.toHaveBeenCalled();
+ expect(d.log).toHaveBeenCalledWith(
+ 'ingress_preflight_rejected',
+ expect.objectContaining({ status: 503, reason: 'plugin restoring', route: 'send-sms' }),
+ );
+ });
+
+ it('accepts the delivery once the restore is done with the plugin', async () => {
+ const restorePending = jest.fn().mockReturnValue(false);
+ const d = depsWith({ restorePending });
+
+ const res = await new IngressService(d).handle(baseReq);
+
+ expect(res.status).toBe(202);
+ expect(d.events.recordOrSkip).toHaveBeenCalledTimes(1);
+ expect(d.enqueue).toHaveBeenCalledTimes(1);
+ });
+
+ it('still answers a failed signature with 401, so an unauthenticated caller learns nothing', async () => {
+ const d = depsWith({
+ restorePending: jest.fn().mockReturnValue(true),
+ manifestRoute: jest.fn().mockReturnValue({
+ route: 'send-sms',
+ maxBodyBytes: 1024,
+ signature: { scheme: 'hmac-sha256', header: 'x-sig' },
+ }),
+ });
+
+ const res = await new IngressService(d).handle({ ...baseReq, headers: { 'x-sig': 'sha256=bad' } });
+
+ expect(res.status).toBe(401);
+ });
+
+ it('charges the bucket before it refuses, so a signed provider that keeps retrying is still bounded', async () => {
+ const admitInstance = jest.fn().mockResolvedValue({ ok: false, headers: { 'Retry-After': '7' } });
+ const d = depsWith({ restorePending: jest.fn().mockReturnValue(true), admitInstance });
+
+ const res = await new IngressService(d).handle(baseReq);
+
+ expect(res.status).toBe(429);
+ expect(admitInstance).toHaveBeenCalledTimes(1);
+ });
+ });
+
it('renders a declared ack status/body/headers', async () => {
const d = depsWith({
manifestRoute: jest.fn().mockReturnValue({
diff --git a/src/modules/integration/ingress.service.ts b/src/modules/integration/ingress.service.ts
index bb28ff1b5..63f423098 100644
--- a/src/modules/integration/ingress.service.ts
+++ b/src/modules/integration/ingress.service.ts
@@ -5,7 +5,7 @@ import { constantTimeEqual } from '../../common/security/constantTimeEqual';
import { PluginIngressRoute } from '../../core/plugins/plugin.interfaces';
import { IngressJobData } from '../queue/processors/ingress.processor';
import type { EngineStatus } from '../../engine/interfaces/whatsapp-engine.interface';
-import { evaluatePreflight } from './ingress-preflight';
+import { evaluatePreflight, pluginRestoringRejection } from './ingress-preflight';
import { renderAck } from './ingress-ack';
import type { IngressAdmission } from './ingress-instance-limit';
import { parseBodyLimitBytes } from '../../config/inflight-body-budget';
@@ -69,6 +69,10 @@ export interface IngressDeps {
// field read); MUST NOT call engine.initialize() or any blocking call. Absent (pure unit tests) → the
// session-alive check skips (passes) rather than false-rejecting.
sessionStatus?: (scope: string) => EngineStatus | undefined;
+ // Optional host-side probe: whether the boot restore has yet to finish with this plugin. A delivery for
+ // such a plugin is refused with a retryable 503 in the preflight stage, since no worker can take it
+ // yet. O(1) (a Set lookup). Absent in pure unit tests: every plugin counts as restored.
+ restorePending?: (pluginId: string) => boolean;
// Optional structured sink for preflight rejections, so operators can audit deliveries that were
// rejected host-side (and therefore leave no dedup/DLQ row). Absent in pure unit tests.
log?: (event: string, meta: Record<string, unknown>) => void;
@@ -183,13 +187,17 @@ export class IngressService {
// cannot probe liveness) and BEFORE the dedup persist (so a 5xx-rejected delivery never writes a
// dedup row that would swallow the provider's retry as a 200 'duplicate' — the dedup trap). A
// rejection leaves no dedup/DLQ row, so log it for operator audit.
- const preflight = evaluatePreflight(route, instance.sessionScope, this.deps.sessionStatus);
+ // A plugin the boot restore has not reached yet is refused in the same stage: no worker can take its delivery.
+ const preflight = this.deps.restorePending?.(req.pluginId)
+ ? pluginRestoringRejection()
+ : evaluatePreflight(route, instance.sessionScope, this.deps.sessionStatus);
if (preflight) {
this.deps.log?.('ingress_preflight_rejected', {
pluginId: req.pluginId,
instanceId: req.instanceId,
route: req.route,
status: preflight.status,
+ reason: preflight.body,
sessionScope: instance.sessionScope,
});
// Returned whole, so a rejection's headers reach the wire. Re-packing the two fields dropped the
diff --git a/src/modules/integration/integration-queue-wiring.spec.ts b/src/modules/integration/integration-queue-wiring.spec.ts
index 95125431e..88bb27884 100644
--- a/src/modules/integration/integration-queue-wiring.spec.ts
+++ b/src/modules/integration/integration-queue-wiring.spec.ts
@@ -190,4 +190,27 @@ describe('IntegrationModule queue wiring (QUEUE_ENABLED conditional)', () => {
await moduleRef.close();
}
});
+
+ // The ingress door refuses a delivery for a plugin the boot restore has not finished with. The
+ // refusal is a dependency of the pure pipeline, so only the real graph can show it is connected.
+ it('hands IngressService the restore state of the real plugin loader', async () => {
+ const { moduleRef } = await compileIntegrationModule('false');
+ try {
+ /* eslint-disable @typescript-eslint/no-require-imports */
+ const { IngressService } = require('./ingress.service') as typeof import('./ingress.service');
+ const { PluginLoaderService } =
+ require('../../core/plugins/plugin-loader.service') as typeof import('../../core/plugins/plugin-loader.service');
+ /* eslint-enable @typescript-eslint/no-require-imports */
+ const ingress = moduleRef.get(IngressService, { strict: false });
+ const loader = moduleRef.get(PluginLoaderService, { strict: false });
+ const { restorePending } = (ingress as unknown as { deps: { restorePending?: (id: string) => boolean } }).deps;
+
+ expect(restorePending?.('adapter')).toBe(false);
+ (loader as unknown as { restorePending: Set<string> }).restorePending.add('adapter');
+ expect(restorePending?.('adapter')).toBe(true);
+ expect(restorePending?.('other')).toBe(false);
+ } finally {
+ await moduleRef.close();
+ }
+ });
});
diff --git a/src/modules/integration/integration.module.ts b/src/modules/integration/integration.module.ts
index 610484692..cc930dfbf 100644
--- a/src/modules/integration/integration.module.ts
+++ b/src/modules/integration/integration.module.ts
@@ -93,6 +93,9 @@ if (process.env.QUEUE_ENABLED === 'true') {
// MUST stay cheap — never call engine.initialize() here (that is the slow/blocking path bounded
// separately by the #667/#696 init-timeout). Undefined = no live engine (stopped/deleted).
sessionStatus: (scope: string) => registry.get(scope)?.getStatus(),
+ // A plugin the boot restore has not reached has no worker to take a delivery: the door answers a
+ // retryable 503 before persisting it, as the port did by staying closed until the restore was done.
+ restorePending: (pluginId: string) => loader.isRestorePending(pluginId),
// Audit sink for preflight rejections (they leave no dedup/DLQ row). The Prometheus counter is
// a separate follow-up (see plan notes); the structured log is the MVP audit surface.
log: (event, meta) => ingressLogger.warn(event, meta),
diff --git a/test/integration-fabric.e2e-spec.ts b/test/integration-fabric.e2e-spec.ts
index 7f862cd1f..c69c02009 100644
--- a/test/integration-fabric.e2e-spec.ts
+++ b/test/integration-fabric.e2e-spec.ts
@@ -169,6 +169,39 @@ describe('Integration Fabric ingress (e2e)', () => {
expect(dispatchWebhookForInstance).not.toHaveBeenCalled();
});
+ it('refuses 503 with Retry-After and stores nothing while the plugin restore is pending, then accepts the retry', async () => {
+ const eventRepo = app.get<Repository<IngressEvent>>(getRepositoryToken(IngressEvent, 'data'));
+ const failureRepo = app.get<Repository<IntegrationDeliveryFailure>>(
+ getRepositoryToken(IntegrationDeliveryFailure, 'data'),
+ );
+ const post = () =>
+ request(app.getHttpServer())
+ .post(INGRESS_PATH)
+ .set('X-Chatwoot-Signature', sig)
+ .set('X-Chatwoot-Delivery', 'delivery-restoring-1')
+ .set('Content-Type', 'application/json')
+ .send(raw);
+ const restoring = jest.spyOn(app.get(PluginLoaderService), 'isRestorePending').mockReturnValue(true);
+
+ try {
+ const refused = await post();
+
+ expect(refused.status).toBe(503);
+ expect(refused.headers['retry-after']).toBe('5');
+ expect(dispatchWebhookForInstance).not.toHaveBeenCalled();
+ // A stored row would answer the provider's retry with a 'duplicate' ack and lose the delivery.
+ expect(await eventRepo.countBy({ providerDeliveryId: 'delivery-restoring-1' })).toBe(0);
+ expect(await failureRepo.countBy({ deliveryId: 'delivery-restoring-1' })).toBe(0);
+ } finally {
+ restoring.mockRestore();
+ }
+
+ const retried = await post();
+
+ expect(retried.status).toBe(202);
+ expect(dispatchWebhookForInstance).toHaveBeenCalledTimes(1);
+ });
+
it('marks the persisted event dispatched once a successful inline dispatch is recorded', async () => {
const eventRepo = app.get<Repository<IngressEvent>>(getRepositoryToken(IngressEvent, 'data'));
const res = await request(app.getHttpServer())