Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(integration): accept ingress for a restoring plugin
Yudhi Armyndharis286cdea3fdbf
diff --git a/src/modules/integration/ingress-enqueue.service.spec.ts b/src/modules/integration/ingress-enqueue.service.spec.tsindex fa292915c..a7b9410ea 100644--- a/src/modules/integration/ingress-enqueue.service.spec.ts+++ b/src/modules/integration/ingress-enqueue.service.spec.ts@@ -190,6 +190,47 @@ describe('IngressEnqueueService', () => {expect(loader.dispatchWebhookForInstance).toHaveBeenCalledWith(data);});+ // A plugin the boot restore has not reached has no worker. With the queue on, the job waits in the queue and+ // IngressProcessor holds it; dispatching inline is the one path that would have to run the plugin right away.+ describe('defersInlineDispatch', () => {+ const restoring = (ids: string[]) => {+ loader.isRestorePending = jest.fn((id: string) => ids.includes(id));+ };++ it('defers a plugin the restore has not reached while dispatch is inline', () => {+ restoring(['chatwoot']);+ (config.get as jest.Mock).mockReturnValue(false);+ const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);++ expect(svc.defersInlineDispatch('chatwoot')).toBe(true);+ expect(svc.defersInlineDispatch('other')).toBe(false);+ });++ it('also defers when the queue is switched on but no queue instance was injected', () => {+ restoring(['chatwoot']);+ (config.get as jest.Mock).mockReturnValue(true);+ const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, undefined);++ expect(svc.defersInlineDispatch('chatwoot')).toBe(true);+ });++ it('does not defer when a queue takes the job', () => {+ restoring(['chatwoot']);+ (config.get as jest.Mock).mockReturnValue(true);+ const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);++ expect(svc.defersInlineDispatch('chatwoot')).toBe(false);+ });++ it('does not defer once the restore is done with the plugin', () => {+ restoring([]);+ (config.get as jest.Mock).mockReturnValue(false);+ const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);++ expect(svc.defersInlineDispatch('chatwoot')).toBe(false);+ });+ });+describe('onApplicationBootstrap (queue-wiring tripwire)', () => {const ORIGINAL = process.env.QUEUE_ENABLED;diff --git a/src/modules/integration/ingress-enqueue.service.ts b/src/modules/integration/ingress-enqueue.service.tsindex e38b91799..abb79803a 100644--- a/src/modules/integration/ingress-enqueue.service.ts+++ b/src/modules/integration/ingress-enqueue.service.ts@@ -138,6 +138,16 @@ export class IngressEnqueueService implements OnApplicationBootstrap {}}+ /**+ * Whether a delivery of this plugin must wait to be dispatched: it would run inline, because the queue is off,+ * in a plugin the boot restore has not reached, which has no worker to run it in. With the queue on, the job+ * is enqueued instead, and IngressProcessor holds it until the plugin is up.+ */+ defersInlineDispatch(pluginId: string): boolean {+ const queued = this.config.get<boolean>('queue.enabled', false) && !!this.ingressQueue;+ return !queued && this.loader.isRestorePending(pluginId);+ }+/*** State of the job an earlier enqueue() of this delivery left in the queue, or undefined when the* queue is off, holds no such job, or cannot answer. add() resolves for a duplicate jobId whateverdiff --git a/src/modules/integration/ingress-preflight.ts b/src/modules/integration/ingress-preflight.tsindex 925b84fb0..df47503f5 100644--- a/src/modules/integration/ingress-preflight.ts+++ b/src/modules/integration/ingress-preflight.ts@@ -24,9 +24,11 @@ const retryable503 = (body: string): PreflightRejection => ({});/**- * The refusal for a delivery whose plugin the boot restore has not finished with. No worker can take it- * yet, so an ack would only dead-letter it after the retry budget, and the provider would never send it- * again. IngressService answers it in the same stage as the preflight, so it leaves no dedup row either.+ * The refusal for a delivery on a route that declares a response contract, to a plugin the boot restore has+ * not finished with. The provider wants to know now (an OTP delivered a minute late is useless), and no+ * worker can take the delivery yet, so it is told to retry. A route without a response contract is not+ * refused: its delivery is accepted and held until the plugin is up. IngressService answers this in the+ * same stage as the preflight, so it leaves no dedup row either.*/export const pluginRestoringRejection = (): PreflightRejection => retryable503('plugin restoring');diff --git a/src/modules/integration/ingress.service.spec.ts b/src/modules/integration/ingress.service.spec.tsindex 3ea91b2d7..4d11f0b8e 100644--- a/src/modules/integration/ingress.service.spec.ts+++ b/src/modules/integration/ingress.service.spec.ts@@ -729,12 +729,26 @@ describe('IngressService.handle — response contract', () => {expect(d.enqueue).toHaveBeenCalled();});- // While the boot restore is still bringing a plugin up, no worker can take its delivery: an ack would- // only dead-letter it once the retry budget is spent, and the provider would never send it again.+ // While the boot restore is still bringing a plugin up there is no worker to run its deliveries. A route that+ // declares a response contract asked for an answer now (an OTP that arrives after the restore is useless), so it+ // is refused with a retryable 503. Any other route is accepted and persisted like every delivery: the queue holds+ // its job (IngressProcessor), or with the queue off the reconciler dispatches the row once the plugin is up.describe('a plugin that is still being restored after boot', () => {- it('refuses 503 with Retry-After BEFORE dedup or enqueue, so the retry is not swallowed as a duplicate', async () => {+ // A response contract that is an ack alone, so a 503 can only come from the restore.+ const answeringRoute = () =>+ jest.fn().mockReturnValue({+ route: 'send-sms',+ mode: 'async',+ verify: 'core',+ maxBodyBytes: 1024,+ signature: { scheme: 'none' },+ dedupHeader: 'x-delivery',+ response: { ack: { status: 204 } },+ });++ it('refuses a route that declares a response contract, BEFORE dedup or enqueue, so the retry is not swallowed', async () => {const restorePending = jest.fn().mockReturnValue(true);- const d = depsWith({ restorePending });+ const d = depsWith({ restorePending, manifestRoute: answeringRoute() });const res = await new IngressService(d).handle(baseReq);@@ -748,13 +762,43 @@ describe('IngressService.handle — response contract', () => {);});- it('accepts the delivery once the restore is done with the plugin', async () => {- const restorePending = jest.fn().mockReturnValue(false);- const d = depsWith({ restorePending });+ it('accepts, persists and enqueues a delivery of a route without a response contract', async () => {+ const d = depsWith({ restorePending: jest.fn().mockReturnValue(true) });++ const res = await new IngressService(d).handle(baseReq);++ expect(res.status).toBe(202);+ expect(d.events.recordOrSkip).toHaveBeenCalledTimes(1);+ expect(d.enqueue).toHaveBeenCalledTimes(1);+ expect(d.log).not.toHaveBeenCalled();+ });++ it('persists and acks a delivery but does not dispatch it while dispatch has to wait', async () => {+ const deferDispatch = jest.fn().mockReturnValue(true);+ const d = depsWith({ restorePending: jest.fn().mockReturnValue(true), deferDispatch });const res = await new IngressService(d).handle(baseReq);expect(res.status).toBe(202);+ expect(deferDispatch).toHaveBeenCalledWith('p');+ expect(d.events.recordOrSkip).toHaveBeenCalledTimes(1);+ expect(d.enqueue).not.toHaveBeenCalled();+ });++ it('dispatches as usual when dispatch need not wait', async () => {+ const d = depsWith({ deferDispatch: jest.fn().mockReturnValue(false) });++ await new IngressService(d).handle(baseReq);++ expect(d.enqueue).toHaveBeenCalledTimes(1);+ });++ it('accepts a route that declares a response contract once the restore is done with the plugin', async () => {+ const d = depsWith({ restorePending: jest.fn().mockReturnValue(false), manifestRoute: answeringRoute() });++ const res = await new IngressService(d).handle(baseReq);++ expect(res.status).toBe(204);expect(d.events.recordOrSkip).toHaveBeenCalledTimes(1);expect(d.enqueue).toHaveBeenCalledTimes(1);});@@ -766,6 +810,7 @@ describe('IngressService.handle — response contract', () => {route: 'send-sms',maxBodyBytes: 1024,signature: { scheme: 'hmac-sha256', header: 'x-sig' },+ response: { ack: { status: 204 } },}),});@@ -776,7 +821,11 @@ describe('IngressService.handle — response contract', () => {it('charges the bucket before it refuses, so a signed provider that keeps retrying is still bounded', async () => {const admitInstance = jest.fn().mockResolvedValue({ ok: false, headers: { 'Retry-After': '7' } });- const d = depsWith({ restorePending: jest.fn().mockReturnValue(true), admitInstance });+ const d = depsWith({+ restorePending: jest.fn().mockReturnValue(true),+ manifestRoute: answeringRoute(),+ admitInstance,+ });const res = await new IngressService(d).handle(baseReq);diff --git a/src/modules/integration/ingress.service.ts b/src/modules/integration/ingress.service.tsindex 63f423098..5d012fa25 100644--- a/src/modules/integration/ingress.service.ts+++ b/src/modules/integration/ingress.service.ts@@ -70,9 +70,15 @@ export interface IngressDeps {// session-alive check skips (passes) rather than false-rejecting.sessionStatus?: (scope: string) => EngineStatus | undefined;// Optional host-side probe: whether the boot restore has yet to finish with this plugin. A delivery for- // such a plugin is refused with a retryable 503 in the preflight stage, since no worker can take it- // yet. O(1) (a Set lookup). Absent in pure unit tests: every plugin counts as restored.+ // such a plugin on a route that declares a response contract is refused with a retryable 503 in the+ // preflight stage, since no worker can take it yet. O(1) (a Set lookup). Absent in pure unit tests: every+ // plugin counts as restored.restorePending?: (pluginId: string) => boolean;+ // Optional host-side probe: whether a delivery that was just persisted must not be dispatched yet. True for a+ // plugin the boot restore has not reached while dispatch runs inline (queue off), where there is no worker+ // to run it in; the row then stays 'pending' for the reconciler. Absent in pure unit tests: every delivery+ // is dispatched.+ deferDispatch?: (pluginId: string) => boolean;// Optional structured sink for preflight rejections, so operators can audit deliveries that were// rejected host-side (and therefore leave no dedup/DLQ row). Absent in pure unit tests.log?: (event: string, meta: Record<string, unknown>) => void;@@ -187,10 +193,13 @@ export class IngressService {// cannot probe liveness) and BEFORE the dedup persist (so a 5xx-rejected delivery never writes a// dedup row that would swallow the provider's retry as a 200 'duplicate' — the dedup trap). A// rejection leaves no dedup/DLQ row, so log it for operator audit.- // A plugin the boot restore has not reached yet is refused in the same stage: no worker can take its delivery.- const preflight = this.deps.restorePending?.(req.pluginId)- ? pluginRestoringRejection()- : evaluatePreflight(route, instance.sessionScope, this.deps.sessionStatus);+ // A route that declares a response contract wants its answer now, so a plugin the boot restore has not+ // reached yet is refused in the same stage. Any other route is accepted and persisted below like every+ // delivery: the queue holds its job until the plugin is up, or the reconciler dispatches the row.+ const preflight =+ route.response && this.deps.restorePending?.(req.pluginId)+ ? pluginRestoringRejection()+ : evaluatePreflight(route, instance.sessionScope, this.deps.sessionStatus);if (preflight) {this.deps.log?.('ingress_preflight_rejected', {pluginId: req.pluginId,@@ -264,6 +273,9 @@ export class IngressService {sessionId: instance.sessionScope,});if (!isNew) return ack; // already persisted/acked; a retry gets the route's ack, not enqueued again+ // Persisted and acked: dispatching it now would run the plugin inline where there is no worker to run it in.+ // The row stays 'pending', which is where a crash here would leave it too, and the reconciler dispatches it.+ if (this.deps.deferDispatch?.(req.pluginId)) return ack;// Best-effort conversation id for P1 ordering. Never throws — a malformed body just yields undefined.const providerConversationId = extractConversationId(route.conversationId, req.headers, req.rawBody);diff --git a/src/modules/integration/integration-queue-wiring.spec.ts b/src/modules/integration/integration-queue-wiring.spec.tsindex 88bb27884..40baef204 100644--- a/src/modules/integration/integration-queue-wiring.spec.ts+++ b/src/modules/integration/integration-queue-wiring.spec.ts@@ -191,8 +191,8 @@ describe('IntegrationModule queue wiring (QUEUE_ENABLED conditional)', () => {}});- // The ingress door refuses a delivery for a plugin the boot restore has not finished with. The- // refusal is a dependency of the pure pipeline, so only the real graph can show it is connected.+ // The ingress door treats a delivery for a plugin the boot restore has not finished with differently. Both+ // decisions are dependencies of the pure pipeline, so only the real graph can show they are connected.it('hands IngressService the restore state of the real plugin loader', async () => {const { moduleRef } = await compileIntegrationModule('false');try {@@ -213,4 +213,57 @@ describe('IntegrationModule queue wiring (QUEUE_ENABLED conditional)', () => {await moduleRef.close();}});++ describe('deferring the dispatch of a persisted delivery for a plugin that is still restoring', () => {+ const ORIGINAL_INTERVAL = process.env.INGRESS_RECONCILE_INTERVAL_MS;+ afterEach(() => {+ if (ORIGINAL_INTERVAL === undefined) delete process.env.INGRESS_RECONCILE_INTERVAL_MS;+ else process.env.INGRESS_RECONCILE_INTERVAL_MS = ORIGINAL_INTERVAL;+ });++ /** The deferDispatch dependency of the real IngressService, with `adapter` marked as restoring. */+ async function deferDispatchFor(queueEnabled: 'true' | 'false') {+ const { moduleRef } = await compileIntegrationModule(queueEnabled);+ /* eslint-disable @typescript-eslint/no-require-imports */+ const { IngressService } = require('./ingress.service') as typeof import('./ingress.service');+ const { PluginLoaderService } =+ require('../../core/plugins/plugin-loader.service') as typeof import('../../core/plugins/plugin-loader.service');+ /* eslint-enable @typescript-eslint/no-require-imports */+ const ingress = moduleRef.get(IngressService, { strict: false });+ const loader = moduleRef.get(PluginLoaderService, { strict: false });+ (loader as unknown as { restorePending: Set<string> }).restorePending.add('adapter');+ const { deferDispatch } = (ingress as unknown as { deps: { deferDispatch?: (id: string) => boolean } }).deps;+ return { moduleRef, deferDispatch: deferDispatch as (id: string) => boolean };+ }++ it('waits for the reconciler when the queue is off and the plugin is restoring', async () => {+ delete process.env.INGRESS_RECONCILE_INTERVAL_MS;+ const { moduleRef, deferDispatch } = await deferDispatchFor('false');+ try {+ expect(deferDispatch('adapter')).toBe(true);+ expect(deferDispatch('other')).toBe(false);+ } finally {+ await moduleRef.close();+ }+ });++ it('does not wait when a queue takes the job, which the processor holds', async () => {+ const { moduleRef, deferDispatch } = await deferDispatchFor('true');+ try {+ expect(deferDispatch('adapter')).toBe(false);+ } finally {+ await moduleRef.close();+ }+ });++ it('does not wait when the reconciler is off, since nothing would ever replay the row', async () => {+ process.env.INGRESS_RECONCILE_INTERVAL_MS = '0';+ const { moduleRef, deferDispatch } = await deferDispatchFor('false');+ try {+ expect(deferDispatch('adapter')).toBe(false);+ } finally {+ await moduleRef.close();+ }+ });+ });});diff --git a/src/modules/integration/integration.module.ts b/src/modules/integration/integration.module.tsindex cc930dfbf..5c53fcaf5 100644--- a/src/modules/integration/integration.module.ts+++ b/src/modules/integration/integration.module.ts@@ -11,7 +11,7 @@ import { IngressService, IngressRouteDescriptor } from './ingress.service';import { IngressController } from './ingress.controller';import { admitIngressInstance } from './ingress-instance-limit';import { IngressEnqueueService, buildIngressDeadLetterRow } from './ingress-enqueue.service';-import { IngressReconcilerService } from './ingress-reconciler.service';+import { IngressReconcilerService, resolveIngressReconcilerOptions } from './ingress-reconciler.service';import { RedriveService } from './redrive.service';import { RedriveController } from './redrive.controller';import { IntegrationRetentionService } from './integration-retention.service';@@ -93,9 +93,13 @@ if (process.env.QUEUE_ENABLED === 'true') {// MUST stay cheap — never call engine.initialize() here (that is the slow/blocking path bounded// separately by the #667/#696 init-timeout). Undefined = no live engine (stopped/deleted).sessionStatus: (scope: string) => registry.get(scope)?.getStatus(),- // A plugin the boot restore has not reached has no worker to take a delivery: the door answers a- // retryable 503 before persisting it, as the port did by staying closed until the restore was done.+ // A plugin the boot restore has not reached has no worker to take a delivery. A route that wants an+ // answer is refused with a retryable 503 before anything is persisted; any other is accepted and held.restorePending: (pluginId: string) => loader.isRestorePending(pluginId),+ // Queue off: the dispatch would run inline, so the persisted row waits for the reconciler. With the+ // reconciler off nothing would ever replay it, so the inline attempt (and its dead letter) stays.+ deferDispatch: (pluginId: string) =>+ ingressEnqueue.defersInlineDispatch(pluginId) && resolveIngressReconcilerOptions().intervalMs > 0,// Audit sink for preflight rejections (they leave no dedup/DLQ row). The Prometheus counter is// a separate follow-up (see plan notes); the structured log is the MVP audit surface.log: (event, meta) => ingressLogger.warn(event, meta),diff --git a/test/integration-fabric.e2e-spec.ts b/test/integration-fabric.e2e-spec.tsindex c69c02009..c25b9e398 100644--- a/test/integration-fabric.e2e-spec.ts+++ b/test/integration-fabric.e2e-spec.ts@@ -11,6 +11,7 @@ import { json, urlencoded, Request } from 'express';import { AppModule } from './../src/app.module';import { applyGlobalValidation } from './../src/config/app-validation';import { PluginLoaderService } from './../src/core/plugins/plugin-loader.service';+import { IngressReconcilerService } from './../src/modules/integration/ingress-reconciler.service';import { PluginInstance } from './../src/modules/integration/entities/plugin-instance.entity';import { IngressEvent } from './../src/modules/integration/entities/ingress-event.entity';import { IntegrationDeliveryFailure } from './../src/modules/integration/entities/integration-delivery-failure.entity';@@ -60,6 +61,10 @@ describe('Integration Fabric ingress (e2e)', () => {},dedupHeader: 'x-chatwoot-delivery',};+ // The same provider contract on a route that declares a response contract (an ack alone), as an adapter whose+ // provider needs an answer now does: while its plugin is restoring, that is the one route a delivery is refused on.+ const answeringManifestRoute = { ...ingressManifestRoute, route: 'send-sms', response: { ack: { status: 204 } } };+ const ANSWERING_PATH = '/api/ingress/chatwoot/acct1/send-sms';beforeAll(async () => {const moduleFixture: TestingModule = await Test.createTestingModule({@@ -97,7 +102,9 @@ describe('Integration Fabric ingress (e2e)', () => {const realGetPlugin = loader.getPlugin.bind(loader);jest.spyOn(loader, 'getPlugin').mockImplementation((pluginId: string) => {if (pluginId === 'chatwoot') {- return { manifest: { ingress: [ingressManifestRoute] } } as unknown as ReturnType<typeof loader.getPlugin>;+ return {+ manifest: { ingress: [ingressManifestRoute, answeringManifestRoute] },+ } as unknown as ReturnType<typeof loader.getPlugin>;}return realGetPlugin(pluginId);});@@ -169,37 +176,108 @@ describe('Integration Fabric ingress (e2e)', () => {expect(dispatchWebhookForInstance).not.toHaveBeenCalled();});- it('refuses 503 with Retry-After and stores nothing while the plugin restore is pending, then accepts the retry', async () => {- const eventRepo = app.get<Repository<IngressEvent>>(getRepositoryToken(IngressEvent, 'data'));- const failureRepo = app.get<Repository<IntegrationDeliveryFailure>>(- getRepositoryToken(IntegrationDeliveryFailure, 'data'),- );- const post = () =>+ // A plugin the boot restore has not reached has no worker. A route without a response contract is accepted and+ // persisted like every delivery; with the queue off the row is not dispatched at the door but left for the+ // reconciler, which dispatches it once the plugin is up.+ describe('while the plugin restore is pending', () => {+ const stateRepos = () => ({+ events: app.get<Repository<IngressEvent>>(getRepositoryToken(IngressEvent, 'data')),+ failures: app.get<Repository<IntegrationDeliveryFailure>>(getRepositoryToken(IntegrationDeliveryFailure, 'data')),+ });+ const post = (path: string, deliveryId: string) =>request(app.getHttpServer())- .post(INGRESS_PATH)+ .post(path).set('X-Chatwoot-Signature', sig)- .set('X-Chatwoot-Delivery', 'delivery-restoring-1')+ .set('X-Chatwoot-Delivery', deliveryId).set('Content-Type', 'application/json').send(raw);- const restoring = jest.spyOn(app.get(PluginLoaderService), 'isRestorePending').mockReturnValue(true);+ const restoring = () => jest.spyOn(app.get(PluginLoaderService), 'isRestorePending').mockReturnValue(true);- try {- const refused = await post();-- expect(refused.status).toBe(503);- expect(refused.headers['retry-after']).toBe('5');- expect(dispatchWebhookForInstance).not.toHaveBeenCalled();- // A stored row would answer the provider's retry with a 'duplicate' ack and lose the delivery.- expect(await eventRepo.countBy({ providerDeliveryId: 'delivery-restoring-1' })).toBe(0);- expect(await failureRepo.countBy({ deliveryId: 'delivery-restoring-1' })).toBe(0);- } finally {- restoring.mockRestore();- }+ it('accepts and persists a delivery of a route without a response contract, and the reconciler delivers it afterwards', async () => {+ const { events, failures } = stateRepos();+ const restore = restoring();+ try {+ const accepted = await post(INGRESS_PATH, 'delivery-held-1');- const retried = await post();+ expect(accepted.status).toBe(202);+ // Not dispatched into a plugin with no worker, and not dead-lettered: the row is the record of it.+ expect(dispatchWebhookForInstance).not.toHaveBeenCalled();+ const held = await events.findOneByOrFail({ providerDeliveryId: 'delivery-held-1' });+ expect(held.dispatchState).toBe('pending');+ expect(held.dispatchAttempts).toBe(0);+ expect(held.payload).not.toBeNull();+ expect(await failures.countBy({ deliveryId: 'delivery-held-1' })).toBe(0);- expect(retried.status).toBe(202);- expect(dispatchWebhookForInstance).toHaveBeenCalledTimes(1);+ // The provider's own retry of it is a duplicate, as for any persisted delivery.+ expect((await post(INGRESS_PATH, 'delivery-held-1')).status).toBe(202);+ expect(await events.countBy({ providerDeliveryId: 'delivery-held-1' })).toBe(1);++ // While the restore lasts the reconciler leaves the row alone.+ const reconciler = app.get(IngressReconcilerService);+ const options = { intervalMs: 60_000, graceMs: 0, batchSize: 50, maxAttempts: 5 };+ await new Promise(resolve => setTimeout(resolve, 5));+ await reconciler.sweep(options);+ expect(dispatchWebhookForInstance).not.toHaveBeenCalled();++ restore.mockRestore();+ const stats = await reconciler.sweep(options);++ expect(stats).toMatchObject({ scanned: 1, replayed: 1 });+ expect(dispatchWebhookForInstance).toHaveBeenCalledTimes(1);+ const delivered = await events.findOneByOrFail({ providerDeliveryId: 'delivery-held-1' });+ expect(delivered.dispatchState).toBe('dispatched');+ expect(delivered.payload).toBeNull();+ } finally {+ restore.mockRestore();+ }+ });++ it('refuses a route that declares a response contract with 503 and Retry-After, stores nothing, then accepts the retry', async () => {+ const { events, failures } = stateRepos();+ const restore = restoring();+ try {+ const refused = await post(ANSWERING_PATH, 'delivery-answer-1');++ expect(refused.status).toBe(503);+ expect(refused.headers['retry-after']).toBe('5');+ expect(dispatchWebhookForInstance).not.toHaveBeenCalled();+ // A stored row would answer the provider's retry with a 'duplicate' ack and lose the delivery.+ expect(await events.countBy({ providerDeliveryId: 'delivery-answer-1' })).toBe(0);+ expect(await failures.countBy({ deliveryId: 'delivery-answer-1' })).toBe(0);+ } finally {+ restore.mockRestore();+ }++ const retried = await post(ANSWERING_PATH, 'delivery-answer-1');++ expect(retried.status).toBe(204);+ expect(dispatchWebhookForInstance).toHaveBeenCalledTimes(1);+ });++ it('still dispatches at the door when the reconciler is off, since nothing would replay a held row', async () => {+ const { events, failures } = stateRepos();+ const original = process.env.INGRESS_RECONCILE_INTERVAL_MS;+ process.env.INGRESS_RECONCILE_INTERVAL_MS = '0';+ const restore = restoring();+ dispatchWebhookForInstance.mockRejectedValueOnce(new Error('no live sandbox host for plugin chatwoot'));+ try {+ const res = await post(INGRESS_PATH, 'delivery-noreconcile-1');++ // The inline dispatch fails and files the redrivable dead letter an operator can see.+ expect(res.status).toBe(202);+ expect(dispatchWebhookForInstance).toHaveBeenCalledTimes(1);+ expect(await failures.countBy({ deliveryId: 'delivery-noreconcile-1', redriven: false })).toBe(1);+ expect((await events.findOneByOrFail({ providerDeliveryId: 'delivery-noreconcile-1' })).dispatchAttempts).toBe(+ 1,+ );+ } finally {+ restore.mockRestore();+ if (original === undefined) delete process.env.INGRESS_RECONCILE_INTERVAL_MS;+ else process.env.INGRESS_RECONCILE_INTERVAL_MS = original;+ // An unused once-rejection must not reach the next test.+ dispatchWebhookForInstance.mockReset().mockResolvedValue(undefined);+ }+ });});it('marks the persisted event dispatched once a successful inline dispatch is recorded', async () => {diff --git a/test/queue-on.e2e-spec.ts b/test/queue-on.e2e-spec.tsindex debbecf3d..e2a08ed65 100644--- a/test/queue-on.e2e-spec.ts+++ b/test/queue-on.e2e-spec.ts@@ -267,6 +267,49 @@ describeQueueOn('Queued dispatch paths (e2e, QUEUE_ENABLED=true)', () => {expect(dispatchedArgs[0][0].deliveryId).toBe(deliveryId);}, 30000);+ it('accepts a delivery for a plugin that is still restoring, holds its job without spending an attempt, then dispatches it', async () => {+ // The held job comes back after the retry delay, at least a second; a short one keeps the test brief.+ const prevDelay = process.env.INGRESS_RETRY_DELAY_MS;+ process.env.INGRESS_RETRY_DELAY_MS = '1000';+ const deliveryId = `queue-on-held-${Date.now()}`;+ const restoring = jest.spyOn(app.get(PluginLoaderService), 'isRestorePending').mockReturnValue(true);+ dispatchWebhookForInstance.mockClear();+ try {+ const res = await request(app.getHttpServer())+ .post(INGRESS_PATH)+ .set('X-Chatwoot-Signature', sig)+ .set('X-Chatwoot-Delivery', deliveryId)+ .set('Content-Type', 'application/json')+ .send(raw);++ // Accepted and queued like any delivery: no refusal for a route without a response contract.+ expect(res.status).toBe(202);+ const jobId = sanitizeIngressJobId(deliveryId, 'chatwoot\u0000acct1');+ // The worker takes the job, sees the plugin is not up, and puts it back in the delayed set. It does this+ // again at every delay, and spends no attempt, so the retry budget is intact when the plugin is up.+ await waitFor(+ async () =>+ (await ingressQueue.getJob(jobId))?.attemptsStarted !== 0 &&+ (await ingressQueue.getJobState(jobId)) === 'delayed',+ );+ await new Promise(resolve => setTimeout(resolve, 2500));+ const held = await ingressQueue.getJob(jobId);+ expect(held?.attemptsStarted).toBeGreaterThan(1);+ expect(held?.attemptsMade).toBe(0);+ expect(dispatchWebhookForInstance).not.toHaveBeenCalled();++ restoring.mockRestore();+ await waitFor(() => dispatchWebhookForInstance.mock.calls.length >= 1, 15000);+ expect((dispatchWebhookForInstance.mock.calls as IngressJobData[][])[0][0].deliveryId).toBe(deliveryId);+ await waitFor(async () => (await ingressQueue.getJobState(jobId)) === 'completed');+ expect((await ingressQueue.getJob(jobId))?.attemptsMade).toBe(1);+ } finally {+ restoring.mockRestore();+ if (prevDelay === undefined) delete process.env.INGRESS_RETRY_DELAY_MS;+ else process.env.INGRESS_RETRY_DELAY_MS = prevDelay;+ }+ }, 60000);+it('enqueues the webhook delivery and the worker POSTs it to the receiver', async () => {const session = await sessionRepo.save(sessionRepo.create({ name: `e2e-queue-on-${Date.now()}` }));await request(app.getHttpServer())