Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(integration): accept ingress for a restoring plugin

Yudhi Armyndharis286cdea3fdbf
diff --git a/src/modules/integration/ingress-enqueue.service.spec.ts b/src/modules/integration/ingress-enqueue.service.spec.ts
index fa292915c..a7b9410ea 100644
--- a/src/modules/integration/ingress-enqueue.service.spec.ts
+++ b/src/modules/integration/ingress-enqueue.service.spec.ts
@@ -190,6 +190,47 @@ describe('IngressEnqueueService', () => {
expect(loader.dispatchWebhookForInstance).toHaveBeenCalledWith(data);
});
+ // A plugin the boot restore has not reached has no worker. With the queue on, the job waits in the queue and
+ // IngressProcessor holds it; dispatching inline is the one path that would have to run the plugin right away.
+ describe('defersInlineDispatch', () => {
+ const restoring = (ids: string[]) => {
+ loader.isRestorePending = jest.fn((id: string) => ids.includes(id));
+ };
+
+ it('defers a plugin the restore has not reached while dispatch is inline', () => {
+ restoring(['chatwoot']);
+ (config.get as jest.Mock).mockReturnValue(false);
+ const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);
+
+ expect(svc.defersInlineDispatch('chatwoot')).toBe(true);
+ expect(svc.defersInlineDispatch('other')).toBe(false);
+ });
+
+ it('also defers when the queue is switched on but no queue instance was injected', () => {
+ restoring(['chatwoot']);
+ (config.get as jest.Mock).mockReturnValue(true);
+ const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, undefined);
+
+ expect(svc.defersInlineDispatch('chatwoot')).toBe(true);
+ });
+
+ it('does not defer when a queue takes the job', () => {
+ restoring(['chatwoot']);
+ (config.get as jest.Mock).mockReturnValue(true);
+ const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);
+
+ expect(svc.defersInlineDispatch('chatwoot')).toBe(false);
+ });
+
+ it('does not defer once the restore is done with the plugin', () => {
+ restoring([]);
+ (config.get as jest.Mock).mockReturnValue(false);
+ const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);
+
+ expect(svc.defersInlineDispatch('chatwoot')).toBe(false);
+ });
+ });
+
describe('onApplicationBootstrap (queue-wiring tripwire)', () => {
const ORIGINAL = process.env.QUEUE_ENABLED;
diff --git a/src/modules/integration/ingress-enqueue.service.ts b/src/modules/integration/ingress-enqueue.service.ts
index e38b91799..abb79803a 100644
--- a/src/modules/integration/ingress-enqueue.service.ts
+++ b/src/modules/integration/ingress-enqueue.service.ts
@@ -138,6 +138,16 @@ export class IngressEnqueueService implements OnApplicationBootstrap {
}
}
+ /**
+ * Whether a delivery of this plugin must wait to be dispatched: it would run inline, because the queue is off,
+ * in a plugin the boot restore has not reached, which has no worker to run it in. With the queue on, the job
+ * is enqueued instead, and IngressProcessor holds it until the plugin is up.
+ */
+ defersInlineDispatch(pluginId: string): boolean {
+ const queued = this.config.get<boolean>('queue.enabled', false) && !!this.ingressQueue;
+ return !queued && this.loader.isRestorePending(pluginId);
+ }
+
/**
* State of the job an earlier enqueue() of this delivery left in the queue, or undefined when the
* queue is off, holds no such job, or cannot answer. add() resolves for a duplicate jobId whatever
diff --git a/src/modules/integration/ingress-preflight.ts b/src/modules/integration/ingress-preflight.ts
index 925b84fb0..df47503f5 100644
--- a/src/modules/integration/ingress-preflight.ts
+++ b/src/modules/integration/ingress-preflight.ts
@@ -24,9 +24,11 @@ const retryable503 = (body: string): PreflightRejection => ({
});
/**
- * The refusal for a delivery whose plugin the boot restore has not finished with. No worker can take it
- * yet, so an ack would only dead-letter it after the retry budget, and the provider would never send it
- * again. IngressService answers it in the same stage as the preflight, so it leaves no dedup row either.
+ * The refusal for a delivery on a route that declares a response contract, to a plugin the boot restore has
+ * not finished with. The provider wants to know now (an OTP delivered a minute late is useless), and no
+ * worker can take the delivery yet, so it is told to retry. A route without a response contract is not
+ * refused: its delivery is accepted and held until the plugin is up. IngressService answers this in the
+ * same stage as the preflight, so it leaves no dedup row either.
*/
export const pluginRestoringRejection = (): PreflightRejection => retryable503('plugin restoring');
diff --git a/src/modules/integration/ingress.service.spec.ts b/src/modules/integration/ingress.service.spec.ts
index 3ea91b2d7..4d11f0b8e 100644
--- a/src/modules/integration/ingress.service.spec.ts
+++ b/src/modules/integration/ingress.service.spec.ts
@@ -729,12 +729,26 @@ describe('IngressService.handle — response contract', () => {
expect(d.enqueue).toHaveBeenCalled();
});
- // While the boot restore is still bringing a plugin up, no worker can take its delivery: an ack would
- // only dead-letter it once the retry budget is spent, and the provider would never send it again.
+ // While the boot restore is still bringing a plugin up there is no worker to run its deliveries. A route that
+ // declares a response contract asked for an answer now (an OTP that arrives after the restore is useless), so it
+ // is refused with a retryable 503. Any other route is accepted and persisted like every delivery: the queue holds
+ // its job (IngressProcessor), or with the queue off the reconciler dispatches the row once the plugin is up.
describe('a plugin that is still being restored after boot', () => {
- it('refuses 503 with Retry-After BEFORE dedup or enqueue, so the retry is not swallowed as a duplicate', async () => {
+ // A response contract that is an ack alone, so a 503 can only come from the restore.
+ const answeringRoute = () =>
+ jest.fn().mockReturnValue({
+ route: 'send-sms',
+ mode: 'async',
+ verify: 'core',
+ maxBodyBytes: 1024,
+ signature: { scheme: 'none' },
+ dedupHeader: 'x-delivery',
+ response: { ack: { status: 204 } },
+ });
+
+ it('refuses a route that declares a response contract, BEFORE dedup or enqueue, so the retry is not swallowed', async () => {
const restorePending = jest.fn().mockReturnValue(true);
- const d = depsWith({ restorePending });
+ const d = depsWith({ restorePending, manifestRoute: answeringRoute() });
const res = await new IngressService(d).handle(baseReq);
@@ -748,13 +762,43 @@ describe('IngressService.handle — response contract', () => {
);
});
- it('accepts the delivery once the restore is done with the plugin', async () => {
- const restorePending = jest.fn().mockReturnValue(false);
- const d = depsWith({ restorePending });
+ it('accepts, persists and enqueues a delivery of a route without a response contract', async () => {
+ const d = depsWith({ restorePending: jest.fn().mockReturnValue(true) });
+
+ const res = await new IngressService(d).handle(baseReq);
+
+ expect(res.status).toBe(202);
+ expect(d.events.recordOrSkip).toHaveBeenCalledTimes(1);
+ expect(d.enqueue).toHaveBeenCalledTimes(1);
+ expect(d.log).not.toHaveBeenCalled();
+ });
+
+ it('persists and acks a delivery but does not dispatch it while dispatch has to wait', async () => {
+ const deferDispatch = jest.fn().mockReturnValue(true);
+ const d = depsWith({ restorePending: jest.fn().mockReturnValue(true), deferDispatch });
const res = await new IngressService(d).handle(baseReq);
expect(res.status).toBe(202);
+ expect(deferDispatch).toHaveBeenCalledWith('p');
+ expect(d.events.recordOrSkip).toHaveBeenCalledTimes(1);
+ expect(d.enqueue).not.toHaveBeenCalled();
+ });
+
+ it('dispatches as usual when dispatch need not wait', async () => {
+ const d = depsWith({ deferDispatch: jest.fn().mockReturnValue(false) });
+
+ await new IngressService(d).handle(baseReq);
+
+ expect(d.enqueue).toHaveBeenCalledTimes(1);
+ });
+
+ it('accepts a route that declares a response contract once the restore is done with the plugin', async () => {
+ const d = depsWith({ restorePending: jest.fn().mockReturnValue(false), manifestRoute: answeringRoute() });
+
+ const res = await new IngressService(d).handle(baseReq);
+
+ expect(res.status).toBe(204);
expect(d.events.recordOrSkip).toHaveBeenCalledTimes(1);
expect(d.enqueue).toHaveBeenCalledTimes(1);
});
@@ -766,6 +810,7 @@ describe('IngressService.handle — response contract', () => {
route: 'send-sms',
maxBodyBytes: 1024,
signature: { scheme: 'hmac-sha256', header: 'x-sig' },
+ response: { ack: { status: 204 } },
}),
});
@@ -776,7 +821,11 @@ describe('IngressService.handle — response contract', () => {
it('charges the bucket before it refuses, so a signed provider that keeps retrying is still bounded', async () => {
const admitInstance = jest.fn().mockResolvedValue({ ok: false, headers: { 'Retry-After': '7' } });
- const d = depsWith({ restorePending: jest.fn().mockReturnValue(true), admitInstance });
+ const d = depsWith({
+ restorePending: jest.fn().mockReturnValue(true),
+ manifestRoute: answeringRoute(),
+ admitInstance,
+ });
const res = await new IngressService(d).handle(baseReq);
diff --git a/src/modules/integration/ingress.service.ts b/src/modules/integration/ingress.service.ts
index 63f423098..5d012fa25 100644
--- a/src/modules/integration/ingress.service.ts
+++ b/src/modules/integration/ingress.service.ts
@@ -70,9 +70,15 @@ export interface IngressDeps {
// session-alive check skips (passes) rather than false-rejecting.
sessionStatus?: (scope: string) => EngineStatus | undefined;
// Optional host-side probe: whether the boot restore has yet to finish with this plugin. A delivery for
- // such a plugin is refused with a retryable 503 in the preflight stage, since no worker can take it
- // yet. O(1) (a Set lookup). Absent in pure unit tests: every plugin counts as restored.
+ // such a plugin on a route that declares a response contract is refused with a retryable 503 in the
+ // preflight stage, since no worker can take it yet. O(1) (a Set lookup). Absent in pure unit tests: every
+ // plugin counts as restored.
restorePending?: (pluginId: string) => boolean;
+ // Optional host-side probe: whether a delivery that was just persisted must not be dispatched yet. True for a
+ // plugin the boot restore has not reached while dispatch runs inline (queue off), where there is no worker
+ // to run it in; the row then stays 'pending' for the reconciler. Absent in pure unit tests: every delivery
+ // is dispatched.
+ deferDispatch?: (pluginId: string) => boolean;
// Optional structured sink for preflight rejections, so operators can audit deliveries that were
// rejected host-side (and therefore leave no dedup/DLQ row). Absent in pure unit tests.
log?: (event: string, meta: Record<string, unknown>) => void;
@@ -187,10 +193,13 @@ export class IngressService {
// cannot probe liveness) and BEFORE the dedup persist (so a 5xx-rejected delivery never writes a
// dedup row that would swallow the provider's retry as a 200 'duplicate' — the dedup trap). A
// rejection leaves no dedup/DLQ row, so log it for operator audit.
- // A plugin the boot restore has not reached yet is refused in the same stage: no worker can take its delivery.
- const preflight = this.deps.restorePending?.(req.pluginId)
- ? pluginRestoringRejection()
- : evaluatePreflight(route, instance.sessionScope, this.deps.sessionStatus);
+ // A route that declares a response contract wants its answer now, so a plugin the boot restore has not
+ // reached yet is refused in the same stage. Any other route is accepted and persisted below like every
+ // delivery: the queue holds its job until the plugin is up, or the reconciler dispatches the row.
+ const preflight =
+ route.response && this.deps.restorePending?.(req.pluginId)
+ ? pluginRestoringRejection()
+ : evaluatePreflight(route, instance.sessionScope, this.deps.sessionStatus);
if (preflight) {
this.deps.log?.('ingress_preflight_rejected', {
pluginId: req.pluginId,
@@ -264,6 +273,9 @@ export class IngressService {
sessionId: instance.sessionScope,
});
if (!isNew) return ack; // already persisted/acked; a retry gets the route's ack, not enqueued again
+ // Persisted and acked: dispatching it now would run the plugin inline where there is no worker to run it in.
+ // The row stays 'pending', which is where a crash here would leave it too, and the reconciler dispatches it.
+ if (this.deps.deferDispatch?.(req.pluginId)) return ack;
// Best-effort conversation id for P1 ordering. Never throws — a malformed body just yields undefined.
const providerConversationId = extractConversationId(route.conversationId, req.headers, req.rawBody);
diff --git a/src/modules/integration/integration-queue-wiring.spec.ts b/src/modules/integration/integration-queue-wiring.spec.ts
index 88bb27884..40baef204 100644
--- a/src/modules/integration/integration-queue-wiring.spec.ts
+++ b/src/modules/integration/integration-queue-wiring.spec.ts
@@ -191,8 +191,8 @@ describe('IntegrationModule queue wiring (QUEUE_ENABLED conditional)', () => {
}
});
- // The ingress door refuses a delivery for a plugin the boot restore has not finished with. The
- // refusal is a dependency of the pure pipeline, so only the real graph can show it is connected.
+ // The ingress door treats a delivery for a plugin the boot restore has not finished with differently. Both
+ // decisions are dependencies of the pure pipeline, so only the real graph can show they are connected.
it('hands IngressService the restore state of the real plugin loader', async () => {
const { moduleRef } = await compileIntegrationModule('false');
try {
@@ -213,4 +213,57 @@ describe('IntegrationModule queue wiring (QUEUE_ENABLED conditional)', () => {
await moduleRef.close();
}
});
+
+ describe('deferring the dispatch of a persisted delivery for a plugin that is still restoring', () => {
+ const ORIGINAL_INTERVAL = process.env.INGRESS_RECONCILE_INTERVAL_MS;
+ afterEach(() => {
+ if (ORIGINAL_INTERVAL === undefined) delete process.env.INGRESS_RECONCILE_INTERVAL_MS;
+ else process.env.INGRESS_RECONCILE_INTERVAL_MS = ORIGINAL_INTERVAL;
+ });
+
+ /** The deferDispatch dependency of the real IngressService, with `adapter` marked as restoring. */
+ async function deferDispatchFor(queueEnabled: 'true' | 'false') {
+ const { moduleRef } = await compileIntegrationModule(queueEnabled);
+ /* eslint-disable @typescript-eslint/no-require-imports */
+ const { IngressService } = require('./ingress.service') as typeof import('./ingress.service');
+ const { PluginLoaderService } =
+ require('../../core/plugins/plugin-loader.service') as typeof import('../../core/plugins/plugin-loader.service');
+ /* eslint-enable @typescript-eslint/no-require-imports */
+ const ingress = moduleRef.get(IngressService, { strict: false });
+ const loader = moduleRef.get(PluginLoaderService, { strict: false });
+ (loader as unknown as { restorePending: Set<string> }).restorePending.add('adapter');
+ const { deferDispatch } = (ingress as unknown as { deps: { deferDispatch?: (id: string) => boolean } }).deps;
+ return { moduleRef, deferDispatch: deferDispatch as (id: string) => boolean };
+ }
+
+ it('waits for the reconciler when the queue is off and the plugin is restoring', async () => {
+ delete process.env.INGRESS_RECONCILE_INTERVAL_MS;
+ const { moduleRef, deferDispatch } = await deferDispatchFor('false');
+ try {
+ expect(deferDispatch('adapter')).toBe(true);
+ expect(deferDispatch('other')).toBe(false);
+ } finally {
+ await moduleRef.close();
+ }
+ });
+
+ it('does not wait when a queue takes the job, which the processor holds', async () => {
+ const { moduleRef, deferDispatch } = await deferDispatchFor('true');
+ try {
+ expect(deferDispatch('adapter')).toBe(false);
+ } finally {
+ await moduleRef.close();
+ }
+ });
+
+ it('does not wait when the reconciler is off, since nothing would ever replay the row', async () => {
+ process.env.INGRESS_RECONCILE_INTERVAL_MS = '0';
+ const { moduleRef, deferDispatch } = await deferDispatchFor('false');
+ try {
+ expect(deferDispatch('adapter')).toBe(false);
+ } finally {
+ await moduleRef.close();
+ }
+ });
+ });
});
diff --git a/src/modules/integration/integration.module.ts b/src/modules/integration/integration.module.ts
index cc930dfbf..5c53fcaf5 100644
--- a/src/modules/integration/integration.module.ts
+++ b/src/modules/integration/integration.module.ts
@@ -11,7 +11,7 @@ import { IngressService, IngressRouteDescriptor } from './ingress.service';
import { IngressController } from './ingress.controller';
import { admitIngressInstance } from './ingress-instance-limit';
import { IngressEnqueueService, buildIngressDeadLetterRow } from './ingress-enqueue.service';
-import { IngressReconcilerService } from './ingress-reconciler.service';
+import { IngressReconcilerService, resolveIngressReconcilerOptions } from './ingress-reconciler.service';
import { RedriveService } from './redrive.service';
import { RedriveController } from './redrive.controller';
import { IntegrationRetentionService } from './integration-retention.service';
@@ -93,9 +93,13 @@ if (process.env.QUEUE_ENABLED === 'true') {
// MUST stay cheap — never call engine.initialize() here (that is the slow/blocking path bounded
// separately by the #667/#696 init-timeout). Undefined = no live engine (stopped/deleted).
sessionStatus: (scope: string) => registry.get(scope)?.getStatus(),
- // A plugin the boot restore has not reached has no worker to take a delivery: the door answers a
- // retryable 503 before persisting it, as the port did by staying closed until the restore was done.
+ // A plugin the boot restore has not reached has no worker to take a delivery. A route that wants an
+ // answer is refused with a retryable 503 before anything is persisted; any other is accepted and held.
restorePending: (pluginId: string) => loader.isRestorePending(pluginId),
+ // Queue off: the dispatch would run inline, so the persisted row waits for the reconciler. With the
+ // reconciler off nothing would ever replay it, so the inline attempt (and its dead letter) stays.
+ deferDispatch: (pluginId: string) =>
+ ingressEnqueue.defersInlineDispatch(pluginId) && resolveIngressReconcilerOptions().intervalMs > 0,
// Audit sink for preflight rejections (they leave no dedup/DLQ row). The Prometheus counter is
// a separate follow-up (see plan notes); the structured log is the MVP audit surface.
log: (event, meta) => ingressLogger.warn(event, meta),
diff --git a/test/integration-fabric.e2e-spec.ts b/test/integration-fabric.e2e-spec.ts
index c69c02009..c25b9e398 100644
--- a/test/integration-fabric.e2e-spec.ts
+++ b/test/integration-fabric.e2e-spec.ts
@@ -11,6 +11,7 @@ import { json, urlencoded, Request } from 'express';
import { AppModule } from './../src/app.module';
import { applyGlobalValidation } from './../src/config/app-validation';
import { PluginLoaderService } from './../src/core/plugins/plugin-loader.service';
+import { IngressReconcilerService } from './../src/modules/integration/ingress-reconciler.service';
import { PluginInstance } from './../src/modules/integration/entities/plugin-instance.entity';
import { IngressEvent } from './../src/modules/integration/entities/ingress-event.entity';
import { IntegrationDeliveryFailure } from './../src/modules/integration/entities/integration-delivery-failure.entity';
@@ -60,6 +61,10 @@ describe('Integration Fabric ingress (e2e)', () => {
},
dedupHeader: 'x-chatwoot-delivery',
};
+ // The same provider contract on a route that declares a response contract (an ack alone), as an adapter whose
+ // provider needs an answer now does: while its plugin is restoring, that is the one route a delivery is refused on.
+ const answeringManifestRoute = { ...ingressManifestRoute, route: 'send-sms', response: { ack: { status: 204 } } };
+ const ANSWERING_PATH = '/api/ingress/chatwoot/acct1/send-sms';
beforeAll(async () => {
const moduleFixture: TestingModule = await Test.createTestingModule({
@@ -97,7 +102,9 @@ describe('Integration Fabric ingress (e2e)', () => {
const realGetPlugin = loader.getPlugin.bind(loader);
jest.spyOn(loader, 'getPlugin').mockImplementation((pluginId: string) => {
if (pluginId === 'chatwoot') {
- return { manifest: { ingress: [ingressManifestRoute] } } as unknown as ReturnType<typeof loader.getPlugin>;
+ return {
+ manifest: { ingress: [ingressManifestRoute, answeringManifestRoute] },
+ } as unknown as ReturnType<typeof loader.getPlugin>;
}
return realGetPlugin(pluginId);
});
@@ -169,37 +176,108 @@ describe('Integration Fabric ingress (e2e)', () => {
expect(dispatchWebhookForInstance).not.toHaveBeenCalled();
});
- it('refuses 503 with Retry-After and stores nothing while the plugin restore is pending, then accepts the retry', async () => {
- const eventRepo = app.get<Repository<IngressEvent>>(getRepositoryToken(IngressEvent, 'data'));
- const failureRepo = app.get<Repository<IntegrationDeliveryFailure>>(
- getRepositoryToken(IntegrationDeliveryFailure, 'data'),
- );
- const post = () =>
+ // A plugin the boot restore has not reached has no worker. A route without a response contract is accepted and
+ // persisted like every delivery; with the queue off the row is not dispatched at the door but left for the
+ // reconciler, which dispatches it once the plugin is up.
+ describe('while the plugin restore is pending', () => {
+ const stateRepos = () => ({
+ events: app.get<Repository<IngressEvent>>(getRepositoryToken(IngressEvent, 'data')),
+ failures: app.get<Repository<IntegrationDeliveryFailure>>(getRepositoryToken(IntegrationDeliveryFailure, 'data')),
+ });
+ const post = (path: string, deliveryId: string) =>
request(app.getHttpServer())
- .post(INGRESS_PATH)
+ .post(path)
.set('X-Chatwoot-Signature', sig)
- .set('X-Chatwoot-Delivery', 'delivery-restoring-1')
+ .set('X-Chatwoot-Delivery', deliveryId)
.set('Content-Type', 'application/json')
.send(raw);
- const restoring = jest.spyOn(app.get(PluginLoaderService), 'isRestorePending').mockReturnValue(true);
+ const restoring = () => jest.spyOn(app.get(PluginLoaderService), 'isRestorePending').mockReturnValue(true);
- try {
- const refused = await post();
-
- expect(refused.status).toBe(503);
- expect(refused.headers['retry-after']).toBe('5');
- expect(dispatchWebhookForInstance).not.toHaveBeenCalled();
- // A stored row would answer the provider's retry with a 'duplicate' ack and lose the delivery.
- expect(await eventRepo.countBy({ providerDeliveryId: 'delivery-restoring-1' })).toBe(0);
- expect(await failureRepo.countBy({ deliveryId: 'delivery-restoring-1' })).toBe(0);
- } finally {
- restoring.mockRestore();
- }
+ it('accepts and persists a delivery of a route without a response contract, and the reconciler delivers it afterwards', async () => {
+ const { events, failures } = stateRepos();
+ const restore = restoring();
+ try {
+ const accepted = await post(INGRESS_PATH, 'delivery-held-1');
- const retried = await post();
+ expect(accepted.status).toBe(202);
+ // Not dispatched into a plugin with no worker, and not dead-lettered: the row is the record of it.
+ expect(dispatchWebhookForInstance).not.toHaveBeenCalled();
+ const held = await events.findOneByOrFail({ providerDeliveryId: 'delivery-held-1' });
+ expect(held.dispatchState).toBe('pending');
+ expect(held.dispatchAttempts).toBe(0);
+ expect(held.payload).not.toBeNull();
+ expect(await failures.countBy({ deliveryId: 'delivery-held-1' })).toBe(0);
- expect(retried.status).toBe(202);
- expect(dispatchWebhookForInstance).toHaveBeenCalledTimes(1);
+ // The provider's own retry of it is a duplicate, as for any persisted delivery.
+ expect((await post(INGRESS_PATH, 'delivery-held-1')).status).toBe(202);
+ expect(await events.countBy({ providerDeliveryId: 'delivery-held-1' })).toBe(1);
+
+ // While the restore lasts the reconciler leaves the row alone.
+ const reconciler = app.get(IngressReconcilerService);
+ const options = { intervalMs: 60_000, graceMs: 0, batchSize: 50, maxAttempts: 5 };
+ await new Promise(resolve => setTimeout(resolve, 5));
+ await reconciler.sweep(options);
+ expect(dispatchWebhookForInstance).not.toHaveBeenCalled();
+
+ restore.mockRestore();
+ const stats = await reconciler.sweep(options);
+
+ expect(stats).toMatchObject({ scanned: 1, replayed: 1 });
+ expect(dispatchWebhookForInstance).toHaveBeenCalledTimes(1);
+ const delivered = await events.findOneByOrFail({ providerDeliveryId: 'delivery-held-1' });
+ expect(delivered.dispatchState).toBe('dispatched');
+ expect(delivered.payload).toBeNull();
+ } finally {
+ restore.mockRestore();
+ }
+ });
+
+ it('refuses a route that declares a response contract with 503 and Retry-After, stores nothing, then accepts the retry', async () => {
+ const { events, failures } = stateRepos();
+ const restore = restoring();
+ try {
+ const refused = await post(ANSWERING_PATH, 'delivery-answer-1');
+
+ expect(refused.status).toBe(503);
+ expect(refused.headers['retry-after']).toBe('5');
+ expect(dispatchWebhookForInstance).not.toHaveBeenCalled();
+ // A stored row would answer the provider's retry with a 'duplicate' ack and lose the delivery.
+ expect(await events.countBy({ providerDeliveryId: 'delivery-answer-1' })).toBe(0);
+ expect(await failures.countBy({ deliveryId: 'delivery-answer-1' })).toBe(0);
+ } finally {
+ restore.mockRestore();
+ }
+
+ const retried = await post(ANSWERING_PATH, 'delivery-answer-1');
+
+ expect(retried.status).toBe(204);
+ expect(dispatchWebhookForInstance).toHaveBeenCalledTimes(1);
+ });
+
+ it('still dispatches at the door when the reconciler is off, since nothing would replay a held row', async () => {
+ const { events, failures } = stateRepos();
+ const original = process.env.INGRESS_RECONCILE_INTERVAL_MS;
+ process.env.INGRESS_RECONCILE_INTERVAL_MS = '0';
+ const restore = restoring();
+ dispatchWebhookForInstance.mockRejectedValueOnce(new Error('no live sandbox host for plugin chatwoot'));
+ try {
+ const res = await post(INGRESS_PATH, 'delivery-noreconcile-1');
+
+ // The inline dispatch fails and files the redrivable dead letter an operator can see.
+ expect(res.status).toBe(202);
+ expect(dispatchWebhookForInstance).toHaveBeenCalledTimes(1);
+ expect(await failures.countBy({ deliveryId: 'delivery-noreconcile-1', redriven: false })).toBe(1);
+ expect((await events.findOneByOrFail({ providerDeliveryId: 'delivery-noreconcile-1' })).dispatchAttempts).toBe(
+ 1,
+ );
+ } finally {
+ restore.mockRestore();
+ if (original === undefined) delete process.env.INGRESS_RECONCILE_INTERVAL_MS;
+ else process.env.INGRESS_RECONCILE_INTERVAL_MS = original;
+ // An unused once-rejection must not reach the next test.
+ dispatchWebhookForInstance.mockReset().mockResolvedValue(undefined);
+ }
+ });
});
it('marks the persisted event dispatched once a successful inline dispatch is recorded', async () => {
diff --git a/test/queue-on.e2e-spec.ts b/test/queue-on.e2e-spec.ts
index debbecf3d..e2a08ed65 100644
--- a/test/queue-on.e2e-spec.ts
+++ b/test/queue-on.e2e-spec.ts
@@ -267,6 +267,49 @@ describeQueueOn('Queued dispatch paths (e2e, QUEUE_ENABLED=true)', () => {
expect(dispatchedArgs[0][0].deliveryId).toBe(deliveryId);
}, 30000);
+ it('accepts a delivery for a plugin that is still restoring, holds its job without spending an attempt, then dispatches it', async () => {
+ // The held job comes back after the retry delay, at least a second; a short one keeps the test brief.
+ const prevDelay = process.env.INGRESS_RETRY_DELAY_MS;
+ process.env.INGRESS_RETRY_DELAY_MS = '1000';
+ const deliveryId = `queue-on-held-${Date.now()}`;
+ const restoring = jest.spyOn(app.get(PluginLoaderService), 'isRestorePending').mockReturnValue(true);
+ dispatchWebhookForInstance.mockClear();
+ try {
+ const res = await request(app.getHttpServer())
+ .post(INGRESS_PATH)
+ .set('X-Chatwoot-Signature', sig)
+ .set('X-Chatwoot-Delivery', deliveryId)
+ .set('Content-Type', 'application/json')
+ .send(raw);
+
+ // Accepted and queued like any delivery: no refusal for a route without a response contract.
+ expect(res.status).toBe(202);
+ const jobId = sanitizeIngressJobId(deliveryId, 'chatwoot\u0000acct1');
+ // The worker takes the job, sees the plugin is not up, and puts it back in the delayed set. It does this
+ // again at every delay, and spends no attempt, so the retry budget is intact when the plugin is up.
+ await waitFor(
+ async () =>
+ (await ingressQueue.getJob(jobId))?.attemptsStarted !== 0 &&
+ (await ingressQueue.getJobState(jobId)) === 'delayed',
+ );
+ await new Promise(resolve => setTimeout(resolve, 2500));
+ const held = await ingressQueue.getJob(jobId);
+ expect(held?.attemptsStarted).toBeGreaterThan(1);
+ expect(held?.attemptsMade).toBe(0);
+ expect(dispatchWebhookForInstance).not.toHaveBeenCalled();
+
+ restoring.mockRestore();
+ await waitFor(() => dispatchWebhookForInstance.mock.calls.length >= 1, 15000);
+ expect((dispatchWebhookForInstance.mock.calls as IngressJobData[][])[0][0].deliveryId).toBe(deliveryId);
+ await waitFor(async () => (await ingressQueue.getJobState(jobId)) === 'completed');
+ expect((await ingressQueue.getJob(jobId))?.attemptsMade).toBe(1);
+ } finally {
+ restoring.mockRestore();
+ if (prevDelay === undefined) delete process.env.INGRESS_RETRY_DELAY_MS;
+ else process.env.INGRESS_RETRY_DELAY_MS = prevDelay;
+ }
+ }, 60000);
+
it('enqueues the webhook delivery and the worker POSTs it to the receiver', async () => {
const session = await sessionRepo.save(sessionRepo.create({ name: `e2e-queue-on-${Date.now()}` }));
await request(app.getHttpServer())