Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(integration): keep unreplayable rows out of the ingress sweep batch

Yudhi Armyndharisf9db2338f67c
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.ts
index cf098eaa5..ffc7d5206 100644
--- a/src/modules/integration/ingress-reconciler.service.spec.ts
+++ b/src/modules/integration/ingress-reconciler.service.spec.ts
@@ -1,6 +1,7 @@
import { DataSource, Repository } from 'typeorm';
import { IngressEvent } from './entities/ingress-event.entity';
import { IntegrationDeliveryFailure } from './entities/integration-delivery-failure.entity';
+import { PluginInstance } from './entities/plugin-instance.entity';
import { IngressReconcilerService, IngressReconcilerOptions } from './ingress-reconciler.service';
import { IngressEnqueueService } from './ingress-enqueue.service';
import { PluginInstanceService } from './plugin-instance.service';
@@ -28,12 +29,15 @@ describe('IngressReconcilerService.sweep', () => {
ds = new DataSource({
type: 'better-sqlite3',
database: ':memory:',
- entities: [IngressEvent, IntegrationDeliveryFailure],
+ entities: [IngressEvent, IntegrationDeliveryFailure, PluginInstance],
synchronize: true,
});
await ds.initialize();
events = ds.getRepository(IngressEvent);
failures = ds.getRepository(IntegrationDeliveryFailure);
+ await ds
+ .getRepository(PluginInstance)
+ .save({ id: 'plug:inst', pluginId: 'plug', instanceId: 'inst', secret: 's', enabled: true });
enqueue = jest.fn().mockResolvedValue({ outcome: 'queued' });
getPlugin = jest.fn().mockReturnValue(undefined);
resolveInstance = jest.fn().mockResolvedValue({ enabled: true });
@@ -115,13 +119,33 @@ describe('IngressReconcilerService.sweep', () => {
const stats = await service.sweep(OPTS);
- expect(stats).toMatchObject({ scanned: 0, skipped: 1 });
+ expect(stats).toMatchObject({ scanned: 0, skipped: 0 }); // filtered out before it can hold a batch slot
expect(enqueue).not.toHaveBeenCalled();
const event = await stored(id);
expect(event.dispatchState).toBe('pending');
expect(event.dispatchAttempts).toBe(0);
});
+ it('does not let rows it cannot replay starve a later row of an enabled instance', async () => {
+ await ds
+ .getRepository(PluginInstance)
+ .save({ id: 'plug:off', pluginId: 'plug', instanceId: 'off', secret: 's', enabled: false });
+ resolveInstance.mockImplementation((_p: string, i: string) =>
+ Promise.resolve(i === 'off' ? { enabled: false } : { enabled: true }),
+ );
+ const disabled = await insertEvent({ instanceId: 'off', createdAt: minutesAgo(10) });
+ await insertEvent({ instanceId: 'gone', createdAt: minutesAgo(9) }); // instance deleted
+ await insertEvent({ payload: null, createdAt: minutesAgo(8) });
+ const live = await insertEvent({ createdAt: minutesAgo(5) });
+
+ const stats = await service.sweep({ ...OPTS, batchSize: 3 });
+
+ expect(stats.replayed).toBe(1);
+ expect(enqueuedJobs().map(([d]) => d.instanceId)).toEqual(['inst']);
+ expect((await stored(live)).dispatchState).toBe('dispatched');
+ expect((await stored(disabled)).dispatchState).toBe('pending'); // replayed once re-enabled
+ });
+
it('re-derives the conversation lane from the manifest route instead of degrading per-instance', async () => {
getPlugin.mockReturnValue({
manifest: { ingress: [{ route: 'chatwoot', conversationId: { jsonPointer: '/conversation/id' } }] },
diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.ts
index 175cd6d27..7e2a35665 100644
--- a/src/modules/integration/ingress-reconciler.service.ts
+++ b/src/modules/integration/ingress-reconciler.service.ts
@@ -1,8 +1,9 @@
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
-import { LessThan, Repository } from 'typeorm';
+import { Repository } from 'typeorm';
import { IngressEvent } from './entities/ingress-event.entity';
import { IntegrationDeliveryFailure } from './entities/integration-delivery-failure.entity';
+import { PluginInstance } from './entities/plugin-instance.entity';
import { IngressEnqueueService, buildIngressDeadLetterRow } from './ingress-enqueue.service';
import { extractConversationId } from './ingress.service';
import { PluginInstanceService } from './plugin-instance.service';
@@ -104,11 +105,24 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
this.sweeping = true;
try {
const cutoff = new Date(now.getTime() - opts.graceMs);
- const rows = await this.events.find({
- where: { dispatchState: 'pending', createdAt: LessThan(cutoff) },
- order: { createdAt: 'ASC' },
- take: opts.batchSize,
- });
+ // Only rows the sweep can act on may take a batch slot: a row of a disabled or deleted instance,
+ // or one without a payload, is skipped without being written, so selecting it would hand the same
+ // oldest rows back every sweep and starve every other instance's stranded deliveries. The join is
+ // 1:1 (plugin_instances is unique on pluginId+instanceId), so limit() bounds rows, not join fan-out.
+ const rows = await this.events
+ .createQueryBuilder('e')
+ .innerJoin(
+ PluginInstance,
+ 'pi',
+ 'pi.pluginId = e.pluginId AND pi.instanceId = e.instanceId AND pi.enabled = :enabled',
+ { enabled: true },
+ )
+ .where('e.dispatchState = :state', { state: 'pending' })
+ .andWhere('e.createdAt < :cutoff', { cutoff })
+ .andWhere('e.payload IS NOT NULL')
+ .orderBy('e.createdAt', 'ASC')
+ .limit(opts.batchSize)
+ .getMany();
for (const row of rows) {
// A row whose latest attempt is still inside the grace window cools down between replays;
// it keeps its batch slot (bounded by maxAttempts, so the leak is capped) but is not hit again.
@@ -117,8 +131,8 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
continue;
}
// A 'pending' row without a payload cannot be replayed (payloads are retired only once an
- // outcome is recorded, so this means an imported/corrupt row). Skip it loudly rather than
- // dispatching an empty delivery or spinning the attempt budget on a row that can never fire.
+ // outcome is recorded, so this means an imported/corrupt row). The query already excludes it;
+ // this narrows the type and still refuses to dispatch an empty delivery.
if (!hasPayload(row)) {
this.logger.error('Ingress event is pending without a payload; cannot replay', undefined, {
pluginId: row.pluginId,