Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(integration): keep restoring plugins out of the replay batch

Yudhi Armyndharis4855c8012209
diff --git a/src/core/plugins/plugin-loader.service.ts b/src/core/plugins/plugin-loader.service.ts
index 120673ca2..d4a57eef5 100644
--- a/src/core/plugins/plugin-loader.service.ts
+++ b/src/core/plugins/plugin-loader.service.ts
@@ -319,6 +319,11 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap
return this.restorePending.has(id);
}
+ /** The ids {@link isRestorePending} answers true for, for a caller that has to leave them out of a query. */
+ restorePendingIds(): string[] {
+ return [...this.restorePending];
+ }
+
private async restorePlugins(): Promise<void> {
const restorable = this.getAllPlugins().filter(
p => !p.builtIn && this.pluginStorage.getPluginEntry(p.manifest.id)?.enabledByOperator === true,
diff --git a/src/core/plugins/plugin-restore-on-boot.spec.ts b/src/core/plugins/plugin-restore-on-boot.spec.ts
index bd70afaf1..e7ab7dd51 100644
--- a/src/core/plugins/plugin-restore-on-boot.spec.ts
+++ b/src/core/plugins/plugin-restore-on-boot.spec.ts
@@ -298,12 +298,14 @@ describe('PluginLoaderService — restoring the operator enable decision across
expect(loader2.isRestorePending(manifest.id)).toBe(true);
expect(loader2.isRestorePending(second.id)).toBe(true);
expect(loader2.isRestorePending('never-queued')).toBe(false);
+ expect(loader2.restorePendingIds().sort()).toEqual([manifest.id, second.id].sort());
releaseFirst();
await jest.advanceTimersByTimeAsync(0);
expect(loader2.isRestorePending(manifest.id)).toBe(false);
expect(loader2.isRestorePending(second.id)).toBe(false);
+ expect(loader2.restorePendingIds()).toEqual([]);
});
// The restore ending clears every id, so only a plugin that finished while a later one is still being
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.ts
index bc2341f3c..80f438762 100644
--- a/src/modules/integration/ingress-reconciler.service.spec.ts
+++ b/src/modules/integration/ingress-reconciler.service.spec.ts
@@ -22,11 +22,21 @@ describe('IngressReconcilerService.sweep', () => {
let failures: Repository<IntegrationDeliveryFailure>;
let enqueue: jest.Mock;
let getPlugin: jest.Mock;
- let restorePending: jest.Mock;
+ let restoring: Set<string>;
+ // What the loader reports as still restoring when a batch is selected. The per-row check reads
+ // `restoring` directly, so a test can make the two disagree, as they do when a restore begins during a pass.
+ let restoringIds: () => string[];
let resolveInstance: jest.Mock;
let service: IngressReconcilerService;
let seq: number;
+ const loaderDouble = (): PluginLoaderService =>
+ ({
+ getPlugin,
+ isRestorePending: (id: string) => restoring.has(id),
+ restorePendingIds: () => restoringIds(),
+ }) as unknown as PluginLoaderService;
+
beforeEach(async () => {
seq = 0;
ds = new DataSource({
@@ -43,13 +53,14 @@ describe('IngressReconcilerService.sweep', () => {
.save({ id: 'plug:inst', pluginId: 'plug', instanceId: 'inst', secret: 's', enabled: true });
enqueue = jest.fn().mockResolvedValue({ outcome: 'queued' });
getPlugin = jest.fn().mockReturnValue(undefined);
- restorePending = jest.fn().mockReturnValue(false);
+ restoring = new Set();
+ restoringIds = () => [...restoring];
resolveInstance = jest.fn().mockResolvedValue({ enabled: true });
service = new IngressReconcilerService(
events,
failures,
{ enqueue, existingJobState: jest.fn().mockResolvedValue(undefined) } as unknown as IngressEnqueueService,
- { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,
+ loaderDouble(),
{ resolve: resolveInstance } as unknown as PluginInstanceService,
);
});
@@ -557,13 +568,9 @@ describe('IngressReconcilerService.sweep', () => {
{ get: jest.fn().mockReturnValue(true) } as unknown as ConfigService,
queue as never,
);
- service = new IngressReconcilerService(
- events,
- failures,
- realEnqueue,
- { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,
- { resolve: resolveInstance } as unknown as PluginInstanceService,
- );
+ service = new IngressReconcilerService(events, failures, realEnqueue, loaderDouble(), {
+ resolve: resolveInstance,
+ } as unknown as PluginInstanceService);
});
const processorDeadLetter = () =>
@@ -800,29 +807,58 @@ describe('IngressReconcilerService.sweep', () => {
expect((await stored(id)).dispatchState).toBe('dispatched');
});
- // The door answers a delivery for a plugin the boot restore has not reached with a 503 (no worker can take
- // it yet). A replay is the same delivery arriving later, and would only spend one of the row's attempts on
- // the same refusal.
+ // A replay is a delivery arriving later than the live one, and a plugin the boot restore has not reached has no
+ // worker to take it: the dispatch would only spend an attempt of the row's budget on a refusal.
it('leaves a row whose plugin is still being restored untouched, and replays it once that is done', async () => {
const id = await insertEvent();
- restorePending.mockReturnValue(true);
+ restoring.add('plug');
const stats = await service.sweep(OPTS);
- expect(restorePending).toHaveBeenCalledWith('plug');
- expect(stats).toMatchObject({ scanned: 0, replayed: 0, skipped: 1 });
+ expect(stats).toMatchObject({ scanned: 0, replayed: 0 });
expect(enqueue).not.toHaveBeenCalled();
const held = await stored(id);
expect(held.dispatchState).toBe('pending');
expect(held.dispatchAttempts).toBe(0);
expect(await failures.count()).toBe(0);
- restorePending.mockReturnValue(false);
+ restoring.delete('plug');
const stats2 = await service.sweep(OPTS);
expect(stats2).toMatchObject({ scanned: 1, replayed: 1 });
expect((await stored(id)).dispatchState).toBe('dispatched');
});
+ // The batch is the oldest rows a pass can act on. Rows of a restoring plugin are skipped, so if they were
+ // selected like the rest, a plugin with a batch's worth of stranded rows would hand the same ones back every
+ // pass and no other plugin's stranded delivery would ever be replayed until its restore ended.
+ it('does not let the stranded rows of a restoring plugin take the whole batch', async () => {
+ await ds
+ .getRepository(PluginInstance)
+ .save({ id: 'other:inst', pluginId: 'other', instanceId: 'inst', secret: 's', enabled: true });
+ for (let i = 0; i < OPTS.batchSize + 1; i++) await insertEvent({ createdAt: minutesAgo(20) });
+ const other = await insertEvent({ pluginId: 'other', createdAt: minutesAgo(10) });
+ restoring.add('plug');
+
+ const stats = await service.sweep(OPTS);
+
+ expect(stats).toMatchObject({ scanned: 1, replayed: 1, skipped: 0 });
+ expect(enqueuedJobs().map(([data]) => data.pluginId)).toEqual(['other']);
+ expect((await stored(other)).dispatchState).toBe('dispatched');
+ expect(await events.countBy({ pluginId: 'plug', dispatchState: 'pending' })).toBe(OPTS.batchSize + 1);
+ });
+
+ it('still leaves a row alone whose plugin began restoring after the batch was read', async () => {
+ const id = await insertEvent();
+ restoringIds = () => []; // the batch is selected before the restore begins...
+ restoring.add('plug'); // ...and the per-row check sees it begin
+
+ const stats = await service.sweep(OPTS);
+
+ expect(stats).toMatchObject({ scanned: 0, replayed: 0, skipped: 1 });
+ expect(enqueue).not.toHaveBeenCalled();
+ expect((await stored(id)).dispatchAttempts).toBe(0);
+ });
+
it('skips a row whose instance was deleted after persist', async () => {
await insertEvent();
resolveInstance.mockResolvedValue(null);
@@ -937,7 +973,7 @@ describe('IngressReconcilerService.sweep', () => {
waitUntilReady: jest.fn().mockResolvedValue(undefined),
} as never,
),
- { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,
+ loaderDouble(),
{ resolve: resolveInstance } as unknown as PluginInstanceService,
);
@@ -994,7 +1030,7 @@ describe('IngressReconcilerService.sweep', () => {
{ get: jest.fn().mockReturnValue(true) } as unknown as ConfigService,
queue as never,
),
- { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,
+ loaderDouble(),
{ resolve: resolveInstance } as unknown as PluginInstanceService,
);
diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.ts
index 61bebe05a..792165670 100644
--- a/src/modules/integration/ingress-reconciler.service.ts
+++ b/src/modules/integration/ingress-reconciler.service.ts
@@ -166,6 +166,10 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
// that cannot be read is kept out of the next batches for a while, so a batch full of them cannot starve
// the rows behind.
const unreadable = this.unreadableRows(now);
+ // A plugin the boot restore has not reached has no worker to take a replay, and the loop below skips its
+ // rows, so they are left out here too: a batch's worth of them would otherwise be handed back by every
+ // pass until the restore ends, with no other plugin's stranded delivery replayed in the meantime.
+ const restoring = this.loader.restorePendingIds();
const query = this.events
.createQueryBuilder('e')
.select('e.id')
@@ -179,6 +183,7 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
.andWhere('e.createdAt < :cutoff', { cutoff })
.andWhere('e.payload IS NOT NULL');
if (unreadable.length) query.andWhere('e.id NOT IN (:...unreadable)', { unreadable });
+ if (restoring.length) query.andWhere('e.pluginId NOT IN (:...restoring)', { restoring });
const batch = await query.orderBy('e.createdAt', 'ASC').limit(opts.batchSize).getMany();
for (const { id } of batch) {
if (this.stop.signal.aborted) break;