Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(integration): keep restoring plugins out of the replay batch
Yudhi Armyndharis4855c8012209
diff --git a/src/core/plugins/plugin-loader.service.ts b/src/core/plugins/plugin-loader.service.tsindex 120673ca2..d4a57eef5 100644--- a/src/core/plugins/plugin-loader.service.ts+++ b/src/core/plugins/plugin-loader.service.ts@@ -319,6 +319,11 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrapreturn this.restorePending.has(id);}+ /** The ids {@link isRestorePending} answers true for, for a caller that has to leave them out of a query. */+ restorePendingIds(): string[] {+ return [...this.restorePending];+ }+private async restorePlugins(): Promise<void> {const restorable = this.getAllPlugins().filter(p => !p.builtIn && this.pluginStorage.getPluginEntry(p.manifest.id)?.enabledByOperator === true,diff --git a/src/core/plugins/plugin-restore-on-boot.spec.ts b/src/core/plugins/plugin-restore-on-boot.spec.tsindex bd70afaf1..e7ab7dd51 100644--- a/src/core/plugins/plugin-restore-on-boot.spec.ts+++ b/src/core/plugins/plugin-restore-on-boot.spec.ts@@ -298,12 +298,14 @@ describe('PluginLoaderService — restoring the operator enable decision acrossexpect(loader2.isRestorePending(manifest.id)).toBe(true);expect(loader2.isRestorePending(second.id)).toBe(true);expect(loader2.isRestorePending('never-queued')).toBe(false);+ expect(loader2.restorePendingIds().sort()).toEqual([manifest.id, second.id].sort());releaseFirst();await jest.advanceTimersByTimeAsync(0);expect(loader2.isRestorePending(manifest.id)).toBe(false);expect(loader2.isRestorePending(second.id)).toBe(false);+ expect(loader2.restorePendingIds()).toEqual([]);});// The restore ending clears every id, so only a plugin that finished while a later one is still beingdiff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.tsindex bc2341f3c..80f438762 100644--- a/src/modules/integration/ingress-reconciler.service.spec.ts+++ b/src/modules/integration/ingress-reconciler.service.spec.ts@@ -22,11 +22,21 @@ describe('IngressReconcilerService.sweep', () => {let failures: Repository<IntegrationDeliveryFailure>;let enqueue: jest.Mock;let getPlugin: jest.Mock;- let restorePending: jest.Mock;+ let restoring: Set<string>;+ // What the loader reports as still restoring when a batch is selected. The per-row check reads+ // `restoring` directly, so a test can make the two disagree, as they do when a restore begins during a pass.+ let restoringIds: () => string[];let resolveInstance: jest.Mock;let service: IngressReconcilerService;let seq: number;+ const loaderDouble = (): PluginLoaderService =>+ ({+ getPlugin,+ isRestorePending: (id: string) => restoring.has(id),+ restorePendingIds: () => restoringIds(),+ }) as unknown as PluginLoaderService;+beforeEach(async () => {seq = 0;ds = new DataSource({@@ -43,13 +53,14 @@ describe('IngressReconcilerService.sweep', () => {.save({ id: 'plug:inst', pluginId: 'plug', instanceId: 'inst', secret: 's', enabled: true });enqueue = jest.fn().mockResolvedValue({ outcome: 'queued' });getPlugin = jest.fn().mockReturnValue(undefined);- restorePending = jest.fn().mockReturnValue(false);+ restoring = new Set();+ restoringIds = () => [...restoring];resolveInstance = jest.fn().mockResolvedValue({ enabled: true });service = new IngressReconcilerService(events,failures,{ enqueue, existingJobState: jest.fn().mockResolvedValue(undefined) } as unknown as IngressEnqueueService,- { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,+ loaderDouble(),{ resolve: resolveInstance } as unknown as PluginInstanceService,);});@@ -557,13 +568,9 @@ describe('IngressReconcilerService.sweep', () => {{ get: jest.fn().mockReturnValue(true) } as unknown as ConfigService,queue as never,);- service = new IngressReconcilerService(- events,- failures,- realEnqueue,- { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,- { resolve: resolveInstance } as unknown as PluginInstanceService,- );+ service = new IngressReconcilerService(events, failures, realEnqueue, loaderDouble(), {+ resolve: resolveInstance,+ } as unknown as PluginInstanceService);});const processorDeadLetter = () =>@@ -800,29 +807,58 @@ describe('IngressReconcilerService.sweep', () => {expect((await stored(id)).dispatchState).toBe('dispatched');});- // The door answers a delivery for a plugin the boot restore has not reached with a 503 (no worker can take- // it yet). A replay is the same delivery arriving later, and would only spend one of the row's attempts on- // the same refusal.+ // A replay is a delivery arriving later than the live one, and a plugin the boot restore has not reached has no+ // worker to take it: the dispatch would only spend an attempt of the row's budget on a refusal.it('leaves a row whose plugin is still being restored untouched, and replays it once that is done', async () => {const id = await insertEvent();- restorePending.mockReturnValue(true);+ restoring.add('plug');const stats = await service.sweep(OPTS);- expect(restorePending).toHaveBeenCalledWith('plug');- expect(stats).toMatchObject({ scanned: 0, replayed: 0, skipped: 1 });+ expect(stats).toMatchObject({ scanned: 0, replayed: 0 });expect(enqueue).not.toHaveBeenCalled();const held = await stored(id);expect(held.dispatchState).toBe('pending');expect(held.dispatchAttempts).toBe(0);expect(await failures.count()).toBe(0);- restorePending.mockReturnValue(false);+ restoring.delete('plug');const stats2 = await service.sweep(OPTS);expect(stats2).toMatchObject({ scanned: 1, replayed: 1 });expect((await stored(id)).dispatchState).toBe('dispatched');});+ // The batch is the oldest rows a pass can act on. Rows of a restoring plugin are skipped, so if they were+ // selected like the rest, a plugin with a batch's worth of stranded rows would hand the same ones back every+ // pass and no other plugin's stranded delivery would ever be replayed until its restore ended.+ it('does not let the stranded rows of a restoring plugin take the whole batch', async () => {+ await ds+ .getRepository(PluginInstance)+ .save({ id: 'other:inst', pluginId: 'other', instanceId: 'inst', secret: 's', enabled: true });+ for (let i = 0; i < OPTS.batchSize + 1; i++) await insertEvent({ createdAt: minutesAgo(20) });+ const other = await insertEvent({ pluginId: 'other', createdAt: minutesAgo(10) });+ restoring.add('plug');++ const stats = await service.sweep(OPTS);++ expect(stats).toMatchObject({ scanned: 1, replayed: 1, skipped: 0 });+ expect(enqueuedJobs().map(([data]) => data.pluginId)).toEqual(['other']);+ expect((await stored(other)).dispatchState).toBe('dispatched');+ expect(await events.countBy({ pluginId: 'plug', dispatchState: 'pending' })).toBe(OPTS.batchSize + 1);+ });++ it('still leaves a row alone whose plugin began restoring after the batch was read', async () => {+ const id = await insertEvent();+ restoringIds = () => []; // the batch is selected before the restore begins...+ restoring.add('plug'); // ...and the per-row check sees it begin++ const stats = await service.sweep(OPTS);++ expect(stats).toMatchObject({ scanned: 0, replayed: 0, skipped: 1 });+ expect(enqueue).not.toHaveBeenCalled();+ expect((await stored(id)).dispatchAttempts).toBe(0);+ });+it('skips a row whose instance was deleted after persist', async () => {await insertEvent();resolveInstance.mockResolvedValue(null);@@ -937,7 +973,7 @@ describe('IngressReconcilerService.sweep', () => {waitUntilReady: jest.fn().mockResolvedValue(undefined),} as never,),- { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,+ loaderDouble(),{ resolve: resolveInstance } as unknown as PluginInstanceService,);@@ -994,7 +1030,7 @@ describe('IngressReconcilerService.sweep', () => {{ get: jest.fn().mockReturnValue(true) } as unknown as ConfigService,queue as never,),- { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,+ loaderDouble(),{ resolve: resolveInstance } as unknown as PluginInstanceService,);diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.tsindex 61bebe05a..792165670 100644--- a/src/modules/integration/ingress-reconciler.service.ts+++ b/src/modules/integration/ingress-reconciler.service.ts@@ -166,6 +166,10 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {// that cannot be read is kept out of the next batches for a while, so a batch full of them cannot starve// the rows behind.const unreadable = this.unreadableRows(now);+ // A plugin the boot restore has not reached has no worker to take a replay, and the loop below skips its+ // rows, so they are left out here too: a batch's worth of them would otherwise be handed back by every+ // pass until the restore ends, with no other plugin's stranded delivery replayed in the meantime.+ const restoring = this.loader.restorePendingIds();const query = this.events.createQueryBuilder('e').select('e.id')@@ -179,6 +183,7 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {.andWhere('e.createdAt < :cutoff', { cutoff }).andWhere('e.payload IS NOT NULL');if (unreadable.length) query.andWhere('e.id NOT IN (:...unreadable)', { unreadable });+ if (restoring.length) query.andWhere('e.pluginId NOT IN (:...restoring)', { restoring });const batch = await query.orderBy('e.createdAt', 'ASC').limit(opts.batchSize).getMany();for (const { id } of batch) {if (this.stop.signal.aborted) break;