Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(plugins): mark restoring plugins before the queue workers start
Yudhi Armyndharisd870f7b8b17a
diff --git a/src/core/plugins/plugin-loader.service.ts b/src/core/plugins/plugin-loader.service.tsindex d4a57eef5..0e5bd71fc 100644--- a/src/core/plugins/plugin-loader.service.ts+++ b/src/core/plugins/plugin-loader.service.ts@@ -254,6 +254,9 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstraponModuleInit(): void {this.scanner.scanAtBoot();+ // Marked now, though restored at bootstrap: the BullMQ workers start in between, and a queued ingress job+ // they take for one of these plugins must be held (isRestorePending), not dispatched to a missing worker.+ this.queueRestore();}/** Hand back the chats a plugin took over. A host without the conversation store has none to hand back. */@@ -305,7 +308,7 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap}}- /** Plugins the boot restore has queued or is enabling, so a lifecycle change cannot interleave with it. */+ /** Plugins the boot restore has queued (from module init on) or is enabling, so no lifecycle change interleaves. */private readonly restorePending = new Set<string>();/**@@ -324,11 +327,17 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrapreturn [...this.restorePending];}+ /** Mark the plugins the operator had enabled as restoring, and return their ids in restore order. */+ private queueRestore(): string[] {+ const ids = this.getAllPlugins()+ .filter(p => !p.builtIn && this.pluginStorage.getPluginEntry(p.manifest.id)?.enabledByOperator === true)+ .map(p => p.manifest.id);+ for (const id of ids) this.restorePending.add(id);+ return ids;+ }+private async restorePlugins(): Promise<void> {- const restorable = this.getAllPlugins().filter(- p => !p.builtIn && this.pluginStorage.getPluginEntry(p.manifest.id)?.enabledByOperator === true,- );- for (const plugin of restorable) this.restorePending.add(plugin.manifest.id);+ const restorable = this.queueRestore();try {for (let i = 0; i < restorable.length; i++) {if (this.shuttingDown || this.shutdownService?.isShuttingDown()) {@@ -337,8 +346,7 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap});return;}- const plugin = restorable[i];- const pluginId = plugin.manifest.id;+ const pluginId = restorable[i];try {await this.enablePlugin(pluginId);} catch (error) {diff --git a/src/core/plugins/plugin-restore-on-boot.spec.ts b/src/core/plugins/plugin-restore-on-boot.spec.tsindex e7ab7dd51..3ef4468e9 100644--- a/src/core/plugins/plugin-restore-on-boot.spec.ts+++ b/src/core/plugins/plugin-restore-on-boot.spec.ts@@ -120,6 +120,43 @@ describe('PluginLoaderService — restoring the operator enable decision acrossexpect(enable).not.toHaveBeenCalled();});+ it('holds the plugins it will restore from module init, before bootstrap starts the restore', async () => {+ // The BullMQ workers start between module init and bootstrap; a queued ingress job they take there has+ // to find its plugin restoring rather than dispatch into a loader with no worker for it.+ const idle: PluginManifest = { ...manifest, id: 'ext-idle', name: 'Ext Idle' };+ const idleDir = path.join(tmpDir, 'plugins', idle.id);+ fs.mkdirSync(idleDir, { recursive: true });+ fs.writeFileSync(path.join(idleDir, 'manifest.json'), JSON.stringify(idle));+ fs.writeFileSync(path.join(idleDir, 'index.js'), 'module.exports = class {};');+ loader.loadPlugin(pluginDir);+ storage.setPluginEnabledByOperator(manifest.id, true);++ // A restart: the boot scan reads the plugins directory, as the real boot does.+ const bootConfig = {+ get: (k: string) => ({ dataDir: tmpDir, 'plugins.dir': path.join(tmpDir, 'plugins') })[k],+ } as unknown as ConfigService;+ const loader2 = new PluginLoaderService(+ bootConfig,+ new HookManager(),+ new PluginStorageService(bootConfig),+ {} as unknown as ModuleRef,+ );+ const enable = jest.spyOn(loader2, 'enablePlugin').mockResolvedValue(undefined);++ loader2.onModuleInit();++ expect(loader2.getPlugin(idle.id)).toBeDefined();+ expect(enable).not.toHaveBeenCalled();+ expect(loader2.isRestorePending(manifest.id)).toBe(true);+ expect(loader2.isRestorePending(idle.id)).toBe(false);++ await loader2.onApplicationBootstrap();++ expect(enable).toHaveBeenCalledTimes(1);+ expect(enable).toHaveBeenCalledWith(manifest.id);+ expect(loader2.restorePendingIds()).toEqual([]);+ });+it('keeps booting when one plugin fails to come back, and still restores the others', async () => {const second: PluginManifest = { ...manifest, id: 'ext-two', name: 'Ext Two' };const secondDir = path.join(tmpDir, 'plugins', second.id);diff --git a/test/queue-on.e2e-spec.ts b/test/queue-on.e2e-spec.tsindex e2a08ed65..b71307cf7 100644--- a/test/queue-on.e2e-spec.ts+++ b/test/queue-on.e2e-spec.ts@@ -43,12 +43,13 @@ if (!REDIS_AVAILABLE && process.env.GITHUB_ACTIONS === 'true') {const describeQueueOn = REDIS_AVAILABLE ? describe : describe.skip;import { createHmac, randomBytes } from 'node:crypto';-import { readFileSync } from 'node:fs';+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';import http from 'node:http';import { AddressInfo } from 'node:net';import { join } from 'node:path';import { Test, TestingModule } from '@nestjs/testing';import { INestApplication } from '@nestjs/common';+import { ConfigService } from '@nestjs/config';import { getRepositoryToken } from '@nestjs/typeorm';import { getQueueToken } from '@nestjs/bullmq';import { Repository } from 'typeorm';@@ -60,10 +61,13 @@ import { json, urlencoded, Request } from 'express';import { AppModule } from '../src/app.module';import { applyGlobalValidation } from '../src/config/app-validation';import { PluginLoaderService } from '../src/core/plugins/plugin-loader.service';+import { PluginStorageService } from '../src/core/plugins/plugin-storage.service';+import { PluginStatus, PluginType } from '../src/core/plugins/plugin.interfaces';import { PluginInstance } from '../src/modules/integration/entities/plugin-instance.entity';import { IngressEvent } from '../src/modules/integration/entities/ingress-event.entity';+import { IntegrationDeliveryFailure } from '../src/modules/integration/entities/integration-delivery-failure.entity';import { WebhookOutboxEvent } from '../src/modules/webhook/entities/webhook-outbox-event.entity';-import { sanitizeIngressJobId } from '../src/modules/integration/ingress-enqueue.service';+import { resolveIngressJobOptions, sanitizeIngressJobId } from '../src/modules/integration/ingress-enqueue.service';import { QUEUE_NAMES } from '../src/modules/queue/queue-names';import { IngressJobData } from '../src/modules/queue/processors/ingress.processor';import { AuthService } from '../src/modules/auth/auth.service';@@ -71,6 +75,14 @@ import { ApiKeyRole } from '../src/modules/auth/entities/api-key.entity';import { Session } from '../src/modules/session/entities/session.entity';import { WebhookService } from '../src/modules/webhook/webhook.service';+const waitFor = async (predicate: () => boolean | Promise<boolean>, timeoutMs = 10000): Promise<void> => {+ const start = Date.now();+ while (!(await predicate())) {+ if (Date.now() - start > timeoutMs) throw new Error('timed out waiting for condition');+ await new Promise(r => setTimeout(r, 10));+ }+};+/*** Queue-on coverage for the dispatch paths every other e2e suite boots with QUEUE_ENABLED=false:* the full AppModule comes up with the BullMQ wiring live against a real Redis, and both the@@ -134,14 +146,6 @@ describeQueueOn('Queued dispatch paths (e2e, QUEUE_ENABLED=true)', () => {dedupHeader: 'x-chatwoot-delivery',};- const waitFor = async (predicate: () => boolean | Promise<boolean>, timeoutMs = 10000): Promise<void> => {- const start = Date.now();- while (!(await predicate())) {- if (Date.now() - start > timeoutMs) throw new Error('timed out waiting for condition');- await new Promise(r => setTimeout(r, 10));- }- };-beforeAll(async () => {// The 127.0.0.1 receiver would trip the SSRF guard at registration AND at delivery time; disable// it for this suite like the webhooks suite does (restored in afterAll).@@ -354,3 +358,156 @@ describeQueueOn('Queued dispatch paths (e2e, QUEUE_ENABLED=true)', () => {}});});++/**+ * The BullMQ workers start while the modules initialise, and PluginLoaderService restores the plugins the+ * operator had enabled only at bootstrap, after that. A job already waiting in Redis can be taken in between+ * (on a busy host the bootstrap comes later than the worker's first fetch), so its plugin has to count as+ * restoring from module init on: held without spending an attempt, while the job of a plugin that is not+ * being restored is dispatched as usual.+ */+describeQueueOn('An ingress job waiting in Redis when the gateway starts (e2e, QUEUE_ENABLED=true)', () => {+ const PLUGIN = 'restore-held';+ const run = `${Date.now()}`;+ const held: IngressJobData = {+ pluginId: PLUGIN,+ instanceId: 'acct1',+ route: 'hook',+ deliveryId: `restore-held-${run}`,+ payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },+ };+ const served: IngressJobData = { ...held, pluginId: 'not-restoring', deliveryId: `not-restoring-${run}` };+ const prevEnv = {+ PLUGINS_DIR: process.env.PLUGINS_DIR,+ INGRESS_MAX_ATTEMPTS: process.env.INGRESS_MAX_ATTEMPTS,+ INGRESS_RETRY_DELAY_MS: process.env.INGRESS_RETRY_DELAY_MS,+ };+ let app: INestApplication<App>;+ let queue: Queue<IngressJobData>;+ let failures: Repository<IntegrationDeliveryFailure>;+ let booting: Promise<unknown>;+ let bootstrap: jest.SpyInstance;+ let enable: jest.SpyInstance;+ let dispatch: jest.SpyInstance;+ // The restore of the plugin stays in flight until the test ends it.+ let restored = false;+ let endRestore!: () => void;+ const restore = new Promise<void>(resolve => (endRestore = resolve));++ beforeAll(async () => {+ // A single attempt: a job that spends it before the plugin is up is dead-lettered.+ process.env.INGRESS_MAX_ATTEMPTS = '1';+ process.env.INGRESS_RETRY_DELAY_MS = '1000';++ // A plugin the operator had enabled before the restart: its package on disk and its decision in the registry.+ const pluginsDir = join(process.env.PLUGIN_STATE_DIR!, 'plugins');+ process.env.PLUGINS_DIR = pluginsDir;+ const manifest = {+ id: PLUGIN,+ name: 'Restore Held',+ version: '1.0.0',+ type: PluginType.EXTENSION,+ main: 'index.js',+ };+ mkdirSync(join(pluginsDir, PLUGIN), { recursive: true });+ writeFileSync(join(pluginsDir, PLUGIN, 'manifest.json'), JSON.stringify(manifest));+ writeFileSync(join(pluginsDir, PLUGIN, 'index.js'), 'module.exports = class {};');+ new PluginStorageService({+ get: (k: string) => (k === 'dataDir' ? process.env.PLUGIN_STATE_DIR : undefined),+ } as unknown as ConfigService).setPluginEntry({+ ...manifest,+ status: PluginStatus.INSTALLED,+ config: {},+ builtIn: false,+ installedAt: new Date(),+ updatedAt: new Date(),+ enabledByOperator: true,+ });++ queue = new Queue<IngressJobData>(QUEUE_NAMES.INGRESS, {+ connection: { host: REDIS_HOST, port: REDIS_PORT },+ prefix: 'openwa-e2e',+ });+ for (const d of [held, served]) {+ await queue.add('ingress', d, { jobId: d.deliveryId, ...resolveIngressJobOptions() });+ }++ // The worker's first fetch, before the bootstrap starts the restore: wait until the worker has decided on the job.+ // eslint-disable-next-line @typescript-eslint/unbound-method -- called below with the loader as this+ const realBootstrap = PluginLoaderService.prototype.onApplicationBootstrap;+ bootstrap = jest.spyOn(PluginLoaderService.prototype, 'onApplicationBootstrap').mockImplementation(async function (+ this: PluginLoaderService,+ ) {+ await waitFor(async () => !['waiting', 'active'].includes(await queue.getJobState(held.deliveryId)));+ return realBootstrap.call(this);+ });+ // eslint-disable-next-line @typescript-eslint/unbound-method -- called below with the loader as this+ const realEnable = PluginLoaderService.prototype.enablePlugin;+ enable = jest.spyOn(PluginLoaderService.prototype, 'enablePlugin').mockImplementation(function (+ this: PluginLoaderService,+ id: string,+ ) {+ return id === PLUGIN ? restore : realEnable.call(this, id);+ });+ // As the real dispatch does, it fails while the plugin has no worker yet.+ dispatch = jest+ .spyOn(PluginLoaderService.prototype, 'dispatchWebhookForInstance')+ .mockImplementation((d: IngressJobData) =>+ d.pluginId === PLUGIN && !restored+ ? Promise.reject(new Error(`no live sandbox host for plugin ${PLUGIN}`))+ : Promise.resolve(),+ );++ app = (await Test.createTestingModule({ imports: [AppModule] }).compile()).createNestApplication();+ failures = app.get(getRepositoryToken(IntegrationDeliveryFailure, 'data'));+ // init() waits for the restore, so it settles only once the test has ended the restore.+ booting = app.init();+ }, 60000);++ afterAll(async () => {+ restored = true;+ endRestore();+ try {+ await booting;+ await app?.close();+ } catch {+ /* ignore teardown-only multi-datasource quirk */+ }+ // A job left behind by a failed run would be dispatched by the next run's first suite.+ await Promise.allSettled([held, served].map(d => queue?.remove(d.deliveryId)));+ await queue?.close();+ bootstrap?.mockRestore();+ enable?.mockRestore();+ dispatch?.mockRestore();+ for (const [key, value] of Object.entries(prevEnv)) {+ if (value === undefined) delete process.env[key];+ else process.env[key] = value;+ }+ });++ it('holds the job of a plugin still being restored without spending its attempt, and serves the others', async () => {+ // The worker has taken the job and put it back while the restore is in flight. A held job comes back at+ // every retry delay, so the state is read once.+ let state = '';+ await waitFor(async () => !['waiting', 'active'].includes((state = await queue.getJobState(held.deliveryId))));+ expect(state).toBe('delayed');+ expect((await queue.getJob(held.deliveryId))?.attemptsStarted).toBeGreaterThanOrEqual(1);+ expect((await queue.getJob(held.deliveryId))?.attemptsMade).toBe(0);+ expect(dispatch.mock.calls.map(([d]: [IngressJobData]) => d.deliveryId)).not.toContain(held.deliveryId);+ expect(await failures.countBy({ deliveryId: held.deliveryId })).toBe(0);++ // A plugin that is not being restored is not held.+ await waitFor(async () => (await queue.getJobState(served.deliveryId)) === 'completed');+ expect((await queue.getJob(served.deliveryId))?.attemptsMade).toBe(1);++ await waitFor(() => enable.mock.calls.some(([id]: [string]) => id === PLUGIN));+ restored = true;+ endRestore();+ await booting;++ await waitFor(async () => (await queue.getJobState(held.deliveryId)) === 'completed', 15000);+ expect((await queue.getJob(held.deliveryId))?.attemptsMade).toBe(1);+ expect(dispatch.mock.calls.filter(([d]: [IngressJobData]) => d.deliveryId === held.deliveryId)).toHaveLength(1);+ expect(await failures.countBy({ deliveryId: held.deliveryId })).toBe(0);+ }, 60000);+});