Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(plugins): mark restoring plugins before the queue workers start

Yudhi Armyndharisd870f7b8b17a
diff --git a/src/core/plugins/plugin-loader.service.ts b/src/core/plugins/plugin-loader.service.ts
index d4a57eef5..0e5bd71fc 100644
--- a/src/core/plugins/plugin-loader.service.ts
+++ b/src/core/plugins/plugin-loader.service.ts
@@ -254,6 +254,9 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap
onModuleInit(): void {
this.scanner.scanAtBoot();
+ // Marked now, though restored at bootstrap: the BullMQ workers start in between, and a queued ingress job
+ // they take for one of these plugins must be held (isRestorePending), not dispatched to a missing worker.
+ this.queueRestore();
}
/** Hand back the chats a plugin took over. A host without the conversation store has none to hand back. */
@@ -305,7 +308,7 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap
}
}
- /** Plugins the boot restore has queued or is enabling, so a lifecycle change cannot interleave with it. */
+ /** Plugins the boot restore has queued (from module init on) or is enabling, so no lifecycle change interleaves. */
private readonly restorePending = new Set<string>();
/**
@@ -324,11 +327,17 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap
return [...this.restorePending];
}
+ /** Mark the plugins the operator had enabled as restoring, and return their ids in restore order. */
+ private queueRestore(): string[] {
+ const ids = this.getAllPlugins()
+ .filter(p => !p.builtIn && this.pluginStorage.getPluginEntry(p.manifest.id)?.enabledByOperator === true)
+ .map(p => p.manifest.id);
+ for (const id of ids) this.restorePending.add(id);
+ return ids;
+ }
+
private async restorePlugins(): Promise<void> {
- const restorable = this.getAllPlugins().filter(
- p => !p.builtIn && this.pluginStorage.getPluginEntry(p.manifest.id)?.enabledByOperator === true,
- );
- for (const plugin of restorable) this.restorePending.add(plugin.manifest.id);
+ const restorable = this.queueRestore();
try {
for (let i = 0; i < restorable.length; i++) {
if (this.shuttingDown || this.shutdownService?.isShuttingDown()) {
@@ -337,8 +346,7 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap
});
return;
}
- const plugin = restorable[i];
- const pluginId = plugin.manifest.id;
+ const pluginId = restorable[i];
try {
await this.enablePlugin(pluginId);
} catch (error) {
diff --git a/src/core/plugins/plugin-restore-on-boot.spec.ts b/src/core/plugins/plugin-restore-on-boot.spec.ts
index e7ab7dd51..3ef4468e9 100644
--- a/src/core/plugins/plugin-restore-on-boot.spec.ts
+++ b/src/core/plugins/plugin-restore-on-boot.spec.ts
@@ -120,6 +120,43 @@ describe('PluginLoaderService — restoring the operator enable decision across
expect(enable).not.toHaveBeenCalled();
});
+ it('holds the plugins it will restore from module init, before bootstrap starts the restore', async () => {
+ // The BullMQ workers start between module init and bootstrap; a queued ingress job they take there has
+ // to find its plugin restoring rather than dispatch into a loader with no worker for it.
+ const idle: PluginManifest = { ...manifest, id: 'ext-idle', name: 'Ext Idle' };
+ const idleDir = path.join(tmpDir, 'plugins', idle.id);
+ fs.mkdirSync(idleDir, { recursive: true });
+ fs.writeFileSync(path.join(idleDir, 'manifest.json'), JSON.stringify(idle));
+ fs.writeFileSync(path.join(idleDir, 'index.js'), 'module.exports = class {};');
+ loader.loadPlugin(pluginDir);
+ storage.setPluginEnabledByOperator(manifest.id, true);
+
+ // A restart: the boot scan reads the plugins directory, as the real boot does.
+ const bootConfig = {
+ get: (k: string) => ({ dataDir: tmpDir, 'plugins.dir': path.join(tmpDir, 'plugins') })[k],
+ } as unknown as ConfigService;
+ const loader2 = new PluginLoaderService(
+ bootConfig,
+ new HookManager(),
+ new PluginStorageService(bootConfig),
+ {} as unknown as ModuleRef,
+ );
+ const enable = jest.spyOn(loader2, 'enablePlugin').mockResolvedValue(undefined);
+
+ loader2.onModuleInit();
+
+ expect(loader2.getPlugin(idle.id)).toBeDefined();
+ expect(enable).not.toHaveBeenCalled();
+ expect(loader2.isRestorePending(manifest.id)).toBe(true);
+ expect(loader2.isRestorePending(idle.id)).toBe(false);
+
+ await loader2.onApplicationBootstrap();
+
+ expect(enable).toHaveBeenCalledTimes(1);
+ expect(enable).toHaveBeenCalledWith(manifest.id);
+ expect(loader2.restorePendingIds()).toEqual([]);
+ });
+
it('keeps booting when one plugin fails to come back, and still restores the others', async () => {
const second: PluginManifest = { ...manifest, id: 'ext-two', name: 'Ext Two' };
const secondDir = path.join(tmpDir, 'plugins', second.id);
diff --git a/test/queue-on.e2e-spec.ts b/test/queue-on.e2e-spec.ts
index e2a08ed65..b71307cf7 100644
--- a/test/queue-on.e2e-spec.ts
+++ b/test/queue-on.e2e-spec.ts
@@ -43,12 +43,13 @@ if (!REDIS_AVAILABLE && process.env.GITHUB_ACTIONS === 'true') {
const describeQueueOn = REDIS_AVAILABLE ? describe : describe.skip;
import { createHmac, randomBytes } from 'node:crypto';
-import { readFileSync } from 'node:fs';
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import http from 'node:http';
import { AddressInfo } from 'node:net';
import { join } from 'node:path';
import { Test, TestingModule } from '@nestjs/testing';
import { INestApplication } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
import { getRepositoryToken } from '@nestjs/typeorm';
import { getQueueToken } from '@nestjs/bullmq';
import { Repository } from 'typeorm';
@@ -60,10 +61,13 @@ import { json, urlencoded, Request } from 'express';
import { AppModule } from '../src/app.module';
import { applyGlobalValidation } from '../src/config/app-validation';
import { PluginLoaderService } from '../src/core/plugins/plugin-loader.service';
+import { PluginStorageService } from '../src/core/plugins/plugin-storage.service';
+import { PluginStatus, PluginType } from '../src/core/plugins/plugin.interfaces';
import { PluginInstance } from '../src/modules/integration/entities/plugin-instance.entity';
import { IngressEvent } from '../src/modules/integration/entities/ingress-event.entity';
+import { IntegrationDeliveryFailure } from '../src/modules/integration/entities/integration-delivery-failure.entity';
import { WebhookOutboxEvent } from '../src/modules/webhook/entities/webhook-outbox-event.entity';
-import { sanitizeIngressJobId } from '../src/modules/integration/ingress-enqueue.service';
+import { resolveIngressJobOptions, sanitizeIngressJobId } from '../src/modules/integration/ingress-enqueue.service';
import { QUEUE_NAMES } from '../src/modules/queue/queue-names';
import { IngressJobData } from '../src/modules/queue/processors/ingress.processor';
import { AuthService } from '../src/modules/auth/auth.service';
@@ -71,6 +75,14 @@ import { ApiKeyRole } from '../src/modules/auth/entities/api-key.entity';
import { Session } from '../src/modules/session/entities/session.entity';
import { WebhookService } from '../src/modules/webhook/webhook.service';
+const waitFor = async (predicate: () => boolean | Promise<boolean>, timeoutMs = 10000): Promise<void> => {
+ const start = Date.now();
+ while (!(await predicate())) {
+ if (Date.now() - start > timeoutMs) throw new Error('timed out waiting for condition');
+ await new Promise(r => setTimeout(r, 10));
+ }
+};
+
/**
* Queue-on coverage for the dispatch paths every other e2e suite boots with QUEUE_ENABLED=false:
* the full AppModule comes up with the BullMQ wiring live against a real Redis, and both the
@@ -134,14 +146,6 @@ describeQueueOn('Queued dispatch paths (e2e, QUEUE_ENABLED=true)', () => {
dedupHeader: 'x-chatwoot-delivery',
};
- const waitFor = async (predicate: () => boolean | Promise<boolean>, timeoutMs = 10000): Promise<void> => {
- const start = Date.now();
- while (!(await predicate())) {
- if (Date.now() - start > timeoutMs) throw new Error('timed out waiting for condition');
- await new Promise(r => setTimeout(r, 10));
- }
- };
-
beforeAll(async () => {
// The 127.0.0.1 receiver would trip the SSRF guard at registration AND at delivery time; disable
// it for this suite like the webhooks suite does (restored in afterAll).
@@ -354,3 +358,156 @@ describeQueueOn('Queued dispatch paths (e2e, QUEUE_ENABLED=true)', () => {
}
});
});
+
+/**
+ * The BullMQ workers start while the modules initialise, and PluginLoaderService restores the plugins the
+ * operator had enabled only at bootstrap, after that. A job already waiting in Redis can be taken in between
+ * (on a busy host the bootstrap comes later than the worker's first fetch), so its plugin has to count as
+ * restoring from module init on: held without spending an attempt, while the job of a plugin that is not
+ * being restored is dispatched as usual.
+ */
+describeQueueOn('An ingress job waiting in Redis when the gateway starts (e2e, QUEUE_ENABLED=true)', () => {
+ const PLUGIN = 'restore-held';
+ const run = `${Date.now()}`;
+ const held: IngressJobData = {
+ pluginId: PLUGIN,
+ instanceId: 'acct1',
+ route: 'hook',
+ deliveryId: `restore-held-${run}`,
+ payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
+ };
+ const served: IngressJobData = { ...held, pluginId: 'not-restoring', deliveryId: `not-restoring-${run}` };
+ const prevEnv = {
+ PLUGINS_DIR: process.env.PLUGINS_DIR,
+ INGRESS_MAX_ATTEMPTS: process.env.INGRESS_MAX_ATTEMPTS,
+ INGRESS_RETRY_DELAY_MS: process.env.INGRESS_RETRY_DELAY_MS,
+ };
+ let app: INestApplication<App>;
+ let queue: Queue<IngressJobData>;
+ let failures: Repository<IntegrationDeliveryFailure>;
+ let booting: Promise<unknown>;
+ let bootstrap: jest.SpyInstance;
+ let enable: jest.SpyInstance;
+ let dispatch: jest.SpyInstance;
+ // The restore of the plugin stays in flight until the test ends it.
+ let restored = false;
+ let endRestore!: () => void;
+ const restore = new Promise<void>(resolve => (endRestore = resolve));
+
+ beforeAll(async () => {
+ // A single attempt: a job that spends it before the plugin is up is dead-lettered.
+ process.env.INGRESS_MAX_ATTEMPTS = '1';
+ process.env.INGRESS_RETRY_DELAY_MS = '1000';
+
+ // A plugin the operator had enabled before the restart: its package on disk and its decision in the registry.
+ const pluginsDir = join(process.env.PLUGIN_STATE_DIR!, 'plugins');
+ process.env.PLUGINS_DIR = pluginsDir;
+ const manifest = {
+ id: PLUGIN,
+ name: 'Restore Held',
+ version: '1.0.0',
+ type: PluginType.EXTENSION,
+ main: 'index.js',
+ };
+ mkdirSync(join(pluginsDir, PLUGIN), { recursive: true });
+ writeFileSync(join(pluginsDir, PLUGIN, 'manifest.json'), JSON.stringify(manifest));
+ writeFileSync(join(pluginsDir, PLUGIN, 'index.js'), 'module.exports = class {};');
+ new PluginStorageService({
+ get: (k: string) => (k === 'dataDir' ? process.env.PLUGIN_STATE_DIR : undefined),
+ } as unknown as ConfigService).setPluginEntry({
+ ...manifest,
+ status: PluginStatus.INSTALLED,
+ config: {},
+ builtIn: false,
+ installedAt: new Date(),
+ updatedAt: new Date(),
+ enabledByOperator: true,
+ });
+
+ queue = new Queue<IngressJobData>(QUEUE_NAMES.INGRESS, {
+ connection: { host: REDIS_HOST, port: REDIS_PORT },
+ prefix: 'openwa-e2e',
+ });
+ for (const d of [held, served]) {
+ await queue.add('ingress', d, { jobId: d.deliveryId, ...resolveIngressJobOptions() });
+ }
+
+ // The worker's first fetch, before the bootstrap starts the restore: wait until the worker has decided on the job.
+ // eslint-disable-next-line @typescript-eslint/unbound-method -- called below with the loader as this
+ const realBootstrap = PluginLoaderService.prototype.onApplicationBootstrap;
+ bootstrap = jest.spyOn(PluginLoaderService.prototype, 'onApplicationBootstrap').mockImplementation(async function (
+ this: PluginLoaderService,
+ ) {
+ await waitFor(async () => !['waiting', 'active'].includes(await queue.getJobState(held.deliveryId)));
+ return realBootstrap.call(this);
+ });
+ // eslint-disable-next-line @typescript-eslint/unbound-method -- called below with the loader as this
+ const realEnable = PluginLoaderService.prototype.enablePlugin;
+ enable = jest.spyOn(PluginLoaderService.prototype, 'enablePlugin').mockImplementation(function (
+ this: PluginLoaderService,
+ id: string,
+ ) {
+ return id === PLUGIN ? restore : realEnable.call(this, id);
+ });
+ // As the real dispatch does, it fails while the plugin has no worker yet.
+ dispatch = jest
+ .spyOn(PluginLoaderService.prototype, 'dispatchWebhookForInstance')
+ .mockImplementation((d: IngressJobData) =>
+ d.pluginId === PLUGIN && !restored
+ ? Promise.reject(new Error(`no live sandbox host for plugin ${PLUGIN}`))
+ : Promise.resolve(),
+ );
+
+ app = (await Test.createTestingModule({ imports: [AppModule] }).compile()).createNestApplication();
+ failures = app.get(getRepositoryToken(IntegrationDeliveryFailure, 'data'));
+ // init() waits for the restore, so it settles only once the test has ended the restore.
+ booting = app.init();
+ }, 60000);
+
+ afterAll(async () => {
+ restored = true;
+ endRestore();
+ try {
+ await booting;
+ await app?.close();
+ } catch {
+ /* ignore teardown-only multi-datasource quirk */
+ }
+ // A job left behind by a failed run would be dispatched by the next run's first suite.
+ await Promise.allSettled([held, served].map(d => queue?.remove(d.deliveryId)));
+ await queue?.close();
+ bootstrap?.mockRestore();
+ enable?.mockRestore();
+ dispatch?.mockRestore();
+ for (const [key, value] of Object.entries(prevEnv)) {
+ if (value === undefined) delete process.env[key];
+ else process.env[key] = value;
+ }
+ });
+
+ it('holds the job of a plugin still being restored without spending its attempt, and serves the others', async () => {
+ // The worker has taken the job and put it back while the restore is in flight. A held job comes back at
+ // every retry delay, so the state is read once.
+ let state = '';
+ await waitFor(async () => !['waiting', 'active'].includes((state = await queue.getJobState(held.deliveryId))));
+ expect(state).toBe('delayed');
+ expect((await queue.getJob(held.deliveryId))?.attemptsStarted).toBeGreaterThanOrEqual(1);
+ expect((await queue.getJob(held.deliveryId))?.attemptsMade).toBe(0);
+ expect(dispatch.mock.calls.map(([d]: [IngressJobData]) => d.deliveryId)).not.toContain(held.deliveryId);
+ expect(await failures.countBy({ deliveryId: held.deliveryId })).toBe(0);
+
+ // A plugin that is not being restored is not held.
+ await waitFor(async () => (await queue.getJobState(served.deliveryId)) === 'completed');
+ expect((await queue.getJob(served.deliveryId))?.attemptsMade).toBe(1);
+
+ await waitFor(() => enable.mock.calls.some(([id]: [string]) => id === PLUGIN));
+ restored = true;
+ endRestore();
+ await booting;
+
+ await waitFor(async () => (await queue.getJobState(held.deliveryId)) === 'completed', 15000);
+ expect((await queue.getJob(held.deliveryId))?.attemptsMade).toBe(1);
+ expect(dispatch.mock.calls.filter(([d]: [IngressJobData]) => d.deliveryId === held.deliveryId)).toHaveLength(1);
+ expect(await failures.countBy({ deliveryId: held.deliveryId })).toBe(0);
+ }, 60000);
+});