Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(plugins): restore plugins without delaying the HTTP port

Yudhi Armyndharise64291c5b3d7
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.md
index dae95be3c..50a7ddcc9 100644
--- a/docs/06-api-specification.md
+++ b/docs/06-api-specification.md
@@ -6205,7 +6205,7 @@ The digest shown is a placeholder: use the SHA-256 of the `.zip` (`sha256sum cha
**Response** `201` — the newly installed `PluginDto`.
-**Errors:** `400` invalid URL / download, integrity check, or package invalid · `401` · `403` · `409` already installed
+**Errors:** `400` invalid URL / download, integrity check, or package invalid · `401` · `403` · `409` already installed, or the plugin is still being restored after startup (retry shortly)
---
@@ -6229,7 +6229,7 @@ Enable a plugin.
Enable failures are returned in-band as `{ "success": false, "message": "…" }` (still HTTP 200).
-**Errors:** `401` · `403` · `404` unknown id
+**Errors:** `401` · `403` · `404` unknown id · `409` the plugin is still being restored after startup (retry shortly)
---
@@ -6256,7 +6256,7 @@ the gateway runs that engine whatever its plugin status says. The request is ref
`{ "success": false, "message": "…" }` (still HTTP 200) and changes nothing; set `ENGINE_TYPE` and
restart to switch engines.
-**Errors:** `401` · `403` · `404` unknown id
+**Errors:** `401` · `403` · `404` unknown id · `409` the plugin is still being restored after startup (retry shortly)
---
@@ -6381,7 +6381,7 @@ Update an installed plugin in place from a URL, preserving config + enabled stat
**Response** `201` — the updated `PluginDto`.
-**Errors:** `400` download/package invalid, manifest id mismatch, or built-in plugin · `401` · `403` · `404` unknown id · `503` shutdown began before the update of a plugin the operator switched on finished applying (the previous version is kept)
+**Errors:** `400` download/package invalid, manifest id mismatch, or built-in plugin · `401` · `403` · `404` unknown id · `409` the plugin is still being restored after startup (retry shortly) · `503` shutdown began before the update of a plugin the operator switched on finished applying (the previous version is kept)
---
@@ -6405,7 +6405,7 @@ Uninstall a plugin: dispatch its `onUnload` lifecycle hook, delete its files, dr
Note: this DELETE returns `200` with a body (not the usual `204`).
-**Errors:** `400` cannot uninstall (e.g. built-in) · `401` · `403` · `404` unknown id
+**Errors:** `400` cannot uninstall (e.g. built-in) · `401` · `403` · `404` unknown id · `409` the plugin is still being restored after startup (retry shortly)
---
diff --git a/openapi.json b/openapi.json
index f83b1e9fd..9b4c00d7b 100644
--- a/openapi.json
+++ b/openapi.json
@@ -16051,7 +16051,7 @@
}
},
"409": {
- "description": "Plugin already installed",
+ "description": "Plugin already installed, or the plugin is still being restored after startup; retry shortly",
"content": {
"application/json": {
"schema": {
@@ -16244,6 +16244,16 @@
}
}
}
+ },
+ "409": {
+ "description": "The plugin is still being restored after startup; retry shortly",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
}
},
"summary": "Uninstall a plugin (removes its files; built-ins are protected)",
@@ -16305,6 +16315,16 @@
}
}
}
+ },
+ "409": {
+ "description": "The plugin is still being restored after startup; retry shortly",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
}
},
"summary": "Enable a plugin",
@@ -16366,6 +16386,16 @@
}
}
}
+ },
+ "409": {
+ "description": "The plugin is still being restored after startup; retry shortly",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
}
},
"summary": "Disable a plugin",
@@ -16760,6 +16790,16 @@
}
}
},
+ "409": {
+ "description": "The plugin is still being restored after startup; retry shortly",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
"503": {
"description": "Gateway shutting down; the previous version was kept",
"content": {
diff --git a/src/core/plugins/plugin-loader.service.ts b/src/core/plugins/plugin-loader.service.ts
index 45c4d4675..120673ca2 100644
--- a/src/core/plugins/plugin-loader.service.ts
+++ b/src/core/plugins/plugin-loader.service.ts
@@ -58,6 +58,13 @@ const SANDBOX_CAP_TIMEOUT_MS = 30000;
/** Total time the plugin shutdown phase may take (disable pass, in-flight enables, second pass). See shutdown-budget.ts. */
export const PLUGIN_SHUTDOWN_WAIT_MS = 10_000;
+/**
+ * Longest the boot waits for the operator-enabled plugins to be restored before the gateway starts
+ * listening. Under the Docker health check's retries and the Helm startup probe, and above what a healthy
+ * plugin needs. See onApplicationBootstrap for what a plugin that is still restoring misses meanwhile.
+ */
+export const PLUGIN_RESTORE_BOOT_BUDGET_MS = 60_000;
+
/**
* How long a worker has to answer the liveness ping the host sends after a dispatch times out. A worker
* stuck in a synchronous loop is terminated about 5 s after its first dispatch timeout (about 10 s after
@@ -270,31 +277,77 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap
* code executes. Built-ins are skipped: an engine is enabled by EngineFactory against the configured
* engine.type, and enabling a non-active engine here would be rejected anyway.
*
- * Best-effort and sequential: a plugin that cannot come back is logged
- * and left in ERROR, and never holds up the gateway.
+ * Best-effort and sequential: a plugin that cannot come back is logged and left in ERROR. The HTTP
+ * port opens only after this hook returns, and one plugin can take up to 90 s here (load, onLoad and
+ * onEnable are each bounded at 30 s), so the wait is capped at PLUGIN_RESTORE_BOOT_BUDGET_MS. Plugins not
+ * yet restored when it ends keep coming up, in order, in the background. The trade-off is deliberate:
+ * until they are up, the hooks they own (a send moderation veto, for one) are not active, so a request
+ * that arrives in that window is not seen by them. Without the cap, plugins whose backends are down after
+ * an outage kept the whole gateway from listening and could fail its health checks into a restart loop.
+ * A plugin the restore has not finished with also takes no lifecycle change from the operator meanwhile
+ * (isRestorePending).
*/
async onApplicationBootstrap(): Promise<void> {
+ const restoring = this.restorePlugins();
+ let timer: NodeJS.Timeout | undefined;
+ const settled = await Promise.race([
+ restoring.then(() => true),
+ new Promise<false>(resolve => {
+ timer = setTimeout(resolve, PLUGIN_RESTORE_BOOT_BUDGET_MS, false);
+ }),
+ ]);
+ clearTimeout(timer);
+ if (!settled) {
+ this.logger.warn(
+ 'Plugin restore is still running; the gateway starts listening and the remaining plugins come up in the background',
+ { waitMs: PLUGIN_RESTORE_BOOT_BUDGET_MS, action: 'plugin_restore_deferred' },
+ );
+ }
+ }
+
+ /** Plugins the boot restore has queued or is enabling, so a lifecycle change cannot interleave with it. */
+ private readonly restorePending = new Set<string>();
+
+ /**
+ * Whether the boot restore still has this plugin ahead of it or in flight. When the restore outlasts the
+ * boot wait it runs while requests are served, and it enables outside the per-plugin operation chain of
+ * PluginsService: a disable would be answered "not enabled" and then overtaken by the enable, and an
+ * uninstall would leave a live worker without a map entry. PluginsService refuses such a change until the
+ * restore is done with the plugin, which is what the closed port did before the boot wait was capped.
+ */
+ isRestorePending(id: string): boolean {
+ return this.restorePending.has(id);
+ }
+
+ private async restorePlugins(): Promise<void> {
const restorable = this.getAllPlugins().filter(
p => !p.builtIn && this.pluginStorage.getPluginEntry(p.manifest.id)?.enabledByOperator === true,
);
- for (let i = 0; i < restorable.length; i++) {
- if (this.shuttingDown || this.shutdownService?.isShuttingDown()) {
- this.logger.log(`Plugin restore stopped at ${i} of ${restorable.length} plugin(s): shutting down`, {
- action: 'plugin_restore_aborted',
- });
- return;
- }
- const plugin = restorable[i];
- const pluginId = plugin.manifest.id;
- try {
- await this.enablePlugin(pluginId);
- } catch (error) {
- this.logger.error(
- `Failed to restore plugin ${pluginId} on startup; it stays disabled until re-enabled`,
- error instanceof Error ? error.message : String(error),
- { pluginId, action: 'plugin_restore_failed' },
- );
+ for (const plugin of restorable) this.restorePending.add(plugin.manifest.id);
+ try {
+ for (let i = 0; i < restorable.length; i++) {
+ if (this.shuttingDown || this.shutdownService?.isShuttingDown()) {
+ this.logger.log(`Plugin restore stopped at ${i} of ${restorable.length} plugin(s): shutting down`, {
+ action: 'plugin_restore_aborted',
+ });
+ return;
+ }
+ const plugin = restorable[i];
+ const pluginId = plugin.manifest.id;
+ try {
+ await this.enablePlugin(pluginId);
+ } catch (error) {
+ this.logger.error(
+ `Failed to restore plugin ${pluginId} on startup; it stays disabled until re-enabled`,
+ error instanceof Error ? error.message : String(error),
+ { pluginId, action: 'plugin_restore_failed' },
+ );
+ } finally {
+ this.restorePending.delete(pluginId);
+ }
}
+ } finally {
+ this.restorePending.clear();
}
}
diff --git a/src/core/plugins/plugin-restore-on-boot.spec.ts b/src/core/plugins/plugin-restore-on-boot.spec.ts
index 5a7c76e7e..30b3ceaed 100644
--- a/src/core/plugins/plugin-restore-on-boot.spec.ts
+++ b/src/core/plugins/plugin-restore-on-boot.spec.ts
@@ -3,7 +3,7 @@ import * as os from 'os';
import * as path from 'path';
import { ConfigService } from '@nestjs/config';
import { ModuleRef } from '@nestjs/core';
-import { PluginLoaderService } from './plugin-loader.service';
+import { PLUGIN_RESTORE_BOOT_BUDGET_MS, PluginLoaderService } from './plugin-loader.service';
import type { ShutdownService } from '../../common/services/shutdown.service';
import { PluginStorageService } from './plugin-storage.service';
import { HookManager } from '../hooks';
@@ -212,6 +212,128 @@ describe('PluginLoaderService — restoring the operator enable decision across
expect(enable).toHaveBeenCalledTimes(1);
});
+ describe('a plugin that takes a long time to come back', () => {
+ const second: PluginManifest = { ...manifest, id: 'ext-two', name: 'Ext Two' };
+ let loader2: PluginLoaderService;
+ let releaseFirst: () => void;
+ let enable: jest.SpyInstance;
+
+ beforeEach(() => {
+ jest.useFakeTimers();
+ const secondDir = path.join(tmpDir, 'plugins', second.id);
+ fs.mkdirSync(secondDir, { recursive: true });
+ fs.writeFileSync(path.join(secondDir, 'manifest.json'), JSON.stringify(second));
+ fs.writeFileSync(path.join(secondDir, 'index.js'), 'module.exports = class {};');
+
+ const storage2 = new PluginStorageService(config);
+ loader2 = makeLoader(storage2);
+ loader2.loadPlugin(pluginDir);
+ loader2.loadPlugin(secondDir);
+ storage2.setPluginEnabledByOperator(manifest.id, true);
+ storage2.setPluginEnabledByOperator(second.id, true);
+ // The first plugin's enable hangs (a backend that is down) until the test lets it go.
+ enable = jest.spyOn(loader2, 'enablePlugin').mockImplementation((id: string) =>
+ id === manifest.id
+ ? new Promise<void>(resolve => {
+ releaseFirst = resolve;
+ })
+ : Promise.resolve(),
+ );
+ });
+
+ afterEach(() => {
+ jest.useRealTimers();
+ });
+
+ it('does not hold the gateway back past the restore budget, and finishes restoring the rest afterwards', async () => {
+ const warn = jest
+ .spyOn((loader2 as unknown as { logger: { warn: (...args: unknown[]) => void } }).logger, 'warn')
+ .mockImplementation(() => undefined);
+
+ const booted = loader2.onApplicationBootstrap();
+ await jest.advanceTimersByTimeAsync(PLUGIN_RESTORE_BOOT_BUDGET_MS);
+
+ await expect(booted).resolves.toBeUndefined();
+ expect(warn).toHaveBeenCalledWith(
+ expect.stringContaining('remaining plugins come up in the background'),
+ expect.objectContaining({ action: 'plugin_restore_deferred' }),
+ );
+ // The first plugin is still being enabled, so the second has not started: order is kept.
+ expect(enable).toHaveBeenCalledTimes(1);
+
+ releaseFirst();
+ await jest.advanceTimersByTimeAsync(0);
+
+ expect(enable).toHaveBeenCalledWith(second.id);
+ });
+
+ it('returns as soon as every plugin is back, without waiting out the budget', async () => {
+ const booted = loader2.onApplicationBootstrap();
+ releaseFirst();
+
+ await expect(booted).resolves.toBeUndefined();
+ expect(enable).toHaveBeenCalledTimes(2);
+ // No timer is left running for the budget.
+ expect(jest.getTimerCount()).toBe(0);
+ });
+
+ it('stops the background restore when the gateway shuts down', async () => {
+ const booted = loader2.onApplicationBootstrap();
+ await jest.advanceTimersByTimeAsync(PLUGIN_RESTORE_BOOT_BUDGET_MS);
+ await booted;
+
+ await loader2.onModuleDestroy();
+ releaseFirst();
+ await jest.advanceTimersByTimeAsync(0);
+
+ expect(enable).toHaveBeenCalledTimes(1);
+ });
+
+ it('keeps a plugin pending from the start of the restore until its own enable settles', async () => {
+ const booted = loader2.onApplicationBootstrap();
+ await jest.advanceTimersByTimeAsync(PLUGIN_RESTORE_BOOT_BUDGET_MS);
+ await booted;
+
+ // The first is being enabled and the second is queued behind it: neither takes a lifecycle change yet.
+ expect(loader2.isRestorePending(manifest.id)).toBe(true);
+ expect(loader2.isRestorePending(second.id)).toBe(true);
+ expect(loader2.isRestorePending('never-queued')).toBe(false);
+
+ releaseFirst();
+ await jest.advanceTimersByTimeAsync(0);
+
+ expect(loader2.isRestorePending(manifest.id)).toBe(false);
+ expect(loader2.isRestorePending(second.id)).toBe(false);
+ });
+
+ it('stops holding a plugin whose restore failed', async () => {
+ enable.mockImplementation((id: string) =>
+ id === manifest.id ? Promise.reject(new Error('backend down')) : Promise.resolve(),
+ );
+ jest
+ .spyOn((loader2 as unknown as { logger: { error: (...args: unknown[]) => void } }).logger, 'error')
+ .mockImplementation(() => undefined);
+
+ await loader2.onApplicationBootstrap();
+
+ expect(loader2.isRestorePending(manifest.id)).toBe(false);
+ expect(loader2.isRestorePending(second.id)).toBe(false);
+ });
+
+ it('lets go of the plugins it never reached when the gateway shuts down', async () => {
+ const booted = loader2.onApplicationBootstrap();
+ await jest.advanceTimersByTimeAsync(PLUGIN_RESTORE_BOOT_BUDGET_MS);
+ await booted;
+
+ await loader2.onModuleDestroy();
+ releaseFirst();
+ await jest.advanceTimersByTimeAsync(0);
+
+ expect(loader2.isRestorePending(manifest.id)).toBe(false);
+ expect(loader2.isRestorePending(second.id)).toBe(false);
+ });
+ });
+
it('does not treat the shutdown teardown as the operator disabling the plugin', async () => {
// The regression that makes this whole design fragile: onModuleDestroy disables every enabled
// plugin on the way out. If teardown cleared the intent, one graceful restart would erase it and
diff --git a/src/modules/plugins/plugins.controller.ts b/src/modules/plugins/plugins.controller.ts
index 8032c7ee9..ef548ff9d 100644
--- a/src/modules/plugins/plugins.controller.ts
+++ b/src/modules/plugins/plugins.controller.ts
@@ -31,6 +31,9 @@ import { ApiKeyRole } from '../auth/entities/api-key.entity';
/** Max accepted upload size for a plugin package (compressed). */
const MAX_PLUGIN_UPLOAD_BYTES = 5 * 1024 * 1024;
+/** A lifecycle change to a plugin the boot restore has not finished with, which can outlast the boot wait. */
+const RESTORING_CONFLICT = 'The plugin is still being restored after startup; retry shortly';
+
@ApiTags('plugins')
@Controller('plugins')
// Plugin installation and lifecycle are deployment-global and execute plugin code as the OpenWA
@@ -81,7 +84,10 @@ export class PluginsController {
@ApiOperation({ summary: 'Install a plugin by downloading its .zip from a URL (SSRF-guarded)' })
@ApiResponse({ status: 201, description: 'Plugin installed', type: PluginDto })
@ApiResponse({ status: 400, description: 'Invalid URL, download failed, or invalid package' })
- @ApiResponse({ status: 409, description: 'Plugin already installed' })
+ @ApiResponse({
+ status: 409,
+ description: 'Plugin already installed, or the plugin is still being restored after startup; retry shortly',
+ })
async installFromUrl(@Body() dto: InstallFromUrlDto): Promise<PluginDto> {
return await this.pluginsService.installFromUrl(dto.url);
}
@@ -114,6 +120,7 @@ export class PluginsController {
@ApiOperation({ summary: 'Enable a plugin' })
@ApiResponse({ status: 200, description: 'Plugin enabled successfully', type: PluginActionResponseDto })
@ApiResponse({ status: 404, description: 'Plugin not found' })
+ @ApiResponse({ status: 409, description: RESTORING_CONFLICT })
async enable(@Param('id') id: string): Promise<{ success: boolean; message: string }> {
return await this.pluginsService.enable(id);
}
@@ -125,6 +132,7 @@ export class PluginsController {
@ApiOperation({ summary: 'Disable a plugin' })
@ApiResponse({ status: 200, description: 'Plugin disabled successfully', type: PluginActionResponseDto })
@ApiResponse({ status: 404, description: 'Plugin not found' })
+ @ApiResponse({ status: 409, description: RESTORING_CONFLICT })
async disable(@Param('id') id: string): Promise<{ success: boolean; message: string }> {
return await this.pluginsService.disable(id);
}
@@ -202,6 +210,7 @@ export class PluginsController {
@ApiResponse({ status: 201, description: 'Plugin updated', type: PluginDto })
@ApiResponse({ status: 400, description: 'Invalid URL/package, id mismatch, or built-in' })
@ApiResponse({ status: 404, description: 'Plugin not found' })
+ @ApiResponse({ status: 409, description: RESTORING_CONFLICT })
@ApiResponse({
status: 503,
description: 'Gateway shutting down; the previous version was kept',
@@ -217,6 +226,7 @@ export class PluginsController {
@ApiResponse({ status: 200, description: 'Plugin uninstalled', type: PluginActionResponseDto })
@ApiResponse({ status: 400, description: 'Cannot uninstall (e.g. built-in)' })
@ApiResponse({ status: 404, description: 'Plugin not found' })
+ @ApiResponse({ status: 409, description: RESTORING_CONFLICT })
async uninstall(@Param('id') id: string): Promise<{ success: boolean; message: string }> {
return await this.pluginsService.uninstall(id);
}
diff --git a/src/modules/plugins/plugins.service.spec.ts b/src/modules/plugins/plugins.service.spec.ts
index 2db3af63f..2bccfdcad 100644
--- a/src/modules/plugins/plugins.service.spec.ts
+++ b/src/modules/plugins/plugins.service.spec.ts
@@ -800,6 +800,70 @@ describe('PluginsService — disable when the plugin is not loaded', () => {
});
});
+describe('PluginsService: a plugin the boot restore has not finished with', () => {
+ let tmpDir: string;
+ let pluginsDir: string;
+ let config: ConfigService;
+
+ beforeEach(() => {
+ tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'owa-pending-'));
+ pluginsDir = path.join(tmpDir, 'plugins');
+ fs.mkdirSync(pluginsDir, { recursive: true });
+ config = {
+ get: (k: string) => (k === 'plugins.dir' ? pluginsDir : k === 'dataDir' ? tmpDir : undefined),
+ } as unknown as ConfigService;
+ });
+ afterEach(() => fs.rmSync(tmpDir, { recursive: true, force: true }));
+
+ // The restore enables outside the per-plugin chain once it outlasts the boot wait. A disable answered
+ // "not enabled" would then be overtaken by the enable, and an uninstall would leave a worker running
+ // under no map entry, so both wait for the restore instead of racing it.
+ it('refuses enable, disable, update and uninstall of it, and leaves every other plugin alone', async () => {
+ const storage = new PluginStorageService(config);
+ const loader = new PluginLoaderService(config, new HookManager(), storage, {} as unknown as ModuleRef);
+ const service = new PluginsService(loader, config);
+ service.install({ buffer: pkg() });
+ service.install({ buffer: pkg({ id: 'other-plg' }) });
+ loader.setOperatorEnabled('svc-plg', true);
+ jest.spyOn(loader, 'isRestorePending').mockImplementation(id => id === 'svc-plg');
+
+ for (const attempt of [
+ () => service.enable('svc-plg'),
+ () => service.disable('svc-plg'),
+ () => service.updatePackage('svc-plg', pkg({ version: '2.0.0' })),
+ () => service.uninstall('svc-plg'),
+ ]) {
+ await expect(attempt()).rejects.toThrow(ConflictException);
+ await expect(attempt()).rejects.toThrow(/still being restored/);
+ }
+
+ // Nothing changed: the operator's decision stands and the plugin is still installed.
+ expect(storage.getPluginEntry('svc-plg')?.enabledByOperator).toBe(true);
+ expect(loader.getPlugin('svc-plg')?.manifest.version).toBe('1.0.0');
+ expect(fs.existsSync(path.join(pluginsDir, 'svc-plg'))).toBe(true);
+
+ await expect(service.disable('other-plg')).resolves.toMatchObject({ success: true });
+ await expect(service.uninstall('other-plg')).resolves.toMatchObject({ success: true });
+ });
+
+ it('takes the same changes again once the restore is done with it', async () => {
+ const loader = new PluginLoaderService(
+ config,
+ new HookManager(),
+ new PluginStorageService(config),
+ {} as unknown as ModuleRef,
+ );
+ const service = new PluginsService(loader, config);
+ service.install({ buffer: pkg() });
+ const pending = jest.spyOn(loader, 'isRestorePending').mockReturnValue(true);
+ await expect(service.uninstall('svc-plg')).rejects.toThrow(ConflictException);
+
+ pending.mockReturnValue(false);
+
+ await expect(service.uninstall('svc-plg')).resolves.toMatchObject({ success: true });
+ });
+});
+
describe('PluginsService: disable an engine plugin', () => {
let tmpDir: string;
diff --git a/src/modules/plugins/plugins.service.ts b/src/modules/plugins/plugins.service.ts
index 43b249dea..c28f367db 100644
--- a/src/modules/plugins/plugins.service.ts
+++ b/src/modules/plugins/plugins.service.ts
@@ -49,6 +49,13 @@ export class PluginsService {
private readonly opChains = new Map<string, Promise<unknown>>();
private serialize<T>(id: string, op: () => Promise<T>): Promise<T> {
+ // A boot restore that outlasted the boot wait enables plugins in the background, outside this chain, so a
+ // change to a plugin it has not finished with would interleave with its enable (see isRestorePending).
+ if (this.pluginLoader.isRestorePending(id)) {
+ return Promise.reject(
+ new ConflictException(`Plugin ${id} is still being restored after startup. Retry in a moment.`),
+ );
+ }
const prior = this.opChains.get(id) ?? Promise.resolve();
const next = prior.catch(() => undefined).then(op);
this.opChains.set(id, next);