Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(plugins): restore plugins without delaying the HTTP port
Yudhi Armyndharise64291c5b3d7
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.mdindex dae95be3c..50a7ddcc9 100644--- a/docs/06-api-specification.md+++ b/docs/06-api-specification.md@@ -6205,7 +6205,7 @@ The digest shown is a placeholder: use the SHA-256 of the `.zip` (`sha256sum cha**Response** `201` — the newly installed `PluginDto`.-**Errors:** `400` invalid URL / download, integrity check, or package invalid · `401` · `403` · `409` already installed+**Errors:** `400` invalid URL / download, integrity check, or package invalid · `401` · `403` · `409` already installed, or the plugin is still being restored after startup (retry shortly)---@@ -6229,7 +6229,7 @@ Enable a plugin.Enable failures are returned in-band as `{ "success": false, "message": "…" }` (still HTTP 200).-**Errors:** `401` · `403` · `404` unknown id+**Errors:** `401` · `403` · `404` unknown id · `409` the plugin is still being restored after startup (retry shortly)---@@ -6256,7 +6256,7 @@ the gateway runs that engine whatever its plugin status says. The request is ref`{ "success": false, "message": "…" }` (still HTTP 200) and changes nothing; set `ENGINE_TYPE` andrestart to switch engines.-**Errors:** `401` · `403` · `404` unknown id+**Errors:** `401` · `403` · `404` unknown id · `409` the plugin is still being restored after startup (retry shortly)---@@ -6381,7 +6381,7 @@ Update an installed plugin in place from a URL, preserving config + enabled stat**Response** `201` — the updated `PluginDto`.-**Errors:** `400` download/package invalid, manifest id mismatch, or built-in plugin · `401` · `403` · `404` unknown id · `503` shutdown began before the update of a plugin the operator switched on finished applying (the previous version is kept)+**Errors:** `400` download/package invalid, manifest id mismatch, or built-in plugin · `401` · `403` · `404` unknown id · `409` the plugin is still being restored after startup (retry shortly) · `503` shutdown began before the update of a plugin the operator switched on finished applying (the previous version is kept)---@@ -6405,7 +6405,7 @@ Uninstall a plugin: dispatch its `onUnload` lifecycle hook, delete its files, drNote: this DELETE returns `200` with a body (not the usual `204`).-**Errors:** `400` cannot uninstall (e.g. built-in) · `401` · `403` · `404` unknown id+**Errors:** `400` cannot uninstall (e.g. built-in) · `401` · `403` · `404` unknown id · `409` the plugin is still being restored after startup (retry shortly)---diff --git a/openapi.json b/openapi.jsonindex f83b1e9fd..9b4c00d7b 100644--- a/openapi.json+++ b/openapi.json@@ -16051,7 +16051,7 @@}},"409": {- "description": "Plugin already installed",+ "description": "Plugin already installed, or the plugin is still being restored after startup; retry shortly","content": {"application/json": {"schema": {@@ -16244,6 +16244,16 @@}}}+ },+ "409": {+ "description": "The plugin is still being restored after startup; retry shortly",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ErrorResponse"+ }+ }+ }}},"summary": "Uninstall a plugin (removes its files; built-ins are protected)",@@ -16305,6 +16315,16 @@}}}+ },+ "409": {+ "description": "The plugin is still being restored after startup; retry shortly",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ErrorResponse"+ }+ }+ }}},"summary": "Enable a plugin",@@ -16366,6 +16386,16 @@}}}+ },+ "409": {+ "description": "The plugin is still being restored after startup; retry shortly",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ErrorResponse"+ }+ }+ }}},"summary": "Disable a plugin",@@ -16760,6 +16790,16 @@}}},+ "409": {+ "description": "The plugin is still being restored after startup; retry shortly",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ErrorResponse"+ }+ }+ }+ },"503": {"description": "Gateway shutting down; the previous version was kept","content": {diff --git a/src/core/plugins/plugin-loader.service.ts b/src/core/plugins/plugin-loader.service.tsindex 45c4d4675..120673ca2 100644--- a/src/core/plugins/plugin-loader.service.ts+++ b/src/core/plugins/plugin-loader.service.ts@@ -58,6 +58,13 @@ const SANDBOX_CAP_TIMEOUT_MS = 30000;/** Total time the plugin shutdown phase may take (disable pass, in-flight enables, second pass). See shutdown-budget.ts. */export const PLUGIN_SHUTDOWN_WAIT_MS = 10_000;+/**+ * Longest the boot waits for the operator-enabled plugins to be restored before the gateway starts+ * listening. Under the Docker health check's retries and the Helm startup probe, and above what a healthy+ * plugin needs. See onApplicationBootstrap for what a plugin that is still restoring misses meanwhile.+ */+export const PLUGIN_RESTORE_BOOT_BUDGET_MS = 60_000;+/*** How long a worker has to answer the liveness ping the host sends after a dispatch times out. A worker* stuck in a synchronous loop is terminated about 5 s after its first dispatch timeout (about 10 s after@@ -270,31 +277,77 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap* code executes. Built-ins are skipped: an engine is enabled by EngineFactory against the configured* engine.type, and enabling a non-active engine here would be rejected anyway.*- * Best-effort and sequential: a plugin that cannot come back is logged- * and left in ERROR, and never holds up the gateway.+ * Best-effort and sequential: a plugin that cannot come back is logged and left in ERROR. The HTTP+ * port opens only after this hook returns, and one plugin can take up to 90 s here (load, onLoad and+ * onEnable are each bounded at 30 s), so the wait is capped at PLUGIN_RESTORE_BOOT_BUDGET_MS. Plugins not+ * yet restored when it ends keep coming up, in order, in the background. The trade-off is deliberate:+ * until they are up, the hooks they own (a send moderation veto, for one) are not active, so a request+ * that arrives in that window is not seen by them. Without the cap, plugins whose backends are down after+ * an outage kept the whole gateway from listening and could fail its health checks into a restart loop.+ * A plugin the restore has not finished with also takes no lifecycle change from the operator meanwhile+ * (isRestorePending).*/async onApplicationBootstrap(): Promise<void> {+ const restoring = this.restorePlugins();+ let timer: NodeJS.Timeout | undefined;+ const settled = await Promise.race([+ restoring.then(() => true),+ new Promise<false>(resolve => {+ timer = setTimeout(resolve, PLUGIN_RESTORE_BOOT_BUDGET_MS, false);+ }),+ ]);+ clearTimeout(timer);+ if (!settled) {+ this.logger.warn(+ 'Plugin restore is still running; the gateway starts listening and the remaining plugins come up in the background',+ { waitMs: PLUGIN_RESTORE_BOOT_BUDGET_MS, action: 'plugin_restore_deferred' },+ );+ }+ }++ /** Plugins the boot restore has queued or is enabling, so a lifecycle change cannot interleave with it. */+ private readonly restorePending = new Set<string>();++ /**+ * Whether the boot restore still has this plugin ahead of it or in flight. When the restore outlasts the+ * boot wait it runs while requests are served, and it enables outside the per-plugin operation chain of+ * PluginsService: a disable would be answered "not enabled" and then overtaken by the enable, and an+ * uninstall would leave a live worker without a map entry. PluginsService refuses such a change until the+ * restore is done with the plugin, which is what the closed port did before the boot wait was capped.+ */+ isRestorePending(id: string): boolean {+ return this.restorePending.has(id);+ }++ private async restorePlugins(): Promise<void> {const restorable = this.getAllPlugins().filter(p => !p.builtIn && this.pluginStorage.getPluginEntry(p.manifest.id)?.enabledByOperator === true,);- for (let i = 0; i < restorable.length; i++) {- if (this.shuttingDown || this.shutdownService?.isShuttingDown()) {- this.logger.log(`Plugin restore stopped at ${i} of ${restorable.length} plugin(s): shutting down`, {- action: 'plugin_restore_aborted',- });- return;- }- const plugin = restorable[i];- const pluginId = plugin.manifest.id;- try {- await this.enablePlugin(pluginId);- } catch (error) {- this.logger.error(- `Failed to restore plugin ${pluginId} on startup; it stays disabled until re-enabled`,- error instanceof Error ? error.message : String(error),- { pluginId, action: 'plugin_restore_failed' },- );+ for (const plugin of restorable) this.restorePending.add(plugin.manifest.id);+ try {+ for (let i = 0; i < restorable.length; i++) {+ if (this.shuttingDown || this.shutdownService?.isShuttingDown()) {+ this.logger.log(`Plugin restore stopped at ${i} of ${restorable.length} plugin(s): shutting down`, {+ action: 'plugin_restore_aborted',+ });+ return;+ }+ const plugin = restorable[i];+ const pluginId = plugin.manifest.id;+ try {+ await this.enablePlugin(pluginId);+ } catch (error) {+ this.logger.error(+ `Failed to restore plugin ${pluginId} on startup; it stays disabled until re-enabled`,+ error instanceof Error ? error.message : String(error),+ { pluginId, action: 'plugin_restore_failed' },+ );+ } finally {+ this.restorePending.delete(pluginId);+ }}+ } finally {+ this.restorePending.clear();}}diff --git a/src/core/plugins/plugin-restore-on-boot.spec.ts b/src/core/plugins/plugin-restore-on-boot.spec.tsindex 5a7c76e7e..30b3ceaed 100644--- a/src/core/plugins/plugin-restore-on-boot.spec.ts+++ b/src/core/plugins/plugin-restore-on-boot.spec.ts@@ -3,7 +3,7 @@ import * as os from 'os';import * as path from 'path';import { ConfigService } from '@nestjs/config';import { ModuleRef } from '@nestjs/core';-import { PluginLoaderService } from './plugin-loader.service';+import { PLUGIN_RESTORE_BOOT_BUDGET_MS, PluginLoaderService } from './plugin-loader.service';import type { ShutdownService } from '../../common/services/shutdown.service';import { PluginStorageService } from './plugin-storage.service';import { HookManager } from '../hooks';@@ -212,6 +212,128 @@ describe('PluginLoaderService — restoring the operator enable decision acrossexpect(enable).toHaveBeenCalledTimes(1);});+ describe('a plugin that takes a long time to come back', () => {+ const second: PluginManifest = { ...manifest, id: 'ext-two', name: 'Ext Two' };+ let loader2: PluginLoaderService;+ let releaseFirst: () => void;+ let enable: jest.SpyInstance;++ beforeEach(() => {+ jest.useFakeTimers();+ const secondDir = path.join(tmpDir, 'plugins', second.id);+ fs.mkdirSync(secondDir, { recursive: true });+ fs.writeFileSync(path.join(secondDir, 'manifest.json'), JSON.stringify(second));+ fs.writeFileSync(path.join(secondDir, 'index.js'), 'module.exports = class {};');++ const storage2 = new PluginStorageService(config);+ loader2 = makeLoader(storage2);+ loader2.loadPlugin(pluginDir);+ loader2.loadPlugin(secondDir);+ storage2.setPluginEnabledByOperator(manifest.id, true);+ storage2.setPluginEnabledByOperator(second.id, true);+ // The first plugin's enable hangs (a backend that is down) until the test lets it go.+ enable = jest.spyOn(loader2, 'enablePlugin').mockImplementation((id: string) =>+ id === manifest.id+ ? new Promise<void>(resolve => {+ releaseFirst = resolve;+ })+ : Promise.resolve(),+ );+ });++ afterEach(() => {+ jest.useRealTimers();+ });++ it('does not hold the gateway back past the restore budget, and finishes restoring the rest afterwards', async () => {+ const warn = jest+ .spyOn((loader2 as unknown as { logger: { warn: (...args: unknown[]) => void } }).logger, 'warn')+ .mockImplementation(() => undefined);++ const booted = loader2.onApplicationBootstrap();+ await jest.advanceTimersByTimeAsync(PLUGIN_RESTORE_BOOT_BUDGET_MS);++ await expect(booted).resolves.toBeUndefined();+ expect(warn).toHaveBeenCalledWith(+ expect.stringContaining('remaining plugins come up in the background'),+ expect.objectContaining({ action: 'plugin_restore_deferred' }),+ );+ // The first plugin is still being enabled, so the second has not started: order is kept.+ expect(enable).toHaveBeenCalledTimes(1);++ releaseFirst();+ await jest.advanceTimersByTimeAsync(0);++ expect(enable).toHaveBeenCalledWith(second.id);+ });++ it('returns as soon as every plugin is back, without waiting out the budget', async () => {+ const booted = loader2.onApplicationBootstrap();+ releaseFirst();++ await expect(booted).resolves.toBeUndefined();+ expect(enable).toHaveBeenCalledTimes(2);+ // No timer is left running for the budget.+ expect(jest.getTimerCount()).toBe(0);+ });++ it('stops the background restore when the gateway shuts down', async () => {+ const booted = loader2.onApplicationBootstrap();+ await jest.advanceTimersByTimeAsync(PLUGIN_RESTORE_BOOT_BUDGET_MS);+ await booted;++ await loader2.onModuleDestroy();+ releaseFirst();+ await jest.advanceTimersByTimeAsync(0);++ expect(enable).toHaveBeenCalledTimes(1);+ });++ it('keeps a plugin pending from the start of the restore until its own enable settles', async () => {+ const booted = loader2.onApplicationBootstrap();+ await jest.advanceTimersByTimeAsync(PLUGIN_RESTORE_BOOT_BUDGET_MS);+ await booted;++ // The first is being enabled and the second is queued behind it: neither takes a lifecycle change yet.+ expect(loader2.isRestorePending(manifest.id)).toBe(true);+ expect(loader2.isRestorePending(second.id)).toBe(true);+ expect(loader2.isRestorePending('never-queued')).toBe(false);++ releaseFirst();+ await jest.advanceTimersByTimeAsync(0);++ expect(loader2.isRestorePending(manifest.id)).toBe(false);+ expect(loader2.isRestorePending(second.id)).toBe(false);+ });++ it('stops holding a plugin whose restore failed', async () => {+ enable.mockImplementation((id: string) =>+ id === manifest.id ? Promise.reject(new Error('backend down')) : Promise.resolve(),+ );+ jest+ .spyOn((loader2 as unknown as { logger: { error: (...args: unknown[]) => void } }).logger, 'error')+ .mockImplementation(() => undefined);++ await loader2.onApplicationBootstrap();++ expect(loader2.isRestorePending(manifest.id)).toBe(false);+ expect(loader2.isRestorePending(second.id)).toBe(false);+ });++ it('lets go of the plugins it never reached when the gateway shuts down', async () => {+ const booted = loader2.onApplicationBootstrap();+ await jest.advanceTimersByTimeAsync(PLUGIN_RESTORE_BOOT_BUDGET_MS);+ await booted;++ await loader2.onModuleDestroy();+ releaseFirst();+ await jest.advanceTimersByTimeAsync(0);++ expect(loader2.isRestorePending(manifest.id)).toBe(false);+ expect(loader2.isRestorePending(second.id)).toBe(false);+ });+ });+it('does not treat the shutdown teardown as the operator disabling the plugin', async () => {// The regression that makes this whole design fragile: onModuleDestroy disables every enabled// plugin on the way out. If teardown cleared the intent, one graceful restart would erase it anddiff --git a/src/modules/plugins/plugins.controller.ts b/src/modules/plugins/plugins.controller.tsindex 8032c7ee9..ef548ff9d 100644--- a/src/modules/plugins/plugins.controller.ts+++ b/src/modules/plugins/plugins.controller.ts@@ -31,6 +31,9 @@ import { ApiKeyRole } from '../auth/entities/api-key.entity';/** Max accepted upload size for a plugin package (compressed). */const MAX_PLUGIN_UPLOAD_BYTES = 5 * 1024 * 1024;+/** A lifecycle change to a plugin the boot restore has not finished with, which can outlast the boot wait. */+const RESTORING_CONFLICT = 'The plugin is still being restored after startup; retry shortly';+@ApiTags('plugins')@Controller('plugins')// Plugin installation and lifecycle are deployment-global and execute plugin code as the OpenWA@@ -81,7 +84,10 @@ export class PluginsController {@ApiOperation({ summary: 'Install a plugin by downloading its .zip from a URL (SSRF-guarded)' })@ApiResponse({ status: 201, description: 'Plugin installed', type: PluginDto })@ApiResponse({ status: 400, description: 'Invalid URL, download failed, or invalid package' })- @ApiResponse({ status: 409, description: 'Plugin already installed' })+ @ApiResponse({+ status: 409,+ description: 'Plugin already installed, or the plugin is still being restored after startup; retry shortly',+ })async installFromUrl(@Body() dto: InstallFromUrlDto): Promise<PluginDto> {return await this.pluginsService.installFromUrl(dto.url);}@@ -114,6 +120,7 @@ export class PluginsController {@ApiOperation({ summary: 'Enable a plugin' })@ApiResponse({ status: 200, description: 'Plugin enabled successfully', type: PluginActionResponseDto })@ApiResponse({ status: 404, description: 'Plugin not found' })+ @ApiResponse({ status: 409, description: RESTORING_CONFLICT })async enable(@Param('id') id: string): Promise<{ success: boolean; message: string }> {return await this.pluginsService.enable(id);}@@ -125,6 +132,7 @@ export class PluginsController {@ApiOperation({ summary: 'Disable a plugin' })@ApiResponse({ status: 200, description: 'Plugin disabled successfully', type: PluginActionResponseDto })@ApiResponse({ status: 404, description: 'Plugin not found' })+ @ApiResponse({ status: 409, description: RESTORING_CONFLICT })async disable(@Param('id') id: string): Promise<{ success: boolean; message: string }> {return await this.pluginsService.disable(id);}@@ -202,6 +210,7 @@ export class PluginsController {@ApiResponse({ status: 201, description: 'Plugin updated', type: PluginDto })@ApiResponse({ status: 400, description: 'Invalid URL/package, id mismatch, or built-in' })@ApiResponse({ status: 404, description: 'Plugin not found' })+ @ApiResponse({ status: 409, description: RESTORING_CONFLICT })@ApiResponse({status: 503,description: 'Gateway shutting down; the previous version was kept',@@ -217,6 +226,7 @@ export class PluginsController {@ApiResponse({ status: 200, description: 'Plugin uninstalled', type: PluginActionResponseDto })@ApiResponse({ status: 400, description: 'Cannot uninstall (e.g. built-in)' })@ApiResponse({ status: 404, description: 'Plugin not found' })+ @ApiResponse({ status: 409, description: RESTORING_CONFLICT })async uninstall(@Param('id') id: string): Promise<{ success: boolean; message: string }> {return await this.pluginsService.uninstall(id);}diff --git a/src/modules/plugins/plugins.service.spec.ts b/src/modules/plugins/plugins.service.spec.tsindex 2db3af63f..2bccfdcad 100644--- a/src/modules/plugins/plugins.service.spec.ts+++ b/src/modules/plugins/plugins.service.spec.ts@@ -800,6 +800,70 @@ describe('PluginsService — disable when the plugin is not loaded', () => {});});+describe('PluginsService: a plugin the boot restore has not finished with', () => {+ let tmpDir: string;+ let pluginsDir: string;+ let config: ConfigService;++ beforeEach(() => {+ tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'owa-pending-'));+ pluginsDir = path.join(tmpDir, 'plugins');+ fs.mkdirSync(pluginsDir, { recursive: true });+ config = {+ get: (k: string) => (k === 'plugins.dir' ? pluginsDir : k === 'dataDir' ? tmpDir : undefined),+ } as unknown as ConfigService;+ });+ afterEach(() => fs.rmSync(tmpDir, { recursive: true, force: true }));++ // The restore enables outside the per-plugin chain once it outlasts the boot wait. A disable answered+ // "not enabled" would then be overtaken by the enable, and an uninstall would leave a worker running+ // under no map entry, so both wait for the restore instead of racing it.+ it('refuses enable, disable, update and uninstall of it, and leaves every other plugin alone', async () => {+ const storage = new PluginStorageService(config);+ const loader = new PluginLoaderService(config, new HookManager(), storage, {} as unknown as ModuleRef);+ const service = new PluginsService(loader, config);+ service.install({ buffer: pkg() });+ service.install({ buffer: pkg({ id: 'other-plg' }) });+ loader.setOperatorEnabled('svc-plg', true);+ jest.spyOn(loader, 'isRestorePending').mockImplementation(id => id === 'svc-plg');++ for (const attempt of [+ () => service.enable('svc-plg'),+ () => service.disable('svc-plg'),+ () => service.updatePackage('svc-plg', pkg({ version: '2.0.0' })),+ () => service.uninstall('svc-plg'),+ ]) {+ await expect(attempt()).rejects.toThrow(ConflictException);+ await expect(attempt()).rejects.toThrow(/still being restored/);+ }++ // Nothing changed: the operator's decision stands and the plugin is still installed.+ expect(storage.getPluginEntry('svc-plg')?.enabledByOperator).toBe(true);+ expect(loader.getPlugin('svc-plg')?.manifest.version).toBe('1.0.0');+ expect(fs.existsSync(path.join(pluginsDir, 'svc-plg'))).toBe(true);++ await expect(service.disable('other-plg')).resolves.toMatchObject({ success: true });+ await expect(service.uninstall('other-plg')).resolves.toMatchObject({ success: true });+ });++ it('takes the same changes again once the restore is done with it', async () => {+ const loader = new PluginLoaderService(+ config,+ new HookManager(),+ new PluginStorageService(config),+ {} as unknown as ModuleRef,+ );+ const service = new PluginsService(loader, config);+ service.install({ buffer: pkg() });+ const pending = jest.spyOn(loader, 'isRestorePending').mockReturnValue(true);+ await expect(service.uninstall('svc-plg')).rejects.toThrow(ConflictException);++ pending.mockReturnValue(false);++ await expect(service.uninstall('svc-plg')).resolves.toMatchObject({ success: true });+ });+});+describe('PluginsService: disable an engine plugin', () => {let tmpDir: string;diff --git a/src/modules/plugins/plugins.service.ts b/src/modules/plugins/plugins.service.tsindex 43b249dea..c28f367db 100644--- a/src/modules/plugins/plugins.service.ts+++ b/src/modules/plugins/plugins.service.ts@@ -49,6 +49,13 @@ export class PluginsService {private readonly opChains = new Map<string, Promise<unknown>>();private serialize<T>(id: string, op: () => Promise<T>): Promise<T> {+ // A boot restore that outlasted the boot wait enables plugins in the background, outside this chain, so a+ // change to a plugin it has not finished with would interleave with its enable (see isRestorePending).+ if (this.pluginLoader.isRestorePending(id)) {+ return Promise.reject(+ new ConflictException(`Plugin ${id} is still being restored after startup. Retry in a moment.`),+ );+ }const prior = this.opChains.get(id) ?? Promise.resolve();const next = prior.catch(() => undefined).then(op);this.opChains.set(id, next);