Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(plugins): wait for in-flight plugin enables at shutdown (#1799)
Yudhi Armyndharisdc945e0da006
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 7da85bd39..bbdce24a6 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Disable plugins whose enable is still running at shutdown, and refuse new plugin enables during teardown.- Stop in-flight webhook replay, ingress replay and pending-message sweeps at shutdown.- Close queue workers before plugins shut down, so jobs taken during shutdown no longer spend attempts or get dead-lettered.- Answer new HTTP requests with `503` once shutdown teardown begins.diff --git a/docs/19-plugin-architecture.md b/docs/19-plugin-architecture.mdindex ed9e1ac73..47f06ad58 100644--- a/docs/19-plugin-architecture.md+++ b/docs/19-plugin-architecture.md@@ -579,7 +579,8 @@ double-enable, and engines must match the configured active engine):**Disable / unload / uninstall.** `disablePlugin` runs `onDisable` (force-terminating the worker for asandboxed plugin, even if `onDisable` hangs or throws) and unregisters the plugin's hooks. `onModuleDestroy`-disables every enabled plugin on graceful shutdown so stateful plugins can flush. `uninstallPlugin`+disables every enabled plugin on graceful shutdown so stateful plugins can flush; it first waits for+enables already in flight, and refuses new ones once teardown starts. `uninstallPlugin`disables + unloads, drops the registry entry, and deletes the plugin's directory and its `ctx.storage`data dir (built-ins are protected and cannot be uninstalled). The unload path dispatches `onUnload`:for a sandboxed plugin it runs in the worker between `onDisable` and terminate (a plain disable doesdiff --git a/src/core/plugins/plugin-loader.service.spec.ts b/src/core/plugins/plugin-loader.service.spec.tsindex 6d0f6c6da..c76e25dc3 100644--- a/src/core/plugins/plugin-loader.service.spec.ts+++ b/src/core/plugins/plugin-loader.service.spec.ts@@ -399,6 +399,33 @@ describe('PluginLoaderService — graceful shutdown (onModuleDestroy)', () => {expect(okDisable).toHaveBeenCalledTimes(1);expect(loader.getPlugin('ok-plg')?.status).toBe(PluginStatus.DISABLED);});++ it('waits for an enable already in flight and disables it, so its onDisable still runs', async () => {+ let release!: () => void;+ const gate = new Promise<void>(resolve => (release = resolve));+ const onDisable = jest.fn(() => Promise.resolve());+ loader.registerBuiltInPlugin(ext('slow-plg'), { onEnable: () => gate, onDisable });++ const enabling = loader.enablePlugin('slow-plg');+ const destroying = loader.onModuleDestroy();+ release();+ await enabling;+ await destroying;++ expect(onDisable).toHaveBeenCalledTimes(1);+ expect(loader.getPlugin('slow-plg')?.status).toBe(PluginStatus.DISABLED);+ });++ it('refuses an enable once teardown has begun', async () => {+ const onEnable = jest.fn(() => Promise.resolve());+ loader.registerBuiltInPlugin(ext('late-plg'), { onEnable });++ await loader.onModuleDestroy();++ await expect(loader.enablePlugin('late-plg')).rejects.toThrow(/shutting down/);+ expect(onEnable).not.toHaveBeenCalled();+ expect(loader.getPlugin('late-plg')?.status).not.toBe(PluginStatus.ENABLED);+ });});describe('PluginLoaderService — enable-failure hook cleanup', () => {diff --git a/src/core/plugins/plugin-loader.service.ts b/src/core/plugins/plugin-loader.service.tsindex aed549ba2..15c0ea838 100644--- a/src/core/plugins/plugin-loader.service.ts+++ b/src/core/plugins/plugin-loader.service.ts@@ -133,6 +133,10 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrapprivate readonly lifecycle: PluginLifecycle;/** Owns the removal job — uninstall of an installed user plugin. */private readonly uninstaller: PluginUninstaller;+ /** Set when shutdown teardown begins; from then on no enable may start. */+ private shuttingDown = false;+ /** Enables in flight, awaited by onModuleDestroy so one that lands during teardown is still disabled. */+ private readonly pendingEnables = new Set<Promise<void>>();constructor(private readonly configService: ConfigService,@@ -252,7 +256,14 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrapconst restorable = this.getAllPlugins().filter(p => !p.builtIn && this.pluginStorage.getPluginEntry(p.manifest.id)?.enabledByOperator === true,);- for (const plugin of restorable) {+ for (let i = 0; i < restorable.length; i++) {+ if (this.shuttingDown) {+ this.logger.log(`Plugin restore stopped at ${i} of ${restorable.length} plugin(s): shutting down`, {+ action: 'plugin_restore_aborted',+ });+ return;+ }+ const plugin = restorable[i];const pluginId = plugin.manifest.id;try {await this.enablePlugin(pluginId);@@ -271,8 +282,13 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap* buffers, close connections, and persist state. Previously onDisable only ran via the REST disable* and uninstall paths, so a normal restart/deploy/scale-down skipped it and stateful plugins lost* in-flight work. Best-effort and sequential: one plugin's failure must not block the others.+ *+ * Enables already in flight are awaited first, so a plugin that finishes enabling mid-teardown is in+ * the snapshot below instead of running on with its onDisable never called; later enables are refused.*/async onModuleDestroy(): Promise<void> {+ this.shuttingDown = true;+ await Promise.allSettled(this.pendingEnables);const enabled = this.getAllPlugins().filter(p => p.status === PluginStatus.ENABLED);for (const plugin of enabled) {try {@@ -312,8 +328,22 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrapreturn this.scanner.loadPlugin(pluginPath);}- enablePlugin(pluginId: string): Promise<void> {- return this.lifecycle.enablePlugin(pluginId);+ /** True once shutdown teardown has begun; enablePlugin refuses from then on. */+ isShuttingDown(): boolean {+ return this.shuttingDown;+ }++ async enablePlugin(pluginId: string): Promise<void> {+ if (this.shuttingDown) {+ throw new Error(`Plugin ${pluginId} cannot be enabled while the gateway is shutting down`);+ }+ const enabling = this.lifecycle.enablePlugin(pluginId);+ this.pendingEnables.add(enabling);+ try {+ await enabling;+ } finally {+ this.pendingEnables.delete(enabling);+ }}disablePlugin(pluginId: string, opts?: { unload?: boolean }): Promise<void> {diff --git a/src/core/plugins/plugin-restore-on-boot.spec.ts b/src/core/plugins/plugin-restore-on-boot.spec.tsindex 147a1ad43..ce6696b01 100644--- a/src/core/plugins/plugin-restore-on-boot.spec.ts+++ b/src/core/plugins/plugin-restore-on-boot.spec.ts@@ -142,6 +142,42 @@ describe('PluginLoaderService — restoring the operator enable decision acrossexpect(enable).toHaveBeenCalledWith(second.id);});+ it('stops restoring plugins once shutdown teardown begins', async () => {+ const second: PluginManifest = { ...manifest, id: 'ext-two', name: 'Ext Two' };+ const secondDir = path.join(tmpDir, 'plugins', second.id);+ fs.mkdirSync(secondDir, { recursive: true });+ fs.writeFileSync(path.join(secondDir, 'manifest.json'), JSON.stringify(second));+ fs.writeFileSync(path.join(secondDir, 'index.js'), 'module.exports = class {};');++ const storage2 = new PluginStorageService(config);+ const loader2 = makeLoader(storage2);+ loader2.loadPlugin(pluginDir);+ loader2.loadPlugin(secondDir);+ storage2.setPluginEnabledByOperator(manifest.id, true);+ storage2.setPluginEnabledByOperator(second.id, true);++ const log = jest+ .spyOn((loader2 as unknown as { logger: { log: (...args: unknown[]) => void } }).logger, 'log')+ .mockImplementation(() => undefined);+ // The gateway starts shutting down while the first plugin is being restored.+ let destroying: Promise<void> | undefined;+ const enable = jest.spyOn(loader2, 'enablePlugin').mockImplementation(() => {+ destroying = loader2.onModuleDestroy();+ return Promise.resolve();+ });++ await loader2.onApplicationBootstrap();+ await destroying;++ expect(enable).toHaveBeenCalledTimes(1);+ expect(enable).not.toHaveBeenCalledWith(second.id);+ // The skipped plugins are reported, so an operator can tell the restore was cut short.+ expect(log).toHaveBeenCalledWith(+ expect.stringContaining('stopped at 1 of 2'),+ expect.objectContaining({ action: 'plugin_restore_aborted' }),+ );+ });+it('does not treat the shutdown teardown as the operator disabling the plugin', async () => {// The regression that makes this whole design fragile: onModuleDestroy disables every enabled// plugin on the way out. If teardown cleared the intent, one graceful restart would erase it anddiff --git a/src/modules/plugins/plugins.service.spec.ts b/src/modules/plugins/plugins.service.spec.tsindex 357a5236d..99f808152 100644--- a/src/modules/plugins/plugins.service.spec.ts+++ b/src/modules/plugins/plugins.service.spec.ts@@ -165,6 +165,25 @@ describe('PluginsService — install / uninstall (real loader + disk)', () => {enableSpy.mockRestore();});+ it('keeps the new version when an update finishes during shutdown instead of rolling back', async () => {+ service.install({ buffer: pkg({ version: '1.0.0' }) });+ pluginStorage.setPluginEnabledByOperator('svc-plg', true);+ // Teardown has begun (nothing was enabled yet), then the update of a running plugin reaches its re-enable.+ await loader.onModuleDestroy();+ loader.getPlugin('svc-plg')!.status = PluginStatus.ENABLED;++ const dto = await service.updatePackage('svc-plg', pkg({ version: '2.0.0' }));++ expect(dto.version).toBe('2.0.0');+ const onDisk = JSON.parse(fs.readFileSync(path.join(pluginsDir, 'svc-plg', 'manifest.json'), 'utf8')) as {+ version: string;+ };+ expect(onDisk.version).toBe('2.0.0');+ expect(fs.existsSync(path.join(pluginsDir, '.svc-plg.bak'))).toBe(false);+ // The operator decision survives, so the next boot restores the new version.+ expect(pluginStorage.getPluginEntry('svc-plg')?.enabledByOperator).toBe(true);+ });+it('cleans up the staging + backup dirs after a successful update (swap happened)', async () => {service.install({ buffer: pkg({ version: '1.0.0' }) });diff --git a/src/modules/plugins/plugins.service.ts b/src/modules/plugins/plugins.service.tsindex 09ca50383..0953f884e 100644--- a/src/modules/plugins/plugins.service.ts+++ b/src/modules/plugins/plugins.service.ts@@ -598,7 +598,14 @@ export class PluginsService {fs.chmodSync(stateFile, 0o600);}this.pluginLoader.loadPlugin(dir);- if (wasEnabled) {+ if (wasEnabled && this.pluginLoader.isShuttingDown()) {+ // Shutdown refuses new enables. That is not a failed update: keep the new version and let the+ // next boot restore it from the persisted operator decision instead of rolling back.+ logger.warn(`Plugin ${id} updated during shutdown; it will be enabled on the next start`, {+ pluginId: id,+ action: 'plugin_update_enable_deferred',+ });+ } else if (wasEnabled) {await this.pluginLoader.enablePlugin(id);}fs.rmSync(backup, { recursive: true, force: true });