Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(plugins): wait for in-flight plugin enables at shutdown (#1799)

Yudhi Armyndharisdc945e0da006
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7da85bd39..bbdce24a6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Disable plugins whose enable is still running at shutdown, and refuse new plugin enables during teardown.
- Stop in-flight webhook replay, ingress replay and pending-message sweeps at shutdown.
- Close queue workers before plugins shut down, so jobs taken during shutdown no longer spend attempts or get dead-lettered.
- Answer new HTTP requests with `503` once shutdown teardown begins.
diff --git a/docs/19-plugin-architecture.md b/docs/19-plugin-architecture.md
index ed9e1ac73..47f06ad58 100644
--- a/docs/19-plugin-architecture.md
+++ b/docs/19-plugin-architecture.md
@@ -579,7 +579,8 @@ double-enable, and engines must match the configured active engine):
**Disable / unload / uninstall.** `disablePlugin` runs `onDisable` (force-terminating the worker for a
sandboxed plugin, even if `onDisable` hangs or throws) and unregisters the plugin's hooks. `onModuleDestroy`
-disables every enabled plugin on graceful shutdown so stateful plugins can flush. `uninstallPlugin`
+disables every enabled plugin on graceful shutdown so stateful plugins can flush; it first waits for
+enables already in flight, and refuses new ones once teardown starts. `uninstallPlugin`
disables + unloads, drops the registry entry, and deletes the plugin's directory and its `ctx.storage`
data dir (built-ins are protected and cannot be uninstalled). The unload path dispatches `onUnload`:
for a sandboxed plugin it runs in the worker between `onDisable` and terminate (a plain disable does
diff --git a/src/core/plugins/plugin-loader.service.spec.ts b/src/core/plugins/plugin-loader.service.spec.ts
index 6d0f6c6da..c76e25dc3 100644
--- a/src/core/plugins/plugin-loader.service.spec.ts
+++ b/src/core/plugins/plugin-loader.service.spec.ts
@@ -399,6 +399,33 @@ describe('PluginLoaderService — graceful shutdown (onModuleDestroy)', () => {
expect(okDisable).toHaveBeenCalledTimes(1);
expect(loader.getPlugin('ok-plg')?.status).toBe(PluginStatus.DISABLED);
});
+
+ it('waits for an enable already in flight and disables it, so its onDisable still runs', async () => {
+ let release!: () => void;
+ const gate = new Promise<void>(resolve => (release = resolve));
+ const onDisable = jest.fn(() => Promise.resolve());
+ loader.registerBuiltInPlugin(ext('slow-plg'), { onEnable: () => gate, onDisable });
+
+ const enabling = loader.enablePlugin('slow-plg');
+ const destroying = loader.onModuleDestroy();
+ release();
+ await enabling;
+ await destroying;
+
+ expect(onDisable).toHaveBeenCalledTimes(1);
+ expect(loader.getPlugin('slow-plg')?.status).toBe(PluginStatus.DISABLED);
+ });
+
+ it('refuses an enable once teardown has begun', async () => {
+ const onEnable = jest.fn(() => Promise.resolve());
+ loader.registerBuiltInPlugin(ext('late-plg'), { onEnable });
+
+ await loader.onModuleDestroy();
+
+ await expect(loader.enablePlugin('late-plg')).rejects.toThrow(/shutting down/);
+ expect(onEnable).not.toHaveBeenCalled();
+ expect(loader.getPlugin('late-plg')?.status).not.toBe(PluginStatus.ENABLED);
+ });
});
describe('PluginLoaderService — enable-failure hook cleanup', () => {
diff --git a/src/core/plugins/plugin-loader.service.ts b/src/core/plugins/plugin-loader.service.ts
index aed549ba2..15c0ea838 100644
--- a/src/core/plugins/plugin-loader.service.ts
+++ b/src/core/plugins/plugin-loader.service.ts
@@ -133,6 +133,10 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap
private readonly lifecycle: PluginLifecycle;
/** Owns the removal job — uninstall of an installed user plugin. */
private readonly uninstaller: PluginUninstaller;
+ /** Set when shutdown teardown begins; from then on no enable may start. */
+ private shuttingDown = false;
+ /** Enables in flight, awaited by onModuleDestroy so one that lands during teardown is still disabled. */
+ private readonly pendingEnables = new Set<Promise<void>>();
constructor(
private readonly configService: ConfigService,
@@ -252,7 +256,14 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap
const restorable = this.getAllPlugins().filter(
p => !p.builtIn && this.pluginStorage.getPluginEntry(p.manifest.id)?.enabledByOperator === true,
);
- for (const plugin of restorable) {
+ for (let i = 0; i < restorable.length; i++) {
+ if (this.shuttingDown) {
+ this.logger.log(`Plugin restore stopped at ${i} of ${restorable.length} plugin(s): shutting down`, {
+ action: 'plugin_restore_aborted',
+ });
+ return;
+ }
+ const plugin = restorable[i];
const pluginId = plugin.manifest.id;
try {
await this.enablePlugin(pluginId);
@@ -271,8 +282,13 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap
* buffers, close connections, and persist state. Previously onDisable only ran via the REST disable
* and uninstall paths, so a normal restart/deploy/scale-down skipped it and stateful plugins lost
* in-flight work. Best-effort and sequential: one plugin's failure must not block the others.
+ *
+ * Enables already in flight are awaited first, so a plugin that finishes enabling mid-teardown is in
+ * the snapshot below instead of running on with its onDisable never called; later enables are refused.
*/
async onModuleDestroy(): Promise<void> {
+ this.shuttingDown = true;
+ await Promise.allSettled(this.pendingEnables);
const enabled = this.getAllPlugins().filter(p => p.status === PluginStatus.ENABLED);
for (const plugin of enabled) {
try {
@@ -312,8 +328,22 @@ export class PluginLoaderService implements OnModuleInit, OnApplicationBootstrap
return this.scanner.loadPlugin(pluginPath);
}
- enablePlugin(pluginId: string): Promise<void> {
- return this.lifecycle.enablePlugin(pluginId);
+ /** True once shutdown teardown has begun; enablePlugin refuses from then on. */
+ isShuttingDown(): boolean {
+ return this.shuttingDown;
+ }
+
+ async enablePlugin(pluginId: string): Promise<void> {
+ if (this.shuttingDown) {
+ throw new Error(`Plugin ${pluginId} cannot be enabled while the gateway is shutting down`);
+ }
+ const enabling = this.lifecycle.enablePlugin(pluginId);
+ this.pendingEnables.add(enabling);
+ try {
+ await enabling;
+ } finally {
+ this.pendingEnables.delete(enabling);
+ }
}
disablePlugin(pluginId: string, opts?: { unload?: boolean }): Promise<void> {
diff --git a/src/core/plugins/plugin-restore-on-boot.spec.ts b/src/core/plugins/plugin-restore-on-boot.spec.ts
index 147a1ad43..ce6696b01 100644
--- a/src/core/plugins/plugin-restore-on-boot.spec.ts
+++ b/src/core/plugins/plugin-restore-on-boot.spec.ts
@@ -142,6 +142,42 @@ describe('PluginLoaderService — restoring the operator enable decision across
expect(enable).toHaveBeenCalledWith(second.id);
});
+ it('stops restoring plugins once shutdown teardown begins', async () => {
+ const second: PluginManifest = { ...manifest, id: 'ext-two', name: 'Ext Two' };
+ const secondDir = path.join(tmpDir, 'plugins', second.id);
+ fs.mkdirSync(secondDir, { recursive: true });
+ fs.writeFileSync(path.join(secondDir, 'manifest.json'), JSON.stringify(second));
+ fs.writeFileSync(path.join(secondDir, 'index.js'), 'module.exports = class {};');
+
+ const storage2 = new PluginStorageService(config);
+ const loader2 = makeLoader(storage2);
+ loader2.loadPlugin(pluginDir);
+ loader2.loadPlugin(secondDir);
+ storage2.setPluginEnabledByOperator(manifest.id, true);
+ storage2.setPluginEnabledByOperator(second.id, true);
+
+ const log = jest
+ .spyOn((loader2 as unknown as { logger: { log: (...args: unknown[]) => void } }).logger, 'log')
+ .mockImplementation(() => undefined);
+ // The gateway starts shutting down while the first plugin is being restored.
+ let destroying: Promise<void> | undefined;
+ const enable = jest.spyOn(loader2, 'enablePlugin').mockImplementation(() => {
+ destroying = loader2.onModuleDestroy();
+ return Promise.resolve();
+ });
+
+ await loader2.onApplicationBootstrap();
+ await destroying;
+
+ expect(enable).toHaveBeenCalledTimes(1);
+ expect(enable).not.toHaveBeenCalledWith(second.id);
+ // The skipped plugins are reported, so an operator can tell the restore was cut short.
+ expect(log).toHaveBeenCalledWith(
+ expect.stringContaining('stopped at 1 of 2'),
+ expect.objectContaining({ action: 'plugin_restore_aborted' }),
+ );
+ });
+
it('does not treat the shutdown teardown as the operator disabling the plugin', async () => {
// The regression that makes this whole design fragile: onModuleDestroy disables every enabled
// plugin on the way out. If teardown cleared the intent, one graceful restart would erase it and
diff --git a/src/modules/plugins/plugins.service.spec.ts b/src/modules/plugins/plugins.service.spec.ts
index 357a5236d..99f808152 100644
--- a/src/modules/plugins/plugins.service.spec.ts
+++ b/src/modules/plugins/plugins.service.spec.ts
@@ -165,6 +165,25 @@ describe('PluginsService — install / uninstall (real loader + disk)', () => {
enableSpy.mockRestore();
});
+ it('keeps the new version when an update finishes during shutdown instead of rolling back', async () => {
+ service.install({ buffer: pkg({ version: '1.0.0' }) });
+ pluginStorage.setPluginEnabledByOperator('svc-plg', true);
+ // Teardown has begun (nothing was enabled yet), then the update of a running plugin reaches its re-enable.
+ await loader.onModuleDestroy();
+ loader.getPlugin('svc-plg')!.status = PluginStatus.ENABLED;
+
+ const dto = await service.updatePackage('svc-plg', pkg({ version: '2.0.0' }));
+
+ expect(dto.version).toBe('2.0.0');
+ const onDisk = JSON.parse(fs.readFileSync(path.join(pluginsDir, 'svc-plg', 'manifest.json'), 'utf8')) as {
+ version: string;
+ };
+ expect(onDisk.version).toBe('2.0.0');
+ expect(fs.existsSync(path.join(pluginsDir, '.svc-plg.bak'))).toBe(false);
+ // The operator decision survives, so the next boot restores the new version.
+ expect(pluginStorage.getPluginEntry('svc-plg')?.enabledByOperator).toBe(true);
+ });
+
it('cleans up the staging + backup dirs after a successful update (swap happened)', async () => {
service.install({ buffer: pkg({ version: '1.0.0' }) });
diff --git a/src/modules/plugins/plugins.service.ts b/src/modules/plugins/plugins.service.ts
index 09ca50383..0953f884e 100644
--- a/src/modules/plugins/plugins.service.ts
+++ b/src/modules/plugins/plugins.service.ts
@@ -598,7 +598,14 @@ export class PluginsService {
fs.chmodSync(stateFile, 0o600);
}
this.pluginLoader.loadPlugin(dir);
- if (wasEnabled) {
+ if (wasEnabled && this.pluginLoader.isShuttingDown()) {
+ // Shutdown refuses new enables. That is not a failed update: keep the new version and let the
+ // next boot restore it from the persisted operator decision instead of rolling back.
+ logger.warn(`Plugin ${id} updated during shutdown; it will be enabled on the next start`, {
+ pluginId: id,
+ action: 'plugin_update_enable_deferred',
+ });
+ } else if (wasEnabled) {
await this.pluginLoader.enablePlugin(id);
}
fs.rmSync(backup, { recursive: true, force: true });