The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(plugins): retry errored plugins at boot instead of leaving them dead (#12054)
Devin Foley627eef7cbde9
diff --git a/server/src/__tests__/plugin-loader-error-retry.test.ts b/server/src/__tests__/plugin-loader-error-retry.test.tsnew file mode 100644index 0000000000..bcde5be425--- /dev/null+++ b/server/src/__tests__/plugin-loader-error-retry.test.ts@@ -0,0 +1,151 @@+/**+ * loadAll boot retry for plugins stranded in error status.+ *+ * An activation failure marks a plugin `error`, and the loader used to skip+ * those rows on every later boot — the row stayed dead until an operator+ * flipped it back to `ready` by hand, even when the underlying cause (missing+ * package dependencies, a stale build output) had long been fixed on disk.+ * loadAll now queues errored plugins for one retry per boot: it flips each row+ * to `ready` first (the error status cannot legally re-enter `error`, so a+ * failed retry could not re-mark itself otherwise) and then activates it like+ * any ready plugin. A retry that fails re-records the error through markError.+ */+import { beforeEach, describe, expect, it, vi } from "vitest";+import type { Db } from "@paperclipai/db";++const mockRegistry = vi.hoisted(() => ({+ getById: vi.fn(),+ getByKey: vi.fn(),+ list: vi.fn(),+ listInstalled: vi.fn(),+ listByStatus: vi.fn(),+ update: vi.fn(),+ updateStatus: vi.fn(),+ upsertConfig: vi.fn(),+ getConfig: vi.fn(),+ delete: vi.fn(),+}));++vi.mock("../services/plugin-registry.js", () => ({+ pluginRegistryService: () => mockRegistry,+}));++import { pluginLoader } from "../services/plugin-loader.js";+import type { PluginRuntimeServices } from "../services/plugin-loader.js";++function createPluginRecord(overrides: Record<string, unknown> = {}) {+ return {+ id: "plugin-err-1",+ pluginKey: "example.broken-plugin",+ packageName: "@example/broken-plugin",+ packagePath: "/nonexistent/broken-plugin",+ version: "1.0.0",+ apiVersion: 1,+ categories: [],+ status: "error",+ lastError: "Activation failed: previous boot failure",+ installOrder: 1,+ manifestJson: {+ id: "example.broken-plugin",+ apiVersion: 1,+ version: "1.0.0",+ displayName: "Broken Plugin",+ description: "Fixture",+ author: "Test",+ categories: [],+ capabilities: [],+ entrypoints: { worker: "dist/worker.js" },+ },+ ...overrides,+ };+}++function createRuntimeServices() {+ return {+ lifecycleManager: {+ markError: vi.fn(async () => createPluginRecord()),+ },+ workerManager: {},+ eventBus: {},+ jobScheduler: {},+ jobStore: {},+ toolDispatcher: {},+ buildHostHandlers: vi.fn(() => ({})),+ instanceInfo: { hostVersion: "0.0.0-test" },+ } as unknown as PluginRuntimeServices;+}++function createLoader(runtimeServices: PluginRuntimeServices) {+ return pluginLoader(+ {} as unknown as Db,+ { localPluginDir: "/nonexistent/local-plugins", enableLocalFilesystem: false, enableNpmDiscovery: false },+ runtimeServices,+ );+}++describe("pluginLoader.loadAll error retry", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("flips an errored plugin to ready and retries its activation at boot", async () => {+ const erroredPlugin = createPluginRecord();+ mockRegistry.listByStatus.mockImplementation(async (status: string) => {+ if (status === "error") return [erroredPlugin];+ return [];+ });+ mockRegistry.updateStatus.mockResolvedValue({ ...erroredPlugin, status: "ready", lastError: null });++ const runtimeServices = createRuntimeServices();+ const loader = createLoader(runtimeServices);++ const result = await loader.loadAll();++ // The flip precedes activation, and clears the stale error.+ expect(mockRegistry.updateStatus).toHaveBeenCalledExactlyOnceWith(erroredPlugin.id, { status: "ready" });+ // The retried plugin joins the boot batch; its package is unresolvable, so+ // the attempt fails and re-records a fresh error via markError.+ expect(result.total).toBe(1);+ expect(result.succeeded).toBe(0);+ expect(result.failed).toBe(1);+ expect(runtimeServices.lifecycleManager.markError).toHaveBeenCalledWith(+ erroredPlugin.id,+ expect.stringContaining("Activation failed"),+ );+ });++ it("keeps loading ready plugins when the errored flip fails", async () => {+ const readyPlugin = createPluginRecord({+ id: "plugin-ready-1",+ pluginKey: "example.ready-plugin",+ status: "ready",+ lastError: null,+ });+ const erroredPlugin = createPluginRecord();+ mockRegistry.listByStatus.mockImplementation(async (status: string) => {+ if (status === "ready") return [readyPlugin];+ if (status === "error") return [erroredPlugin];+ return [];+ });+ mockRegistry.updateStatus.mockRejectedValue(new Error("db write refused"));++ const loader = createLoader(createRuntimeServices());++ const result = await loader.loadAll();++ // The failed flip skips the retry but never aborts the boot load.+ expect(mockRegistry.updateStatus).toHaveBeenCalledExactlyOnceWith(erroredPlugin.id, { status: "ready" });+ expect(result.total).toBe(1);+ expect(result.results[0]?.plugin.id).toBe(readyPlugin.id);+ });++ it("returns the empty result when no plugin is ready or errored", async () => {+ mockRegistry.listByStatus.mockResolvedValue([]);++ const loader = createLoader(createRuntimeServices());++ const result = await loader.loadAll();++ expect(result).toEqual({ total: 0, succeeded: 0, failed: 0, results: [] });+ });+});diff --git a/server/src/services/plugin-loader.ts b/server/src/services/plugin-loader.tsindex 7e24046e38..52d7843031 100644--- a/server/src/services/plugin-loader.ts+++ b/server/src/services/plugin-loader.ts@@ -1945,25 +1945,61 @@ export function pluginLoader(// Fetch all plugins in ready status, ordered by installOrderconst readyPlugins = (await registry.listByStatus("ready")) as PluginRecord[];- if (readyPlugins.length === 0) {+ // Retry plugins stranded in error status. An activation failure is often+ // environmental — missing package dependencies, a stale build output, a+ // module that moved under a pull — and the fix lands on disk without any+ // write to the plugin row, so the row would otherwise stay dead until an+ // operator flips it back by hand. One attempt per boot cannot crash-loop+ // within a running process, and a failed attempt re-records the error+ // through the normal markError path. The flip to ready must run BEFORE+ // activation: the error status only legally transitions to ready or+ // uninstalled, so a retry that failed while still in error status could+ // not re-mark itself as errored.+ const erroredPlugins = (await registry.listByStatus("error")) as PluginRecord[];+ const retriedPlugins: PluginRecord[] = [];+ for (const plugin of erroredPlugins) {+ try {+ const flipped = (await registry.updateStatus(plugin.id, { status: "ready" })) as PluginRecord | null;+ if (flipped) retriedPlugins.push(flipped);+ } catch (err) {+ log.warn(+ {+ pluginId: plugin.id,+ pluginKey: plugin.pluginKey,+ err: err instanceof Error ? err.message : String(err),+ },+ "plugin-loader: could not queue errored plugin for a boot retry",+ );+ }+ }+ if (retriedPlugins.length > 0) {+ log.info(+ { count: retriedPlugins.length, pluginKeys: retriedPlugins.map((plugin) => plugin.pluginKey) },+ "plugin-loader: retrying plugins that failed activation on a previous boot",+ );+ }++ const pluginsToLoad = [...readyPlugins, ...retriedPlugins];++ if (pluginsToLoad.length === 0) {log.info("plugin-loader: no ready plugins to load");return { total: 0, succeeded: 0, failed: 0, results: [] };}log.info(- { count: readyPlugins.length },+ { count: pluginsToLoad.length },"plugin-loader: found ready plugins to load",);// Load plugins in parallelconst results = await Promise.allSettled(- readyPlugins.map((plugin) => activatePlugin(plugin))+ pluginsToLoad.map((plugin) => activatePlugin(plugin)));const loadResults = results.map((r, i) => {if (r.status === "fulfilled") return r.value;return {- plugin: readyPlugins[i]!,+ plugin: pluginsToLoad[i]!,success: false,error: String(r.reason),registered: { worker: false, eventSubscriptions: 0, jobs: 0, webhooks: 0, tools: 0 },@@ -1975,7 +2011,7 @@ export function pluginLoader(log.info({- total: readyPlugins.length,+ total: pluginsToLoad.length,succeeded,failed,},@@ -1983,7 +2019,7 @@ export function pluginLoader();return {- total: readyPlugins.length,+ total: pluginsToLoad.length,succeeded,failed,results: loadResults,@@ -2447,8 +2483,9 @@ export function pluginLoader();if (options.markErrorOnFailure) {- // Mark the plugin as errored in the database so it is not retried- // automatically on next startup without operator intervention.+ // Mark the plugin as errored in the database. The running process+ // leaves it inactive; the next boot's loadAll retries it once, and the+ // lifecycle enable() path can revive it sooner by hand.// markError also deactivates the plugin runtime in this process.try {await lifecycleManager.markError(pluginId, `Activation failed: ${errorMessage}`);