Skip to content

The open-source app everyone uses to manage agents at work

fix(plugins): retry errored plugins at boot instead of leaving them dead (#12054)

Devin Foley627eef7cbde9
diff --git a/server/src/__tests__/plugin-loader-error-retry.test.ts b/server/src/__tests__/plugin-loader-error-retry.test.ts
new file mode 100644
index 0000000000..bcde5be425
--- /dev/null
+++ b/server/src/__tests__/plugin-loader-error-retry.test.ts
@@ -0,0 +1,151 @@
+/**
+ * loadAll boot retry for plugins stranded in error status.
+ *
+ * An activation failure marks a plugin `error`, and the loader used to skip
+ * those rows on every later boot — the row stayed dead until an operator
+ * flipped it back to `ready` by hand, even when the underlying cause (missing
+ * package dependencies, a stale build output) had long been fixed on disk.
+ * loadAll now queues errored plugins for one retry per boot: it flips each row
+ * to `ready` first (the error status cannot legally re-enter `error`, so a
+ * failed retry could not re-mark itself otherwise) and then activates it like
+ * any ready plugin. A retry that fails re-records the error through markError.
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { Db } from "@paperclipai/db";
+
+const mockRegistry = vi.hoisted(() => ({
+ getById: vi.fn(),
+ getByKey: vi.fn(),
+ list: vi.fn(),
+ listInstalled: vi.fn(),
+ listByStatus: vi.fn(),
+ update: vi.fn(),
+ updateStatus: vi.fn(),
+ upsertConfig: vi.fn(),
+ getConfig: vi.fn(),
+ delete: vi.fn(),
+}));
+
+vi.mock("../services/plugin-registry.js", () => ({
+ pluginRegistryService: () => mockRegistry,
+}));
+
+import { pluginLoader } from "../services/plugin-loader.js";
+import type { PluginRuntimeServices } from "../services/plugin-loader.js";
+
+function createPluginRecord(overrides: Record<string, unknown> = {}) {
+ return {
+ id: "plugin-err-1",
+ pluginKey: "example.broken-plugin",
+ packageName: "@example/broken-plugin",
+ packagePath: "/nonexistent/broken-plugin",
+ version: "1.0.0",
+ apiVersion: 1,
+ categories: [],
+ status: "error",
+ lastError: "Activation failed: previous boot failure",
+ installOrder: 1,
+ manifestJson: {
+ id: "example.broken-plugin",
+ apiVersion: 1,
+ version: "1.0.0",
+ displayName: "Broken Plugin",
+ description: "Fixture",
+ author: "Test",
+ categories: [],
+ capabilities: [],
+ entrypoints: { worker: "dist/worker.js" },
+ },
+ ...overrides,
+ };
+}
+
+function createRuntimeServices() {
+ return {
+ lifecycleManager: {
+ markError: vi.fn(async () => createPluginRecord()),
+ },
+ workerManager: {},
+ eventBus: {},
+ jobScheduler: {},
+ jobStore: {},
+ toolDispatcher: {},
+ buildHostHandlers: vi.fn(() => ({})),
+ instanceInfo: { hostVersion: "0.0.0-test" },
+ } as unknown as PluginRuntimeServices;
+}
+
+function createLoader(runtimeServices: PluginRuntimeServices) {
+ return pluginLoader(
+ {} as unknown as Db,
+ { localPluginDir: "/nonexistent/local-plugins", enableLocalFilesystem: false, enableNpmDiscovery: false },
+ runtimeServices,
+ );
+}
+
+describe("pluginLoader.loadAll error retry", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("flips an errored plugin to ready and retries its activation at boot", async () => {
+ const erroredPlugin = createPluginRecord();
+ mockRegistry.listByStatus.mockImplementation(async (status: string) => {
+ if (status === "error") return [erroredPlugin];
+ return [];
+ });
+ mockRegistry.updateStatus.mockResolvedValue({ ...erroredPlugin, status: "ready", lastError: null });
+
+ const runtimeServices = createRuntimeServices();
+ const loader = createLoader(runtimeServices);
+
+ const result = await loader.loadAll();
+
+ // The flip precedes activation, and clears the stale error.
+ expect(mockRegistry.updateStatus).toHaveBeenCalledExactlyOnceWith(erroredPlugin.id, { status: "ready" });
+ // The retried plugin joins the boot batch; its package is unresolvable, so
+ // the attempt fails and re-records a fresh error via markError.
+ expect(result.total).toBe(1);
+ expect(result.succeeded).toBe(0);
+ expect(result.failed).toBe(1);
+ expect(runtimeServices.lifecycleManager.markError).toHaveBeenCalledWith(
+ erroredPlugin.id,
+ expect.stringContaining("Activation failed"),
+ );
+ });
+
+ it("keeps loading ready plugins when the errored flip fails", async () => {
+ const readyPlugin = createPluginRecord({
+ id: "plugin-ready-1",
+ pluginKey: "example.ready-plugin",
+ status: "ready",
+ lastError: null,
+ });
+ const erroredPlugin = createPluginRecord();
+ mockRegistry.listByStatus.mockImplementation(async (status: string) => {
+ if (status === "ready") return [readyPlugin];
+ if (status === "error") return [erroredPlugin];
+ return [];
+ });
+ mockRegistry.updateStatus.mockRejectedValue(new Error("db write refused"));
+
+ const loader = createLoader(createRuntimeServices());
+
+ const result = await loader.loadAll();
+
+ // The failed flip skips the retry but never aborts the boot load.
+ expect(mockRegistry.updateStatus).toHaveBeenCalledExactlyOnceWith(erroredPlugin.id, { status: "ready" });
+ expect(result.total).toBe(1);
+ expect(result.results[0]?.plugin.id).toBe(readyPlugin.id);
+ });
+
+ it("returns the empty result when no plugin is ready or errored", async () => {
+ mockRegistry.listByStatus.mockResolvedValue([]);
+
+ const loader = createLoader(createRuntimeServices());
+
+ const result = await loader.loadAll();
+
+ expect(result).toEqual({ total: 0, succeeded: 0, failed: 0, results: [] });
+ });
+});
diff --git a/server/src/services/plugin-loader.ts b/server/src/services/plugin-loader.ts
index 7e24046e38..52d7843031 100644
--- a/server/src/services/plugin-loader.ts
+++ b/server/src/services/plugin-loader.ts
@@ -1945,25 +1945,61 @@ export function pluginLoader(
// Fetch all plugins in ready status, ordered by installOrder
const readyPlugins = (await registry.listByStatus("ready")) as PluginRecord[];
- if (readyPlugins.length === 0) {
+ // Retry plugins stranded in error status. An activation failure is often
+ // environmental — missing package dependencies, a stale build output, a
+ // module that moved under a pull — and the fix lands on disk without any
+ // write to the plugin row, so the row would otherwise stay dead until an
+ // operator flips it back by hand. One attempt per boot cannot crash-loop
+ // within a running process, and a failed attempt re-records the error
+ // through the normal markError path. The flip to ready must run BEFORE
+ // activation: the error status only legally transitions to ready or
+ // uninstalled, so a retry that failed while still in error status could
+ // not re-mark itself as errored.
+ const erroredPlugins = (await registry.listByStatus("error")) as PluginRecord[];
+ const retriedPlugins: PluginRecord[] = [];
+ for (const plugin of erroredPlugins) {
+ try {
+ const flipped = (await registry.updateStatus(plugin.id, { status: "ready" })) as PluginRecord | null;
+ if (flipped) retriedPlugins.push(flipped);
+ } catch (err) {
+ log.warn(
+ {
+ pluginId: plugin.id,
+ pluginKey: plugin.pluginKey,
+ err: err instanceof Error ? err.message : String(err),
+ },
+ "plugin-loader: could not queue errored plugin for a boot retry",
+ );
+ }
+ }
+ if (retriedPlugins.length > 0) {
+ log.info(
+ { count: retriedPlugins.length, pluginKeys: retriedPlugins.map((plugin) => plugin.pluginKey) },
+ "plugin-loader: retrying plugins that failed activation on a previous boot",
+ );
+ }
+
+ const pluginsToLoad = [...readyPlugins, ...retriedPlugins];
+
+ if (pluginsToLoad.length === 0) {
log.info("plugin-loader: no ready plugins to load");
return { total: 0, succeeded: 0, failed: 0, results: [] };
}
log.info(
- { count: readyPlugins.length },
+ { count: pluginsToLoad.length },
"plugin-loader: found ready plugins to load",
);
// Load plugins in parallel
const results = await Promise.allSettled(
- readyPlugins.map((plugin) => activatePlugin(plugin))
+ pluginsToLoad.map((plugin) => activatePlugin(plugin))
);
const loadResults = results.map((r, i) => {
if (r.status === "fulfilled") return r.value;
return {
- plugin: readyPlugins[i]!,
+ plugin: pluginsToLoad[i]!,
success: false,
error: String(r.reason),
registered: { worker: false, eventSubscriptions: 0, jobs: 0, webhooks: 0, tools: 0 },
@@ -1975,7 +2011,7 @@ export function pluginLoader(
log.info(
{
- total: readyPlugins.length,
+ total: pluginsToLoad.length,
succeeded,
failed,
},
@@ -1983,7 +2019,7 @@ export function pluginLoader(
);
return {
- total: readyPlugins.length,
+ total: pluginsToLoad.length,
succeeded,
failed,
results: loadResults,
@@ -2447,8 +2483,9 @@ export function pluginLoader(
);
if (options.markErrorOnFailure) {
- // Mark the plugin as errored in the database so it is not retried
- // automatically on next startup without operator intervention.
+ // Mark the plugin as errored in the database. The running process
+ // leaves it inactive; the next boot's loadAll retries it once, and the
+ // lifecycle enable() path can revive it sooner by hand.
// markError also deactivates the plugin runtime in this process.
try {
await lifecycleManager.markError(pluginId, `Activation failed: ${errorMessage}`);