The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
Separate plugin workspace completion from maintenance inspection (#17329)
Nicky Leacha708a133363a
diff --git a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md b/doc/plugins/PLUGIN_RUNNER_EXECUTION.mdindex 46dafc05e9..8b53736907 100644--- a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md+++ b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md@@ -139,3 +139,23 @@ after it saves the agent. Core does not require a computer before this save.The Runner fixture uses a synthetic Codex provider and local sockets.It does not qualify a deployed plugin, live account, or production WSS proxy.++## Workspace completion and maintenance++A storage provider can declare `workspaceProvider.finalizationVersion: 1`.+Core then sends `workspaceOperation` with `operation.kind: "finalize"`, the retained+binding, lifecycle generation, and originating `runId` and `agentId`. The operation+must verify workspace identity and make the run's retained writes durable before+returning `{ kind: "finalized", headSha, branchName }`. Git workspaces must still+report the assigned branch. Non-Git workspaces return null Git fields.++Completion permits dirty, untracked, ignored, and unpublished work. It does not+prove remote recoverability or authorize deletion. The provider chooses how to+meet this contract; Core does not prescribe a VM, filesystem, or Git layout.+Core gives completion a 30-second deadline. An unavailable execution host or an+unconfirmed durability barrier must fail retryably, never return success.+Recovery receives the original run and agent context.++The existing agent-independent `inspect` and `remove` operations continue to+protect storage maintenance. Providers that do not declare finalization version 1+retain the inspection-based completion path. No stored binding migration is required.diff --git a/packages/plugins/sdk/src/workspace-provider.ts b/packages/plugins/sdk/src/workspace-provider.tsindex 456411cbf9..bd28c5610f 100644--- a/packages/plugins/sdk/src/workspace-provider.ts+++ b/packages/plugins/sdk/src/workspace-provider.ts@@ -44,6 +44,9 @@ export const workspaceProviderOperationSchema = z.discriminatedUnion("kind", [fresh: freshWorkspaceSchema.optional(),}).strict(),+ // Verify the retained workspace and flush run writes. This does not certify+ // remote recoverability or authorize removal; dirty work is valid output.+ z.object({ kind: z.literal("finalize"), binding: workspaceProviderBindingSchema }).strict(),z.object({kind: z.literal("inspect"),@@ -81,6 +84,8 @@ export const workspaceProviderResultSchema = z.discriminatedUnion("kind", [remotelyRecoverable: z.boolean().nullable(),}).strict(),+ // Return only after retained files are durable and their identity is verified.+ z.object({ kind: z.literal("finalized"), headSha: sha.nullable(), branchName: z.string().nullable() }).strict(),// Return only after removal is confirmed. Busy, failed and uncertain operations must throw.z.object({ kind: z.literal("removed") }).strict(),]);@@ -105,7 +110,7 @@ export const pluginWorkspaceParamsSchema = zoperation: workspaceProviderOperationSchema,}).strict();-/** Run and agent are optional context. Storage operations never require an execution lease. */+/** Finalization receives the originating run/agent context; maintenance remains agent-independent. */export type PluginWorkspaceParams = z.infer<typeof pluginWorkspaceParamsSchema>;export function parseWorkspaceProviderResult(@@ -118,7 +123,7 @@ export function parseWorkspaceProviderResult(? "prepared": params.operation.kind === "inspect"? "inspection"- : "removed";+ : params.operation.kind === "finalize" ? "finalized" : "removed";if (result.kind !== expected)throw new Error("Workspace provider returned the wrong operation result");if (diff --git a/packages/plugins/sdk/tests/workspace-provider.test.ts b/packages/plugins/sdk/tests/workspace-provider.test.tsindex 5f79260f6d..b21f732085 100644--- a/packages/plugins/sdk/tests/workspace-provider.test.ts+++ b/packages/plugins/sdk/tests/workspace-provider.test.ts@@ -50,6 +50,16 @@ const result = {branchCreatedByRuntime: true,};describe("workspace provider SDK", () => {+ it("separates completion acknowledgement from removal safety", () => {+ const input: PluginWorkspaceParams = { ...params, agentId: id(6), runId: id(7),+ operation: { kind: "finalize", binding: result.binding } };+ expect(workspaceProviderOperationSchema.parse(input.operation)).toEqual(input.operation);+ expect(parseWorkspaceProviderResult(input, { kind: "finalized", headSha: "a".repeat(40), branchName: "task" }))+ .toEqual({ kind: "finalized", headSha: "a".repeat(40), branchName: "task" });+ expect(() => parseWorkspaceProviderResult(input, { kind: "removed" })).toThrow();+ expect(() => parseWorkspaceProviderResult(input, { kind: "finalized", headSha: "bad", branchName: "task" })).toThrow();+ expect(() => workspaceProviderOperationSchema.parse({ ...input.operation, localPath: "/untrusted" })).toThrow();+ });it("prepares without a VM, lease, or host path", async () => {const harness = createWorkspaceProviderHarness(async (input) => {expect(input.agentId).toBeNull();diff --git a/packages/shared/src/types/plugin.ts b/packages/shared/src/types/plugin.tsindex 89b70b6519..acfebb4395 100644--- a/packages/shared/src/types/plugin.ts+++ b/packages/shared/src/types/plugin.ts@@ -697,6 +697,8 @@ export interface PaperclipPluginManifestV1 {agentLifecycle?: true;/** Own project storage and durable workspace operations as one contract. */workspaceProvider?: {+ /** Supports bounded run completion separately from removal-safety inspection. */+ finalizationVersion?: 1;freshWorkspaceVersion?: 1;modes: Array<"shared_workspace" | "isolated_workspace" | "operator_branch">;strategies: Array<"project_primary" | "git_worktree">;diff --git a/packages/shared/src/validators/plugin.ts b/packages/shared/src/validators/plugin.tsindex 70a688978d..16b8092f0c 100644--- a/packages/shared/src/validators/plugin.ts+++ b/packages/shared/src/validators/plugin.ts@@ -792,6 +792,7 @@ export const pluginManifestV1Schema = z.object({capabilities: z.array(z.enum(PLUGIN_CAPABILITIES)).min(1),agentLifecycle: z.literal(true).optional(),workspaceProvider: z.object({+ finalizationVersion: z.literal(1).optional(),freshWorkspaceVersion: z.literal(1).optional(),modes: z.array(z.enum(["shared_workspace", "isolated_workspace", "operator_branch"])).min(1).max(3),strategies: z.array(z.enum(["project_primary", "git_worktree"])).min(1).max(2),diff --git a/server/src/__tests__/plugin-workspaces.integration.test.ts b/server/src/__tests__/plugin-workspaces.integration.test.tsindex 7f3952b5f8..0d6a92f3c9 100644--- a/server/src/__tests__/plugin-workspaces.integration.test.ts+++ b/server/src/__tests__/plugin-workspaces.integration.test.ts@@ -14,6 +14,7 @@ import {import {preparePluginWorkspace,inspectPluginWorkspace,+ finalizePluginWorkspace,removePluginWorkspace,type WorkspaceWorkers,} from "../services/plugin-workspaces.js";@@ -42,6 +43,34 @@ const support = await getEmbeddedPostgresTestSupport();});const fixture = (companyId?: string) =>createPluginWorkspaceFixture(db, companyId);+ it("uses bounded finalization only for opted-in providers, preserving maintenance inspection", async () => {+ const f = await fixture();+ const workspace = (await preparePluginWorkspace(db, f.workers, f.input))!;+ const context = { runId: f.runId, agentId: f.input.agentId };+ expect((await finalizePluginWorkspace(db, f.workers, workspace, context)).kind).toBe("inspection");+ const [plugin] = await db.select().from(plugins).where(eq(plugins.id, f.pluginId));+ await db.update(plugins).set({ manifestJson: { ...plugin.manifestJson,+ workspaceProvider: { ...plugin.manifestJson.workspaceProvider!, finalizationVersion: 1 } } }).where(eq(plugins.id, f.pluginId));+ const completionIds: string[] = [];+ const call = f.workers.call.bind(f.workers);+ f.workers.call = async (...args) => {+ const p = args[2] as any;+ if (p.operation.kind !== "finalize") return call(...args);+ expect(p.agentId).toBe(context.agentId);+ completionIds.push(p.operationId);+ expect(Date.parse(p.deadline) - Date.now()).toBeLessThanOrEqual(30_000);+ expect(args[3]).toBe(40_000);+ return { kind: "finalized", branchName: workspace.branchName, headSha: "a".repeat(40) };+ };+ expect((await finalizePluginWorkspace(db, f.workers, workspace, context)).kind).toBe("finalized");+ await finalizePluginWorkspace(db, f.workers, workspace, context);+ await finalizePluginWorkspace(db, f.workers, workspace, { ...context, runId: randomUUID() });+ expect(completionIds[1]).toBe(completionIds[0]);+ expect(completionIds[2]).not.toBe(completionIds[0]);+ expect((await inspectPluginWorkspace(db, f.workers, workspace)).kind).toBe("inspection");+ f.workers.call = async () => { throw new Error("sync failed"); };+ await expect(finalizePluginWorkspace(db, f.workers, workspace, context)).rejects.toThrow("incomplete");+ });it("keeps the single-connection pool responsive while provider preparation holds its lifecycle lock", async () => {const f = await fixture();let entered!: () => void, release!: () => void;diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.tsindex 2b4e91c57c..c080ba515f 100644--- a/server/src/services/heartbeat/runtime-execution.test.ts+++ b/server/src/services/heartbeat/runtime-execution.test.ts@@ -209,7 +209,7 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () =name: "Plugin worktree", providerType: "plugin", providerRef: "provider:opaque-worktree", cwd: "/workspaces/plugin-only",mode: "isolated_workspace", strategyType: "git_worktree", branchName: "task-branch" }).returning();input.workspace.persistedExecutionWorkspace = workspace;- const inspect = vi.spyOn(pluginWorkspaces, "inspectPluginWorkspace").mockResolvedValue({ kind: "inspection",+ const inspect = vi.spyOn(pluginWorkspaces, "finalizePluginWorkspace").mockResolvedValue({ kind: "inspection",branchName: branch === "expected" ? "task-branch" : "other-branch", headSha: "a".repeat(40),hasDirtyTrackedFiles: true, hasUntrackedFiles: false, hasLocalOnlyData: true, isMergedIntoBase: false, remotelyRecoverable: false });try {@@ -221,7 +221,7 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () =await expect(result).rejects.toThrow("plugin_workspace_branch_mismatch");expect((await finalizations()).some(row => row.status === "succeeded")).toBe(false);}- expect(inspect).toHaveBeenCalledWith(db, undefined, expect.objectContaining({ id: workspace.id, providerRef: workspace.providerRef }));+ expect(inspect).toHaveBeenCalledWith(db, undefined, expect.objectContaining({ id: workspace.id, providerRef: workspace.providerRef }), { agentId: input.agent.id, runId: input.run.id });} finally { inspect.mockRestore(); }});diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.tsindex 039dbe6934..c080a24cae 100644--- a/server/src/services/heartbeat/runtime-execution.ts+++ b/server/src/services/heartbeat/runtime-execution.ts@@ -284,7 +284,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe): persistedExecutionWorkspace;pluginWorkspaceFinalization = workspaceRecord- ? await inspectPluginWorkspaceForFinalization(db, options.pluginWorkerManager, workspaceRecord, agent.id)+ ? await inspectPluginWorkspaceForFinalization(db, options.pluginWorkerManager, workspaceRecord, agent.id, run.id): null;if (pluginWorkspaceFinalization || workspaceRecord?.strategyType !== "git_worktree") return null;diff --git a/server/src/services/native-runtime/native-plugin-workspace-finalization.test.ts b/server/src/services/native-runtime/native-plugin-workspace-finalization.test.tsnew file mode 100644index 0000000000..f8c461f674--- /dev/null+++ b/server/src/services/native-runtime/native-plugin-workspace-finalization.test.ts@@ -0,0 +1,20 @@+import { describe, it, expect, vi } from "vitest";+import { inspectPluginWorkspaceForFinalization } from "./native-plugin-workspace-finalization.js";+import { finalizePluginWorkspace } from "../plugin-workspaces.js";+vi.mock("../plugin-workspaces.js", () => ({ finalizePluginWorkspace: vi.fn() }));+const db = {} as Parameters<typeof inspectPluginWorkspaceForFinalization>[0];+const workspace = { id: "workspace", providerType: "plugin", strategyType: "git_worktree", branchName: "task" } as Parameters<typeof inspectPluginWorkspaceForFinalization>[2];+describe("provider run completion", () => {+ it("passes original run ownership and accepts retained unpublished work", async () => {+ vi.mocked(finalizePluginWorkspace).mockResolvedValue({kind:"finalized",headSha:"a".repeat(40),branchName:"task"});+ expect(await inspectPluginWorkspaceForFinalization(db,undefined,workspace,"agent","run"))+ .toMatchObject({providerType:"plugin",inspection:{kind:"finalized"}});+ expect(finalizePluginWorkspace).toHaveBeenLastCalledWith(db,undefined,workspace,{agentId:"agent",runId:"run"});+ });+ it("fails closed on changed branch and durability failure", async () => {+ vi.mocked(finalizePluginWorkspace).mockResolvedValue({kind:"finalized",headSha:"a".repeat(40),branchName:"wrong"});+ await expect(inspectPluginWorkspaceForFinalization(db,undefined,workspace,"agent","run")).rejects.toThrow("branch_mismatch");+ vi.mocked(finalizePluginWorkspace).mockRejectedValue(new Error("sync incomplete"));+ await expect(inspectPluginWorkspaceForFinalization(db,undefined,workspace,"agent","run")).rejects.toThrow("sync incomplete");+ });+});diff --git a/server/src/services/native-runtime/native-plugin-workspace-finalization.ts b/server/src/services/native-runtime/native-plugin-workspace-finalization.tsindex 3d94e756df..5eec1afccb 100644--- a/server/src/services/native-runtime/native-plugin-workspace-finalization.ts+++ b/server/src/services/native-runtime/native-plugin-workspace-finalization.ts@@ -1,5 +1,5 @@import type { Db, executionWorkspaces } from "@paperclipai/db";-import { inspectPluginWorkspace, type WorkspaceWorkers } from "../plugin-workspaces.js";+import { finalizePluginWorkspace, type WorkspaceWorkers } from "../plugin-workspaces.js";/** Validate retained plugin files without interpreting a provider reference as a Core path. */export async function inspectPluginWorkspaceForFinalization(@@ -7,9 +7,10 @@ export async function inspectPluginWorkspaceForFinalization(workers: WorkspaceWorkers | undefined,workspace: typeof executionWorkspaces.$inferSelect,agentId: string,+ runId: string,): Promise<Record<string, unknown> | null> {if (workspace.providerType === "plugin") {- const inspection = await inspectPluginWorkspace(db, workers, workspace);+ const inspection = await finalizePluginWorkspace(db, workers, workspace, { agentId, runId });if (workspace.strategyType === "git_worktree"&& (!workspace.branchName || inspection.branchName !== workspace.branchName)) {throw new Error("plugin_workspace_branch_mismatch");diff --git a/server/src/services/native-runtime/native-workspace-finalizer.test.ts b/server/src/services/native-runtime/native-workspace-finalizer.test.tsindex c674bcfb4e..f79698d5b9 100644--- a/server/src/services/native-runtime/native-workspace-finalizer.test.ts+++ b/server/src/services/native-runtime/native-workspace-finalizer.test.ts@@ -730,12 +730,12 @@ describe("native workspace finalization recovery", () => {mode: "isolated_workspace", strategyType: providerType === "plugin" ? "git_worktree" : "project_primary",branchName: "task-branch", metadata: providerType === "plugin" ? {} : { runnerTaskWorkspaceOwner: { agentId } } }).returning();const seeded = await seedRun({ executionWorkspaceId: workspace.id, title: "Plugin result before controller restart" });- const inspect = vi.spyOn(pluginWorkspaces, "inspectPluginWorkspace").mockResolvedValue({ kind: "inspection",+ const inspect = vi.spyOn(pluginWorkspaces, "finalizePluginWorkspace").mockResolvedValue({ kind: "inspection",branchName: "task-branch", headSha: "a".repeat(40), hasDirtyTrackedFiles: true, hasUntrackedFiles: false,hasLocalOnlyData: true, isMergedIntoBase: false, remotelyRecoverable: false });try {expect(await resumeNativeWorkspaceFinalization({ db, runId: seeded.runId })).toMatchObject({ status: "succeeded" });- if (providerType === "plugin") expect(inspect).toHaveBeenCalledWith(db, undefined, expect.objectContaining({ id: workspace.id }));+ if (providerType === "plugin") expect(inspect).toHaveBeenCalledWith(db, undefined, expect.objectContaining({ id: workspace.id }), { agentId, runId: seeded.runId });else expect(inspect).not.toHaveBeenCalled();} finally { inspect.mockRestore(); }});@@ -749,7 +749,7 @@ describe("native workspace finalization recovery", () => {]).returning();const blocked = await seedRun({ executionWorkspaceId: unavailable.id, title: "Plugin is unavailable after restart" });const ready = await seedRun({ executionWorkspaceId: healthy.id, title: "Other result can recover" });- const inspect = vi.spyOn(pluginWorkspaces, "inspectPluginWorkspace").mockRejectedValue(new Error(message));+ const inspect = vi.spyOn(pluginWorkspaces, "finalizePluginWorkspace").mockRejectedValue(new Error(message));const code = message.startsWith("plugin_workspace_") ? message : "plugin_workspace_inspection_failed";try {await expect(reconcileNativeFinalizations(db, [blocked.runId, ready.runId])).resolves.toBeDefined();diff --git a/server/src/services/native-runtime/native-workspace-finalizer.ts b/server/src/services/native-runtime/native-workspace-finalizer.tsindex 9e0220004c..0607284b97 100644--- a/server/src/services/native-runtime/native-workspace-finalizer.ts+++ b/server/src/services/native-runtime/native-workspace-finalizer.ts@@ -244,7 +244,7 @@ export async function resumeNativeWorkspaceFinalization(input: {let pluginWorkspace: Record<string, unknown> | null;try {pluginWorkspace = workspace- ? await inspectPluginWorkspaceForFinalization(input.db, input.pluginWorkerManager, workspace, bound.agentId)+ ? await inspectPluginWorkspaceForFinalization(input.db, input.pluginWorkerManager, workspace, bound.agentId, input.runId): null;} catch (error) {await ownership.assertHeld();diff --git a/server/src/services/plugin-workspaces.ts b/server/src/services/plugin-workspaces.tsindex 9e97724144..e2383dcfc7 100644--- a/server/src/services/plugin-workspaces.ts+++ b/server/src/services/plugin-workspaces.ts@@ -90,6 +90,9 @@ export async function callWorkspaceProvider(const selectedProvider = await provider(db, workers, workspace.companyId, pluginId);if (input.operation.kind === "prepare" && input.operation.fresh && selectedProvider.manifestJson.workspaceProvider?.freshWorkspaceVersion !== 1)throw conflict("Fresh workspace preparation is unavailable");+ if (input.operation.kind === "finalize" &&+ (selectedProvider.manifestJson.workspaceProvider?.finalizationVersion !== 1 || !input.runId || !input.agentId))+ throw conflict("Workspace finalization is unavailable");const generation =typeof workspace.metadata?.lifecycleGeneration === "number"? workspace.metadata.lifecycleGeneration@@ -111,11 +114,12 @@ export async function callWorkspaceProvider(? randomUUID(): createHash("sha256").update(- JSON.stringify([workspace.id, operationGeneration, operation]),+ JSON.stringify([workspace.id, operationGeneration, operation,+ ...(operation.kind === "finalize" ? [input.runId, input.agentId] : [])]),).digest("hex"),generation: operationGeneration,- deadline: new Date(Date.now() + 300_000).toISOString(),+ deadline: new Date(Date.now() + (operation.kind === "finalize" ? 30_000 : 300_000)).toISOString(),runId: input.runId ?? null,agentId: input.agentId ?? null,operation,@@ -126,7 +130,7 @@ export async function callWorkspaceProvider(pluginId,"workspaceOperation",params,- 310_000,+ operation.kind === "finalize" ? 40_000 : 310_000,);} catch {throw conflict(@@ -604,25 +608,42 @@ async function inspectBinding(return result;}-export async function inspectPluginWorkspace(- db: Db,- workers: WorkspaceWorkers | undefined,- workspace: Workspace,-) {+/** Reconcile durable lifecycle intent under the same lock for either read path. */+async function withRetainedWorkspace<T>(db: Db, workers: WorkspaceWorkers | undefined,+ workspace: Workspace, operation: (workspace: Workspace) => Promise<T>) {return withExecutionWorkspaceLifecycleLock(db, workspace.id, async () => {- // Keep recovery intent/results durable across an ambiguous provider response.- workspace = await reconcilePluginWorkspaceReservation(- db,- workers,- workspace,- );+ workspace = await reconcilePluginWorkspaceReservation(db, workers, workspace);if (workspace.metadata?.workspaceProviderRestoration)workspace = await finishPendingRestoration(db, workers, workspace);if (workspace.metadata?.workspaceProviderRemoval)workspace = await finishPendingRemoval(db, workers, workspace);- if (workspace.metadata?.workspaceProviderRemoved === true)- return removedInspection();- return inspectBinding(db, workers, workspace);+ return operation(workspace);+ });+}++export async function inspectPluginWorkspace(db: Db, workers: WorkspaceWorkers | undefined, workspace: Workspace) {+ return withRetainedWorkspace(db, workers, workspace, (current) =>+ current.metadata?.workspaceProviderRemoved === true+ ? Promise.resolve(removedInspection()) : inspectBinding(db, workers, current));+}++/** Completion does not need the deletion-safety proof supplied by inspect. */+export async function finalizePluginWorkspace(db: Db, workers: WorkspaceWorkers | undefined,+ workspace: Workspace, context: { runId: string; agentId: string }) {+ return withRetainedWorkspace(db, workers, workspace, async (current) => {+ if (current.metadata?.workspaceProviderRemoved === true)+ throw conflict("Workspace was removed before finalization");+ const binding = readWorkspaceProviderBinding(current.metadata);+ if (!binding) throw conflict("Workspace storage ownership is missing");+ const selected = await provider(db, workers, current.companyId, binding.pluginId);+ // Existing providers continue to use their established inspection contract.+ if (selected.manifestJson.workspaceProvider?.finalizationVersion !== 1)+ return inspectBinding(db, workers, current);+ const result = await callWorkspaceProvider(db, workers, {+ workspace: current, ...context, operation: { kind: "finalize", binding },+ });+ if (result.kind !== "finalized") throw conflict("Workspace finalization is incomplete");+ return result;});}