The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(server): retain stopped ephemeral workspaces after failed copy-back (#15790)
Devin Foley7e10dbe652ef
diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.mdindex 469514c278..2be7f1b67d 100644--- a/doc/native-workspace-finalization-recovery.md+++ b/doc/native-workspace-finalization-recovery.md@@ -219,3 +219,28 @@ plugin RPC. Its intent pins the registered built-in provider and its receipt say`builtin.stopLease`. This provider owns no real process or filesystem; the receiptmodels its lifecycle. Missing built-in stop support or an unconfirmed receiptalso remains pending through restart, without calling release or destroy.++### Legacy copy-back after a confirmed stop++A legacy adapter can report failed workspace copy-back after cancellation has+already stopped and released its sandbox lease. The host records the exact lease+snapshot from before adapter dispatch. For an originally ephemeral source, the+recorder can preserve the released allocation for repair only while the current+lease is still ephemeral and its company, run, issue, workspace, provider,+allocation, acquisition time, plugin, and realized root still match. It requires+both the dedicated stop-and-retain receipt and the matching stopped-execution+receipt. A competing owner or changed source fails closed. The recorder leaves+release status and receipts intact, excludes the allocation from reuse, and+creates the existing board-owned file-repair obligation without replaying work+or changing a newer run or the task's status.++This late-recording path does not admit reusable leases. Provider resume can+precede the database handoff, so an old stop receipt and a database row lock+cannot prove that reuse has not started. Existing active-source and previously+recorded recovery paths retain their behavior. A missing or rejected original+snapshot does not authorize adopting another released lease.++A retained source receipt records control-plane evidence; it does not prove the+provider still holds the original bytes or that copy-back succeeded. Operators+must verify and recover the exact source separately. Later successful runs and+completed tasks do not certify recovery of earlier workspace files.diff --git a/server/src/__tests__/legacy-workspace-restore-recovery.test.ts b/server/src/__tests__/legacy-workspace-restore-recovery.test.tsindex 24d12c6674..6278de9819 100644--- a/server/src/__tests__/legacy-workspace-restore-recovery.test.ts+++ b/server/src/__tests__/legacy-workspace-restore-recovery.test.ts@@ -1,12 +1,13 @@import { randomUUID } from "node:crypto";import { and, eq, sql } from "drizzle-orm";import { afterAll, beforeAll, describe, expect, it } from "vitest";-import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db";+import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db";import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";import { recordLegacyWorkspaceRestoreFailure, legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";import { hasConversationContinuationPolicy, conversationRecoveryActionPredicate } from "../services/conversation-continuation.js";import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../services/legacy-workspace-restore-recovery.js";-import { settleStopOnlyCleanup } from "../services/sandbox-stop-and-retain.js";+import { hasConfirmedSandboxStopAndRetain, prepareSandboxStopAndRetain, settleStopOnlyCleanup } from "../services/sandbox-stop-and-retain.js";+import { environmentService } from "../services/environments.js";import { hasRequiredWorkspaceRecovery } from "../services/workspace-restore-recovery-state.js";const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();@@ -22,14 +23,20 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbeddeawait db.insert(companies).values({ id: companyId, name: "Workspace recovery", issuePrefix: companyId.slice(0, 8) });await db.insert(agents).values({ id: agentId, companyId, name: "Engineer", role: "engineer", adapterType: "codex_local" });await db.insert(issues).values({ id: issueId, companyId, title: "Restore files", status: options.taskDone ? "done" : "in_progress", assigneeAgentId: agentId });- const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "legacy", contextSnapshot: { issueId } }).returning();+ const [project] = await db.insert(projects).values({ companyId, name: "Workspace project" }).returning();+ const [workspace] = await db.insert(executionWorkspaces).values({ companyId, projectId: project.id,+ sourceIssueId: issueId, mode: "isolated", strategyType: "git_worktree", name: "Fixture workspace", cwd: "/work/fixture",+ }).returning();+ const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "running", runtimeMode: "legacy",+ contextSnapshot: { issueId, executionWorkspaceId: workspace.id },+ }).returning();if (options.local) await db.insert(environments).values({ name: "Local", driver: "local" }).onConflictDoNothing();const [environment] = options.local ? await db.select().from(environments).where(eq(environments.driver, "local")): await db.insert(environments).values({ name: `Isolated test sandbox ${runId}`, driver: "sandbox" }).returning();- const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, heartbeatRunId: runId, environmentId: environment.id,+ const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, executionWorkspaceId: workspace.id, heartbeatRunId: runId, environmentId: environment.id,status: "active", leasePolicy: options.reusable ? "reuse_by_environment" : "ephemeral",provider: options.local ? "local" : "daytona", providerLeaseId: randomUUID(),- metadata: { driver: options.local ? "local" : "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: !options.local },+ metadata: { remoteCwd: "/work/fixture", workspaceRealization: { authoritativeRoot: "/work/fixture" }, driver: options.local ? "local" : "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: !options.local },}).returning();return { companyId, runId, issueId, run, lease };}@@ -38,6 +45,22 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde} };const readLease = async (id: string) => (await db.select().from(environmentLeases).where(eq(environmentLeases.id, id)))[0];+ async function stopAndRetain(f: Awaited<ReturnType<typeof seed>>, retry = false) {+ let stopping = (await prepareSandboxStopAndRetain(db, f.lease))!;+ expect(stopping.status).toBe("pending_cleanup");+ if (retry) {+ [stopping] = await db.update(environmentLeases).set({ metadata: {+ ...stopping.metadata, pendingCleanupAttemptId: randomUUID(),+ } }).where(eq(environmentLeases.id, f.lease.id)).returning();+ }+ const stopped = (await settleStopOnlyCleanup(db, stopping, {+ attemptId: String(stopping.metadata!.pendingCleanupAttemptId),+ receipt: { providerLeaseId: f.lease.providerLeaseId, state: "stopped" },+ }))!;+ expect(stopped.status).toBe("released");+ return stopped;+ }+it.each([false, true])("persists terminal status, stop-only source and board action atomically (task done=%s)", async taskDone => {const f = await seed({ taskDone, reusable: true });expect(await legacyExecutionNeedsReconciliationWithEvidence(db, { ...f.run, status: "failed", ...patch })).toBe(true);@@ -69,6 +92,239 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbeddeexpect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)))).toHaveLength(1);});+ it("records the exact stopped source when provider cancellation finishes before copy-back", async () => {+ const f = await seed({ taskDone: true });+ const stopping = await prepareSandboxStopAndRetain(db, f.lease);+ expect(stopping).not.toBeNull();+ const stopped = await settleStopOnlyCleanup(db, stopping!, {+ attemptId: String(stopping!.metadata?.pendingCleanupAttemptId),+ receipt: { providerLeaseId: f.lease.providerLeaseId, state: "stopped" },+ });+ expect(stopped?.status).toBe("released");+ await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(),+ resultJson: { executionCancellation: { state: "acknowledged" } },+ }).where(eq(heartbeatRuns.id, f.runId));+ // This is the lease captured by the host before invoking the adapter.+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);+ const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));+ expect(run.status).toBe("cancelled");+ expect(run.resultJson?.executionCancellation).toEqual({ state: "acknowledged" });+ expect(run.resultJson?.workspaceRestoreRecovery).toMatchObject({ leaseIds: [f.lease.id] });+ const retained = await readLease(f.lease.id);+ expect(retained).toMatchObject({ status: "released", leasePolicy: "retain_on_failure", cleanupStatus: "success",+ releasedAt: stopped!.releasedAt, metadata: { remoteExecutionTermination: stopped!.metadata!.remoteExecutionTermination,+ sandboxStopAndRetainReceipt: stopped!.metadata!.sandboxStopAndRetainReceipt } });+ const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));+ expect(action).toMatchObject({ status: "resolved", outcome: "blocked", ownerType: "board",+ evidence: { automaticRecovery: { replay: "blocked" }, workspaceRestoreRecovery: { leaseIds: [f.lease.id] } } });+ expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("done");+ // A lost commit response can cause the exact adapter receipt to be recorded twice.+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);+ const repeated = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));+ expect(repeated).toHaveLength(1);+ expect(repeated[0]).toMatchObject({ id: action.id, status: "resolved", outcome: "blocked", attemptCount: 1,+ evidence: { automaticRecovery: { replay: "blocked" }, workspaceRestoreRecovery: { leaseIds: [f.lease.id] } } });+ expect(await readLease(f.lease.id)).toEqual(retained);+ });++ it.each(["retry", "builtin"])("keeps the original stop proof after %s dispatch", async scenario => {+ const f = await seed();+ if (scenario === "builtin") {+ [f.lease] = await db.update(environmentLeases).set({ provider: "fake", metadata: {+ driver: "sandbox", remoteCwd: "/work/fixture",+ } }).where(eq(environmentLeases.id, f.lease.id)).returning();+ }+ const stopped = await stopAndRetain(f, scenario === "retry");+ if (scenario === "retry") expect((stopped.metadata!.sandboxStopAndRetainReceipt as Record<string, unknown>).requestId)+ .not.toBe(stopped.metadata!.pendingCleanupAttemptId);+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);+ expect(await readLease(f.lease.id)).toMatchObject({ leasePolicy: "retain_on_failure", status: "released",+ metadata: { sandboxStopAndRetainReceipt: stopped.metadata!.sandboxStopAndRetainReceipt } });+ });++ it.each(["id", "companyId", "heartbeatRunId", "environmentId", "issueId", "executionWorkspaceId", "provider", "providerLeaseId", "acquiredAt", "pluginId", "remoteCwd", "workspaceRealization"])(+ "rejects a stopped source with a different original %s", async field => {+ const f = await seed();+ const stopped = await stopAndRetain(f);+ const source = { ...f.lease, metadata: { ...f.lease.metadata } };+ if (["pluginId", "remoteCwd", "workspaceRealization"].includes(field)) source.metadata[field] = "different-original-value";+ else if (field === "acquiredAt") source.acquiredAt = new Date(source.acquiredAt.getTime() - 1);+ else Object.assign(source, { [field]: randomUUID() });+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, source);+ expect(await readLease(f.lease.id)).toEqual(stopped);+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);+ },+ );++ it.each(["missing_dedicated", "missing_termination", "destroyed", "foreign_receipt", "wrong_plugin", "wrong_method", "empty_request", "invalid_time", "before_acquisition", "after_release", "unequal_confirmation", "pending_intent", "cleanup_in_flight", "expired_allocation", "missing_release_time", "retained_without_stop", "failed_cleanup"])(+ "does not infer retention from %s", async scenario => {+ const f = await seed();+ const stopped = await stopAndRetain(f);+ const metadata = structuredClone(stopped.metadata!);+ const dedicated = metadata.sandboxStopAndRetainReceipt as Record<string, unknown>;+ const termination = metadata.remoteExecutionTermination as Record<string, unknown>;+ if (scenario === "missing_dedicated") delete metadata.sandboxStopAndRetainReceipt;+ if (scenario === "missing_termination") delete metadata.remoteExecutionTermination;+ if (scenario === "destroyed") termination.state = "destroyed";+ if (scenario === "foreign_receipt") dedicated.runId = randomUUID();+ if (scenario === "wrong_plugin") dedicated.pluginId = randomUUID();+ if (scenario === "wrong_method") dedicated.method = "environmentReleaseLease";+ if (scenario === "empty_request") dedicated.requestId = " ";+ if (scenario === "invalid_time") dedicated.confirmedAt = termination.confirmedAt = "not-a-time";+ if (scenario === "before_acquisition") dedicated.confirmedAt = termination.confirmedAt = new Date(stopped.acquiredAt.getTime() - 1).toISOString();+ if (scenario === "after_release") dedicated.confirmedAt = termination.confirmedAt = new Date(stopped.releasedAt!.getTime() + 1).toISOString();+ if (scenario === "unequal_confirmation") dedicated.confirmedAt = new Date(stopped.releasedAt!.getTime() - 1).toISOString();+ if (scenario === "pending_intent") metadata.sandboxStopAndRetain = {};+ if (scenario === "cleanup_in_flight") metadata.pendingCleanupInFlight = true;+ const [invalid] = await db.update(environmentLeases).set({ metadata,+ ...(scenario === "retained_without_stop" ? { status: "retained" } : {}),+ ...(scenario === "expired_allocation" ? { expiresAt: new Date(stopped.acquiredAt.getTime() - 1) } : {}),+ ...(scenario === "missing_release_time" ? { releasedAt: null } : {}),+ ...(scenario === "failed_cleanup" ? { cleanupStatus: "failed" } : {}),+ }).where(eq(environmentLeases.id, f.lease.id)).returning();+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);+ expect(await readLease(f.lease.id)).toEqual(invalid);+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);+ },+ );++ it("rejects invalid lease timestamps instead of treating receipt metadata as proof", async () => {+ const f = await seed();+ const stopped = await stopAndRetain(f);+ expect(hasConfirmedSandboxStopAndRetain(stopped)).toBe(true);+ for (const field of ["acquiredAt", "releasedAt", "expiresAt"] as const) {+ expect(hasConfirmedSandboxStopAndRetain({ ...stopped, [field]: new Date(Number.NaN) })).toBe(false);+ }+ });++ it.each(["missing_root", "empty_root", "invalid_realized_root", "run_workspace_mismatch"])(+ "requires the original execution root after %s", async scenario => {+ const f = await seed();+ const metadata = { ...f.lease.metadata };+ if (scenario === "missing_root") delete metadata.remoteCwd;+ if (scenario === "empty_root") metadata.remoteCwd = " ";+ if (scenario === "invalid_realized_root") metadata.workspaceRealization = { authoritativeRoot: " " };+ [f.lease] = await db.update(environmentLeases).set({ metadata }).where(eq(environmentLeases.id, f.lease.id)).returning();+ if (scenario === "run_workspace_mismatch") await db.update(heartbeatRuns).set({ contextSnapshot: {+ ...f.run.contextSnapshot, executionWorkspaceId: randomUUID(),+ } }).where(eq(heartbeatRuns.id, f.runId));+ const stopped = await stopAndRetain(f);+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);+ expect(await readLease(f.lease.id)).toEqual(stopped);+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);+ },+ );++ it("does not search arbitrary stopped leases when the original host snapshot is missing", async () => {+ const f = await seed();+ const stopped = await stopAndRetain(f);+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson);+ expect(await readLease(f.lease.id)).toEqual(stopped);+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);+ });++ it.each(["active", "stopped", "older_stopped", "older_destroyed"])("rejects a competing %s allocation owner across companies", async state => {+ const f = await seed(), other = await seed();+ const stopped = await stopAndRetain(f);+ [other.lease] = await db.update(environmentLeases).set({ providerLeaseId: f.lease.providerLeaseId,+ acquiredAt: new Date(f.lease.acquiredAt.getTime() + (state.startsWith("older_") ? -1_000 : 1)),+ }).where(eq(environmentLeases.id, other.lease.id)).returning();+ if (state !== "active") {+ const stoppedOther = await stopAndRetain(other);+ if (state === "older_destroyed") await db.update(environmentLeases).set({ metadata: {+ ...stoppedOther.metadata, remoteExecutionTermination: {+ ...(stoppedOther.metadata!.remoteExecutionTermination as Record<string, unknown>), state: "destroyed",+ },+ } }).where(eq(environmentLeases.id, other.lease.id));+ }+ const competing = await readLease(other.lease.id);+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);+ expect(await readLease(f.lease.id)).toEqual(stopped);+ expect(await readLease(other.lease.id)).toEqual(competing);+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);+ });++ it("does not claim a released reusable source from a stop receipt that may precede resume", async () => {+ const f = await seed({ reusable: true });+ const stopped = await stopAndRetain(f);+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);+ expect(await readLease(f.lease.id)).toEqual(stopped);+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);+ });++ it.each(["original_reusable", "current_reusable", "expired", "root_rebound", "run_rebound"])(+ "leaves a stopped source unchanged after %s", async scenario => {+ const f = await seed({ reusable: scenario === "original_reusable" });+ const other = await seed();+ const stopped = await stopAndRetain(f);+ const [changed] = await db.update(environmentLeases).set({+ ...(scenario === "original_reusable" ? { leasePolicy: "ephemeral" } : {}),+ ...(scenario === "current_reusable" ? { leasePolicy: "reuse_by_environment" } : {}),+ ...(scenario === "expired" ? { status: "expired" } : {}),+ ...(scenario === "root_rebound" ? { metadata: { ...stopped.metadata, remoteCwd: "/work/replacement" } } : {}),+ ...(scenario === "run_rebound" ? { heartbeatRunId: other.runId } : {}),+ }).where(eq(environmentLeases.id, f.lease.id)).returning();+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);+ expect(await readLease(f.lease.id)).toEqual(changed);+ expect(await readLease(other.lease.id)).toEqual(other.lease);+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);+ },+ );++ it("does not adopt the expired source when a normal reusable handoff wins first", async () => {+ const f = await seed({ reusable: true });+ await stopAndRetain(f);+ const newerRunId = randomUUID();+ await db.insert(heartbeatRuns).values({ id: newerRunId, companyId: f.companyId, agentId: f.run.agentId,+ status: "running", runtimeMode: "legacy", contextSnapshot: { issueId: f.issueId } });+ // The production handoff also checks the immutable task/agent scope.+ [f.lease] = await db.update(environmentLeases).set({ metadata: {+ ...(await readLease(f.lease.id)).metadata, agentId: f.run.agentId,+ } }).where(eq(environmentLeases.id, f.lease.id)).returning();+ const newer = await environmentService(db).acquireLease({ companyId: f.companyId,+ environmentId: f.lease.environmentId!, issueId: f.issueId, executionWorkspaceId: f.lease.executionWorkspaceId, heartbeatRunId: newerRunId,+ leasePolicy: "reuse_by_environment", provider: f.lease.provider!, providerLeaseId: f.lease.providerLeaseId,+ metadata: { ...f.lease.metadata, agentId: f.run.agentId }, replacesReusableLeaseId: f.lease.id,+ });+ const expired = await readLease(f.lease.id);+ expect(expired.status).toBe("expired");+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);+ expect(await readLease(f.lease.id)).toEqual(expired);+ expect(await readLease(newer.id)).toEqual(newer);+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);+ });++ it("rolls back the stopped-source hold and action if recording cannot commit", async () => {+ const f = await seed();+ const stopped = await stopAndRetain(f);+ await expect(db.transaction(async tx => {+ await recordLegacyWorkspaceRestoreFailure(tx as unknown as typeof db, f.run, patch.resultJson, f.lease);+ throw new Error("recording transaction failed");+ })).rejects.toThrow("recording transaction failed");+ expect(await readLease(f.lease.id)).toEqual(stopped);+ expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0].resultJson).toBeNull();+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);+ });++ it("preserves newer task execution while retaining only the original stopped source", async () => {+ const f = await seed(), newerRunId = randomUUID();+ await stopAndRetain(f);+ await db.insert(heartbeatRuns).values({ id: newerRunId, companyId: f.companyId, agentId: f.run.agentId,+ status: "running", runtimeMode: "legacy", contextSnapshot: { issueId: f.issueId } });+ await db.update(issues).set({ executionRunId: newerRunId, checkoutRunId: newerRunId }).where(eq(issues.id, f.issueId));+ const [newerLease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId,+ heartbeatRunId: newerRunId, environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: randomUUID(),+ status: "active", leasePolicy: "ephemeral", metadata: { driver: "sandbox", pluginId: randomUUID() },+ }).returning();+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);+ expect(await readLease(f.lease.id)).toMatchObject({ leasePolicy: "retain_on_failure" });+ expect(await readLease(newerLease.id)).toEqual(newerLease);+ expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0])+ .toMatchObject({ status: "in_progress", executionRunId: newerRunId, checkoutRunId: newerRunId });+ const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));+ expect(action.evidence.automaticRecovery).toMatchObject({ replay: "blocked" });+ });+it("leaves local lock retry and successful remote cleanup policy unchanged", async () => {const local = await seed({ local: true });const failure = { ...patch, resultJson: { ...patch.resultJson, workspaceRestoreFailure: "restore_lock_timeout" } };diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex 4df871c3d5..a549ee7ce4 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -4738,6 +4738,9 @@ export function heartbeatService(await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);const workspaceRealization = realizationResult.workspaceRealization;const executionTarget = realizationResult.executionTarget;+ // Preserve the host-owned source before adapter context can share lease+ // metadata. A later copy-back failure must not adopt a rebound source.+ const workspaceRestoreSource = structuredClone(realizationResult.lease);let instructionCopy: Awaited<ReturnType<typeof instructionCopies.prepare>> = null;let instructionSave: Record<string, unknown> | null = null;const instructionPreparationKey = createHash("sha256").update(JSON.stringify({@@ -7157,7 +7160,7 @@ export function heartbeatService(};// Retention is the fallback even if recording this receipt fails.providerResourceDispositionForRun = "stop_and_retain";- await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence);+ await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence, workspaceRestoreSource);}for (const stream of ["stdout", "stderr"] as const) {const tail = identityRedactor.finish(stream);diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.tsindex 9d56e1891e..828ad95155 100644--- a/server/src/services/legacy-execution-recovery.ts+++ b/server/src/services/legacy-execution-recovery.ts@@ -1,5 +1,5 @@import { hasRequiredWorkspaceRecovery, preserveWorkspaceRestoreRecoveryMetadata, LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js";-import { hasUnrestoredRemoteWorkspace, preserveLegacyWorkspaceRestoreSources } from "./legacy-workspace-restore-recovery.js";+import { hasUnrestoredRemoteWorkspace, preserveLegacyWorkspaceRestoreSources, type LegacyWorkspaceRestoreSource } from "./legacy-workspace-restore-recovery.js";import { isPreDispatchReviewWaitVerified } from "./pre-dispatch-review-wait.js";import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";import { normalizeMaxTurnStopReason } from "./heartbeat-stop-metadata.js";@@ -74,6 +74,7 @@ export async function terminalizeLegacyExecution(input: {writeConditions?: SQL[];/** Adapter settlement records a copy-back failure before host finalization. */recordRestoreFailureOnly?: boolean;+ workspaceRestoreSource?: LegacyWorkspaceRestoreSource;}) {const { db, run, status } = input;let patch = input.patch;@@ -122,7 +123,8 @@ export async function terminalizeLegacyExecution(input: {).returning();if (!updated) return null;- const retainedLeaseIds = await preserveLegacyWorkspaceRestoreSources(tx as unknown as Db, input.recordRestoreFailureOnly ? { ...updated, status: "failed" } : updated);+ const retainedLeaseIds = await preserveLegacyWorkspaceRestoreSources(tx as unknown as Db,+ input.recordRestoreFailureOnly ? { ...updated, status: "failed" } : updated, input.workspaceRestoreSource);if (retainedLeaseIds.length) {[updated] = await tx.update(heartbeatRuns).set({ resultJson: {...updated.resultJson,@@ -310,9 +312,9 @@ export async function settleInterruptedNativeBootstrap(/** Capture the required-file obligation before unrelated host writes. Keeps* execution ownership while the adapter's host finalization is still running. */-export async function recordLegacyWorkspaceRestoreFailure(db: Db, run: Run, evidence: Record<string, unknown>) {+export async function recordLegacyWorkspaceRestoreFailure(db: Db, run: Run, evidence: Record<string, unknown>, source?: LegacyWorkspaceRestoreSource) {if (run.runtimeMode !== "legacy" || !hasWorkspaceRestoreFailure(evidence)) return;await terminalizeLegacyExecution({ db, run, status: run.status,- recordRestoreFailureOnly: true, patch: { resultJson: evidence },+ recordRestoreFailureOnly: true, patch: { resultJson: evidence }, workspaceRestoreSource: source,});}diff --git a/server/src/services/legacy-workspace-restore-recovery.ts b/server/src/services/legacy-workspace-restore-recovery.tsindex 9f7822375a..fc1419ddfb 100644--- a/server/src/services/legacy-workspace-restore-recovery.ts+++ b/server/src/services/legacy-workspace-restore-recovery.ts@@ -1,28 +1,57 @@+import { isDeepStrictEqual } from "node:util";import { and, eq, inArray, isNotNull, ne, or, sql } from "drizzle-orm";import { environmentLeases, environments, heartbeatRuns, type Db } from "@paperclipai/db";import { hasWorkspaceRestoreFailure, WORKSPACE_RESTORE_FAILURE_CODES } from "@paperclipai/shared";-import { prepareSandboxStopAndRetain } from "./sandbox-stop-and-retain.js";+import { hasConfirmedSandboxStopAndRetain, prepareSandboxStopAndRetain } from "./sandbox-stop-and-retain.js";import { hasRequiredWorkspaceRecovery, LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js";type Run = typeof heartbeatRuns.$inferSelect;+/** The host's realized lease snapshot from before adapter execution. Never+ * reconstruct this identity from a terminal status or a search of old leases. */+export type LegacyWorkspaceRestoreSource = Pick<typeof environmentLeases.$inferSelect,+ "id" | "companyId" | "heartbeatRunId" | "environmentId" | "issueId" | "executionWorkspaceId"+ | "provider" | "providerLeaseId" | "acquiredAt" | "leasePolicy" | "metadata">;function needsRemoteRestore(run: Run): boolean {return run.runtimeMode === "legacy" && ["failed", "timed_out", "interrupted", "cancelled"].includes(run.status)&& hasWorkspaceRestoreFailure(run.resultJson);}-function leaseScope(run: Run) {+function leaseScope(run: Run, source?: LegacyWorkspaceRestoreSource) {return and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),or(eq(environmentLeases.status, "active"), and(sql`${environmentLeases.metadata}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`,- sql`${environmentLeases.metadata}->'workspaceRestoreRecovery'->>'runId' = ${run.id}`)),+ sql`${environmentLeases.metadata}->'workspaceRestoreRecovery'->>'runId' = ${run.id}`),+ source ? and(eq(environmentLeases.id, source.id), eq(environmentLeases.status, "released")) : undefined),isNotNull(environmentLeases.providerLeaseId),isNotNull(environmentLeases.provider), ne(environmentLeases.provider, "local"),or(eq(environments.driver, "sandbox"), sql`${environmentLeases.metadata}->>'driver' = 'sandbox'`,sql`${environmentLeases.metadata}->>'sandboxProviderPlugin' = 'true'`));}+function matchesOriginalSource(lease: typeof environmentLeases.$inferSelect, run: Run, source: LegacyWorkspaceRestoreSource) {+ const issueId = run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId ?? null;+ const workspaceId = run.contextSnapshot?.executionWorkspaceId;+ const remoteCwd = source.metadata?.remoteCwd;+ const realization = source.metadata?.workspaceRealization;+ if (typeof remoteCwd !== "string" || !remoteCwd.trim()+ || (typeof workspaceId === "string" && source.executionWorkspaceId !== workspaceId)) return false;+ if (realization !== undefined && realization !== null) {+ if (typeof realization !== "object" || Array.isArray(realization)) return false;+ const root = (realization as Record<string, unknown>).authoritativeRoot;+ if (root !== undefined && (typeof root !== "string" || !root.trim())) return false;+ }+ return source.leasePolicy === "ephemeral"+ && lease.leasePolicy === source.leasePolicy+ && source.companyId === run.companyId && source.heartbeatRunId === run.id && source.issueId === issueId+ && (["id", "companyId", "heartbeatRunId", "environmentId", "issueId", "executionWorkspaceId", "provider", "providerLeaseId"] as const)+ .every(key => lease[key] === source[key])+ && lease.acquiredAt.getTime() === source.acquiredAt.getTime()+ && (["pluginId", "sandboxProviderPlugin", "remoteCwd", "workspaceRealization"] as const)+ .every(key => isDeepStrictEqual(lease.metadata?.[key], source.metadata?.[key]));+}+/** A local lock timeout has no remote source to retain and keeps its old policy. */export async function hasUnrestoredRemoteWorkspace(db: Db, run: Run): Promise<boolean> {if (!needsRemoteRestore(run)) return false;@@ -35,14 +64,40 @@ export async function hasUnrestoredRemoteWorkspace(db: Db, run: Run): Promise<bo/** Call in the terminal-status transaction. No cleanup worker may observe the* failed run without its exact stop-only intent. The original allocation must* not enter warm reuse; it belongs to the board's workspace repair action. */-export async function preserveLegacyWorkspaceRestoreSources(db: Db, run: Run): Promise<string[]> {+export async function preserveLegacyWorkspaceRestoreSources(db: Db, run: Run, source?: LegacyWorkspaceRestoreSource): Promise<string[]> {if (!needsRemoteRestore(run)) return [];const rows = await db.select({ lease: environmentLeases }).from(environmentLeases).leftJoin(environments, eq(environments.id, environmentLeases.environmentId))- .where(leaseScope(run)).for("update", { of: environmentLeases });+ .where(leaseScope(run, source)).for("update", { of: environmentLeases });const retained: string[] = [];for (const { lease } of rows) {- if (lease.status !== "active") { retained.push(lease.id); continue; }+ if (lease.status !== "active") {+ const recovery = lease.metadata?.workspaceRestoreRecovery as Record<string, unknown> | undefined;+ if (recovery?.schema === LEGACY_WORKSPACE_RECOVERY_SCHEMA && recovery.runId === run.id) {+ retained.push(lease.id); continue;+ }+ // Cancellation can finish the dedicated stop before copy-back reports its+ // failure. Only the host's original source and two matching receipts can+ // turn that released row into a file-repair hold.+ if (!source || !matchesOriginalSource(lease, run, source) || !hasConfirmedSandboxStopAndRetain(lease)) continue;+ const [other] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and(+ ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider!),+ eq(environmentLeases.providerLeaseId, lease.providerLeaseId!),+ )).limit(1);+ // An ephemeral source has no legitimate reuse history. Any other physical+ // owner makes retention ambiguous, including an older owner that later+ // destroyed the allocation. Do not expose cross-company owner identity.+ if (other) continue;+ await db.update(environmentLeases).set({ leasePolicy: "retain_on_failure",+ metadata: sql`${environmentLeases.metadata} || ${JSON.stringify({ workspaceRestoreRecovery: {+ schema: LEGACY_WORKSPACE_RECOVERY_SCHEMA, runId: run.id,+ } })}::jsonb`,+ }).where(eq(environmentLeases.id, lease.id));+ // Reusable sources are excluded: provider resume precedes the database+ // handoff, so a row lock cannot fence an in-flight resume. Keep the release+ // status and receipts; this repair hold grants no execute right.+ retained.push(lease.id); continue;+ }// The stop-only worker rechecks exact allocation ownership, including// competing leases, before dispatch. Persist the intent even when cleanup// must wait, so no terminal-run sweep can fall back to destruction.diff --git a/server/src/services/sandbox-stop-and-retain.ts b/server/src/services/sandbox-stop-and-retain.tsindex e9823dcabd..d90c2d3e62 100644--- a/server/src/services/sandbox-stop-and-retain.ts+++ b/server/src/services/sandbox-stop-and-retain.ts@@ -2,13 +2,46 @@ import { randomUUID } from "node:crypto";import { and, eq, sql, type SQL } from "drizzle-orm";import { environmentLeases, type Db } from "@paperclipai/db";import { isBuiltinSandboxProvider } from "./sandbox-provider-runtime.js";-import { remoteTerminationReceipt } from "./remote-execution-termination.js";+import { hasRemoteTerminationReceipt, remoteTerminationReceipt } from "./remote-execution-termination.js";import { hasNativeWorkspaceExportResume, readNativeWorkspaceExportResume, settleNativeWorkspaceExportResume } from "./native-runtime/native-workspace-export-resume.js";type Lease = Pick<typeof environmentLeases.$inferSelect,"id" | "companyId" | "heartbeatRunId" | "provider" | "providerLeaseId" | "metadata">;export const SANDBOX_STOP_AND_RETAIN_KEY = "sandboxStopAndRetain";+/** A successful stop is not by itself proof that the allocation was retained.+ * Require the dedicated stop-only dispatch receipt and its independent stop+ * confirmation, both bound to the original lease. No provider work is allowed. */+export function hasConfirmedSandboxStopAndRetain(lease: typeof environmentLeases.$inferSelect): boolean {+ const acquiredAt = lease.acquiredAt.getTime();+ const releasedAt = lease.releasedAt?.getTime();+ const expiresAt = lease.expiresAt?.getTime();+ if (!Number.isFinite(acquiredAt) || releasedAt === undefined || !Number.isFinite(releasedAt)+ || (expiresAt !== undefined && (!Number.isFinite(expiresAt) || expiresAt <= Date.now()))) return false;+ const value = lease.metadata?.sandboxStopAndRetainReceipt;+ if (!value || typeof value !== "object" || Array.isArray(value)) return false;+ const receipt = value as Record<string, unknown>;+ const termination = lease.metadata?.remoteExecutionTermination as Record<string, unknown> | undefined;+ if (lease.status !== "released" || !hasRemoteTerminationReceipt(lease)+ || termination?.state !== "stopped" || hasStopOnlyCleanup(lease)+ || lease.metadata?.pendingCleanupInFlight === true+ || receipt.schema !== "paperclip.sandbox-stop-and-retain-receipt.v1"+ || receipt.companyId !== lease.companyId || receipt.runId !== lease.heartbeatRunId+ || receipt.leaseId !== lease.id || receipt.provider !== lease.provider+ || receipt.providerLeaseId !== lease.providerLeaseId+ || typeof receipt.requestId !== "string" || !receipt.requestId.trim()+ || typeof receipt.confirmedAt !== "string" || receipt.confirmedAt !== termination.confirmedAt) return false;+ const confirmedAt = Date.parse(receipt.confirmedAt);+ if (!Number.isFinite(confirmedAt) || confirmedAt < acquiredAt || confirmedAt > releasedAt) return false;+ if (receipt.builtinProvider !== undefined) {+ return receipt.builtinProvider === lease.provider && isBuiltinSandboxProvider(lease.provider!)+ && receipt.method === "builtin.stopLease" && receipt.pluginId === undefined+ && lease.metadata?.pluginId == null && !lease.metadata?.sandboxProviderPlugin;+ }+ return receipt.method === "environmentStopLease" && typeof receipt.pluginId === "string"+ && !!receipt.pluginId.trim() && receipt.pluginId === lease.metadata?.pluginId;+}+export function hasStopOnlyCleanup(lease: Pick<Lease, "metadata">): boolean {return hasNativeWorkspaceExportResume(lease)|| Object.prototype.hasOwnProperty.call(lease.metadata ?? {}, SANDBOX_STOP_AND_RETAIN_KEY);