Skip to content

The open-source app everyone uses to manage agents at work

fix(server): retain stopped ephemeral workspaces after failed copy-back (#15790)

Devin Foley7e10dbe652ef
diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.md
index 469514c278..2be7f1b67d 100644
--- a/doc/native-workspace-finalization-recovery.md
+++ b/doc/native-workspace-finalization-recovery.md
@@ -219,3 +219,28 @@ plugin RPC. Its intent pins the registered built-in provider and its receipt say
`builtin.stopLease`. This provider owns no real process or filesystem; the receipt
models its lifecycle. Missing built-in stop support or an unconfirmed receipt
also remains pending through restart, without calling release or destroy.
+
+### Legacy copy-back after a confirmed stop
+
+A legacy adapter can report failed workspace copy-back after cancellation has
+already stopped and released its sandbox lease. The host records the exact lease
+snapshot from before adapter dispatch. For an originally ephemeral source, the
+recorder can preserve the released allocation for repair only while the current
+lease is still ephemeral and its company, run, issue, workspace, provider,
+allocation, acquisition time, plugin, and realized root still match. It requires
+both the dedicated stop-and-retain receipt and the matching stopped-execution
+receipt. A competing owner or changed source fails closed. The recorder leaves
+release status and receipts intact, excludes the allocation from reuse, and
+creates the existing board-owned file-repair obligation without replaying work
+or changing a newer run or the task's status.
+
+This late-recording path does not admit reusable leases. Provider resume can
+precede the database handoff, so an old stop receipt and a database row lock
+cannot prove that reuse has not started. Existing active-source and previously
+recorded recovery paths retain their behavior. A missing or rejected original
+snapshot does not authorize adopting another released lease.
+
+A retained source receipt records control-plane evidence; it does not prove the
+provider still holds the original bytes or that copy-back succeeded. Operators
+must verify and recover the exact source separately. Later successful runs and
+completed tasks do not certify recovery of earlier workspace files.
diff --git a/server/src/__tests__/legacy-workspace-restore-recovery.test.ts b/server/src/__tests__/legacy-workspace-restore-recovery.test.ts
index 24d12c6674..6278de9819 100644
--- a/server/src/__tests__/legacy-workspace-restore-recovery.test.ts
+++ b/server/src/__tests__/legacy-workspace-restore-recovery.test.ts
@@ -1,12 +1,13 @@
import { randomUUID } from "node:crypto";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db";
+import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { recordLegacyWorkspaceRestoreFailure, legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
import { hasConversationContinuationPolicy, conversationRecoveryActionPredicate } from "../services/conversation-continuation.js";
import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../services/legacy-workspace-restore-recovery.js";
-import { settleStopOnlyCleanup } from "../services/sandbox-stop-and-retain.js";
+import { hasConfirmedSandboxStopAndRetain, prepareSandboxStopAndRetain, settleStopOnlyCleanup } from "../services/sandbox-stop-and-retain.js";
+import { environmentService } from "../services/environments.js";
import { hasRequiredWorkspaceRecovery } from "../services/workspace-restore-recovery-state.js";
const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();
@@ -22,14 +23,20 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
await db.insert(companies).values({ id: companyId, name: "Workspace recovery", issuePrefix: companyId.slice(0, 8) });
await db.insert(agents).values({ id: agentId, companyId, name: "Engineer", role: "engineer", adapterType: "codex_local" });
await db.insert(issues).values({ id: issueId, companyId, title: "Restore files", status: options.taskDone ? "done" : "in_progress", assigneeAgentId: agentId });
- const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "legacy", contextSnapshot: { issueId } }).returning();
+ const [project] = await db.insert(projects).values({ companyId, name: "Workspace project" }).returning();
+ const [workspace] = await db.insert(executionWorkspaces).values({ companyId, projectId: project.id,
+ sourceIssueId: issueId, mode: "isolated", strategyType: "git_worktree", name: "Fixture workspace", cwd: "/work/fixture",
+ }).returning();
+ const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "running", runtimeMode: "legacy",
+ contextSnapshot: { issueId, executionWorkspaceId: workspace.id },
+ }).returning();
if (options.local) await db.insert(environments).values({ name: "Local", driver: "local" }).onConflictDoNothing();
const [environment] = options.local ? await db.select().from(environments).where(eq(environments.driver, "local"))
: await db.insert(environments).values({ name: `Isolated test sandbox ${runId}`, driver: "sandbox" }).returning();
- const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, heartbeatRunId: runId, environmentId: environment.id,
+ const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, executionWorkspaceId: workspace.id, heartbeatRunId: runId, environmentId: environment.id,
status: "active", leasePolicy: options.reusable ? "reuse_by_environment" : "ephemeral",
provider: options.local ? "local" : "daytona", providerLeaseId: randomUUID(),
- metadata: { driver: options.local ? "local" : "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: !options.local },
+ metadata: { remoteCwd: "/work/fixture", workspaceRealization: { authoritativeRoot: "/work/fixture" }, driver: options.local ? "local" : "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: !options.local },
}).returning();
return { companyId, runId, issueId, run, lease };
}
@@ -38,6 +45,22 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
} };
const readLease = async (id: string) => (await db.select().from(environmentLeases).where(eq(environmentLeases.id, id)))[0];
+ async function stopAndRetain(f: Awaited<ReturnType<typeof seed>>, retry = false) {
+ let stopping = (await prepareSandboxStopAndRetain(db, f.lease))!;
+ expect(stopping.status).toBe("pending_cleanup");
+ if (retry) {
+ [stopping] = await db.update(environmentLeases).set({ metadata: {
+ ...stopping.metadata, pendingCleanupAttemptId: randomUUID(),
+ } }).where(eq(environmentLeases.id, f.lease.id)).returning();
+ }
+ const stopped = (await settleStopOnlyCleanup(db, stopping, {
+ attemptId: String(stopping.metadata!.pendingCleanupAttemptId),
+ receipt: { providerLeaseId: f.lease.providerLeaseId, state: "stopped" },
+ }))!;
+ expect(stopped.status).toBe("released");
+ return stopped;
+ }
+
it.each([false, true])("persists terminal status, stop-only source and board action atomically (task done=%s)", async taskDone => {
const f = await seed({ taskDone, reusable: true });
expect(await legacyExecutionNeedsReconciliationWithEvidence(db, { ...f.run, status: "failed", ...patch })).toBe(true);
@@ -69,6 +92,239 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)))).toHaveLength(1);
});
+ it("records the exact stopped source when provider cancellation finishes before copy-back", async () => {
+ const f = await seed({ taskDone: true });
+ const stopping = await prepareSandboxStopAndRetain(db, f.lease);
+ expect(stopping).not.toBeNull();
+ const stopped = await settleStopOnlyCleanup(db, stopping!, {
+ attemptId: String(stopping!.metadata?.pendingCleanupAttemptId),
+ receipt: { providerLeaseId: f.lease.providerLeaseId, state: "stopped" },
+ });
+ expect(stopped?.status).toBe("released");
+ await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(),
+ resultJson: { executionCancellation: { state: "acknowledged" } },
+ }).where(eq(heartbeatRuns.id, f.runId));
+ // This is the lease captured by the host before invoking the adapter.
+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);
+ const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
+ expect(run.status).toBe("cancelled");
+ expect(run.resultJson?.executionCancellation).toEqual({ state: "acknowledged" });
+ expect(run.resultJson?.workspaceRestoreRecovery).toMatchObject({ leaseIds: [f.lease.id] });
+ const retained = await readLease(f.lease.id);
+ expect(retained).toMatchObject({ status: "released", leasePolicy: "retain_on_failure", cleanupStatus: "success",
+ releasedAt: stopped!.releasedAt, metadata: { remoteExecutionTermination: stopped!.metadata!.remoteExecutionTermination,
+ sandboxStopAndRetainReceipt: stopped!.metadata!.sandboxStopAndRetainReceipt } });
+ const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
+ expect(action).toMatchObject({ status: "resolved", outcome: "blocked", ownerType: "board",
+ evidence: { automaticRecovery: { replay: "blocked" }, workspaceRestoreRecovery: { leaseIds: [f.lease.id] } } });
+ expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("done");
+ // A lost commit response can cause the exact adapter receipt to be recorded twice.
+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);
+ const repeated = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
+ expect(repeated).toHaveLength(1);
+ expect(repeated[0]).toMatchObject({ id: action.id, status: "resolved", outcome: "blocked", attemptCount: 1,
+ evidence: { automaticRecovery: { replay: "blocked" }, workspaceRestoreRecovery: { leaseIds: [f.lease.id] } } });
+ expect(await readLease(f.lease.id)).toEqual(retained);
+ });
+
+ it.each(["retry", "builtin"])("keeps the original stop proof after %s dispatch", async scenario => {
+ const f = await seed();
+ if (scenario === "builtin") {
+ [f.lease] = await db.update(environmentLeases).set({ provider: "fake", metadata: {
+ driver: "sandbox", remoteCwd: "/work/fixture",
+ } }).where(eq(environmentLeases.id, f.lease.id)).returning();
+ }
+ const stopped = await stopAndRetain(f, scenario === "retry");
+ if (scenario === "retry") expect((stopped.metadata!.sandboxStopAndRetainReceipt as Record<string, unknown>).requestId)
+ .not.toBe(stopped.metadata!.pendingCleanupAttemptId);
+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);
+ expect(await readLease(f.lease.id)).toMatchObject({ leasePolicy: "retain_on_failure", status: "released",
+ metadata: { sandboxStopAndRetainReceipt: stopped.metadata!.sandboxStopAndRetainReceipt } });
+ });
+
+ it.each(["id", "companyId", "heartbeatRunId", "environmentId", "issueId", "executionWorkspaceId", "provider", "providerLeaseId", "acquiredAt", "pluginId", "remoteCwd", "workspaceRealization"])(
+ "rejects a stopped source with a different original %s", async field => {
+ const f = await seed();
+ const stopped = await stopAndRetain(f);
+ const source = { ...f.lease, metadata: { ...f.lease.metadata } };
+ if (["pluginId", "remoteCwd", "workspaceRealization"].includes(field)) source.metadata[field] = "different-original-value";
+ else if (field === "acquiredAt") source.acquiredAt = new Date(source.acquiredAt.getTime() - 1);
+ else Object.assign(source, { [field]: randomUUID() });
+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, source);
+ expect(await readLease(f.lease.id)).toEqual(stopped);
+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);
+ },
+ );
+
+ it.each(["missing_dedicated", "missing_termination", "destroyed", "foreign_receipt", "wrong_plugin", "wrong_method", "empty_request", "invalid_time", "before_acquisition", "after_release", "unequal_confirmation", "pending_intent", "cleanup_in_flight", "expired_allocation", "missing_release_time", "retained_without_stop", "failed_cleanup"])(
+ "does not infer retention from %s", async scenario => {
+ const f = await seed();
+ const stopped = await stopAndRetain(f);
+ const metadata = structuredClone(stopped.metadata!);
+ const dedicated = metadata.sandboxStopAndRetainReceipt as Record<string, unknown>;
+ const termination = metadata.remoteExecutionTermination as Record<string, unknown>;
+ if (scenario === "missing_dedicated") delete metadata.sandboxStopAndRetainReceipt;
+ if (scenario === "missing_termination") delete metadata.remoteExecutionTermination;
+ if (scenario === "destroyed") termination.state = "destroyed";
+ if (scenario === "foreign_receipt") dedicated.runId = randomUUID();
+ if (scenario === "wrong_plugin") dedicated.pluginId = randomUUID();
+ if (scenario === "wrong_method") dedicated.method = "environmentReleaseLease";
+ if (scenario === "empty_request") dedicated.requestId = " ";
+ if (scenario === "invalid_time") dedicated.confirmedAt = termination.confirmedAt = "not-a-time";
+ if (scenario === "before_acquisition") dedicated.confirmedAt = termination.confirmedAt = new Date(stopped.acquiredAt.getTime() - 1).toISOString();
+ if (scenario === "after_release") dedicated.confirmedAt = termination.confirmedAt = new Date(stopped.releasedAt!.getTime() + 1).toISOString();
+ if (scenario === "unequal_confirmation") dedicated.confirmedAt = new Date(stopped.releasedAt!.getTime() - 1).toISOString();
+ if (scenario === "pending_intent") metadata.sandboxStopAndRetain = {};
+ if (scenario === "cleanup_in_flight") metadata.pendingCleanupInFlight = true;
+ const [invalid] = await db.update(environmentLeases).set({ metadata,
+ ...(scenario === "retained_without_stop" ? { status: "retained" } : {}),
+ ...(scenario === "expired_allocation" ? { expiresAt: new Date(stopped.acquiredAt.getTime() - 1) } : {}),
+ ...(scenario === "missing_release_time" ? { releasedAt: null } : {}),
+ ...(scenario === "failed_cleanup" ? { cleanupStatus: "failed" } : {}),
+ }).where(eq(environmentLeases.id, f.lease.id)).returning();
+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);
+ expect(await readLease(f.lease.id)).toEqual(invalid);
+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);
+ },
+ );
+
+ it("rejects invalid lease timestamps instead of treating receipt metadata as proof", async () => {
+ const f = await seed();
+ const stopped = await stopAndRetain(f);
+ expect(hasConfirmedSandboxStopAndRetain(stopped)).toBe(true);
+ for (const field of ["acquiredAt", "releasedAt", "expiresAt"] as const) {
+ expect(hasConfirmedSandboxStopAndRetain({ ...stopped, [field]: new Date(Number.NaN) })).toBe(false);
+ }
+ });
+
+ it.each(["missing_root", "empty_root", "invalid_realized_root", "run_workspace_mismatch"])(
+ "requires the original execution root after %s", async scenario => {
+ const f = await seed();
+ const metadata = { ...f.lease.metadata };
+ if (scenario === "missing_root") delete metadata.remoteCwd;
+ if (scenario === "empty_root") metadata.remoteCwd = " ";
+ if (scenario === "invalid_realized_root") metadata.workspaceRealization = { authoritativeRoot: " " };
+ [f.lease] = await db.update(environmentLeases).set({ metadata }).where(eq(environmentLeases.id, f.lease.id)).returning();
+ if (scenario === "run_workspace_mismatch") await db.update(heartbeatRuns).set({ contextSnapshot: {
+ ...f.run.contextSnapshot, executionWorkspaceId: randomUUID(),
+ } }).where(eq(heartbeatRuns.id, f.runId));
+ const stopped = await stopAndRetain(f);
+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);
+ expect(await readLease(f.lease.id)).toEqual(stopped);
+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);
+ },
+ );
+
+ it("does not search arbitrary stopped leases when the original host snapshot is missing", async () => {
+ const f = await seed();
+ const stopped = await stopAndRetain(f);
+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson);
+ expect(await readLease(f.lease.id)).toEqual(stopped);
+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);
+ });
+
+ it.each(["active", "stopped", "older_stopped", "older_destroyed"])("rejects a competing %s allocation owner across companies", async state => {
+ const f = await seed(), other = await seed();
+ const stopped = await stopAndRetain(f);
+ [other.lease] = await db.update(environmentLeases).set({ providerLeaseId: f.lease.providerLeaseId,
+ acquiredAt: new Date(f.lease.acquiredAt.getTime() + (state.startsWith("older_") ? -1_000 : 1)),
+ }).where(eq(environmentLeases.id, other.lease.id)).returning();
+ if (state !== "active") {
+ const stoppedOther = await stopAndRetain(other);
+ if (state === "older_destroyed") await db.update(environmentLeases).set({ metadata: {
+ ...stoppedOther.metadata, remoteExecutionTermination: {
+ ...(stoppedOther.metadata!.remoteExecutionTermination as Record<string, unknown>), state: "destroyed",
+ },
+ } }).where(eq(environmentLeases.id, other.lease.id));
+ }
+ const competing = await readLease(other.lease.id);
+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);
+ expect(await readLease(f.lease.id)).toEqual(stopped);
+ expect(await readLease(other.lease.id)).toEqual(competing);
+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);
+ });
+
+ it("does not claim a released reusable source from a stop receipt that may precede resume", async () => {
+ const f = await seed({ reusable: true });
+ const stopped = await stopAndRetain(f);
+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);
+ expect(await readLease(f.lease.id)).toEqual(stopped);
+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);
+ });
+
+ it.each(["original_reusable", "current_reusable", "expired", "root_rebound", "run_rebound"])(
+ "leaves a stopped source unchanged after %s", async scenario => {
+ const f = await seed({ reusable: scenario === "original_reusable" });
+ const other = await seed();
+ const stopped = await stopAndRetain(f);
+ const [changed] = await db.update(environmentLeases).set({
+ ...(scenario === "original_reusable" ? { leasePolicy: "ephemeral" } : {}),
+ ...(scenario === "current_reusable" ? { leasePolicy: "reuse_by_environment" } : {}),
+ ...(scenario === "expired" ? { status: "expired" } : {}),
+ ...(scenario === "root_rebound" ? { metadata: { ...stopped.metadata, remoteCwd: "/work/replacement" } } : {}),
+ ...(scenario === "run_rebound" ? { heartbeatRunId: other.runId } : {}),
+ }).where(eq(environmentLeases.id, f.lease.id)).returning();
+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);
+ expect(await readLease(f.lease.id)).toEqual(changed);
+ expect(await readLease(other.lease.id)).toEqual(other.lease);
+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);
+ },
+ );
+
+ it("does not adopt the expired source when a normal reusable handoff wins first", async () => {
+ const f = await seed({ reusable: true });
+ await stopAndRetain(f);
+ const newerRunId = randomUUID();
+ await db.insert(heartbeatRuns).values({ id: newerRunId, companyId: f.companyId, agentId: f.run.agentId,
+ status: "running", runtimeMode: "legacy", contextSnapshot: { issueId: f.issueId } });
+ // The production handoff also checks the immutable task/agent scope.
+ [f.lease] = await db.update(environmentLeases).set({ metadata: {
+ ...(await readLease(f.lease.id)).metadata, agentId: f.run.agentId,
+ } }).where(eq(environmentLeases.id, f.lease.id)).returning();
+ const newer = await environmentService(db).acquireLease({ companyId: f.companyId,
+ environmentId: f.lease.environmentId!, issueId: f.issueId, executionWorkspaceId: f.lease.executionWorkspaceId, heartbeatRunId: newerRunId,
+ leasePolicy: "reuse_by_environment", provider: f.lease.provider!, providerLeaseId: f.lease.providerLeaseId,
+ metadata: { ...f.lease.metadata, agentId: f.run.agentId }, replacesReusableLeaseId: f.lease.id,
+ });
+ const expired = await readLease(f.lease.id);
+ expect(expired.status).toBe("expired");
+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);
+ expect(await readLease(f.lease.id)).toEqual(expired);
+ expect(await readLease(newer.id)).toEqual(newer);
+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);
+ });
+
+ it("rolls back the stopped-source hold and action if recording cannot commit", async () => {
+ const f = await seed();
+ const stopped = await stopAndRetain(f);
+ await expect(db.transaction(async tx => {
+ await recordLegacyWorkspaceRestoreFailure(tx as unknown as typeof db, f.run, patch.resultJson, f.lease);
+ throw new Error("recording transaction failed");
+ })).rejects.toThrow("recording transaction failed");
+ expect(await readLease(f.lease.id)).toEqual(stopped);
+ expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0].resultJson).toBeNull();
+ expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]);
+ });
+
+ it("preserves newer task execution while retaining only the original stopped source", async () => {
+ const f = await seed(), newerRunId = randomUUID();
+ await stopAndRetain(f);
+ await db.insert(heartbeatRuns).values({ id: newerRunId, companyId: f.companyId, agentId: f.run.agentId,
+ status: "running", runtimeMode: "legacy", contextSnapshot: { issueId: f.issueId } });
+ await db.update(issues).set({ executionRunId: newerRunId, checkoutRunId: newerRunId }).where(eq(issues.id, f.issueId));
+ const [newerLease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId,
+ heartbeatRunId: newerRunId, environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: randomUUID(),
+ status: "active", leasePolicy: "ephemeral", metadata: { driver: "sandbox", pluginId: randomUUID() },
+ }).returning();
+ await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease);
+ expect(await readLease(f.lease.id)).toMatchObject({ leasePolicy: "retain_on_failure" });
+ expect(await readLease(newerLease.id)).toEqual(newerLease);
+ expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0])
+ .toMatchObject({ status: "in_progress", executionRunId: newerRunId, checkoutRunId: newerRunId });
+ const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
+ expect(action.evidence.automaticRecovery).toMatchObject({ replay: "blocked" });
+ });
+
it("leaves local lock retry and successful remote cleanup policy unchanged", async () => {
const local = await seed({ local: true });
const failure = { ...patch, resultJson: { ...patch.resultJson, workspaceRestoreFailure: "restore_lock_timeout" } };
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 4df871c3d5..a549ee7ce4 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -4738,6 +4738,9 @@ export function heartbeatService(
await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);
const workspaceRealization = realizationResult.workspaceRealization;
const executionTarget = realizationResult.executionTarget;
+ // Preserve the host-owned source before adapter context can share lease
+ // metadata. A later copy-back failure must not adopt a rebound source.
+ const workspaceRestoreSource = structuredClone(realizationResult.lease);
let instructionCopy: Awaited<ReturnType<typeof instructionCopies.prepare>> = null;
let instructionSave: Record<string, unknown> | null = null;
const instructionPreparationKey = createHash("sha256").update(JSON.stringify({
@@ -7157,7 +7160,7 @@ export function heartbeatService(
};
// Retention is the fallback even if recording this receipt fails.
providerResourceDispositionForRun = "stop_and_retain";
- await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence);
+ await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence, workspaceRestoreSource);
}
for (const stream of ["stdout", "stderr"] as const) {
const tail = identityRedactor.finish(stream);
diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts
index 9d56e1891e..828ad95155 100644
--- a/server/src/services/legacy-execution-recovery.ts
+++ b/server/src/services/legacy-execution-recovery.ts
@@ -1,5 +1,5 @@
import { hasRequiredWorkspaceRecovery, preserveWorkspaceRestoreRecoveryMetadata, LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js";
-import { hasUnrestoredRemoteWorkspace, preserveLegacyWorkspaceRestoreSources } from "./legacy-workspace-restore-recovery.js";
+import { hasUnrestoredRemoteWorkspace, preserveLegacyWorkspaceRestoreSources, type LegacyWorkspaceRestoreSource } from "./legacy-workspace-restore-recovery.js";
import { isPreDispatchReviewWaitVerified } from "./pre-dispatch-review-wait.js";
import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";
import { normalizeMaxTurnStopReason } from "./heartbeat-stop-metadata.js";
@@ -74,6 +74,7 @@ export async function terminalizeLegacyExecution(input: {
writeConditions?: SQL[];
/** Adapter settlement records a copy-back failure before host finalization. */
recordRestoreFailureOnly?: boolean;
+ workspaceRestoreSource?: LegacyWorkspaceRestoreSource;
}) {
const { db, run, status } = input;
let patch = input.patch;
@@ -122,7 +123,8 @@ export async function terminalizeLegacyExecution(input: {
)
.returning();
if (!updated) return null;
- const retainedLeaseIds = await preserveLegacyWorkspaceRestoreSources(tx as unknown as Db, input.recordRestoreFailureOnly ? { ...updated, status: "failed" } : updated);
+ const retainedLeaseIds = await preserveLegacyWorkspaceRestoreSources(tx as unknown as Db,
+ input.recordRestoreFailureOnly ? { ...updated, status: "failed" } : updated, input.workspaceRestoreSource);
if (retainedLeaseIds.length) {
[updated] = await tx.update(heartbeatRuns).set({ resultJson: {
...updated.resultJson,
@@ -310,9 +312,9 @@ export async function settleInterruptedNativeBootstrap(
/** Capture the required-file obligation before unrelated host writes. Keeps
* execution ownership while the adapter's host finalization is still running. */
-export async function recordLegacyWorkspaceRestoreFailure(db: Db, run: Run, evidence: Record<string, unknown>) {
+export async function recordLegacyWorkspaceRestoreFailure(db: Db, run: Run, evidence: Record<string, unknown>, source?: LegacyWorkspaceRestoreSource) {
if (run.runtimeMode !== "legacy" || !hasWorkspaceRestoreFailure(evidence)) return;
await terminalizeLegacyExecution({ db, run, status: run.status,
- recordRestoreFailureOnly: true, patch: { resultJson: evidence },
+ recordRestoreFailureOnly: true, patch: { resultJson: evidence }, workspaceRestoreSource: source,
});
}
diff --git a/server/src/services/legacy-workspace-restore-recovery.ts b/server/src/services/legacy-workspace-restore-recovery.ts
index 9f7822375a..fc1419ddfb 100644
--- a/server/src/services/legacy-workspace-restore-recovery.ts
+++ b/server/src/services/legacy-workspace-restore-recovery.ts
@@ -1,28 +1,57 @@
+import { isDeepStrictEqual } from "node:util";
import { and, eq, inArray, isNotNull, ne, or, sql } from "drizzle-orm";
import { environmentLeases, environments, heartbeatRuns, type Db } from "@paperclipai/db";
import { hasWorkspaceRestoreFailure, WORKSPACE_RESTORE_FAILURE_CODES } from "@paperclipai/shared";
-import { prepareSandboxStopAndRetain } from "./sandbox-stop-and-retain.js";
+import { hasConfirmedSandboxStopAndRetain, prepareSandboxStopAndRetain } from "./sandbox-stop-and-retain.js";
import { hasRequiredWorkspaceRecovery, LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js";
type Run = typeof heartbeatRuns.$inferSelect;
+/** The host's realized lease snapshot from before adapter execution. Never
+ * reconstruct this identity from a terminal status or a search of old leases. */
+export type LegacyWorkspaceRestoreSource = Pick<typeof environmentLeases.$inferSelect,
+ "id" | "companyId" | "heartbeatRunId" | "environmentId" | "issueId" | "executionWorkspaceId"
+ | "provider" | "providerLeaseId" | "acquiredAt" | "leasePolicy" | "metadata">;
function needsRemoteRestore(run: Run): boolean {
return run.runtimeMode === "legacy" && ["failed", "timed_out", "interrupted", "cancelled"].includes(run.status)
&& hasWorkspaceRestoreFailure(run.resultJson);
}
-function leaseScope(run: Run) {
+function leaseScope(run: Run, source?: LegacyWorkspaceRestoreSource) {
return and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
or(eq(environmentLeases.status, "active"), and(
sql`${environmentLeases.metadata}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`,
- sql`${environmentLeases.metadata}->'workspaceRestoreRecovery'->>'runId' = ${run.id}`)),
+ sql`${environmentLeases.metadata}->'workspaceRestoreRecovery'->>'runId' = ${run.id}`),
+ source ? and(eq(environmentLeases.id, source.id), eq(environmentLeases.status, "released")) : undefined),
isNotNull(environmentLeases.providerLeaseId),
isNotNull(environmentLeases.provider), ne(environmentLeases.provider, "local"),
or(eq(environments.driver, "sandbox"), sql`${environmentLeases.metadata}->>'driver' = 'sandbox'`,
sql`${environmentLeases.metadata}->>'sandboxProviderPlugin' = 'true'`));
}
+function matchesOriginalSource(lease: typeof environmentLeases.$inferSelect, run: Run, source: LegacyWorkspaceRestoreSource) {
+ const issueId = run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId ?? null;
+ const workspaceId = run.contextSnapshot?.executionWorkspaceId;
+ const remoteCwd = source.metadata?.remoteCwd;
+ const realization = source.metadata?.workspaceRealization;
+ if (typeof remoteCwd !== "string" || !remoteCwd.trim()
+ || (typeof workspaceId === "string" && source.executionWorkspaceId !== workspaceId)) return false;
+ if (realization !== undefined && realization !== null) {
+ if (typeof realization !== "object" || Array.isArray(realization)) return false;
+ const root = (realization as Record<string, unknown>).authoritativeRoot;
+ if (root !== undefined && (typeof root !== "string" || !root.trim())) return false;
+ }
+ return source.leasePolicy === "ephemeral"
+ && lease.leasePolicy === source.leasePolicy
+ && source.companyId === run.companyId && source.heartbeatRunId === run.id && source.issueId === issueId
+ && (["id", "companyId", "heartbeatRunId", "environmentId", "issueId", "executionWorkspaceId", "provider", "providerLeaseId"] as const)
+ .every(key => lease[key] === source[key])
+ && lease.acquiredAt.getTime() === source.acquiredAt.getTime()
+ && (["pluginId", "sandboxProviderPlugin", "remoteCwd", "workspaceRealization"] as const)
+ .every(key => isDeepStrictEqual(lease.metadata?.[key], source.metadata?.[key]));
+}
+
/** A local lock timeout has no remote source to retain and keeps its old policy. */
export async function hasUnrestoredRemoteWorkspace(db: Db, run: Run): Promise<boolean> {
if (!needsRemoteRestore(run)) return false;
@@ -35,14 +64,40 @@ export async function hasUnrestoredRemoteWorkspace(db: Db, run: Run): Promise<bo
/** Call in the terminal-status transaction. No cleanup worker may observe the
* failed run without its exact stop-only intent. The original allocation must
* not enter warm reuse; it belongs to the board's workspace repair action. */
-export async function preserveLegacyWorkspaceRestoreSources(db: Db, run: Run): Promise<string[]> {
+export async function preserveLegacyWorkspaceRestoreSources(db: Db, run: Run, source?: LegacyWorkspaceRestoreSource): Promise<string[]> {
if (!needsRemoteRestore(run)) return [];
const rows = await db.select({ lease: environmentLeases }).from(environmentLeases)
.leftJoin(environments, eq(environments.id, environmentLeases.environmentId))
- .where(leaseScope(run)).for("update", { of: environmentLeases });
+ .where(leaseScope(run, source)).for("update", { of: environmentLeases });
const retained: string[] = [];
for (const { lease } of rows) {
- if (lease.status !== "active") { retained.push(lease.id); continue; }
+ if (lease.status !== "active") {
+ const recovery = lease.metadata?.workspaceRestoreRecovery as Record<string, unknown> | undefined;
+ if (recovery?.schema === LEGACY_WORKSPACE_RECOVERY_SCHEMA && recovery.runId === run.id) {
+ retained.push(lease.id); continue;
+ }
+ // Cancellation can finish the dedicated stop before copy-back reports its
+ // failure. Only the host's original source and two matching receipts can
+ // turn that released row into a file-repair hold.
+ if (!source || !matchesOriginalSource(lease, run, source) || !hasConfirmedSandboxStopAndRetain(lease)) continue;
+ const [other] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
+ ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider!),
+ eq(environmentLeases.providerLeaseId, lease.providerLeaseId!),
+ )).limit(1);
+ // An ephemeral source has no legitimate reuse history. Any other physical
+ // owner makes retention ambiguous, including an older owner that later
+ // destroyed the allocation. Do not expose cross-company owner identity.
+ if (other) continue;
+ await db.update(environmentLeases).set({ leasePolicy: "retain_on_failure",
+ metadata: sql`${environmentLeases.metadata} || ${JSON.stringify({ workspaceRestoreRecovery: {
+ schema: LEGACY_WORKSPACE_RECOVERY_SCHEMA, runId: run.id,
+ } })}::jsonb`,
+ }).where(eq(environmentLeases.id, lease.id));
+ // Reusable sources are excluded: provider resume precedes the database
+ // handoff, so a row lock cannot fence an in-flight resume. Keep the release
+ // status and receipts; this repair hold grants no execute right.
+ retained.push(lease.id); continue;
+ }
// The stop-only worker rechecks exact allocation ownership, including
// competing leases, before dispatch. Persist the intent even when cleanup
// must wait, so no terminal-run sweep can fall back to destruction.
diff --git a/server/src/services/sandbox-stop-and-retain.ts b/server/src/services/sandbox-stop-and-retain.ts
index e9823dcabd..d90c2d3e62 100644
--- a/server/src/services/sandbox-stop-and-retain.ts
+++ b/server/src/services/sandbox-stop-and-retain.ts
@@ -2,13 +2,46 @@ import { randomUUID } from "node:crypto";
import { and, eq, sql, type SQL } from "drizzle-orm";
import { environmentLeases, type Db } from "@paperclipai/db";
import { isBuiltinSandboxProvider } from "./sandbox-provider-runtime.js";
-import { remoteTerminationReceipt } from "./remote-execution-termination.js";
+import { hasRemoteTerminationReceipt, remoteTerminationReceipt } from "./remote-execution-termination.js";
import { hasNativeWorkspaceExportResume, readNativeWorkspaceExportResume, settleNativeWorkspaceExportResume } from "./native-runtime/native-workspace-export-resume.js";
type Lease = Pick<typeof environmentLeases.$inferSelect,
"id" | "companyId" | "heartbeatRunId" | "provider" | "providerLeaseId" | "metadata">;
export const SANDBOX_STOP_AND_RETAIN_KEY = "sandboxStopAndRetain";
+/** A successful stop is not by itself proof that the allocation was retained.
+ * Require the dedicated stop-only dispatch receipt and its independent stop
+ * confirmation, both bound to the original lease. No provider work is allowed. */
+export function hasConfirmedSandboxStopAndRetain(lease: typeof environmentLeases.$inferSelect): boolean {
+ const acquiredAt = lease.acquiredAt.getTime();
+ const releasedAt = lease.releasedAt?.getTime();
+ const expiresAt = lease.expiresAt?.getTime();
+ if (!Number.isFinite(acquiredAt) || releasedAt === undefined || !Number.isFinite(releasedAt)
+ || (expiresAt !== undefined && (!Number.isFinite(expiresAt) || expiresAt <= Date.now()))) return false;
+ const value = lease.metadata?.sandboxStopAndRetainReceipt;
+ if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+ const receipt = value as Record<string, unknown>;
+ const termination = lease.metadata?.remoteExecutionTermination as Record<string, unknown> | undefined;
+ if (lease.status !== "released" || !hasRemoteTerminationReceipt(lease)
+ || termination?.state !== "stopped" || hasStopOnlyCleanup(lease)
+ || lease.metadata?.pendingCleanupInFlight === true
+ || receipt.schema !== "paperclip.sandbox-stop-and-retain-receipt.v1"
+ || receipt.companyId !== lease.companyId || receipt.runId !== lease.heartbeatRunId
+ || receipt.leaseId !== lease.id || receipt.provider !== lease.provider
+ || receipt.providerLeaseId !== lease.providerLeaseId
+ || typeof receipt.requestId !== "string" || !receipt.requestId.trim()
+ || typeof receipt.confirmedAt !== "string" || receipt.confirmedAt !== termination.confirmedAt) return false;
+ const confirmedAt = Date.parse(receipt.confirmedAt);
+ if (!Number.isFinite(confirmedAt) || confirmedAt < acquiredAt || confirmedAt > releasedAt) return false;
+ if (receipt.builtinProvider !== undefined) {
+ return receipt.builtinProvider === lease.provider && isBuiltinSandboxProvider(lease.provider!)
+ && receipt.method === "builtin.stopLease" && receipt.pluginId === undefined
+ && lease.metadata?.pluginId == null && !lease.metadata?.sandboxProviderPlugin;
+ }
+ return receipt.method === "environmentStopLease" && typeof receipt.pluginId === "string"
+ && !!receipt.pluginId.trim() && receipt.pluginId === lease.metadata?.pluginId;
+}
+
export function hasStopOnlyCleanup(lease: Pick<Lease, "metadata">): boolean {
return hasNativeWorkspaceExportResume(lease)
|| Object.prototype.hasOwnProperty.call(lease.metadata ?? {}, SANDBOX_STOP_AND_RETAIN_KEY);