The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
feat: reuse Daytona sandbox leases (#8513)
Devin Foleycd38c150b0e7
diff --git a/packages/plugins/sandbox-providers/daytona/src/manifest.ts b/packages/plugins/sandbox-providers/daytona/src/manifest.tsindex dcc684342e..c78ef2c00f 100644--- a/packages/plugins/sandbox-providers/daytona/src/manifest.ts+++ b/packages/plugins/sandbox-providers/daytona/src/manifest.ts@@ -23,6 +23,7 @@ const manifest: PaperclipPluginManifestV1 = {displayName: "Daytona Sandbox",description:"Provisions Daytona sandboxes with configurable image or snapshot selection, startup timeouts, and lease reuse.",+ supportsReusableLeases: true,configSchema: {type: "object",properties: {diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.tsindex a9f0022989..1620ca96f3 100644--- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts@@ -174,6 +174,9 @@ describe("Daytona sandbox provider plugin", () => {companyId: "company-1",environmentId: "env-1",runId: "run-1",+ agentId: "agent-1",+ executionWorkspaceId: "workspace-1",+ adapterType: "codex_local",config: {image: "node:20",timeoutMs: 300000,@@ -189,8 +192,21 @@ describe("Daytona sandbox provider plugin", () => {sandboxId: "sandbox-123",remoteCwd: "/home/daytona/paperclip-workspace",reuseLease: true,+ workspaceSentinel: {+ path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",+ result: "written",+ },},});+ expect(sandbox.fs.createFolder).toHaveBeenCalledWith(+ "/home/daytona/paperclip-workspace/.paperclip-runtime",+ "755",+ );+ expect(sandbox.fs.uploadFile).toHaveBeenCalledWith(+ expect.any(Buffer),+ "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",+ 300,+ );});it("deletes the sandbox if lease setup throws after sandbox creation", async () => {@@ -287,6 +303,94 @@ describe("Daytona sandbox provider plugin", () => {});});+ it("resumes a reusable lease when the workspace sentinel matches", async () => {+ process.env.DAYTONA_API_KEY = "host-key";+ const sandbox = createMockSandbox({ id: "sandbox-reuse", state: "stopped" });+ sandbox.process.executeCommand+ .mockResolvedValueOnce({+ exitCode: 0,+ result: JSON.stringify({ token: "sentinel-token" }),+ artifacts: { stdout: JSON.stringify({ token: "sentinel-token" }) },+ })+ .mockResolvedValueOnce({+ exitCode: 0,+ result: "bash",+ artifacts: { stdout: "bash" },+ });+ mockGet.mockResolvedValue(sandbox);++ const lease = await plugin.definition.onEnvironmentResumeLease?.({+ driverKey: "daytona",+ companyId: "company-1",+ environmentId: "env-1",+ providerLeaseId: "sandbox-reuse",+ config: {+ timeoutMs: 300000,+ reuseLease: true,+ },+ leaseMetadata: {+ workspaceSentinel: {+ path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",+ token: "sentinel-token",+ result: "written",+ },+ },+ });++ expect(sandbox.start).toHaveBeenCalledWith(300);+ expect(lease).toMatchObject({+ providerLeaseId: "sandbox-reuse",+ metadata: {+ resumedLease: true,+ workspaceSentinel: {+ result: "matched",+ token: "sentinel-token",+ },+ },+ });+ });++ it("expires a reusable lease when the workspace sentinel does not match", async () => {+ process.env.DAYTONA_API_KEY = "host-key";+ const sandbox = createMockSandbox({ id: "sandbox-reuse", state: "stopped" });+ sandbox.process.executeCommand.mockResolvedValueOnce({+ exitCode: 0,+ result: JSON.stringify({ token: "other-token" }),+ artifacts: { stdout: JSON.stringify({ token: "other-token" }) },+ });+ mockGet.mockResolvedValue(sandbox);++ await expect(plugin.definition.onEnvironmentResumeLease?.({+ driverKey: "daytona",+ companyId: "company-1",+ environmentId: "env-1",+ providerLeaseId: "sandbox-reuse",+ config: {+ timeoutMs: 300000,+ reuseLease: true,+ },+ leaseMetadata: {+ workspaceSentinel: {+ path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",+ token: "sentinel-token",+ result: "written",+ },+ },+ })).resolves.toEqual({+ providerLeaseId: null,+ metadata: {+ expired: true,+ workspaceSentinel: {+ path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",+ token: "sentinel-token",+ result: "mismatch",+ },+ },+ });++ expect(sandbox.process.executeCommand).toHaveBeenCalledTimes(1);+ });+it("stops reusable leases and deletes ephemeral leases on release", async () => {process.env.DAYTONA_API_KEY = "host-key";const reusable = createMockSandbox({ id: "sandbox-reusable" });diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.tsindex 2ea53ba648..cc5b294c1e 100644--- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts@@ -1,5 +1,5 @@import path from "node:path";-import { randomUUID } from "node:crypto";+import { createHash, randomUUID } from "node:crypto";import { Daytona, DaytonaNotFoundError, DaytonaTimeoutError } from "@daytonaio/sdk";import type {CreateSandboxBaseParams,@@ -44,6 +44,14 @@ interface DaytonaDriverConfig {reuseLease: boolean;}+type WorkspaceSentinelResult = {+ path: string;+ token: string | null;+ result: "written" | "matched" | "missing" | "mismatch" | "skipped";+};++const WORKSPACE_SENTINEL_RELATIVE_PATH = ".paperclip-runtime/reusable-sandbox-lease.json";+function parseOptionalString(value: unknown): string | null {return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;}@@ -166,6 +174,20 @@ function formatErrorMessage(error: unknown): string {return error instanceof Error ? error.message : String(error);}+function isRecord(value: unknown): value is Record<string, unknown> {+ return Boolean(value) && typeof value === "object" && !Array.isArray(value);+}++function stableStringify(value: unknown): string {+ if (Array.isArray(value)) {+ return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;+ }+ if (isRecord(value)) {+ return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;+ }+ return JSON.stringify(value) ?? "null";+}+function isValidUrl(value: string): boolean {try {new URL(value);@@ -210,12 +232,109 @@ async function detectSandboxShellCommand(sandbox: Sandbox, timeoutSeconds: numbe}}+function workspaceSentinelToken(input: {+ params: Pick<PluginEnvironmentAcquireLeaseParams, "companyId" | "environmentId" | "agentId" | "executionWorkspaceId" | "adapterType">;+ config: DaytonaDriverConfig;+}): string | null {+ if (!input.config.reuseLease || !input.params.agentId || !input.params.executionWorkspaceId) {+ return null;+ }+ return createHash("sha256")+ .update(stableStringify({+ provider: "daytona",+ companyId: input.params.companyId,+ environmentId: input.params.environmentId,+ agentId: input.params.agentId,+ executionWorkspaceId: input.params.executionWorkspaceId,+ adapterType: input.params.adapterType ?? null,+ image: input.config.image,+ snapshot: input.config.snapshot,+ target: input.config.target,+ }))+ .digest("hex");+}++function workspaceSentinelPath(remoteCwd: string): string {+ return path.posix.join(remoteCwd, WORKSPACE_SENTINEL_RELATIVE_PATH);+}++async function writeWorkspaceSentinel(input: {+ sandbox: Sandbox;+ remoteCwd: string;+ params: PluginEnvironmentAcquireLeaseParams;+ config: DaytonaDriverConfig;+ timeoutSeconds: number;+}): Promise<WorkspaceSentinelResult> {+ const sentinelPath = workspaceSentinelPath(input.remoteCwd);+ const token = workspaceSentinelToken({ params: input.params, config: input.config });+ if (!token) {+ return { path: sentinelPath, token: null, result: "skipped" };+ }+ await input.sandbox.fs.createFolder(path.posix.dirname(sentinelPath), "755");+ await input.sandbox.fs.uploadFile(+ Buffer.from(JSON.stringify({+ version: 1,+ token,+ companyId: input.params.companyId,+ environmentId: input.params.environmentId,+ agentId: input.params.agentId,+ executionWorkspaceId: input.params.executionWorkspaceId,+ adapterType: input.params.adapterType ?? null,+ provider: "daytona",+ writtenAt: new Date().toISOString(),+ }, null, 2), "utf8"),+ sentinelPath,+ input.timeoutSeconds,+ );+ return { path: sentinelPath, token, result: "written" };+}++async function verifyWorkspaceSentinel(input: {+ sandbox: Sandbox;+ remoteCwd: string;+ leaseMetadata?: Record<string, unknown>;+ timeoutSeconds: number;+}): Promise<WorkspaceSentinelResult> {+ const metadataSentinel = isRecord(input.leaseMetadata?.workspaceSentinel)+ ? input.leaseMetadata.workspaceSentinel+ : null;+ const sentinelPath = typeof metadataSentinel?.path === "string"+ ? metadataSentinel.path+ : workspaceSentinelPath(input.remoteCwd);+ const expectedToken = typeof metadataSentinel?.token === "string" ? metadataSentinel.token : null;+ if (!expectedToken) {+ return { path: sentinelPath, token: null, result: "missing" };+ }++ const result = await input.sandbox.process.executeCommand(+ `cat ${shellQuote(sentinelPath)}`,+ undefined,+ undefined,+ input.timeoutSeconds,+ );+ if (result.exitCode !== 0) {+ return { path: sentinelPath, token: expectedToken, result: "missing" };+ }+ try {+ const parsed = JSON.parse(result.result ?? result.artifacts?.stdout ?? "") as unknown;+ const actualToken = isRecord(parsed) && typeof parsed.token === "string" ? parsed.token : null;+ return {+ path: sentinelPath,+ token: expectedToken,+ result: actualToken === expectedToken ? "matched" : "mismatch",+ };+ } catch {+ return { path: sentinelPath, token: expectedToken, result: "mismatch" };+ }+}+function leaseMetadata(input: {config: DaytonaDriverConfig;sandbox: Sandbox;shellCommand: "bash" | "sh";remoteCwd: string;resumedLease: boolean;+ workspaceSentinel?: WorkspaceSentinelResult;}) {return {provider: "daytona",@@ -230,6 +349,7 @@ function leaseMetadata(input: {reuseLease: input.config.reuseLease,remoteCwd: input.remoteCwd,resumedLease: input.resumedLease,+ ...(input.workspaceSentinel ? { workspaceSentinel: input.workspaceSentinel } : {}),};}@@ -499,9 +619,16 @@ const plugin = definePlugin({try {const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs));+ const workspaceSentinel = await writeWorkspaceSentinel({+ sandbox,+ remoteCwd,+ params,+ config,+ timeoutSeconds: toTimeoutSeconds(config.timeoutMs),+ });return {providerLeaseId: sandbox.id,- metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: false }),+ metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: false, workspaceSentinel }),};} catch (error) {await sandbox.delete(toTimeoutSeconds(config.timeoutMs)).catch(() => undefined);@@ -521,10 +648,19 @@ const plugin = definePlugin({await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs));try {const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);+ const workspaceSentinel = await verifyWorkspaceSentinel({+ sandbox,+ remoteCwd,+ leaseMetadata: params.leaseMetadata,+ timeoutSeconds: toTimeoutSeconds(config.timeoutMs),+ });+ if (workspaceSentinel.result !== "matched") {+ return { providerLeaseId: null, metadata: { expired: true, workspaceSentinel } };+ }const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs));return {providerLeaseId: sandbox.id,- metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: true }),+ metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: true, workspaceSentinel }),};} catch (error) {await sandbox.delete(toTimeoutSeconds(config.timeoutMs)).catch(() => undefined);diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.tsindex 5ec4681888..954fa76a9d 100644--- a/packages/shared/src/constants.ts+++ b/packages/shared/src/constants.ts@@ -421,7 +421,7 @@ export type EnvironmentDriver = (typeof ENVIRONMENT_DRIVERS)[number];export const ENVIRONMENT_STATUSES = ["active", "archived"] as const;export type EnvironmentStatus = (typeof ENVIRONMENT_STATUSES)[number];-export const ENVIRONMENT_LEASE_STATUSES = ["active", "released", "expired", "failed", "retained"] as const;+export const ENVIRONMENT_LEASE_STATUSES = ["active", "released", "expired", "failed", "retained", "pending_cleanup"] as const;export type EnvironmentLeaseStatus = (typeof ENVIRONMENT_LEASE_STATUSES)[number];export const ENVIRONMENT_LEASE_POLICIES = [diff --git a/packages/shared/src/types/plugin.ts b/packages/shared/src/types/plugin.tsindex 351b2dda0e..366a60ba63 100644--- a/packages/shared/src/types/plugin.ts+++ b/packages/shared/src/types/plugin.ts@@ -138,6 +138,12 @@ export interface PluginEnvironmentDriverDeclaration {displayName: string;/** Optional description for operator-facing docs or UI affordances. */description?: string;+ /**+ * Sandbox providers must opt in before the host retains and resumes provider+ * leases across runs. Providers without this flag keep per-run acquire/release+ * behavior even if their config schema exposes a reuse-like setting.+ */+ supportsReusableLeases?: boolean;/** JSON Schema describing the driver's provider-specific configuration. */configSchema: JsonSchema;}diff --git a/packages/shared/src/validators/plugin.ts b/packages/shared/src/validators/plugin.tsindex 0a857b44d8..45a1e3a845 100644--- a/packages/shared/src/validators/plugin.ts+++ b/packages/shared/src/validators/plugin.ts@@ -124,6 +124,7 @@ export const pluginEnvironmentDriverDeclarationSchema = z.object({kind: z.enum(["environment_driver", "sandbox_provider"]).optional(),displayName: z.string().min(1).max(100),description: z.string().max(500).optional(),+ supportsReusableLeases: z.boolean().optional(),configSchema: jsonSchemaSchema,});diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.tsindex e7ff1f9c98..4ec89cbe1c 100644--- a/server/src/__tests__/environment-runtime.test.ts+++ b/server/src/__tests__/environment-runtime.test.ts@@ -1,4 +1,4 @@-import { randomUUID } from "node:crypto";+import { createHash, randomUUID } from "node:crypto";import { mkdtemp, rm } from "node:fs/promises";import os from "node:os";import path from "node:path";@@ -42,6 +42,25 @@ if (!embeddedPostgresSupport.supported) {);}+function stableStringify(value: unknown): string {+ if (Array.isArray(value)) {+ return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;+ }+ if (value && typeof value === "object") {+ const record = value as Record<string, unknown>;+ return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`).join(",")}}`;+ }+ return JSON.stringify(value) ?? "null";+}++function reusableRuntimeFingerprint(input: {+ provider: string;+ adapterType: string | null;+ config: Record<string, unknown>;+}) {+ return createHash("sha256").update(stableStringify(input)).digest("hex");+}+describe("findReusableSandboxLeaseId", () => {it("matches reusable plugin-backed sandbox leases by provider", () => {const selected = findReusableSandboxLeaseId({@@ -258,6 +277,124 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {};}+ async function seedReusablePluginSandboxLease() {+ const pluginId = randomUUID();+ const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();+ const providerConfig = {+ provider: "fake-plugin",+ image: "fake:test",+ timeoutMs: 1234,+ reuseLease: true,+ };+ const environment = {+ ...baseEnvironment,+ name: "Reusable Plugin Sandbox",+ driver: "sandbox",+ config: providerConfig,+ };+ await environmentService(db).update(environment.id, {+ driver: "sandbox",+ name: environment.name,+ config: providerConfig,+ });+ await db.insert(plugins).values({+ id: pluginId,+ pluginKey: "acme.reusable-sandbox-provider",+ packageName: "@acme/reusable-sandbox-provider",+ version: "1.0.0",+ apiVersion: 1,+ categories: ["automation"],+ manifestJson: {+ id: "acme.reusable-sandbox-provider",+ apiVersion: 1,+ version: "1.0.0",+ displayName: "Reusable Sandbox Provider",+ description: "Test provider with reusable lease support",+ author: "Paperclip",+ categories: ["automation"],+ capabilities: ["environment.drivers.register"],+ entrypoints: { worker: "dist/worker.js" },+ environmentDrivers: [+ {+ driverKey: "fake-plugin",+ kind: "sandbox_provider",+ displayName: "Fake Plugin",+ supportsReusableLeases: true,+ configSchema: {+ type: "object",+ properties: {+ image: { type: "string" },+ timeoutMs: { type: "number" },+ reuseLease: { type: "boolean" },+ },+ },+ },+ ],+ },+ status: "ready",+ installOrder: 1,+ updatedAt: new Date(),+ } as any);+ const executionWorkspaceId = randomUUID();+ const projectId = randomUUID();+ await db.insert(projects).values({+ id: projectId,+ companyId,+ name: `Workspace ${projectId.slice(0, 8)}`,+ status: "active",+ createdAt: new Date(),+ updatedAt: new Date(),+ });+ await db.insert(executionWorkspaces).values({+ id: executionWorkspaceId,+ companyId,+ projectId,+ mode: "shared_workspace",+ strategyType: "project_primary",+ name: "Reusable workspace",+ status: "active",+ providerType: "local_fs",+ createdAt: new Date(),+ updatedAt: new Date(),+ });+ const reusableLease = await environmentService(db).acquireLease({+ companyId,+ environmentId: environment.id,+ executionWorkspaceId,+ heartbeatRunId: runId,+ leasePolicy: "reuse_by_environment",+ provider: "fake-plugin",+ providerLeaseId: "reusable-plugin-lease",+ metadata: {+ agentId,+ driver: "sandbox",+ pluginId,+ pluginKey: "acme.reusable-sandbox-provider",+ sandboxProviderPlugin: true,+ provider: "fake-plugin",+ image: "fake:test",+ timeoutMs: 1234,+ reuseLease: true,+ reusableSandboxLease: {+ version: 1,+ companyId,+ environmentId: environment.id,+ executionWorkspaceId,+ agentId,+ adapterType: null,+ provider: "fake-plugin",+ runtimeFingerprint: reusableRuntimeFingerprint({+ provider: "fake-plugin",+ adapterType: null,+ config: providerConfig,+ }),+ },+ },+ });++ return { pluginId, companyId, executionWorkspaceId, reusableLease };+ }+it("acquires and releases a local run lease through the runtime seam", async () => {const { companyId, environment, runId } = await seedEnvironment();@@ -415,7 +552,8 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {});expect(acquired.lease.status).toBe("active");- expect(acquired.lease.providerLeaseId).toBe(`sandbox://fake/${environment.id}/workspace/agent`);+ expect(acquired.lease.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`));+ expect(acquired.lease.leasePolicy).toBe("ephemeral");expect(acquired.lease.metadata).toMatchObject({driver: "sandbox",provider: "fake",@@ -942,6 +1080,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {driverKey: "fake-plugin",kind: "sandbox_provider",displayName: "Fake Plugin",+ supportsReusableLeases: true,configSchema: {type: "object",properties: {@@ -979,7 +1118,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {createdAt: new Date(),updatedAt: new Date(),});- await environmentService(db).acquireLease({+ const staleLease = await environmentService(db).acquireLease({companyId,environmentId: environment.id,executionWorkspaceId,@@ -989,10 +1128,28 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {providerLeaseId: "stale-plugin-lease",metadata: {agentId,+ driver: "sandbox",+ pluginId,+ pluginKey: "acme.fake-sandbox-provider",+ sandboxProviderPlugin: true,provider: "fake-plugin",image: "fake:test",timeoutMs: 1234,reuseLease: true,+ reusableSandboxLease: {+ version: 1,+ companyId,+ environmentId: environment.id,+ executionWorkspaceId,+ agentId,+ adapterType: null,+ provider: "fake-plugin",+ runtimeFingerprint: reusableRuntimeFingerprint({+ provider: "fake-plugin",+ adapterType: null,+ config: providerConfig,+ }),+ },},});@@ -1002,6 +1159,9 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {if (method === "environmentResumeLease") {throw new Error("stale sandbox");}+ if (method === "environmentDestroyLease") {+ return undefined;+ }if (method === "environmentAcquireLease") {return {providerLeaseId: "fresh-plugin-lease",@@ -1036,7 +1196,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {driverKey: "fake-plugin",providerLeaseId: "stale-plugin-lease",}), 31234);- expect(workerManager.call).toHaveBeenNthCalledWith(2, pluginId, "environmentAcquireLease", expect.objectContaining({+ expect(workerManager.call).toHaveBeenNthCalledWith(2, pluginId, "environmentDestroyLease", expect.objectContaining({+ driverKey: "fake-plugin",+ providerLeaseId: "stale-plugin-lease",+ }), 31234);+ expect(workerManager.call).toHaveBeenNthCalledWith(3, pluginId, "environmentAcquireLease", expect.objectContaining({driverKey: "fake-plugin",config: {image: "fake:test",@@ -1047,6 +1211,257 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {executionWorkspaceId,runId,}), 31234);+ await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({+ status: "expired",+ cleanupStatus: "success",+ });+ });++ it("does not retain or resume plugin-backed sandbox leases unless the provider opts in", async () => {+ const pluginId = randomUUID();+ const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();+ const providerConfig = {+ provider: "fake-plugin",+ image: "fake:test",+ timeoutMs: 1234,+ reuseLease: true,+ };+ const environment = {+ ...baseEnvironment,+ name: "Non-reusable Plugin Sandbox",+ driver: "sandbox",+ config: providerConfig,+ };+ await environmentService(db).update(environment.id, {+ driver: "sandbox",+ name: environment.name,+ config: providerConfig,+ });+ await db.insert(plugins).values({+ id: pluginId,+ pluginKey: "acme.non-reusable-sandbox-provider",+ packageName: "@acme/non-reusable-sandbox-provider",+ version: "1.0.0",+ apiVersion: 1,+ categories: ["automation"],+ manifestJson: {+ id: "acme.non-reusable-sandbox-provider",+ apiVersion: 1,+ version: "1.0.0",+ displayName: "Non-reusable Sandbox Provider",+ description: "Test provider without reusable lease support",+ author: "Paperclip",+ categories: ["automation"],+ capabilities: ["environment.drivers.register"],+ entrypoints: { worker: "dist/worker.js" },+ environmentDrivers: [+ {+ driverKey: "fake-plugin",+ kind: "sandbox_provider",+ displayName: "Fake Plugin",+ configSchema: { type: "object" },+ },+ ],+ },+ status: "ready",+ installOrder: 1,+ updatedAt: new Date(),+ } as any);+ const executionWorkspaceId = randomUUID();+ const projectId = randomUUID();+ await db.insert(projects).values({+ id: projectId,+ companyId,+ name: `Workspace ${projectId.slice(0, 8)}`,+ status: "active",+ createdAt: new Date(),+ updatedAt: new Date(),+ });+ await db.insert(executionWorkspaces).values({+ id: executionWorkspaceId,+ companyId,+ projectId,+ mode: "shared_workspace",+ strategyType: "project_primary",+ name: "Non-reusable workspace",+ status: "active",+ providerType: "local_fs",+ createdAt: new Date(),+ updatedAt: new Date(),+ });+ await environmentService(db).acquireLease({+ companyId,+ environmentId: environment.id,+ executionWorkspaceId,+ heartbeatRunId: runId,+ leasePolicy: "reuse_by_environment",+ provider: "fake-plugin",+ providerLeaseId: "old-plugin-lease",+ metadata: {+ agentId,+ provider: "fake-plugin",+ image: "fake:test",+ timeoutMs: 1234,+ reuseLease: true,+ reusableSandboxLease: {+ version: 1,+ companyId,+ environmentId: environment.id,+ executionWorkspaceId,+ agentId,+ adapterType: null,+ provider: "fake-plugin",+ runtimeFingerprint: reusableRuntimeFingerprint({+ provider: "fake-plugin",+ adapterType: null,+ config: providerConfig,+ }),+ },+ },+ });++ const workerManager = {+ isRunning: vi.fn((id: string) => id === pluginId),+ call: vi.fn(async (_pluginId: string, method: string) => {+ if (method === "environmentAcquireLease") {+ return {+ providerLeaseId: "fresh-plugin-lease",+ metadata: {+ provider: "fake-plugin",+ image: "fake:test",+ timeoutMs: 1234,+ reuseLease: true,+ remoteCwd: "/workspace",+ },+ };+ }+ throw new Error(`Unexpected plugin method: ${method}`);+ }),+ } as unknown as PluginWorkerManager;+ const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });++ const acquired = await runtimeWithPlugin.acquireRunLease({+ companyId,+ environment,+ issueId: null,+ agentId,+ heartbeatRunId: runId,+ persistedExecutionWorkspace: {+ id: executionWorkspaceId,+ mode: "shared_workspace",+ },+ });++ expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease");+ expect(acquired.lease.leasePolicy).toBe("ephemeral");+ expect(workerManager.call).toHaveBeenCalledTimes(1);+ expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234);+ });++ it("destroys scoped reusable plugin-backed sandbox leases", async () => {+ const { pluginId, companyId, executionWorkspaceId, reusableLease } =+ await seedReusablePluginSandboxLease();++ const workerManager = {+ isRunning: vi.fn((id: string) => id === pluginId),+ call: vi.fn(async (_pluginId: string, method: string) => {+ if (method === "environmentDestroyLease") {+ return undefined;+ }+ throw new Error(`Unexpected plugin method: ${method}`);+ }),+ } as unknown as PluginWorkerManager;+ const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });++ const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({+ companyId,+ executionWorkspaceId,+ failureReason: "execution_workspace_closed",+ });++ expect(destroyed).toHaveLength(1);+ expect(destroyed[0]?.lease.id).toBe(reusableLease.id);+ expect(destroyed[0]?.lease.status).toBe("expired");+ expect(workerManager.call).toHaveBeenCalledWith(+ pluginId,+ "environmentDestroyLease",+ expect.objectContaining({+ driverKey: "fake-plugin",+ providerLeaseId: "reusable-plugin-lease",+ }),+ 31234,+ );+ await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({+ status: "expired",+ failureReason: "execution_workspace_closed",+ cleanupStatus: "success",+ });+ });++ it("retries reusable plugin-backed sandbox destroy when the worker is unavailable", async () => {+ const { pluginId, companyId, executionWorkspaceId, reusableLease } =+ await seedReusablePluginSandboxLease();++ const offlineWorkerManager = {+ isRunning: vi.fn(() => false),+ call: vi.fn(),+ } as unknown as PluginWorkerManager;+ const runtimeWithOfflinePlugin = environmentRuntimeService(db, {+ pluginWorkerManager: offlineWorkerManager,+ });++ const pending = await runtimeWithOfflinePlugin.destroyReusableSandboxLeases({+ companyId,+ executionWorkspaceId,+ failureReason: "execution_workspace_closed",+ });++ expect(pending).toHaveLength(1);+ expect(pending[0]?.lease.id).toBe(reusableLease.id);+ expect(pending[0]?.lease.status).toBe("pending_cleanup");+ expect(offlineWorkerManager.call).not.toHaveBeenCalled();+ await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({+ status: "pending_cleanup",+ failureReason: "execution_workspace_closed",+ cleanupStatus: "failed",+ });++ const recoveredWorkerManager = {+ isRunning: vi.fn((id: string) => id === pluginId),+ call: vi.fn(async (_pluginId: string, method: string) => {+ if (method === "environmentDestroyLease") {+ return undefined;+ }+ throw new Error(`Unexpected plugin method: ${method}`);+ }),+ } as unknown as PluginWorkerManager;+ const runtimeWithRecoveredPlugin = environmentRuntimeService(db, {+ pluginWorkerManager: recoveredWorkerManager,+ });++ const retried = await runtimeWithRecoveredPlugin.destroyReusableSandboxLeases({+ companyId,+ executionWorkspaceId,+ failureReason: "cleanup_retry",+ });++ expect(retried).toHaveLength(1);+ expect(retried[0]?.lease.id).toBe(reusableLease.id);+ expect(retried[0]?.lease.status).toBe("expired");+ expect(recoveredWorkerManager.call).toHaveBeenCalledWith(+ pluginId,+ "environmentDestroyLease",+ expect.objectContaining({+ driverKey: "fake-plugin",+ providerLeaseId: "reusable-plugin-lease",+ }),+ 31234,+ );+ await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({+ status: "expired",+ failureReason: "cleanup_retry",+ cleanupStatus: "success",+ });});it("releases a sandbox run lease from metadata after the environment config changes", async () => {diff --git a/server/src/__tests__/sandbox-provider-runtime.test.ts b/server/src/__tests__/sandbox-provider-runtime.test.tsindex 191975230e..3552f395f7 100644--- a/server/src/__tests__/sandbox-provider-runtime.test.ts+++ b/server/src/__tests__/sandbox-provider-runtime.test.ts@@ -44,7 +44,7 @@ describe("sandbox provider runtime", () => {})).rejects.toThrow('Sandbox provider "fake-plugin" is not registered as a built-in provider.');});- it("acquires and resumes fake leases deterministically", async () => {+ it("does not resume fake leases because the built-in fake provider does not opt in", async () => {const lease = await acquireSandboxProviderLease({config: {provider: "fake",@@ -56,7 +56,7 @@ describe("sandbox provider runtime", () => {issueId: "issue-1",});- expect(lease.providerLeaseId).toBe("sandbox://fake/env-1/workspace/agent");+ expect(lease.providerLeaseId).toMatch(/^sandbox:\/\/fake\/run-1\/[0-9a-f-]+$/);expect(lease.metadata).toEqual(expect.objectContaining({provider: "fake",image: "ubuntu:24.04",@@ -75,8 +75,9 @@ describe("sandbox provider runtime", () => {reusableProviderLeaseId: lease.providerLeaseId,});- expect(resumed.providerLeaseId).toBe(lease.providerLeaseId);- expect(resumed.metadata).toEqual(expect.objectContaining({ resumedLease: true }));+ expect(resumed.providerLeaseId).toMatch(/^sandbox:\/\/fake\/run-2\/[0-9a-f-]+$/);+ expect(resumed.providerLeaseId).not.toBe(lease.providerLeaseId);+ expect(resumed.metadata).not.toEqual(expect.objectContaining({ resumedLease: true }));});it("matches reusable fake leases through the selected provider implementation", () => {diff --git a/server/src/app.ts b/server/src/app.tsindex 6658ecfa2b..efd9a74c9a 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -233,7 +233,7 @@ export async function createApp(api.use(fileResourceRoutes(db));api.use(routineRoutes(db, { pluginWorkerManager: workerManager }));api.use(environmentRoutes(db, { pluginWorkerManager: workerManager }));- api.use(executionWorkspaceRoutes(db));+ api.use(executionWorkspaceRoutes(db, { pluginWorkerManager: workerManager }));api.use(goalRoutes(db));api.use(boardChatRoutes(db, { deploymentMode: opts.deploymentMode }));api.use(approvalRoutes(db, { pluginWorkerManager: workerManager }));diff --git a/server/src/routes/execution-workspaces.ts b/server/src/routes/execution-workspaces.tsindex bbe80687cb..bf79b44455 100644--- a/server/src/routes/execution-workspaces.ts+++ b/server/src/routes/execution-workspaces.ts@@ -30,14 +30,19 @@ import {} from "./workspace-command-authz.js";import { assertCanManageExecutionWorkspaceRuntimeServices } from "./workspace-runtime-service-authz.js";import { appendWithCap } from "../adapters/utils.js";+import { environmentRuntimeService } from "../services/environment-runtime.js";+import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";const WORKSPACE_CONTROL_OUTPUT_MAX_CHARS = 256 * 1024;-export function executionWorkspaceRoutes(db: Db) {+export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: PluginWorkerManager } = {}) {const router = Router();const svc = executionWorkspaceService(db);const access = accessService(db);const workspaceOperationsSvc = workspaceOperationService(db);+ const environmentRuntime = environmentRuntimeService(db, {+ pluginWorkerManager: opts.pluginWorkerManager,+ });async function assertExecutionWorkspaceReadAllowed(req: Request, res: Response, companyId: string) {const decision = await access.decide({@@ -538,6 +543,12 @@ export function executionWorkspaceRoutes(db: Db) {}workspace = archivedWorkspace;+ await environmentRuntime.destroyReusableSandboxLeases({+ companyId: existing.companyId,+ executionWorkspaceId: existing.id,+ failureReason: "execution_workspace_closed",+ });+if (existing.mode === "shared_workspace") {await db.update(issues)diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.tsindex fbce62ccd7..21b7d01932 100644--- a/server/src/routes/issues.ts+++ b/server/src/routes/issues.ts@@ -121,6 +121,7 @@ import { feedbackService } from "../services/feedback.js";import { instanceSettingsService } from "../services/instance-settings.js";import { readAcceptedPlanConfirmationTarget } from "../services/issues.js";import { environmentService } from "../services/environments.js";+import { environmentRuntimeService } from "../services/environment-runtime.js";import { redactSensitiveText } from "../redaction.js";import {createCompanySearchRateLimiter,@@ -1107,6 +1108,9 @@ export function issueRoutes(const routinesSvc = routineService(db, {pluginWorkerManager: opts.pluginWorkerManager,});+ const environmentRuntime = environmentRuntimeService(db, {+ pluginWorkerManager: opts.pluginWorkerManager,+ });const issueTreeControlFactory = Object.prototype.hasOwnProperty.call(serviceIndex,"issueTreeControlService",@@ -2793,6 +2797,27 @@ export function issueRoutes(});}+ async function destroyReusableSandboxLeasesForTerminalIssue(issue: {+ id: string;+ companyId: string;+ status: string;+ executionWorkspaceId?: string | null;+ }) {+ try {+ await environmentRuntime.destroyReusableSandboxLeases({+ companyId: issue.companyId,+ issueId: issue.id,+ executionWorkspaceId: issue.executionWorkspaceId ?? null,+ failureReason: `issue_terminal_${issue.status}`,+ });+ } catch (err) {+ logger.warn(+ { err, issueId: issue.id, executionWorkspaceId: issue.executionWorkspaceId ?? null },+ "failed to destroy reusable sandbox leases for terminal issue",+ );+ }+ }+async function resolveIssueRouteId(rawId: string): Promise<string> {const identifier = normalizeIssueReferenceIdentifier(rawId);if (identifier) {@@ -6492,6 +6517,9 @@ export function issueRoutes(const becameTerminal =!["done", "cancelled"].includes(existing.status) && ["done", "cancelled"].includes(issue.status);+ if (becameTerminal) {+ await destroyReusableSandboxLeasesForTerminalIssue(issue);+ }if (becameTerminal && issue.parentId) {const parent = await svc.getWakeableParentAfterChildCompletion(issue.parentId);if (parent) {@@ -7883,6 +7911,9 @@ export function issueRoutes(const becameTerminal =!["done", "cancelled"].includes(issueBeforeCommentDecision.status) &&["done", "cancelled"].includes(currentIssue.status);+ if (becameTerminal) {+ await destroyReusableSandboxLeasesForTerminalIssue(currentIssue);+ }if (becameTerminal && currentIssue.parentId) {const parent = await svc.getWakeableParentAfterChildCompletion(currentIssue.parentId);if (parent) {diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.tsindex 496c85c20b..c5323085e5 100644--- a/server/src/services/environment-runtime.ts+++ b/server/src/services/environment-runtime.ts@@ -1,4 +1,4 @@-import { randomUUID } from "node:crypto";+import { createHash, randomUUID } from "node:crypto";import { and, eq, inArray } from "drizzle-orm";import type { Db } from "@paperclipai/db";import { environmentLeases } from "@paperclipai/db";@@ -24,7 +24,9 @@ import {} from "./environment-config.js";import {acquireSandboxProviderLease,+ destroySandboxProviderLease,findReusableSandboxProviderLeaseId,+ getSandboxProvider as getBuiltinSandboxProvider,isBuiltinSandboxProvider,releaseSandboxProviderLease,sandboxConfigFromLeaseMetadata,@@ -149,6 +151,7 @@ function resolvePluginSandboxRpcTimeoutMs(config: Record<string, unknown>): numbexport interface EnvironmentDriverLeaseInput {environment: Environment;lease: EnvironmentLease;+ failureReason?: string;}export interface EnvironmentDriverRealizeWorkspaceInput extends EnvironmentDriverLeaseInput {@@ -197,6 +200,118 @@ function getLeaseDriverKey(lease: Pick<EnvironmentLease, "metadata">, environmenreturn leaseDriver ?? environment.driver;}+function toEnvironmentLeaseSnapshot(row: typeof environmentLeases.$inferSelect): EnvironmentLease {+ return {+ id: row.id,+ companyId: row.companyId,+ environmentId: row.environmentId,+ executionWorkspaceId: row.executionWorkspaceId ?? null,+ issueId: row.issueId ?? null,+ heartbeatRunId: row.heartbeatRunId ?? null,+ status: row.status as EnvironmentLease["status"],+ leasePolicy: row.leasePolicy as EnvironmentLease["leasePolicy"],+ provider: row.provider ?? null,+ providerLeaseId: row.providerLeaseId ?? null,+ acquiredAt: row.acquiredAt,+ lastUsedAt: row.lastUsedAt,+ expiresAt: row.expiresAt ?? null,+ releasedAt: row.releasedAt ?? null,+ failureReason: row.failureReason ?? null,+ cleanupStatus: row.cleanupStatus as EnvironmentLease["cleanupStatus"],+ metadata: (row.metadata as Record<string, unknown> | null) ?? null,+ createdAt: row.createdAt,+ updatedAt: row.updatedAt,+ };+}++function isRecord(value: unknown): value is Record<string, unknown> {+ return Boolean(value) && typeof value === "object" && !Array.isArray(value);+}++function stableStringify(value: unknown): string {+ if (Array.isArray(value)) {+ return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;+ }+ if (isRecord(value)) {+ return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;+ }+ return JSON.stringify(value) ?? "null";+}++function reusableRuntimeFingerprint(input: {+ provider: string;+ adapterType: string | null;+ config: Record<string, unknown>;+}): string {+ return createHash("sha256")+ .update(stableStringify(input))+ .digest("hex");+}++function buildReusableSandboxLeaseScope(input: {+ companyId: string;+ environmentId: string;+ executionWorkspaceId: string | null;+ agentId: string | null;+ adapterType: string | null;+ provider: string;+ config: Record<string, unknown>;+ providerMetadata?: Record<string, unknown> | null;+}): Record<string, unknown> | null {+ if (!input.executionWorkspaceId || !input.agentId) return null;+ const providerMetadata = input.providerMetadata ?? {};+ const adapterType = input.adapterType ?? null;+ const remoteCwd = readString(providerMetadata.remoteCwd);+ const workspaceSentinel = isRecord(providerMetadata.workspaceSentinel)+ ? { ...providerMetadata.workspaceSentinel }+ : null;+ return {+ version: 1,+ companyId: input.companyId,+ environmentId: input.environmentId,+ executionWorkspaceId: input.executionWorkspaceId,+ agentId: input.agentId,+ adapterType,+ provider: input.provider,+ runtimeFingerprint: reusableRuntimeFingerprint({+ provider: input.provider,+ adapterType,+ config: input.config,+ }),+ ...(remoteCwd ? { remoteCwd } : {}),+ ...(workspaceSentinel ? { workspaceSentinel } : {}),+ };+}++function reusableSandboxLeaseScopeMatches(input: {+ lease: Pick<EnvironmentLease, "metadata">;+ companyId: string;+ environmentId: string;+ executionWorkspaceId: string | null;+ agentId: string | null;+ adapterType: string | null;+ provider: string;+ config: Record<string, unknown>;+}): boolean {+ if (!input.executionWorkspaceId || !input.agentId) return false;+ const scope = input.lease.metadata?.reusableSandboxLease;+ if (!isRecord(scope)) return false;+ const adapterType = input.adapterType ?? null;+ return (+ scope.companyId === input.companyId &&+ scope.environmentId === input.environmentId &&+ scope.executionWorkspaceId === input.executionWorkspaceId &&+ scope.agentId === input.agentId &&+ scope.adapterType === adapterType &&+ scope.provider === input.provider &&+ scope.runtimeFingerprint === reusableRuntimeFingerprint({+ provider: input.provider,+ adapterType,+ config: input.config,+ })+ );+}+export function findReusableSandboxLeaseId(input: {config: SandboxEnvironmentConfig;leases: Array<Pick<EnvironmentLease, "providerLeaseId" | "metadata">>;@@ -449,13 +564,15 @@ function createSandboxEnvironmentDriver(const workerConfig = stripSandboxProviderEnvelope(parsed.config);const storedConfig = storedParsed.config;+ const supportsReusableLeases = pluginProvider.resolved.driver.supportsReusableLeases === true;// Ad-hoc tests (heartbeatRunId === null) must never resume an existing// provider lease. If they did, releasing the test lease at the end of// the probe would tear down the live heartbeat run that owns it.// We also filter out leases whose policy is not reuse_by_environment- // so any non-reusable lease (including ad-hoc test leases that- // landed in the table from older code paths) cannot be matched.+ // and whose status is not reusable so non-reusable, cleanup-pending,+ // or terminal rows cannot be matched.const reusableExistingLeases =+ supportsReusableLeases &&parsed.config.reuseLease &&input.heartbeatRunId !== null &&input.executionWorkspaceId !== null &&@@ -463,11 +580,23 @@ function createSandboxEnvironmentDriver(? (await environmentsSvc.listLeases(input.environment.id)).filter((lease) =>lease.leasePolicy === "reuse_by_environment" &&+ ["active", "released", "retained"].includes(lease.status) &&lease.executionWorkspaceId === input.executionWorkspaceId &&- lease.metadata?.agentId === input.agentId,+ lease.metadata?.agentId === input.agentId &&+ reusableSandboxLeaseScopeMatches({+ lease,+ companyId: input.companyId,+ environmentId: input.environment.id,+ executionWorkspaceId: input.executionWorkspaceId,+ agentId: input.agentId,+ adapterType: input.adapterType,+ provider: parsed.config.provider,+ config: sandboxConfigForLeaseMetadata(storedConfig),+ }),): [];const reusableProviderLeaseId =+ supportsReusableLeases &&parsed.config.reuseLease &&input.heartbeatRunId !== null &&input.executionWorkspaceId !== null &&@@ -478,26 +607,38 @@ function createSandboxEnvironmentDriver(? reusableExistingLeases.find((lease) => lease.providerLeaseId === reusableProviderLeaseId): null;- const providerLease = reusableLease?.providerLeaseId- ? await pluginWorkerManager.call(- pluginProvider.resolved.plugin.id,- "environmentResumeLease",- {- driverKey: parsed.config.provider,- companyId: input.companyId,- environmentId: input.environment.id,- issueId: input.issueId,- config: workerConfig,- providerLeaseId: reusableLease.providerLeaseId,- leaseMetadata: reusableLease.metadata ?? undefined,- },- resolvePluginSandboxRpcTimeoutMs(workerConfig),- ).then((resumed) =>+ let providerLease: PluginEnvironmentLease | null = null;+ if (reusableLease?.providerLeaseId) {+ try {+ const resumed = await pluginWorkerManager.call(+ pluginProvider.resolved.plugin.id,+ "environmentResumeLease",+ {+ driverKey: parsed.config.provider,+ companyId: input.companyId,+ environmentId: input.environment.id,+ issueId: input.issueId,+ config: workerConfig,+ providerLeaseId: reusableLease.providerLeaseId,+ leaseMetadata: reusableLease.metadata ?? undefined,+ },+ resolvePluginSandboxRpcTimeoutMs(workerConfig),+ );+ providerLease =typeof resumed.providerLeaseId === "string" && resumed.providerLeaseId.length > 0? resumed- : null,- ).catch(() => null)- : null;+ : null;+ } catch {+ providerLease = null;+ }+ if (!providerLease) {+ await destroyReusableSandboxLease({+ environment: input.environment,+ lease: reusableLease,+ failureReason: "resume_failed",+ });+ }+ }const acquiredLease = providerLease ?? await pluginWorkerManager.call(pluginProvider.resolved.plugin.id,"environmentAcquireLease",@@ -529,9 +670,25 @@ function createSandboxEnvironmentDriver(// Ad-hoc test leases are never publishable for reuse: storing them// as `reuse_by_environment` would let a concurrent heartbeat resume// the test's provider lease and lose its sandbox when the test ends.- const resolvedLeasePolicy = parsed.config.reuseLease && input.heartbeatRunId !== null+ const resolvedLeasePolicy = supportsReusableLeases && parsed.config.reuseLease && input.heartbeatRunId !== null? "reuse_by_environment": "ephemeral";+ const sanitizedProviderMetadata = stripSecretRefValuesFromPluginLeaseMetadata({+ metadata: acquiredLease.metadata,+ schema: pluginProvider.resolved.driver.configSchema as Record<string, unknown> | null | undefined,+ });+ const reusableScope = resolvedLeasePolicy === "reuse_by_environment"+ ? buildReusableSandboxLeaseScope({+ companyId: input.companyId,+ environmentId: input.environment.id,+ executionWorkspaceId: input.executionWorkspaceId,+ agentId: input.agentId,+ adapterType: input.adapterType,+ provider: parsed.config.provider,+ config: sandboxConfigForLeaseMetadata(storedConfig),+ providerMetadata: sanitizedProviderMetadata,+ })+ : null;return await environmentsSvc.acquireLease({companyId: input.companyId,@@ -551,10 +708,8 @@ function createSandboxEnvironmentDriver(pluginKey: pluginProvider.resolved.plugin.pluginKey,sandboxProviderPlugin: true,...sandboxConfigForLeaseMetadata(storedConfig),- ...stripSecretRefValuesFromPluginLeaseMetadata({- metadata: acquiredLease.metadata,- schema: pluginProvider.resolved.driver.configSchema as Record<string, unknown> | null | undefined,- }),+ ...sanitizedProviderMetadata,+ ...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),},});}@@ -562,9 +717,12 @@ function createSandboxEnvironmentDriver(// Built-in sandbox provider path. Same guard as the plugin-backed path:// ad-hoc tests (heartbeatRunId === null) must never resume an existing// provider lease, or releasing the test lease will terminate the live- // heartbeat run that shares it. Filter to leases whose policy is- // reuse_by_environment so non-reusable rows can never be matched.+ // heartbeat run that shares it. Filter to reusable policies and statuses+ // so non-reusable, cleanup-pending, or terminal rows can never be matched.+ const builtinSandboxProvider = getBuiltinSandboxProvider(parsed.config.provider);+ const supportsReusableLeases = builtinSandboxProvider?.supportsReusableLeases === true;const reusableProviderLeaseId =+ supportsReusableLeases &&parsed.config.reuseLease &&input.heartbeatRunId !== null &&input.executionWorkspaceId !== null &&@@ -576,28 +734,73 @@ function createSandboxEnvironmentDriver(config: parsed.config,leases: leases.filter((lease) =>lease.leasePolicy === "reuse_by_environment" &&+ ["active", "released", "retained"].includes(lease.status) &&lease.executionWorkspaceId === input.executionWorkspaceId &&- lease.metadata?.agentId === input.agentId,+ lease.metadata?.agentId === input.agentId &&+ reusableSandboxLeaseScopeMatches({+ lease,+ companyId: input.companyId,+ environmentId: input.environment.id,+ executionWorkspaceId: input.executionWorkspaceId,+ agentId: input.agentId,+ adapterType: input.adapterType,+ provider: parsed.config.provider,+ config: sandboxConfigForLeaseMetadata(parsed.config),+ }),),}),)): null;+ const reusableLease = reusableProviderLeaseId+ ? (await environmentsSvc.listLeases(input.environment.id)).find((lease) => lease.providerLeaseId === reusableProviderLeaseId)+ : null;- const providerLease = await acquireSandboxProviderLease({- config: parsed.config,- environmentId: input.environment.id,- heartbeatRunId: input.heartbeatRunId ?? randomUUID(),- issueId: input.issueId,- agentId: input.agentId,- executionWorkspaceId: input.executionWorkspaceId,- reusableProviderLeaseId,- });+ let providerLease;+ try {+ providerLease = await acquireSandboxProviderLease({+ config: parsed.config,+ environmentId: input.environment.id,+ heartbeatRunId: input.heartbeatRunId ?? randomUUID(),+ issueId: input.issueId,+ agentId: input.agentId,+ executionWorkspaceId: input.executionWorkspaceId,+ reusableProviderLeaseId,+ });+ } catch (error) {+ if (reusableLease) {+ await destroyReusableSandboxLease({+ environment: input.environment,+ lease: reusableLease,+ failureReason: "resume_failed",+ });+ }+ throw error;+ }+ if (reusableLease && providerLease.providerLeaseId !== reusableLease.providerLeaseId) {+ await destroyReusableSandboxLease({+ environment: input.environment,+ lease: reusableLease,+ failureReason: "resume_failed",+ });+ }// Same ephemeral-policy-for-tests guard as the plugin-backed path:// ad-hoc test leases must not be publishable for reuse.- const resolvedLeasePolicy = parsed.config.reuseLease && input.heartbeatRunId !== null+ const resolvedLeasePolicy = supportsReusableLeases && parsed.config.reuseLease && input.heartbeatRunId !== null? "reuse_by_environment": "ephemeral";+ const reusableScope = resolvedLeasePolicy === "reuse_by_environment"+ ? buildReusableSandboxLeaseScope({+ companyId: input.companyId,+ environmentId: input.environment.id,+ executionWorkspaceId: input.executionWorkspaceId,+ agentId: input.agentId,+ adapterType: input.adapterType,+ provider: parsed.config.provider,+ config: sandboxConfigForLeaseMetadata(parsed.config),+ providerMetadata: providerLease.metadata,+ })+ : null;return await environmentsSvc.acquireLease({companyId: input.companyId,@@ -613,11 +816,20 @@ function createSandboxEnvironmentDriver(driver: input.environment.driver,executionWorkspaceMode: input.executionWorkspaceMode,...providerLease.metadata,+ ...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),},});},async releaseRunLease(input) {+ if (input.status === "expired" && input.lease.leasePolicy === "reuse_by_environment") {+ return await destroyReusableSandboxLease({+ environment: input.environment,+ lease: input.lease,+ failureReason: "lease_expired",+ });+ }+// Check if this lease was acquired through a plugin.if (input.lease.metadata?.sandboxProviderPlugin) {return await releasePluginBackedSandboxLease(input);@@ -748,6 +960,14 @@ function createSandboxEnvironmentDriver(}throw new Error("Sandbox driver does not support direct command execution for built-in providers.");},++ async destroyRunLease(input) {+ return await destroyReusableSandboxLease({+ environment: input.environment,+ lease: input.lease,+ failureReason: input.failureReason ?? "lease_destroyed",+ });+ },};async function releasePluginBackedSandboxLease(@@ -790,6 +1010,68 @@ function createSandboxEnvironmentDriver(cleanupStatus,});}++ async function destroyReusableSandboxLease(input: {+ environment: Environment;+ lease: EnvironmentLease;+ failureReason: string;+ }): Promise<EnvironmentLease | null> {+ let cleanupStatus: "success" | "failed" = "success";+ const metadata = input.lease.metadata ?? {};++ try {+ if (metadata.sandboxProviderPlugin) {+ const pluginId = readString(metadata.pluginId);+ const providerKey = readString(metadata.provider);+ if (!pluginId || !providerKey || !pluginWorkerManager?.isRunning(pluginId)) {+ cleanupStatus = "failed";+ } else {+ const config = await resolvePluginSandboxRuntimeConfig({+ environment: input.environment,+ lease: input.lease,+ provider: providerKey,+ });+ await pluginWorkerManager.call(pluginId, "environmentDestroyLease", {+ driverKey: providerKey,+ companyId: input.lease.companyId,+ environmentId: input.environment.id,+ issueId: input.lease.issueId,+ config: stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig),+ providerLeaseId: input.lease.providerLeaseId,+ leaseMetadata: metadata,+ }, resolvePluginSandboxRpcTimeoutMs(stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig)));+ }+ } else {+ const metadataConfig = sandboxConfigFromLeaseMetadata(input.lease);+ const parsed = metadataConfig+ ? await resolveEnvironmentDriverConfigForRuntime(db, input.lease.companyId, {+ id: input.environment.id,+ driver: "sandbox",+ config: metadataConfig as unknown as Record<string, unknown>,+ })+ : await resolveEnvironmentDriverConfigForRuntime(db, input.lease.companyId, input.environment);+ if (parsed.driver !== "sandbox") {+ cleanupStatus = "failed";+ } else {+ await destroySandboxProviderLease({+ config: parsed.config,+ providerLeaseId: input.lease.providerLeaseId,+ });+ }+ }+ } catch {+ cleanupStatus = "failed";+ }++ return await environmentsSvc.releaseLease(+ input.lease.id,+ cleanupStatus === "success" ? "expired" : "pending_cleanup",+ {+ failureReason: input.failureReason,+ cleanupStatus,+ },+ );+ }}function parseExpiresAt(value: string | null | undefined): Date | null {@@ -1026,7 +1308,9 @@ function createPluginEnvironmentDriver(providerLeaseId: input.lease.providerLeaseId,leaseMetadata: input.lease.metadata ?? undefined,});- return await environmentsSvc.releaseLease(input.lease.id, "failed");+ return await environmentsSvc.releaseLease(input.lease.id, "failed", {+ failureReason: input.failureReason ?? "lease_destroyed",+ });},async realizeWorkspace(input) {@@ -1213,27 +1497,7 @@ export function environmentRuntimeService(const environment = await environmentsSvc.getById(leaseRow.environmentId);if (!environment) continue;- const leaseSnapshot: EnvironmentLease = {- id: leaseRow.id,- companyId: leaseRow.companyId,- environmentId: leaseRow.environmentId,- executionWorkspaceId: leaseRow.executionWorkspaceId ?? null,- issueId: leaseRow.issueId ?? null,- heartbeatRunId: leaseRow.heartbeatRunId ?? null,- status: leaseRow.status as EnvironmentLease["status"],- leasePolicy: leaseRow.leasePolicy as EnvironmentLease["leasePolicy"],- provider: leaseRow.provider ?? null,- providerLeaseId: leaseRow.providerLeaseId ?? null,- acquiredAt: leaseRow.acquiredAt,- lastUsedAt: leaseRow.lastUsedAt,- expiresAt: leaseRow.expiresAt ?? null,- releasedAt: leaseRow.releasedAt ?? null,- failureReason: leaseRow.failureReason ?? null,- cleanupStatus: leaseRow.cleanupStatus as EnvironmentLease["cleanupStatus"],- metadata: (leaseRow.metadata as Record<string, unknown> | null) ?? null,- createdAt: leaseRow.createdAt,- updatedAt: leaseRow.updatedAt,- };+ const leaseSnapshot = toEnvironmentLeaseSnapshot(leaseRow);const driver = getDriver(getLeaseDriverKey(leaseSnapshot, environment));const lease = driver? await driver.releaseRunLease({@@ -1258,6 +1522,60 @@ export function environmentRuntimeService(return released;},+ async destroyReusableSandboxLeases(input: {+ companyId: string;+ issueId?: string | null;+ executionWorkspaceId?: string | null;+ failureReason?: string;+ }): Promise<EnvironmentRuntimeLeaseRecord[]> {+ const scopeConditions = [+ input.issueId ? eq(environmentLeases.issueId, input.issueId) : undefined,+ input.executionWorkspaceId ? eq(environmentLeases.executionWorkspaceId, input.executionWorkspaceId) : undefined,+ ].filter((condition): condition is NonNullable<typeof condition> => Boolean(condition));+ if (scopeConditions.length === 0) return [];++ const leaseRows = await db+ .select()+ .from(environmentLeases)+ .where(+ and(+ eq(environmentLeases.companyId, input.companyId),+ eq(environmentLeases.leasePolicy, "reuse_by_environment"),+ inArray(environmentLeases.status, ["active", "released", "retained", "pending_cleanup"]),+ ...scopeConditions,+ ),+ );++ const destroyed: EnvironmentRuntimeLeaseRecord[] = [];+ for (const leaseRow of leaseRows) {+ const environment = await environmentsSvc.getById(leaseRow.environmentId);+ if (!environment) continue;+ const leaseSnapshot = toEnvironmentLeaseSnapshot(leaseRow);+ const driver = getDriver(getLeaseDriverKey(leaseSnapshot, environment));+ const lease = driver?.destroyRunLease+ ? await driver.destroyRunLease({+ environment,+ lease: leaseSnapshot,+ failureReason: input.failureReason ?? "reusable_lease_destroyed",+ })+ : await environmentsSvc.releaseLease(leaseSnapshot.id, "pending_cleanup", {+ failureReason: input.failureReason ?? "reusable_lease_destroyed",+ cleanupStatus: "failed",+ });+ if (!lease) continue;+ destroyed.push({+ environment,+ lease,+ leaseContext: {+ executionWorkspaceId: lease.executionWorkspaceId,+ executionWorkspaceMode:+ (lease.metadata?.executionWorkspaceMode as ExecutionWorkspace["mode"] | null | undefined) ?? null,+ },+ });+ }+ return destroyed;+ },+async resumeRunLease(input: EnvironmentDriverLeaseInput): Promise<PluginEnvironmentLease | EnvironmentLease | null> {const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment));if (!driver.resumeRunLease) {diff --git a/server/src/services/environments.ts b/server/src/services/environments.tsindex c746389a6c..c68dad20e9 100644--- a/server/src/services/environments.ts+++ b/server/src/services/environments.ts@@ -499,7 +499,7 @@ export function environmentService(db: Db) {releaseLease: async (id: string,- status: Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed" | "retained"> = "released",+ status: Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed" | "retained" | "pending_cleanup"> = "released",options?: {failureReason?: string;cleanupStatus?: EnvironmentLeaseCleanupStatus;diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex 3544d3307d..f097ecff1a 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -841,6 +841,7 @@ async function resolveRunScopedMentionedSkillKeys(input: {function leaseReleaseStatusForRunStatus(status: string | null | undefined,): Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed"> {+ if (status === "cancelled") return "expired";return status === "failed" || status === "timed_out" ? "failed" : "released";}diff --git a/server/src/services/sandbox-provider-runtime.ts b/server/src/services/sandbox-provider-runtime.tsindex e664fe00d5..0b089fdd6f 100644--- a/server/src/services/sandbox-provider-runtime.ts+++ b/server/src/services/sandbox-provider-runtime.ts@@ -78,6 +78,7 @@ export interface SandboxExecuteResult {export interface SandboxProvider {readonly provider: SandboxEnvironmentProvider;+ readonly supportsReusableLeases?: boolean;validateConfig(config: SandboxEnvironmentConfig): Promise<SandboxProviderValidationResult>;probe(config: SandboxEnvironmentConfig): Promise<EnvironmentProbeResult>;acquireLease(input: AcquireSandboxLeaseInput): Promise<SandboxLeaseHandle>;@@ -117,6 +118,7 @@ function buildFakeSandboxProbe(config: FakeSandboxEnvironmentConfig): Environmenclass FakeSandboxProvider implements SandboxProvider {readonly provider = "fake" as const;+ readonly supportsReusableLeases = false;async validateConfig(config: SandboxEnvironmentConfig): Promise<SandboxProviderValidationResult> {assertProviderConfig<FakeSandboxEnvironmentConfig>(this.provider, config);@@ -138,7 +140,7 @@ class FakeSandboxProvider implements SandboxProvider {async acquireLease(input: AcquireSandboxLeaseInput): Promise<SandboxLeaseHandle> {assertProviderConfig<FakeSandboxEnvironmentConfig>(this.provider, input.config);- const providerLeaseId = input.config.reuseLease+ const providerLeaseId = input.config.reuseLease && this.supportsReusableLeases? `sandbox://fake/${input.environmentId}/${input.executionWorkspaceId ?? "workspace"}/${input.agentId ?? "agent"}`: `sandbox://fake/${input.heartbeatRunId}/${randomUUID()}`;@@ -336,7 +338,7 @@ export async function acquireSandboxProviderLease(input: {reusableProviderLeaseId?: string | null;}): Promise<SandboxLeaseHandle> {const provider = requireSandboxProvider(input.config.provider);- if (input.config.reuseLease && input.reusableProviderLeaseId) {+ if (provider.supportsReusableLeases && input.config.reuseLease && input.reusableProviderLeaseId) {const resumedLease = await provider.resumeLease({config: input.config,providerLeaseId: input.reusableProviderLeaseId,