Skip to content

The open-source app everyone uses to manage agents at work

feat: reuse Daytona sandbox leases (#8513)

Devin Foleycd38c150b0e7
diff --git a/packages/plugins/sandbox-providers/daytona/src/manifest.ts b/packages/plugins/sandbox-providers/daytona/src/manifest.ts
index dcc684342e..c78ef2c00f 100644
--- a/packages/plugins/sandbox-providers/daytona/src/manifest.ts
+++ b/packages/plugins/sandbox-providers/daytona/src/manifest.ts
@@ -23,6 +23,7 @@ const manifest: PaperclipPluginManifestV1 = {
displayName: "Daytona Sandbox",
description:
"Provisions Daytona sandboxes with configurable image or snapshot selection, startup timeouts, and lease reuse.",
+ supportsReusableLeases: true,
configSchema: {
type: "object",
properties: {
diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
index a9f0022989..1620ca96f3 100644
--- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
@@ -174,6 +174,9 @@ describe("Daytona sandbox provider plugin", () => {
companyId: "company-1",
environmentId: "env-1",
runId: "run-1",
+ agentId: "agent-1",
+ executionWorkspaceId: "workspace-1",
+ adapterType: "codex_local",
config: {
image: "node:20",
timeoutMs: 300000,
@@ -189,8 +192,21 @@ describe("Daytona sandbox provider plugin", () => {
sandboxId: "sandbox-123",
remoteCwd: "/home/daytona/paperclip-workspace",
reuseLease: true,
+ workspaceSentinel: {
+ path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",
+ result: "written",
+ },
},
});
+ expect(sandbox.fs.createFolder).toHaveBeenCalledWith(
+ "/home/daytona/paperclip-workspace/.paperclip-runtime",
+ "755",
+ );
+ expect(sandbox.fs.uploadFile).toHaveBeenCalledWith(
+ expect.any(Buffer),
+ "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",
+ 300,
+ );
});
it("deletes the sandbox if lease setup throws after sandbox creation", async () => {
@@ -287,6 +303,94 @@ describe("Daytona sandbox provider plugin", () => {
});
});
+ it("resumes a reusable lease when the workspace sentinel matches", async () => {
+ process.env.DAYTONA_API_KEY = "host-key";
+ const sandbox = createMockSandbox({ id: "sandbox-reuse", state: "stopped" });
+ sandbox.process.executeCommand
+ .mockResolvedValueOnce({
+ exitCode: 0,
+ result: JSON.stringify({ token: "sentinel-token" }),
+ artifacts: { stdout: JSON.stringify({ token: "sentinel-token" }) },
+ })
+ .mockResolvedValueOnce({
+ exitCode: 0,
+ result: "bash",
+ artifacts: { stdout: "bash" },
+ });
+ mockGet.mockResolvedValue(sandbox);
+
+ const lease = await plugin.definition.onEnvironmentResumeLease?.({
+ driverKey: "daytona",
+ companyId: "company-1",
+ environmentId: "env-1",
+ providerLeaseId: "sandbox-reuse",
+ config: {
+ timeoutMs: 300000,
+ reuseLease: true,
+ },
+ leaseMetadata: {
+ workspaceSentinel: {
+ path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",
+ token: "sentinel-token",
+ result: "written",
+ },
+ },
+ });
+
+ expect(sandbox.start).toHaveBeenCalledWith(300);
+ expect(lease).toMatchObject({
+ providerLeaseId: "sandbox-reuse",
+ metadata: {
+ resumedLease: true,
+ workspaceSentinel: {
+ result: "matched",
+ token: "sentinel-token",
+ },
+ },
+ });
+ });
+
+ it("expires a reusable lease when the workspace sentinel does not match", async () => {
+ process.env.DAYTONA_API_KEY = "host-key";
+ const sandbox = createMockSandbox({ id: "sandbox-reuse", state: "stopped" });
+ sandbox.process.executeCommand.mockResolvedValueOnce({
+ exitCode: 0,
+ result: JSON.stringify({ token: "other-token" }),
+ artifacts: { stdout: JSON.stringify({ token: "other-token" }) },
+ });
+ mockGet.mockResolvedValue(sandbox);
+
+ await expect(plugin.definition.onEnvironmentResumeLease?.({
+ driverKey: "daytona",
+ companyId: "company-1",
+ environmentId: "env-1",
+ providerLeaseId: "sandbox-reuse",
+ config: {
+ timeoutMs: 300000,
+ reuseLease: true,
+ },
+ leaseMetadata: {
+ workspaceSentinel: {
+ path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",
+ token: "sentinel-token",
+ result: "written",
+ },
+ },
+ })).resolves.toEqual({
+ providerLeaseId: null,
+ metadata: {
+ expired: true,
+ workspaceSentinel: {
+ path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",
+ token: "sentinel-token",
+ result: "mismatch",
+ },
+ },
+ });
+
+ expect(sandbox.process.executeCommand).toHaveBeenCalledTimes(1);
+ });
+
it("stops reusable leases and deletes ephemeral leases on release", async () => {
process.env.DAYTONA_API_KEY = "host-key";
const reusable = createMockSandbox({ id: "sandbox-reusable" });
diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts
index 2ea53ba648..cc5b294c1e 100644
--- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts
+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts
@@ -1,5 +1,5 @@
import path from "node:path";
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
import { Daytona, DaytonaNotFoundError, DaytonaTimeoutError } from "@daytonaio/sdk";
import type {
CreateSandboxBaseParams,
@@ -44,6 +44,14 @@ interface DaytonaDriverConfig {
reuseLease: boolean;
}
+type WorkspaceSentinelResult = {
+ path: string;
+ token: string | null;
+ result: "written" | "matched" | "missing" | "mismatch" | "skipped";
+};
+
+const WORKSPACE_SENTINEL_RELATIVE_PATH = ".paperclip-runtime/reusable-sandbox-lease.json";
+
function parseOptionalString(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
@@ -166,6 +174,20 @@ function formatErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function stableStringify(value: unknown): string {
+ if (Array.isArray(value)) {
+ return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
+ }
+ if (isRecord(value)) {
+ return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
+ }
+ return JSON.stringify(value) ?? "null";
+}
+
function isValidUrl(value: string): boolean {
try {
new URL(value);
@@ -210,12 +232,109 @@ async function detectSandboxShellCommand(sandbox: Sandbox, timeoutSeconds: numbe
}
}
+function workspaceSentinelToken(input: {
+ params: Pick<PluginEnvironmentAcquireLeaseParams, "companyId" | "environmentId" | "agentId" | "executionWorkspaceId" | "adapterType">;
+ config: DaytonaDriverConfig;
+}): string | null {
+ if (!input.config.reuseLease || !input.params.agentId || !input.params.executionWorkspaceId) {
+ return null;
+ }
+ return createHash("sha256")
+ .update(stableStringify({
+ provider: "daytona",
+ companyId: input.params.companyId,
+ environmentId: input.params.environmentId,
+ agentId: input.params.agentId,
+ executionWorkspaceId: input.params.executionWorkspaceId,
+ adapterType: input.params.adapterType ?? null,
+ image: input.config.image,
+ snapshot: input.config.snapshot,
+ target: input.config.target,
+ }))
+ .digest("hex");
+}
+
+function workspaceSentinelPath(remoteCwd: string): string {
+ return path.posix.join(remoteCwd, WORKSPACE_SENTINEL_RELATIVE_PATH);
+}
+
+async function writeWorkspaceSentinel(input: {
+ sandbox: Sandbox;
+ remoteCwd: string;
+ params: PluginEnvironmentAcquireLeaseParams;
+ config: DaytonaDriverConfig;
+ timeoutSeconds: number;
+}): Promise<WorkspaceSentinelResult> {
+ const sentinelPath = workspaceSentinelPath(input.remoteCwd);
+ const token = workspaceSentinelToken({ params: input.params, config: input.config });
+ if (!token) {
+ return { path: sentinelPath, token: null, result: "skipped" };
+ }
+ await input.sandbox.fs.createFolder(path.posix.dirname(sentinelPath), "755");
+ await input.sandbox.fs.uploadFile(
+ Buffer.from(JSON.stringify({
+ version: 1,
+ token,
+ companyId: input.params.companyId,
+ environmentId: input.params.environmentId,
+ agentId: input.params.agentId,
+ executionWorkspaceId: input.params.executionWorkspaceId,
+ adapterType: input.params.adapterType ?? null,
+ provider: "daytona",
+ writtenAt: new Date().toISOString(),
+ }, null, 2), "utf8"),
+ sentinelPath,
+ input.timeoutSeconds,
+ );
+ return { path: sentinelPath, token, result: "written" };
+}
+
+async function verifyWorkspaceSentinel(input: {
+ sandbox: Sandbox;
+ remoteCwd: string;
+ leaseMetadata?: Record<string, unknown>;
+ timeoutSeconds: number;
+}): Promise<WorkspaceSentinelResult> {
+ const metadataSentinel = isRecord(input.leaseMetadata?.workspaceSentinel)
+ ? input.leaseMetadata.workspaceSentinel
+ : null;
+ const sentinelPath = typeof metadataSentinel?.path === "string"
+ ? metadataSentinel.path
+ : workspaceSentinelPath(input.remoteCwd);
+ const expectedToken = typeof metadataSentinel?.token === "string" ? metadataSentinel.token : null;
+ if (!expectedToken) {
+ return { path: sentinelPath, token: null, result: "missing" };
+ }
+
+ const result = await input.sandbox.process.executeCommand(
+ `cat ${shellQuote(sentinelPath)}`,
+ undefined,
+ undefined,
+ input.timeoutSeconds,
+ );
+ if (result.exitCode !== 0) {
+ return { path: sentinelPath, token: expectedToken, result: "missing" };
+ }
+ try {
+ const parsed = JSON.parse(result.result ?? result.artifacts?.stdout ?? "") as unknown;
+ const actualToken = isRecord(parsed) && typeof parsed.token === "string" ? parsed.token : null;
+ return {
+ path: sentinelPath,
+ token: expectedToken,
+ result: actualToken === expectedToken ? "matched" : "mismatch",
+ };
+ } catch {
+ return { path: sentinelPath, token: expectedToken, result: "mismatch" };
+ }
+}
+
function leaseMetadata(input: {
config: DaytonaDriverConfig;
sandbox: Sandbox;
shellCommand: "bash" | "sh";
remoteCwd: string;
resumedLease: boolean;
+ workspaceSentinel?: WorkspaceSentinelResult;
}) {
return {
provider: "daytona",
@@ -230,6 +349,7 @@ function leaseMetadata(input: {
reuseLease: input.config.reuseLease,
remoteCwd: input.remoteCwd,
resumedLease: input.resumedLease,
+ ...(input.workspaceSentinel ? { workspaceSentinel: input.workspaceSentinel } : {}),
};
}
@@ -499,9 +619,16 @@ const plugin = definePlugin({
try {
const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs));
+ const workspaceSentinel = await writeWorkspaceSentinel({
+ sandbox,
+ remoteCwd,
+ params,
+ config,
+ timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
+ });
return {
providerLeaseId: sandbox.id,
- metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: false }),
+ metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: false, workspaceSentinel }),
};
} catch (error) {
await sandbox.delete(toTimeoutSeconds(config.timeoutMs)).catch(() => undefined);
@@ -521,10 +648,19 @@ const plugin = definePlugin({
await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs));
try {
const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
+ const workspaceSentinel = await verifyWorkspaceSentinel({
+ sandbox,
+ remoteCwd,
+ leaseMetadata: params.leaseMetadata,
+ timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
+ });
+ if (workspaceSentinel.result !== "matched") {
+ return { providerLeaseId: null, metadata: { expired: true, workspaceSentinel } };
+ }
const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs));
return {
providerLeaseId: sandbox.id,
- metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: true }),
+ metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: true, workspaceSentinel }),
};
} catch (error) {
await sandbox.delete(toTimeoutSeconds(config.timeoutMs)).catch(() => undefined);
diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts
index 5ec4681888..954fa76a9d 100644
--- a/packages/shared/src/constants.ts
+++ b/packages/shared/src/constants.ts
@@ -421,7 +421,7 @@ export type EnvironmentDriver = (typeof ENVIRONMENT_DRIVERS)[number];
export const ENVIRONMENT_STATUSES = ["active", "archived"] as const;
export type EnvironmentStatus = (typeof ENVIRONMENT_STATUSES)[number];
-export const ENVIRONMENT_LEASE_STATUSES = ["active", "released", "expired", "failed", "retained"] as const;
+export const ENVIRONMENT_LEASE_STATUSES = ["active", "released", "expired", "failed", "retained", "pending_cleanup"] as const;
export type EnvironmentLeaseStatus = (typeof ENVIRONMENT_LEASE_STATUSES)[number];
export const ENVIRONMENT_LEASE_POLICIES = [
diff --git a/packages/shared/src/types/plugin.ts b/packages/shared/src/types/plugin.ts
index 351b2dda0e..366a60ba63 100644
--- a/packages/shared/src/types/plugin.ts
+++ b/packages/shared/src/types/plugin.ts
@@ -138,6 +138,12 @@ export interface PluginEnvironmentDriverDeclaration {
displayName: string;
/** Optional description for operator-facing docs or UI affordances. */
description?: string;
+ /**
+ * Sandbox providers must opt in before the host retains and resumes provider
+ * leases across runs. Providers without this flag keep per-run acquire/release
+ * behavior even if their config schema exposes a reuse-like setting.
+ */
+ supportsReusableLeases?: boolean;
/** JSON Schema describing the driver's provider-specific configuration. */
configSchema: JsonSchema;
}
diff --git a/packages/shared/src/validators/plugin.ts b/packages/shared/src/validators/plugin.ts
index 0a857b44d8..45a1e3a845 100644
--- a/packages/shared/src/validators/plugin.ts
+++ b/packages/shared/src/validators/plugin.ts
@@ -124,6 +124,7 @@ export const pluginEnvironmentDriverDeclarationSchema = z.object({
kind: z.enum(["environment_driver", "sandbox_provider"]).optional(),
displayName: z.string().min(1).max(100),
description: z.string().max(500).optional(),
+ supportsReusableLeases: z.boolean().optional(),
configSchema: jsonSchemaSchema,
});
diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts
index e7ff1f9c98..4ec89cbe1c 100644
--- a/server/src/__tests__/environment-runtime.test.ts
+++ b/server/src/__tests__/environment-runtime.test.ts
@@ -1,4 +1,4 @@
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
@@ -42,6 +42,25 @@ if (!embeddedPostgresSupport.supported) {
);
}
+function stableStringify(value: unknown): string {
+ if (Array.isArray(value)) {
+ return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
+ }
+ if (value && typeof value === "object") {
+ const record = value as Record<string, unknown>;
+ return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`).join(",")}}`;
+ }
+ return JSON.stringify(value) ?? "null";
+}
+
+function reusableRuntimeFingerprint(input: {
+ provider: string;
+ adapterType: string | null;
+ config: Record<string, unknown>;
+}) {
+ return createHash("sha256").update(stableStringify(input)).digest("hex");
+}
+
describe("findReusableSandboxLeaseId", () => {
it("matches reusable plugin-backed sandbox leases by provider", () => {
const selected = findReusableSandboxLeaseId({
@@ -258,6 +277,124 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
};
}
+ async function seedReusablePluginSandboxLease() {
+ const pluginId = randomUUID();
+ const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
+ const providerConfig = {
+ provider: "fake-plugin",
+ image: "fake:test",
+ timeoutMs: 1234,
+ reuseLease: true,
+ };
+ const environment = {
+ ...baseEnvironment,
+ name: "Reusable Plugin Sandbox",
+ driver: "sandbox",
+ config: providerConfig,
+ };
+ await environmentService(db).update(environment.id, {
+ driver: "sandbox",
+ name: environment.name,
+ config: providerConfig,
+ });
+ await db.insert(plugins).values({
+ id: pluginId,
+ pluginKey: "acme.reusable-sandbox-provider",
+ packageName: "@acme/reusable-sandbox-provider",
+ version: "1.0.0",
+ apiVersion: 1,
+ categories: ["automation"],
+ manifestJson: {
+ id: "acme.reusable-sandbox-provider",
+ apiVersion: 1,
+ version: "1.0.0",
+ displayName: "Reusable Sandbox Provider",
+ description: "Test provider with reusable lease support",
+ author: "Paperclip",
+ categories: ["automation"],
+ capabilities: ["environment.drivers.register"],
+ entrypoints: { worker: "dist/worker.js" },
+ environmentDrivers: [
+ {
+ driverKey: "fake-plugin",
+ kind: "sandbox_provider",
+ displayName: "Fake Plugin",
+ supportsReusableLeases: true,
+ configSchema: {
+ type: "object",
+ properties: {
+ image: { type: "string" },
+ timeoutMs: { type: "number" },
+ reuseLease: { type: "boolean" },
+ },
+ },
+ },
+ ],
+ },
+ status: "ready",
+ installOrder: 1,
+ updatedAt: new Date(),
+ } as any);
+ const executionWorkspaceId = randomUUID();
+ const projectId = randomUUID();
+ await db.insert(projects).values({
+ id: projectId,
+ companyId,
+ name: `Workspace ${projectId.slice(0, 8)}`,
+ status: "active",
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ });
+ await db.insert(executionWorkspaces).values({
+ id: executionWorkspaceId,
+ companyId,
+ projectId,
+ mode: "shared_workspace",
+ strategyType: "project_primary",
+ name: "Reusable workspace",
+ status: "active",
+ providerType: "local_fs",
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ });
+ const reusableLease = await environmentService(db).acquireLease({
+ companyId,
+ environmentId: environment.id,
+ executionWorkspaceId,
+ heartbeatRunId: runId,
+ leasePolicy: "reuse_by_environment",
+ provider: "fake-plugin",
+ providerLeaseId: "reusable-plugin-lease",
+ metadata: {
+ agentId,
+ driver: "sandbox",
+ pluginId,
+ pluginKey: "acme.reusable-sandbox-provider",
+ sandboxProviderPlugin: true,
+ provider: "fake-plugin",
+ image: "fake:test",
+ timeoutMs: 1234,
+ reuseLease: true,
+ reusableSandboxLease: {
+ version: 1,
+ companyId,
+ environmentId: environment.id,
+ executionWorkspaceId,
+ agentId,
+ adapterType: null,
+ provider: "fake-plugin",
+ runtimeFingerprint: reusableRuntimeFingerprint({
+ provider: "fake-plugin",
+ adapterType: null,
+ config: providerConfig,
+ }),
+ },
+ },
+ });
+
+ return { pluginId, companyId, executionWorkspaceId, reusableLease };
+ }
+
it("acquires and releases a local run lease through the runtime seam", async () => {
const { companyId, environment, runId } = await seedEnvironment();
@@ -415,7 +552,8 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
});
expect(acquired.lease.status).toBe("active");
- expect(acquired.lease.providerLeaseId).toBe(`sandbox://fake/${environment.id}/workspace/agent`);
+ expect(acquired.lease.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`));
+ expect(acquired.lease.leasePolicy).toBe("ephemeral");
expect(acquired.lease.metadata).toMatchObject({
driver: "sandbox",
provider: "fake",
@@ -942,6 +1080,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
driverKey: "fake-plugin",
kind: "sandbox_provider",
displayName: "Fake Plugin",
+ supportsReusableLeases: true,
configSchema: {
type: "object",
properties: {
@@ -979,7 +1118,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
createdAt: new Date(),
updatedAt: new Date(),
});
- await environmentService(db).acquireLease({
+ const staleLease = await environmentService(db).acquireLease({
companyId,
environmentId: environment.id,
executionWorkspaceId,
@@ -989,10 +1128,28 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
providerLeaseId: "stale-plugin-lease",
metadata: {
agentId,
+ driver: "sandbox",
+ pluginId,
+ pluginKey: "acme.fake-sandbox-provider",
+ sandboxProviderPlugin: true,
provider: "fake-plugin",
image: "fake:test",
timeoutMs: 1234,
reuseLease: true,
+ reusableSandboxLease: {
+ version: 1,
+ companyId,
+ environmentId: environment.id,
+ executionWorkspaceId,
+ agentId,
+ adapterType: null,
+ provider: "fake-plugin",
+ runtimeFingerprint: reusableRuntimeFingerprint({
+ provider: "fake-plugin",
+ adapterType: null,
+ config: providerConfig,
+ }),
+ },
},
});
@@ -1002,6 +1159,9 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
if (method === "environmentResumeLease") {
throw new Error("stale sandbox");
}
+ if (method === "environmentDestroyLease") {
+ return undefined;
+ }
if (method === "environmentAcquireLease") {
return {
providerLeaseId: "fresh-plugin-lease",
@@ -1036,7 +1196,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
driverKey: "fake-plugin",
providerLeaseId: "stale-plugin-lease",
}), 31234);
- expect(workerManager.call).toHaveBeenNthCalledWith(2, pluginId, "environmentAcquireLease", expect.objectContaining({
+ expect(workerManager.call).toHaveBeenNthCalledWith(2, pluginId, "environmentDestroyLease", expect.objectContaining({
+ driverKey: "fake-plugin",
+ providerLeaseId: "stale-plugin-lease",
+ }), 31234);
+ expect(workerManager.call).toHaveBeenNthCalledWith(3, pluginId, "environmentAcquireLease", expect.objectContaining({
driverKey: "fake-plugin",
config: {
image: "fake:test",
@@ -1047,6 +1211,257 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
executionWorkspaceId,
runId,
}), 31234);
+ await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({
+ status: "expired",
+ cleanupStatus: "success",
+ });
+ });
+
+ it("does not retain or resume plugin-backed sandbox leases unless the provider opts in", async () => {
+ const pluginId = randomUUID();
+ const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
+ const providerConfig = {
+ provider: "fake-plugin",
+ image: "fake:test",
+ timeoutMs: 1234,
+ reuseLease: true,
+ };
+ const environment = {
+ ...baseEnvironment,
+ name: "Non-reusable Plugin Sandbox",
+ driver: "sandbox",
+ config: providerConfig,
+ };
+ await environmentService(db).update(environment.id, {
+ driver: "sandbox",
+ name: environment.name,
+ config: providerConfig,
+ });
+ await db.insert(plugins).values({
+ id: pluginId,
+ pluginKey: "acme.non-reusable-sandbox-provider",
+ packageName: "@acme/non-reusable-sandbox-provider",
+ version: "1.0.0",
+ apiVersion: 1,
+ categories: ["automation"],
+ manifestJson: {
+ id: "acme.non-reusable-sandbox-provider",
+ apiVersion: 1,
+ version: "1.0.0",
+ displayName: "Non-reusable Sandbox Provider",
+ description: "Test provider without reusable lease support",
+ author: "Paperclip",
+ categories: ["automation"],
+ capabilities: ["environment.drivers.register"],
+ entrypoints: { worker: "dist/worker.js" },
+ environmentDrivers: [
+ {
+ driverKey: "fake-plugin",
+ kind: "sandbox_provider",
+ displayName: "Fake Plugin",
+ configSchema: { type: "object" },
+ },
+ ],
+ },
+ status: "ready",
+ installOrder: 1,
+ updatedAt: new Date(),
+ } as any);
+ const executionWorkspaceId = randomUUID();
+ const projectId = randomUUID();
+ await db.insert(projects).values({
+ id: projectId,
+ companyId,
+ name: `Workspace ${projectId.slice(0, 8)}`,
+ status: "active",
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ });
+ await db.insert(executionWorkspaces).values({
+ id: executionWorkspaceId,
+ companyId,
+ projectId,
+ mode: "shared_workspace",
+ strategyType: "project_primary",
+ name: "Non-reusable workspace",
+ status: "active",
+ providerType: "local_fs",
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ });
+ await environmentService(db).acquireLease({
+ companyId,
+ environmentId: environment.id,
+ executionWorkspaceId,
+ heartbeatRunId: runId,
+ leasePolicy: "reuse_by_environment",
+ provider: "fake-plugin",
+ providerLeaseId: "old-plugin-lease",
+ metadata: {
+ agentId,
+ provider: "fake-plugin",
+ image: "fake:test",
+ timeoutMs: 1234,
+ reuseLease: true,
+ reusableSandboxLease: {
+ version: 1,
+ companyId,
+ environmentId: environment.id,
+ executionWorkspaceId,
+ agentId,
+ adapterType: null,
+ provider: "fake-plugin",
+ runtimeFingerprint: reusableRuntimeFingerprint({
+ provider: "fake-plugin",
+ adapterType: null,
+ config: providerConfig,
+ }),
+ },
+ },
+ });
+
+ const workerManager = {
+ isRunning: vi.fn((id: string) => id === pluginId),
+ call: vi.fn(async (_pluginId: string, method: string) => {
+ if (method === "environmentAcquireLease") {
+ return {
+ providerLeaseId: "fresh-plugin-lease",
+ metadata: {
+ provider: "fake-plugin",
+ image: "fake:test",
+ timeoutMs: 1234,
+ reuseLease: true,
+ remoteCwd: "/workspace",
+ },
+ };
+ }
+ throw new Error(`Unexpected plugin method: ${method}`);
+ }),
+ } as unknown as PluginWorkerManager;
+ const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
+
+ const acquired = await runtimeWithPlugin.acquireRunLease({
+ companyId,
+ environment,
+ issueId: null,
+ agentId,
+ heartbeatRunId: runId,
+ persistedExecutionWorkspace: {
+ id: executionWorkspaceId,
+ mode: "shared_workspace",
+ },
+ });
+
+ expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease");
+ expect(acquired.lease.leasePolicy).toBe("ephemeral");
+ expect(workerManager.call).toHaveBeenCalledTimes(1);
+ expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234);
+ });
+
+ it("destroys scoped reusable plugin-backed sandbox leases", async () => {
+ const { pluginId, companyId, executionWorkspaceId, reusableLease } =
+ await seedReusablePluginSandboxLease();
+
+ const workerManager = {
+ isRunning: vi.fn((id: string) => id === pluginId),
+ call: vi.fn(async (_pluginId: string, method: string) => {
+ if (method === "environmentDestroyLease") {
+ return undefined;
+ }
+ throw new Error(`Unexpected plugin method: ${method}`);
+ }),
+ } as unknown as PluginWorkerManager;
+ const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
+
+ const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({
+ companyId,
+ executionWorkspaceId,
+ failureReason: "execution_workspace_closed",
+ });
+
+ expect(destroyed).toHaveLength(1);
+ expect(destroyed[0]?.lease.id).toBe(reusableLease.id);
+ expect(destroyed[0]?.lease.status).toBe("expired");
+ expect(workerManager.call).toHaveBeenCalledWith(
+ pluginId,
+ "environmentDestroyLease",
+ expect.objectContaining({
+ driverKey: "fake-plugin",
+ providerLeaseId: "reusable-plugin-lease",
+ }),
+ 31234,
+ );
+ await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
+ status: "expired",
+ failureReason: "execution_workspace_closed",
+ cleanupStatus: "success",
+ });
+ });
+
+ it("retries reusable plugin-backed sandbox destroy when the worker is unavailable", async () => {
+ const { pluginId, companyId, executionWorkspaceId, reusableLease } =
+ await seedReusablePluginSandboxLease();
+
+ const offlineWorkerManager = {
+ isRunning: vi.fn(() => false),
+ call: vi.fn(),
+ } as unknown as PluginWorkerManager;
+ const runtimeWithOfflinePlugin = environmentRuntimeService(db, {
+ pluginWorkerManager: offlineWorkerManager,
+ });
+
+ const pending = await runtimeWithOfflinePlugin.destroyReusableSandboxLeases({
+ companyId,
+ executionWorkspaceId,
+ failureReason: "execution_workspace_closed",
+ });
+
+ expect(pending).toHaveLength(1);
+ expect(pending[0]?.lease.id).toBe(reusableLease.id);
+ expect(pending[0]?.lease.status).toBe("pending_cleanup");
+ expect(offlineWorkerManager.call).not.toHaveBeenCalled();
+ await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
+ status: "pending_cleanup",
+ failureReason: "execution_workspace_closed",
+ cleanupStatus: "failed",
+ });
+
+ const recoveredWorkerManager = {
+ isRunning: vi.fn((id: string) => id === pluginId),
+ call: vi.fn(async (_pluginId: string, method: string) => {
+ if (method === "environmentDestroyLease") {
+ return undefined;
+ }
+ throw new Error(`Unexpected plugin method: ${method}`);
+ }),
+ } as unknown as PluginWorkerManager;
+ const runtimeWithRecoveredPlugin = environmentRuntimeService(db, {
+ pluginWorkerManager: recoveredWorkerManager,
+ });
+
+ const retried = await runtimeWithRecoveredPlugin.destroyReusableSandboxLeases({
+ companyId,
+ executionWorkspaceId,
+ failureReason: "cleanup_retry",
+ });
+
+ expect(retried).toHaveLength(1);
+ expect(retried[0]?.lease.id).toBe(reusableLease.id);
+ expect(retried[0]?.lease.status).toBe("expired");
+ expect(recoveredWorkerManager.call).toHaveBeenCalledWith(
+ pluginId,
+ "environmentDestroyLease",
+ expect.objectContaining({
+ driverKey: "fake-plugin",
+ providerLeaseId: "reusable-plugin-lease",
+ }),
+ 31234,
+ );
+ await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
+ status: "expired",
+ failureReason: "cleanup_retry",
+ cleanupStatus: "success",
+ });
});
it("releases a sandbox run lease from metadata after the environment config changes", async () => {
diff --git a/server/src/__tests__/sandbox-provider-runtime.test.ts b/server/src/__tests__/sandbox-provider-runtime.test.ts
index 191975230e..3552f395f7 100644
--- a/server/src/__tests__/sandbox-provider-runtime.test.ts
+++ b/server/src/__tests__/sandbox-provider-runtime.test.ts
@@ -44,7 +44,7 @@ describe("sandbox provider runtime", () => {
})).rejects.toThrow('Sandbox provider "fake-plugin" is not registered as a built-in provider.');
});
- it("acquires and resumes fake leases deterministically", async () => {
+ it("does not resume fake leases because the built-in fake provider does not opt in", async () => {
const lease = await acquireSandboxProviderLease({
config: {
provider: "fake",
@@ -56,7 +56,7 @@ describe("sandbox provider runtime", () => {
issueId: "issue-1",
});
- expect(lease.providerLeaseId).toBe("sandbox://fake/env-1/workspace/agent");
+ expect(lease.providerLeaseId).toMatch(/^sandbox:\/\/fake\/run-1\/[0-9a-f-]+$/);
expect(lease.metadata).toEqual(expect.objectContaining({
provider: "fake",
image: "ubuntu:24.04",
@@ -75,8 +75,9 @@ describe("sandbox provider runtime", () => {
reusableProviderLeaseId: lease.providerLeaseId,
});
- expect(resumed.providerLeaseId).toBe(lease.providerLeaseId);
- expect(resumed.metadata).toEqual(expect.objectContaining({ resumedLease: true }));
+ expect(resumed.providerLeaseId).toMatch(/^sandbox:\/\/fake\/run-2\/[0-9a-f-]+$/);
+ expect(resumed.providerLeaseId).not.toBe(lease.providerLeaseId);
+ expect(resumed.metadata).not.toEqual(expect.objectContaining({ resumedLease: true }));
});
it("matches reusable fake leases through the selected provider implementation", () => {
diff --git a/server/src/app.ts b/server/src/app.ts
index 6658ecfa2b..efd9a74c9a 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -233,7 +233,7 @@ export async function createApp(
api.use(fileResourceRoutes(db));
api.use(routineRoutes(db, { pluginWorkerManager: workerManager }));
api.use(environmentRoutes(db, { pluginWorkerManager: workerManager }));
- api.use(executionWorkspaceRoutes(db));
+ api.use(executionWorkspaceRoutes(db, { pluginWorkerManager: workerManager }));
api.use(goalRoutes(db));
api.use(boardChatRoutes(db, { deploymentMode: opts.deploymentMode }));
api.use(approvalRoutes(db, { pluginWorkerManager: workerManager }));
diff --git a/server/src/routes/execution-workspaces.ts b/server/src/routes/execution-workspaces.ts
index bbe80687cb..bf79b44455 100644
--- a/server/src/routes/execution-workspaces.ts
+++ b/server/src/routes/execution-workspaces.ts
@@ -30,14 +30,19 @@ import {
} from "./workspace-command-authz.js";
import { assertCanManageExecutionWorkspaceRuntimeServices } from "./workspace-runtime-service-authz.js";
import { appendWithCap } from "../adapters/utils.js";
+import { environmentRuntimeService } from "../services/environment-runtime.js";
+import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
const WORKSPACE_CONTROL_OUTPUT_MAX_CHARS = 256 * 1024;
-export function executionWorkspaceRoutes(db: Db) {
+export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: PluginWorkerManager } = {}) {
const router = Router();
const svc = executionWorkspaceService(db);
const access = accessService(db);
const workspaceOperationsSvc = workspaceOperationService(db);
+ const environmentRuntime = environmentRuntimeService(db, {
+ pluginWorkerManager: opts.pluginWorkerManager,
+ });
async function assertExecutionWorkspaceReadAllowed(req: Request, res: Response, companyId: string) {
const decision = await access.decide({
@@ -538,6 +543,12 @@ export function executionWorkspaceRoutes(db: Db) {
}
workspace = archivedWorkspace;
+ await environmentRuntime.destroyReusableSandboxLeases({
+ companyId: existing.companyId,
+ executionWorkspaceId: existing.id,
+ failureReason: "execution_workspace_closed",
+ });
+
if (existing.mode === "shared_workspace") {
await db
.update(issues)
diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts
index fbce62ccd7..21b7d01932 100644
--- a/server/src/routes/issues.ts
+++ b/server/src/routes/issues.ts
@@ -121,6 +121,7 @@ import { feedbackService } from "../services/feedback.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { readAcceptedPlanConfirmationTarget } from "../services/issues.js";
import { environmentService } from "../services/environments.js";
+import { environmentRuntimeService } from "../services/environment-runtime.js";
import { redactSensitiveText } from "../redaction.js";
import {
createCompanySearchRateLimiter,
@@ -1107,6 +1108,9 @@ export function issueRoutes(
const routinesSvc = routineService(db, {
pluginWorkerManager: opts.pluginWorkerManager,
});
+ const environmentRuntime = environmentRuntimeService(db, {
+ pluginWorkerManager: opts.pluginWorkerManager,
+ });
const issueTreeControlFactory = Object.prototype.hasOwnProperty.call(
serviceIndex,
"issueTreeControlService",
@@ -2793,6 +2797,27 @@ export function issueRoutes(
});
}
+ async function destroyReusableSandboxLeasesForTerminalIssue(issue: {
+ id: string;
+ companyId: string;
+ status: string;
+ executionWorkspaceId?: string | null;
+ }) {
+ try {
+ await environmentRuntime.destroyReusableSandboxLeases({
+ companyId: issue.companyId,
+ issueId: issue.id,
+ executionWorkspaceId: issue.executionWorkspaceId ?? null,
+ failureReason: `issue_terminal_${issue.status}`,
+ });
+ } catch (err) {
+ logger.warn(
+ { err, issueId: issue.id, executionWorkspaceId: issue.executionWorkspaceId ?? null },
+ "failed to destroy reusable sandbox leases for terminal issue",
+ );
+ }
+ }
+
async function resolveIssueRouteId(rawId: string): Promise<string> {
const identifier = normalizeIssueReferenceIdentifier(rawId);
if (identifier) {
@@ -6492,6 +6517,9 @@ export function issueRoutes(
const becameTerminal =
!["done", "cancelled"].includes(existing.status) && ["done", "cancelled"].includes(issue.status);
+ if (becameTerminal) {
+ await destroyReusableSandboxLeasesForTerminalIssue(issue);
+ }
if (becameTerminal && issue.parentId) {
const parent = await svc.getWakeableParentAfterChildCompletion(issue.parentId);
if (parent) {
@@ -7883,6 +7911,9 @@ export function issueRoutes(
const becameTerminal =
!["done", "cancelled"].includes(issueBeforeCommentDecision.status) &&
["done", "cancelled"].includes(currentIssue.status);
+ if (becameTerminal) {
+ await destroyReusableSandboxLeasesForTerminalIssue(currentIssue);
+ }
if (becameTerminal && currentIssue.parentId) {
const parent = await svc.getWakeableParentAfterChildCompletion(currentIssue.parentId);
if (parent) {
diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts
index 496c85c20b..c5323085e5 100644
--- a/server/src/services/environment-runtime.ts
+++ b/server/src/services/environment-runtime.ts
@@ -1,4 +1,4 @@
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
import { and, eq, inArray } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { environmentLeases } from "@paperclipai/db";
@@ -24,7 +24,9 @@ import {
} from "./environment-config.js";
import {
acquireSandboxProviderLease,
+ destroySandboxProviderLease,
findReusableSandboxProviderLeaseId,
+ getSandboxProvider as getBuiltinSandboxProvider,
isBuiltinSandboxProvider,
releaseSandboxProviderLease,
sandboxConfigFromLeaseMetadata,
@@ -149,6 +151,7 @@ function resolvePluginSandboxRpcTimeoutMs(config: Record<string, unknown>): numb
export interface EnvironmentDriverLeaseInput {
environment: Environment;
lease: EnvironmentLease;
+ failureReason?: string;
}
export interface EnvironmentDriverRealizeWorkspaceInput extends EnvironmentDriverLeaseInput {
@@ -197,6 +200,118 @@ function getLeaseDriverKey(lease: Pick<EnvironmentLease, "metadata">, environmen
return leaseDriver ?? environment.driver;
}
+function toEnvironmentLeaseSnapshot(row: typeof environmentLeases.$inferSelect): EnvironmentLease {
+ return {
+ id: row.id,
+ companyId: row.companyId,
+ environmentId: row.environmentId,
+ executionWorkspaceId: row.executionWorkspaceId ?? null,
+ issueId: row.issueId ?? null,
+ heartbeatRunId: row.heartbeatRunId ?? null,
+ status: row.status as EnvironmentLease["status"],
+ leasePolicy: row.leasePolicy as EnvironmentLease["leasePolicy"],
+ provider: row.provider ?? null,
+ providerLeaseId: row.providerLeaseId ?? null,
+ acquiredAt: row.acquiredAt,
+ lastUsedAt: row.lastUsedAt,
+ expiresAt: row.expiresAt ?? null,
+ releasedAt: row.releasedAt ?? null,
+ failureReason: row.failureReason ?? null,
+ cleanupStatus: row.cleanupStatus as EnvironmentLease["cleanupStatus"],
+ metadata: (row.metadata as Record<string, unknown> | null) ?? null,
+ createdAt: row.createdAt,
+ updatedAt: row.updatedAt,
+ };
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function stableStringify(value: unknown): string {
+ if (Array.isArray(value)) {
+ return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
+ }
+ if (isRecord(value)) {
+ return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
+ }
+ return JSON.stringify(value) ?? "null";
+}
+
+function reusableRuntimeFingerprint(input: {
+ provider: string;
+ adapterType: string | null;
+ config: Record<string, unknown>;
+}): string {
+ return createHash("sha256")
+ .update(stableStringify(input))
+ .digest("hex");
+}
+
+function buildReusableSandboxLeaseScope(input: {
+ companyId: string;
+ environmentId: string;
+ executionWorkspaceId: string | null;
+ agentId: string | null;
+ adapterType: string | null;
+ provider: string;
+ config: Record<string, unknown>;
+ providerMetadata?: Record<string, unknown> | null;
+}): Record<string, unknown> | null {
+ if (!input.executionWorkspaceId || !input.agentId) return null;
+ const providerMetadata = input.providerMetadata ?? {};
+ const adapterType = input.adapterType ?? null;
+ const remoteCwd = readString(providerMetadata.remoteCwd);
+ const workspaceSentinel = isRecord(providerMetadata.workspaceSentinel)
+ ? { ...providerMetadata.workspaceSentinel }
+ : null;
+ return {
+ version: 1,
+ companyId: input.companyId,
+ environmentId: input.environmentId,
+ executionWorkspaceId: input.executionWorkspaceId,
+ agentId: input.agentId,
+ adapterType,
+ provider: input.provider,
+ runtimeFingerprint: reusableRuntimeFingerprint({
+ provider: input.provider,
+ adapterType,
+ config: input.config,
+ }),
+ ...(remoteCwd ? { remoteCwd } : {}),
+ ...(workspaceSentinel ? { workspaceSentinel } : {}),
+ };
+}
+
+function reusableSandboxLeaseScopeMatches(input: {
+ lease: Pick<EnvironmentLease, "metadata">;
+ companyId: string;
+ environmentId: string;
+ executionWorkspaceId: string | null;
+ agentId: string | null;
+ adapterType: string | null;
+ provider: string;
+ config: Record<string, unknown>;
+}): boolean {
+ if (!input.executionWorkspaceId || !input.agentId) return false;
+ const scope = input.lease.metadata?.reusableSandboxLease;
+ if (!isRecord(scope)) return false;
+ const adapterType = input.adapterType ?? null;
+ return (
+ scope.companyId === input.companyId &&
+ scope.environmentId === input.environmentId &&
+ scope.executionWorkspaceId === input.executionWorkspaceId &&
+ scope.agentId === input.agentId &&
+ scope.adapterType === adapterType &&
+ scope.provider === input.provider &&
+ scope.runtimeFingerprint === reusableRuntimeFingerprint({
+ provider: input.provider,
+ adapterType,
+ config: input.config,
+ })
+ );
+}
+
export function findReusableSandboxLeaseId(input: {
config: SandboxEnvironmentConfig;
leases: Array<Pick<EnvironmentLease, "providerLeaseId" | "metadata">>;
@@ -449,13 +564,15 @@ function createSandboxEnvironmentDriver(
const workerConfig = stripSandboxProviderEnvelope(parsed.config);
const storedConfig = storedParsed.config;
+ const supportsReusableLeases = pluginProvider.resolved.driver.supportsReusableLeases === true;
// Ad-hoc tests (heartbeatRunId === null) must never resume an existing
// provider lease. If they did, releasing the test lease at the end of
// the probe would tear down the live heartbeat run that owns it.
// We also filter out leases whose policy is not reuse_by_environment
- // so any non-reusable lease (including ad-hoc test leases that
- // landed in the table from older code paths) cannot be matched.
+ // and whose status is not reusable so non-reusable, cleanup-pending,
+ // or terminal rows cannot be matched.
const reusableExistingLeases =
+ supportsReusableLeases &&
parsed.config.reuseLease &&
input.heartbeatRunId !== null &&
input.executionWorkspaceId !== null &&
@@ -463,11 +580,23 @@ function createSandboxEnvironmentDriver(
? (await environmentsSvc.listLeases(input.environment.id))
.filter((lease) =>
lease.leasePolicy === "reuse_by_environment" &&
+ ["active", "released", "retained"].includes(lease.status) &&
lease.executionWorkspaceId === input.executionWorkspaceId &&
- lease.metadata?.agentId === input.agentId,
+ lease.metadata?.agentId === input.agentId &&
+ reusableSandboxLeaseScopeMatches({
+ lease,
+ companyId: input.companyId,
+ environmentId: input.environment.id,
+ executionWorkspaceId: input.executionWorkspaceId,
+ agentId: input.agentId,
+ adapterType: input.adapterType,
+ provider: parsed.config.provider,
+ config: sandboxConfigForLeaseMetadata(storedConfig),
+ }),
)
: [];
const reusableProviderLeaseId =
+ supportsReusableLeases &&
parsed.config.reuseLease &&
input.heartbeatRunId !== null &&
input.executionWorkspaceId !== null &&
@@ -478,26 +607,38 @@ function createSandboxEnvironmentDriver(
? reusableExistingLeases.find((lease) => lease.providerLeaseId === reusableProviderLeaseId)
: null;
- const providerLease = reusableLease?.providerLeaseId
- ? await pluginWorkerManager.call(
- pluginProvider.resolved.plugin.id,
- "environmentResumeLease",
- {
- driverKey: parsed.config.provider,
- companyId: input.companyId,
- environmentId: input.environment.id,
- issueId: input.issueId,
- config: workerConfig,
- providerLeaseId: reusableLease.providerLeaseId,
- leaseMetadata: reusableLease.metadata ?? undefined,
- },
- resolvePluginSandboxRpcTimeoutMs(workerConfig),
- ).then((resumed) =>
+ let providerLease: PluginEnvironmentLease | null = null;
+ if (reusableLease?.providerLeaseId) {
+ try {
+ const resumed = await pluginWorkerManager.call(
+ pluginProvider.resolved.plugin.id,
+ "environmentResumeLease",
+ {
+ driverKey: parsed.config.provider,
+ companyId: input.companyId,
+ environmentId: input.environment.id,
+ issueId: input.issueId,
+ config: workerConfig,
+ providerLeaseId: reusableLease.providerLeaseId,
+ leaseMetadata: reusableLease.metadata ?? undefined,
+ },
+ resolvePluginSandboxRpcTimeoutMs(workerConfig),
+ );
+ providerLease =
typeof resumed.providerLeaseId === "string" && resumed.providerLeaseId.length > 0
? resumed
- : null,
- ).catch(() => null)
- : null;
+ : null;
+ } catch {
+ providerLease = null;
+ }
+ if (!providerLease) {
+ await destroyReusableSandboxLease({
+ environment: input.environment,
+ lease: reusableLease,
+ failureReason: "resume_failed",
+ });
+ }
+ }
const acquiredLease = providerLease ?? await pluginWorkerManager.call(
pluginProvider.resolved.plugin.id,
"environmentAcquireLease",
@@ -529,9 +670,25 @@ function createSandboxEnvironmentDriver(
// Ad-hoc test leases are never publishable for reuse: storing them
// as `reuse_by_environment` would let a concurrent heartbeat resume
// the test's provider lease and lose its sandbox when the test ends.
- const resolvedLeasePolicy = parsed.config.reuseLease && input.heartbeatRunId !== null
+ const resolvedLeasePolicy = supportsReusableLeases && parsed.config.reuseLease && input.heartbeatRunId !== null
? "reuse_by_environment"
: "ephemeral";
+ const sanitizedProviderMetadata = stripSecretRefValuesFromPluginLeaseMetadata({
+ metadata: acquiredLease.metadata,
+ schema: pluginProvider.resolved.driver.configSchema as Record<string, unknown> | null | undefined,
+ });
+ const reusableScope = resolvedLeasePolicy === "reuse_by_environment"
+ ? buildReusableSandboxLeaseScope({
+ companyId: input.companyId,
+ environmentId: input.environment.id,
+ executionWorkspaceId: input.executionWorkspaceId,
+ agentId: input.agentId,
+ adapterType: input.adapterType,
+ provider: parsed.config.provider,
+ config: sandboxConfigForLeaseMetadata(storedConfig),
+ providerMetadata: sanitizedProviderMetadata,
+ })
+ : null;
return await environmentsSvc.acquireLease({
companyId: input.companyId,
@@ -551,10 +708,8 @@ function createSandboxEnvironmentDriver(
pluginKey: pluginProvider.resolved.plugin.pluginKey,
sandboxProviderPlugin: true,
...sandboxConfigForLeaseMetadata(storedConfig),
- ...stripSecretRefValuesFromPluginLeaseMetadata({
- metadata: acquiredLease.metadata,
- schema: pluginProvider.resolved.driver.configSchema as Record<string, unknown> | null | undefined,
- }),
+ ...sanitizedProviderMetadata,
+ ...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
},
});
}
@@ -562,9 +717,12 @@ function createSandboxEnvironmentDriver(
// Built-in sandbox provider path. Same guard as the plugin-backed path:
// ad-hoc tests (heartbeatRunId === null) must never resume an existing
// provider lease, or releasing the test lease will terminate the live
- // heartbeat run that shares it. Filter to leases whose policy is
- // reuse_by_environment so non-reusable rows can never be matched.
+ // heartbeat run that shares it. Filter to reusable policies and statuses
+ // so non-reusable, cleanup-pending, or terminal rows can never be matched.
+ const builtinSandboxProvider = getBuiltinSandboxProvider(parsed.config.provider);
+ const supportsReusableLeases = builtinSandboxProvider?.supportsReusableLeases === true;
const reusableProviderLeaseId =
+ supportsReusableLeases &&
parsed.config.reuseLease &&
input.heartbeatRunId !== null &&
input.executionWorkspaceId !== null &&
@@ -576,28 +734,73 @@ function createSandboxEnvironmentDriver(
config: parsed.config,
leases: leases.filter((lease) =>
lease.leasePolicy === "reuse_by_environment" &&
+ ["active", "released", "retained"].includes(lease.status) &&
lease.executionWorkspaceId === input.executionWorkspaceId &&
- lease.metadata?.agentId === input.agentId,
+ lease.metadata?.agentId === input.agentId &&
+ reusableSandboxLeaseScopeMatches({
+ lease,
+ companyId: input.companyId,
+ environmentId: input.environment.id,
+ executionWorkspaceId: input.executionWorkspaceId,
+ agentId: input.agentId,
+ adapterType: input.adapterType,
+ provider: parsed.config.provider,
+ config: sandboxConfigForLeaseMetadata(parsed.config),
+ }),
),
}),
))
: null;
+ const reusableLease = reusableProviderLeaseId
+ ? (await environmentsSvc.listLeases(input.environment.id)).find((lease) => lease.providerLeaseId === reusableProviderLeaseId)
+ : null;
- const providerLease = await acquireSandboxProviderLease({
- config: parsed.config,
- environmentId: input.environment.id,
- heartbeatRunId: input.heartbeatRunId ?? randomUUID(),
- issueId: input.issueId,
- agentId: input.agentId,
- executionWorkspaceId: input.executionWorkspaceId,
- reusableProviderLeaseId,
- });
+ let providerLease;
+ try {
+ providerLease = await acquireSandboxProviderLease({
+ config: parsed.config,
+ environmentId: input.environment.id,
+ heartbeatRunId: input.heartbeatRunId ?? randomUUID(),
+ issueId: input.issueId,
+ agentId: input.agentId,
+ executionWorkspaceId: input.executionWorkspaceId,
+ reusableProviderLeaseId,
+ });
+ } catch (error) {
+ if (reusableLease) {
+ await destroyReusableSandboxLease({
+ environment: input.environment,
+ lease: reusableLease,
+ failureReason: "resume_failed",
+ });
+ }
+ throw error;
+ }
+ if (reusableLease && providerLease.providerLeaseId !== reusableLease.providerLeaseId) {
+ await destroyReusableSandboxLease({
+ environment: input.environment,
+ lease: reusableLease,
+ failureReason: "resume_failed",
+ });
+ }
// Same ephemeral-policy-for-tests guard as the plugin-backed path:
// ad-hoc test leases must not be publishable for reuse.
- const resolvedLeasePolicy = parsed.config.reuseLease && input.heartbeatRunId !== null
+ const resolvedLeasePolicy = supportsReusableLeases && parsed.config.reuseLease && input.heartbeatRunId !== null
? "reuse_by_environment"
: "ephemeral";
+ const reusableScope = resolvedLeasePolicy === "reuse_by_environment"
+ ? buildReusableSandboxLeaseScope({
+ companyId: input.companyId,
+ environmentId: input.environment.id,
+ executionWorkspaceId: input.executionWorkspaceId,
+ agentId: input.agentId,
+ adapterType: input.adapterType,
+ provider: parsed.config.provider,
+ config: sandboxConfigForLeaseMetadata(parsed.config),
+ providerMetadata: providerLease.metadata,
+ })
+ : null;
return await environmentsSvc.acquireLease({
companyId: input.companyId,
@@ -613,11 +816,20 @@ function createSandboxEnvironmentDriver(
driver: input.environment.driver,
executionWorkspaceMode: input.executionWorkspaceMode,
...providerLease.metadata,
+ ...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
},
});
},
async releaseRunLease(input) {
+ if (input.status === "expired" && input.lease.leasePolicy === "reuse_by_environment") {
+ return await destroyReusableSandboxLease({
+ environment: input.environment,
+ lease: input.lease,
+ failureReason: "lease_expired",
+ });
+ }
+
// Check if this lease was acquired through a plugin.
if (input.lease.metadata?.sandboxProviderPlugin) {
return await releasePluginBackedSandboxLease(input);
@@ -748,6 +960,14 @@ function createSandboxEnvironmentDriver(
}
throw new Error("Sandbox driver does not support direct command execution for built-in providers.");
},
+
+ async destroyRunLease(input) {
+ return await destroyReusableSandboxLease({
+ environment: input.environment,
+ lease: input.lease,
+ failureReason: input.failureReason ?? "lease_destroyed",
+ });
+ },
};
async function releasePluginBackedSandboxLease(
@@ -790,6 +1010,68 @@ function createSandboxEnvironmentDriver(
cleanupStatus,
});
}
+
+ async function destroyReusableSandboxLease(input: {
+ environment: Environment;
+ lease: EnvironmentLease;
+ failureReason: string;
+ }): Promise<EnvironmentLease | null> {
+ let cleanupStatus: "success" | "failed" = "success";
+ const metadata = input.lease.metadata ?? {};
+
+ try {
+ if (metadata.sandboxProviderPlugin) {
+ const pluginId = readString(metadata.pluginId);
+ const providerKey = readString(metadata.provider);
+ if (!pluginId || !providerKey || !pluginWorkerManager?.isRunning(pluginId)) {
+ cleanupStatus = "failed";
+ } else {
+ const config = await resolvePluginSandboxRuntimeConfig({
+ environment: input.environment,
+ lease: input.lease,
+ provider: providerKey,
+ });
+ await pluginWorkerManager.call(pluginId, "environmentDestroyLease", {
+ driverKey: providerKey,
+ companyId: input.lease.companyId,
+ environmentId: input.environment.id,
+ issueId: input.lease.issueId,
+ config: stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig),
+ providerLeaseId: input.lease.providerLeaseId,
+ leaseMetadata: metadata,
+ }, resolvePluginSandboxRpcTimeoutMs(stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig)));
+ }
+ } else {
+ const metadataConfig = sandboxConfigFromLeaseMetadata(input.lease);
+ const parsed = metadataConfig
+ ? await resolveEnvironmentDriverConfigForRuntime(db, input.lease.companyId, {
+ id: input.environment.id,
+ driver: "sandbox",
+ config: metadataConfig as unknown as Record<string, unknown>,
+ })
+ : await resolveEnvironmentDriverConfigForRuntime(db, input.lease.companyId, input.environment);
+ if (parsed.driver !== "sandbox") {
+ cleanupStatus = "failed";
+ } else {
+ await destroySandboxProviderLease({
+ config: parsed.config,
+ providerLeaseId: input.lease.providerLeaseId,
+ });
+ }
+ }
+ } catch {
+ cleanupStatus = "failed";
+ }
+
+ return await environmentsSvc.releaseLease(
+ input.lease.id,
+ cleanupStatus === "success" ? "expired" : "pending_cleanup",
+ {
+ failureReason: input.failureReason,
+ cleanupStatus,
+ },
+ );
+ }
}
function parseExpiresAt(value: string | null | undefined): Date | null {
@@ -1026,7 +1308,9 @@ function createPluginEnvironmentDriver(
providerLeaseId: input.lease.providerLeaseId,
leaseMetadata: input.lease.metadata ?? undefined,
});
- return await environmentsSvc.releaseLease(input.lease.id, "failed");
+ return await environmentsSvc.releaseLease(input.lease.id, "failed", {
+ failureReason: input.failureReason ?? "lease_destroyed",
+ });
},
async realizeWorkspace(input) {
@@ -1213,27 +1497,7 @@ export function environmentRuntimeService(
const environment = await environmentsSvc.getById(leaseRow.environmentId);
if (!environment) continue;
- const leaseSnapshot: EnvironmentLease = {
- id: leaseRow.id,
- companyId: leaseRow.companyId,
- environmentId: leaseRow.environmentId,
- executionWorkspaceId: leaseRow.executionWorkspaceId ?? null,
- issueId: leaseRow.issueId ?? null,
- heartbeatRunId: leaseRow.heartbeatRunId ?? null,
- status: leaseRow.status as EnvironmentLease["status"],
- leasePolicy: leaseRow.leasePolicy as EnvironmentLease["leasePolicy"],
- provider: leaseRow.provider ?? null,
- providerLeaseId: leaseRow.providerLeaseId ?? null,
- acquiredAt: leaseRow.acquiredAt,
- lastUsedAt: leaseRow.lastUsedAt,
- expiresAt: leaseRow.expiresAt ?? null,
- releasedAt: leaseRow.releasedAt ?? null,
- failureReason: leaseRow.failureReason ?? null,
- cleanupStatus: leaseRow.cleanupStatus as EnvironmentLease["cleanupStatus"],
- metadata: (leaseRow.metadata as Record<string, unknown> | null) ?? null,
- createdAt: leaseRow.createdAt,
- updatedAt: leaseRow.updatedAt,
- };
+ const leaseSnapshot = toEnvironmentLeaseSnapshot(leaseRow);
const driver = getDriver(getLeaseDriverKey(leaseSnapshot, environment));
const lease = driver
? await driver.releaseRunLease({
@@ -1258,6 +1522,60 @@ export function environmentRuntimeService(
return released;
},
+ async destroyReusableSandboxLeases(input: {
+ companyId: string;
+ issueId?: string | null;
+ executionWorkspaceId?: string | null;
+ failureReason?: string;
+ }): Promise<EnvironmentRuntimeLeaseRecord[]> {
+ const scopeConditions = [
+ input.issueId ? eq(environmentLeases.issueId, input.issueId) : undefined,
+ input.executionWorkspaceId ? eq(environmentLeases.executionWorkspaceId, input.executionWorkspaceId) : undefined,
+ ].filter((condition): condition is NonNullable<typeof condition> => Boolean(condition));
+ if (scopeConditions.length === 0) return [];
+
+ const leaseRows = await db
+ .select()
+ .from(environmentLeases)
+ .where(
+ and(
+ eq(environmentLeases.companyId, input.companyId),
+ eq(environmentLeases.leasePolicy, "reuse_by_environment"),
+ inArray(environmentLeases.status, ["active", "released", "retained", "pending_cleanup"]),
+ ...scopeConditions,
+ ),
+ );
+
+ const destroyed: EnvironmentRuntimeLeaseRecord[] = [];
+ for (const leaseRow of leaseRows) {
+ const environment = await environmentsSvc.getById(leaseRow.environmentId);
+ if (!environment) continue;
+ const leaseSnapshot = toEnvironmentLeaseSnapshot(leaseRow);
+ const driver = getDriver(getLeaseDriverKey(leaseSnapshot, environment));
+ const lease = driver?.destroyRunLease
+ ? await driver.destroyRunLease({
+ environment,
+ lease: leaseSnapshot,
+ failureReason: input.failureReason ?? "reusable_lease_destroyed",
+ })
+ : await environmentsSvc.releaseLease(leaseSnapshot.id, "pending_cleanup", {
+ failureReason: input.failureReason ?? "reusable_lease_destroyed",
+ cleanupStatus: "failed",
+ });
+ if (!lease) continue;
+ destroyed.push({
+ environment,
+ lease,
+ leaseContext: {
+ executionWorkspaceId: lease.executionWorkspaceId,
+ executionWorkspaceMode:
+ (lease.metadata?.executionWorkspaceMode as ExecutionWorkspace["mode"] | null | undefined) ?? null,
+ },
+ });
+ }
+ return destroyed;
+ },
+
async resumeRunLease(input: EnvironmentDriverLeaseInput): Promise<PluginEnvironmentLease | EnvironmentLease | null> {
const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment));
if (!driver.resumeRunLease) {
diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts
index c746389a6c..c68dad20e9 100644
--- a/server/src/services/environments.ts
+++ b/server/src/services/environments.ts
@@ -499,7 +499,7 @@ export function environmentService(db: Db) {
releaseLease: async (
id: string,
- status: Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed" | "retained"> = "released",
+ status: Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed" | "retained" | "pending_cleanup"> = "released",
options?: {
failureReason?: string;
cleanupStatus?: EnvironmentLeaseCleanupStatus;
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 3544d3307d..f097ecff1a 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -841,6 +841,7 @@ async function resolveRunScopedMentionedSkillKeys(input: {
function leaseReleaseStatusForRunStatus(
status: string | null | undefined,
): Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed"> {
+ if (status === "cancelled") return "expired";
return status === "failed" || status === "timed_out" ? "failed" : "released";
}
diff --git a/server/src/services/sandbox-provider-runtime.ts b/server/src/services/sandbox-provider-runtime.ts
index e664fe00d5..0b089fdd6f 100644
--- a/server/src/services/sandbox-provider-runtime.ts
+++ b/server/src/services/sandbox-provider-runtime.ts
@@ -78,6 +78,7 @@ export interface SandboxExecuteResult {
export interface SandboxProvider {
readonly provider: SandboxEnvironmentProvider;
+ readonly supportsReusableLeases?: boolean;
validateConfig(config: SandboxEnvironmentConfig): Promise<SandboxProviderValidationResult>;
probe(config: SandboxEnvironmentConfig): Promise<EnvironmentProbeResult>;
acquireLease(input: AcquireSandboxLeaseInput): Promise<SandboxLeaseHandle>;
@@ -117,6 +118,7 @@ function buildFakeSandboxProbe(config: FakeSandboxEnvironmentConfig): Environmen
class FakeSandboxProvider implements SandboxProvider {
readonly provider = "fake" as const;
+ readonly supportsReusableLeases = false;
async validateConfig(config: SandboxEnvironmentConfig): Promise<SandboxProviderValidationResult> {
assertProviderConfig<FakeSandboxEnvironmentConfig>(this.provider, config);
@@ -138,7 +140,7 @@ class FakeSandboxProvider implements SandboxProvider {
async acquireLease(input: AcquireSandboxLeaseInput): Promise<SandboxLeaseHandle> {
assertProviderConfig<FakeSandboxEnvironmentConfig>(this.provider, input.config);
- const providerLeaseId = input.config.reuseLease
+ const providerLeaseId = input.config.reuseLease && this.supportsReusableLeases
? `sandbox://fake/${input.environmentId}/${input.executionWorkspaceId ?? "workspace"}/${input.agentId ?? "agent"}`
: `sandbox://fake/${input.heartbeatRunId}/${randomUUID()}`;
@@ -336,7 +338,7 @@ export async function acquireSandboxProviderLease(input: {
reusableProviderLeaseId?: string | null;
}): Promise<SandboxLeaseHandle> {
const provider = requireSandboxProvider(input.config.provider);
- if (input.config.reuseLease && input.reusableProviderLeaseId) {
+ if (provider.supportsReusableLeases && input.config.reuseLease && input.reusableProviderLeaseId) {
const resumedLease = await provider.resumeLease({
config: input.config,
providerLeaseId: input.reusableProviderLeaseId,