The Open Framework for Building Agents
fix(eve): repair Vercel sandbox lifecycle (#3235)
Casey Gowrie05d2047f6cab
diff --git a/.changeset/repair-vercel-sandbox-lifecycle.md b/.changeset/repair-vercel-sandbox-lifecycle.mdnew file mode 100644index 000000000..7f9f95e6b--- /dev/null+++ b/.changeset/repair-vercel-sandbox-lifecycle.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Keep Vercel sandbox template records persistent and replace session sandboxes whose saved filesystem snapshot is no longer available. Ambiguous session creation failures no longer delete the shared template record.diff --git a/docs/sandbox.mdx b/docs/sandbox.mdxindex 7dca8b56a..d167e4860 100644--- a/docs/sandbox.mdx+++ b/docs/sandbox.mdx@@ -291,7 +291,7 @@ Session sandboxes are keyed per durable session, not per deployment, so redeployFor sandboxes with `bootstrap`, unrelated agent changes preserve the template and session sandbox keys even when there are no skill or workspace seed files. You do not need to add a seed file or `revalidationKey` solely to preserve reuse across those changes.-Reattachment still depends on the backend retaining its physical sandbox state. If a persisted Vercel sandbox is no longer available, eve creates a replacement, using the current template when one is configured. Files and other changes made after the original sandbox was created are not restored automatically. Because the durable session still has the same sandbox key, `onSession` does not run again for this replacement. Persist important artifacts outside the sandbox, and do not rely on `onSession` as the only place that applies security-critical configuration.+Reattachment still depends on the backend retaining its physical sandbox state. If a persisted Vercel sandbox is missing or its saved filesystem snapshot is no longer available, eve deletes the unusable record and creates a replacement from the current template when one is configured. Files and other changes made after the original sandbox was created are not restored automatically. Because the durable session still has the same sandbox key, `onSession` does not run again for this replacement. Persist important artifacts outside the sandbox, and do not rely on `onSession` as the only place that applies security-critical configuration.When the eve server stops, no sandbox compute outlives it. `eve dev` stops the sandboxes it started when the dev server closes, and a self-hosted production server stops every open sandbox on shutdown (`SIGTERM`/`SIGINT`). Session state persists across the stop — the next server start reattaches each durable session from its stopped container, VM, or snapshot. Custom `SandboxBackend` adapters implement `stop()` for authored runtime calls and `shutdown()` for server teardown. Both stop the underlying compute while keeping the session reattachable from persisted state where the backend supports it; authored `stop()` failures reject, while process-wide shutdown collects and logs failures without blocking teardown.diff --git a/packages/eve/src/execution/sandbox/bindings/vercel-lifecycle.ts b/packages/eve/src/execution/sandbox/bindings/vercel-lifecycle.tsindex 7f71ac97f..48455e87b 100644--- a/packages/eve/src/execution/sandbox/bindings/vercel-lifecycle.ts+++ b/packages/eve/src/execution/sandbox/bindings/vercel-lifecycle.ts@@ -17,6 +17,23 @@ export async function deleteVercelSandbox(input: {readonly signal?: AbortSignal;}): Promise<void> {await stopVercelSandbox(input.sandbox);+ await deleteVercelSandboxRecord(input);+}++export async function deleteUnusableVercelSandbox(input: {+ readonly createOptions: VercelCreateOptions;+ readonly loadDeleteSandboxModule: () => Promise<VercelDeleteModule>;+ readonly sandbox: VercelSandbox;+}): Promise<void> {+ await deleteVercelSandboxRecord(input);+}++async function deleteVercelSandboxRecord(input: {+ readonly createOptions: VercelCreateOptions;+ readonly loadDeleteSandboxModule: () => Promise<VercelDeleteModule>;+ readonly sandbox: VercelSandbox;+ readonly signal?: AbortSignal;+}): Promise<void> {const credentials = await resolveVercelSandboxCredentials(input.createOptions);const sandboxModule = await input.loadDeleteSandboxModule();const sandbox = await sandboxModule.Sandbox.get({diff --git a/packages/eve/src/execution/sandbox/bindings/vercel.test.ts b/packages/eve/src/execution/sandbox/bindings/vercel.test.tsindex ad9facd64..08507ffe6 100644--- a/packages/eve/src/execution/sandbox/bindings/vercel.test.ts+++ b/packages/eve/src/execution/sandbox/bindings/vercel.test.ts@@ -191,7 +191,7 @@ describe("createVercelSandbox", () => {image: VERCEL_EVE_SANDBOX_IMAGE,name: "template-key",networkPolicy: "allow-all",- persistent: false,+ persistent: true,ports: [3000],projectId: "prj_123",teamId: "team_123",@@ -529,7 +529,7 @@ describe("createVercelSandbox", () => {expect(sandboxModule.Sandbox.create).toHaveBeenCalledWith(expect.objectContaining({name: "template-key",- persistent: false,+ persistent: true,}),);expect(freshTemplate.snapshot).toHaveBeenCalledTimes(1);@@ -672,7 +672,7 @@ describe("createVercelSandbox", () => {const [templateArgs, sessionArgs] = create.mock.calls;expect(templateArgs?.[0]).toMatchObject({name: "template-key",- persistent: false,+ persistent: true,timeout: 30 * 60 * 1_000,});expect(sessionArgs?.[0]).toMatchObject({@@ -794,7 +794,7 @@ describe("createVercelSandbox", () => {expect(templateArgs?.[0]).toMatchObject({name: "template-key",networkPolicy: "allow-all",- persistent: false,+ persistent: true,ports: [3000, 4000],resources: { vcpus: 2 },timeout: 600_000,@@ -1045,7 +1045,7 @@ describe("createVercelSandbox", () => {});expect(create.mock.calls[1]?.[0]).toMatchObject({name: "template-key",- persistent: false,+ persistent: true,});expect(create.mock.calls[2]?.[0]).toMatchObject({name: "session-key",@@ -1053,6 +1053,72 @@ describe("createVercelSandbox", () => {});});+ it("does not invalidate the shared template when a fresh session initialization returns 410", async () => {+ const templateSandbox = createMockSandbox({+ name: "template-key",+ snapshotId: "template-snapshot",+ });+ const freshSession = createMockSandbox({ name: "session-key" });+ const snapshotUnavailableError = Object.assign(new Error("Cannot initialize sandbox"), {+ response: { status: 410 },+ });+ vi.mocked(freshSession.runCommand).mockRejectedValueOnce(snapshotUnavailableError);+ const sandboxModule = {+ Sandbox: {+ create: vi.fn().mockResolvedValueOnce(freshSession),+ get: vi.fn().mockImplementation(async ({ name }: { name: string }) => {+ if (name === "template-key") return templateSandbox;+ return null;+ }),+ },+ };+ const backend = createTestVercelSandbox({+ loadSandboxModule: async () => sandboxModule as never,+ });++ await expect(+ backend.create({+ runtimeContext: { appRoot: "/tmp/test-app-root" },+ sessionKey: "session-key",+ templateKey: "template-key",+ }),+ ).rejects.toThrow('Failed to initialize sandbox session "session-key"');++ expect(templateSandbox.delete).not.toHaveBeenCalled();+ });++ it("does not invalidate the shared template for an ambiguous session-create 404", async () => {+ const templateSandbox = createMockSandbox({+ name: "template-key",+ snapshotId: "template-snapshot",+ });+ const imageNotFoundError = Object.assign(new Error("Image not found"), {+ response: { status: 404 },+ });+ const sandboxModule = {+ Sandbox: {+ create: vi.fn().mockRejectedValueOnce(imageNotFoundError),+ get: vi.fn().mockImplementation(async ({ name }: { name: string }) => {+ if (name === "template-key") return templateSandbox;+ return null;+ }),+ },+ };+ const backend = createTestVercelSandbox({+ loadSandboxModule: async () => sandboxModule as never,+ });++ await expect(+ backend.create({+ runtimeContext: { appRoot: "/tmp/test-app-root" },+ sessionKey: "session-key",+ templateKey: "template-key",+ }),+ ).rejects.toThrow('Failed to create sandbox session "session-key": Image not found');++ expect(templateSandbox.delete).not.toHaveBeenCalled();+ });+it("rebuilds a Vercel template when the named sandbox disappears during prewarm", async () => {const staleTemplate = createMockSandbox({ name: "template-key" });const freshTemplate = createMockSandbox({ name: "template-key" });@@ -1143,6 +1209,65 @@ describe("createVercelSandbox", () => {expect(state.metadata).toEqual({ sandboxName: "persisted-sandbox-name" });});+ it("replaces a persisted session whose snapshot is unavailable", async () => {+ const templateSandbox = createMockSandbox({+ name: "template-key",+ snapshotId: "template-snapshot",+ });+ const staleSession = createMockSandbox({ name: "persisted-sandbox-name", status: "stopped" });+ const replacementSession = createMockSandbox({ name: "session-key" });+ const snapshotUnavailableError = Object.assign(new Error("Cannot resume sandbox"), {+ response: { status: 410 },+ });+ vi.mocked(staleSession.runCommand).mockRejectedValueOnce(snapshotUnavailableError);++ const sandboxModule = {+ Sandbox: {+ create: vi.fn().mockResolvedValueOnce(replacementSession),+ get: vi.fn().mockImplementation(async ({ name }: { name: string }) => {+ if (name === "template-key") return templateSandbox;+ if (name === "persisted-sandbox-name") return staleSession;+ if (name === "session-key") return null;+ return null;+ }),+ },+ };+ const stableDelete = vi.fn().mockResolvedValue(undefined);+ const stableGet = vi.fn().mockResolvedValue({ delete: stableDelete });+ const backend = createTestVercelSandbox({+ loadDeleteSandboxModule: async () => ({ Sandbox: { get: stableGet } }) as never,+ loadSandboxModule: async () => sandboxModule as never,+ });++ const handle = await backend.create({+ existingMetadata: { sandboxName: "persisted-sandbox-name" },+ runtimeContext: { appRoot: "/tmp/test-app-root" },+ sessionKey: "session-key",+ templateKey: "template-key",+ });++ expect(staleSession.delete).not.toHaveBeenCalled();+ expect(stableGet).toHaveBeenCalledWith({+ fetch: expect.any(Function),+ name: "persisted-sandbox-name",+ resume: false,+ signal: undefined,+ });+ expect(stableDelete).toHaveBeenCalledWith({+ deleteOrphanSnapshots: true,+ signal: undefined,+ });+ expect(templateSandbox.delete).not.toHaveBeenCalled();+ expect(sandboxModule.Sandbox.create).toHaveBeenCalledWith(+ expect.objectContaining({+ name: "session-key",+ persistent: true,+ source: { snapshotId: "template-snapshot", type: "snapshot" },+ }),+ );+ expect((await handle.captureState()).metadata).toEqual({ sandboxName: "session-key" });+ });+it("stops the session sandbox on shutdown so no VM outlives the server", async () => {const { handle, sessionSandbox } = await createTestVercelSession();@@ -1389,7 +1514,7 @@ describe("createVercelSandbox", () => {const [templateArgs, sessionArgs] = create.mock.calls;expect(templateArgs?.[0]).toMatchObject({name: "template-key",- persistent: false,+ persistent: true,});expect(sessionArgs?.[0]).toMatchObject({name: "session-key",diff --git a/packages/eve/src/execution/sandbox/bindings/vercel.ts b/packages/eve/src/execution/sandbox/bindings/vercel.tsindex 8cad28ce1..0f6af3830 100644--- a/packages/eve/src/execution/sandbox/bindings/vercel.ts+++ b/packages/eve/src/execution/sandbox/bindings/vercel.ts@@ -51,6 +51,7 @@ import {} from "#execution/sandbox/bindings/vercel-errors.js";import { getNamedVercelSandbox } from "#execution/sandbox/bindings/vercel-lookup.js";import {+ deleteUnusableVercelSandbox,deleteVercelSandbox,stopVercelSandbox,} from "#execution/sandbox/bindings/vercel-lifecycle.js";@@ -116,24 +117,22 @@ export function createVercelSandbox(});const sandboxModule = await loadSandboxModule();+ const ensureSessionInput: EnsureSessionInput = {+ createOptions,+ createSandbox,+ existingMetadata: createInput.existingMetadata,+ resolveSessionCreateOptions: input.resolveSessionCreateOptions,+ sandboxModule,+ sessionId: createInput.tags?.sessionId ?? createInput.sessionKey,+ sessionKey: createInput.sessionKey,+ snapshotId: template?.snapshotId,+ tags,+ };let session: VercelSandboxSessionCreateResult;try {- session = await ensureSession({- createOptions,- createSandbox,- existingMetadata: createInput.existingMetadata,- resolveSessionCreateOptions: input.resolveSessionCreateOptions,- sandboxModule,- sessionId: createInput.tags?.sessionId ?? createInput.sessionKey,- sessionKey: createInput.sessionKey,- snapshotId: template?.snapshotId,- tags,- });+ session = await ensureSession(ensureSessionInput);} catch (error) {- if (- template !== null &&- (isVercelSnapshotUnavailableError(error) || isVercelSandboxMissingError(error))- ) {+ if (template !== null && isVercelSnapshotUnavailableError(error)) {prewarmedTemplates.delete(template.templateKey);const staleTemplate = await getNamedVercelSandbox({createOptions,@@ -152,9 +151,20 @@ export function createVercelSandbox();}- await ensureVercelSandboxBaseRuntime(session.sandbox);- if (template === null && session.created) {- await applyInitialVercelNetworkPolicy(session.sandbox, createOptions.networkPolicy);+ try {+ session = await ensureUsableSession({+ input: ensureSessionInput,+ loadDeleteSandboxModule,+ session,+ });+ if (template === null && session.created) {+ await applyInitialVercelNetworkPolicy(session.sandbox, createOptions.networkPolicy);+ }+ } catch (error) {+ throw new Error(+ `Failed to initialize sandbox session "${createInput.sessionKey}": ${errorMessage(error)}`,+ { cause: error },+ );}return createHandle({@@ -306,7 +316,7 @@ async function ensureTemplate(input: EnsureTemplateInput): Promise<EnsureTemplatcreateOptions: withBaseSetupNetworkPolicy({...input.createOptions,name: input.templateKey,- persistent: false,+ persistent: true,tags: tags,}),});@@ -398,6 +408,30 @@ interface VercelSandboxSessionCreateResult {readonly sandbox: VercelSandbox;}+async function ensureUsableSession(input: {+ readonly input: EnsureSessionInput;+ readonly loadDeleteSandboxModule: () => Promise<VercelDeleteModule>;+ readonly session: VercelSandboxSessionCreateResult;+}): Promise<VercelSandboxSessionCreateResult> {+ try {+ await ensureVercelSandboxBaseRuntime(input.session.sandbox);+ return input.session;+ } catch (error) {+ if (input.session.created || !isVercelSnapshotUnavailableError(error)) {+ throw error;+ }+ }++ await deleteUnusableVercelSandbox({+ createOptions: input.input.createOptions,+ loadDeleteSandboxModule: input.loadDeleteSandboxModule,+ sandbox: input.session.sandbox,+ });+ const replacement = await ensureSession({ ...input.input, existingMetadata: undefined });+ await ensureVercelSandboxBaseRuntime(replacement.sandbox);+ return replacement;+}+async function ensureSession(input: EnsureSessionInput): Promise<VercelSandboxSessionCreateResult> {const sandboxName = getVercelSandboxName(input.existingMetadata) ?? input.sessionKey;const existing = await getNamedVercelSandbox({