The Open Framework for Building Agents
Handle missing sandbox backend state (#20)
JJ Kasper16f5a52f1000
diff --git a/.changeset/vercel-sandbox-missing-template.md b/.changeset/vercel-sandbox-missing-template.mdnew file mode 100644index 000000000..0dc135898--- /dev/null+++ b/.changeset/vercel-sandbox-missing-template.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Handle missing sandbox template and session state more gracefully across Vercel, Microsandbox, and Docker backends. Eve now treats stale Vercel template references, missing Microsandbox session/template snapshots, and Docker template image races as recoverable provisioning misses so the runtime can rebuild or create a fresh sandbox automatically.diff --git a/packages/eve/src/execution/sandbox/bindings/docker.integration.test.ts b/packages/eve/src/execution/sandbox/bindings/docker.integration.test.tsindex 82cb2af99..a297259e4 100644--- a/packages/eve/src/execution/sandbox/bindings/docker.integration.test.ts+++ b/packages/eve/src/execution/sandbox/bindings/docker.integration.test.ts@@ -262,6 +262,30 @@ describe("createDockerSandboxBackend create", () => {).rejects.toThrow(SandboxTemplateNotProvisionedError);});+ it("throws SandboxTemplateNotProvisionedError when a template-backed container fails to start", async () => {+ const appRoot = await createScratchDirectory("eve-docker-sandbox-");+ const { cli } = createFakeDockerCli((args) => {+ if (isContainerInspect(args)) {+ return { exitCode: 1, stderr: "No such container" };+ }+ if (isImageInspect(args, TEMPLATE_IMAGE)) {+ return { exitCode: 0, stdout: "sha256:abc\n" };+ }+ if (args[0] === "run") {+ return { exitCode: 1, stderr: "template-backed container failed to start" };+ }+ return undefined;+ });++ await expect(+ createEngine({ cli }).create({+ runtimeContext: { appRoot },+ sessionKey: SESSION_KEY,+ templateKey: TEMPLATE_KEY,+ }),+ ).rejects.toThrow(SandboxTemplateNotProvisionedError);+ });+it("creates a session container from the template image with labels and tags", async () => {const appRoot = await createScratchDirectory("eve-docker-sandbox-");const previousRunId = process.env[EVE_DEVELOPMENT_SANDBOX_RUN_ID_ENV];diff --git a/packages/eve/src/execution/sandbox/bindings/docker.ts b/packages/eve/src/execution/sandbox/bindings/docker.tsindex 02028f82b..0c11220d5 100644--- a/packages/eve/src/execution/sandbox/bindings/docker.ts+++ b/packages/eve/src/execution/sandbox/bindings/docker.ts@@ -233,16 +233,26 @@ export function createDockerSandboxBackend();}- await startDockerContainer({- cli,- containerName,- image,- initialNetworkPolicy:- createInput.templateKey === null ? "allow-all" : options.networkPolicy,- options,- role: "session",- tags: createInput.tags,- });+ try {+ await startDockerContainer({+ cli,+ containerName,+ image,+ initialNetworkPolicy:+ createInput.templateKey === null ? "allow-all" : options.networkPolicy,+ options,+ role: "session",+ tags: createInput.tags,+ });+ } catch (error) {+ if (createInput.templateKey !== null) {+ throw new SandboxTemplateNotProvisionedError({+ backendName: DOCKER_BACKEND_NAME,+ templateKey: createInput.templateKey,+ });+ }+ throw error;+ }if (createInput.templateKey === null) {await runDockerBaseSetup(cli, containerName);diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.test.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.test.tsindex 7f7075f5e..cd9be1c97 100644--- a/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.test.ts+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.test.ts@@ -5,12 +5,14 @@ import {createMicrosandboxHandle,prewarmMicrosandboxTemplate,} from "#execution/sandbox/bindings/microsandbox-lifecycle.js";+import { SandboxTemplateNotProvisionedError } from "#public/definitions/sandbox-backend.js";import {MICROSANDBOX_DEFAULT_IMAGE,resolveMicrosandboxOptions,} from "#execution/sandbox/bindings/microsandbox-options.js";const runtimeMocks = vi.hoisted(() => ({+ connectMicrosandbox: vi.fn(),createPreparedMicrosandbox: vi.fn(),createProviderName: vi.fn((prefix: string, key: string) => `${prefix}-${key}`),doesPathExist: vi.fn(async () => false),@@ -57,6 +59,7 @@ describe("createMicrosandboxHandle", () => {clearActiveMicrosandboxSessionHandlesForTest();vi.clearAllMocks();runtimeMocks.loadMicrosandboxModule.mockResolvedValue({} as never);+ runtimeMocks.connectMicrosandbox.mockReset();runtimeMocks.sandboxExists.mockResolvedValue(false);runtimeMocks.snapshotExists.mockResolvedValue(true);metadataMocks.readSessionMetadata.mockResolvedValue(null);@@ -106,6 +109,65 @@ describe("createMicrosandboxHandle", () => {expect(secondHandle).toBe(firstHandle);expect(runtimeMocks.createPreparedMicrosandbox).toHaveBeenCalledTimes(1);});++ it("creates fresh from the template when persisted session state disappeared", async () => {+ const vm = createFakeMicrosandboxVm("session-key");+ runtimeMocks.connectMicrosandbox.mockResolvedValueOnce(null);+ runtimeMocks.createPreparedMicrosandbox.mockResolvedValue(vm);+ runtimeMocks.snapshotExists.mockResolvedValue(true);+ const options = resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE });++ const handle = await createMicrosandboxHandle({+ backendName: "microsandbox",+ createInput: {+ existingMetadata: {+ optionsHash: "options-hash",+ sandboxName: "deleted-sandbox",+ stateSnapshotName: "deleted-session-snapshot",+ version: 2,+ },+ runtimeContext: { appRoot: "/tmp/eve-app" },+ sessionKey: "session-key",+ templateKey: "template-key",+ },+ options,+ optionsHash: "options-hash",+ });++ expect(runtimeMocks.connectMicrosandbox).toHaveBeenCalledTimes(1);+ expect(runtimeMocks.createPreparedMicrosandbox).toHaveBeenCalledWith(+ expect.objectContaining({+ fromSnapshot: "template-snapshot",+ sessionKey: "session-key",+ setupBaseRuntime: false,+ }),+ );+ await expect(handle.captureState()).resolves.toMatchObject({+ backendName: "microsandbox",+ sessionKey: "session-key",+ });+ });++ it("reports a missing template snapshot race as not provisioned", async () => {+ runtimeMocks.createPreparedMicrosandbox.mockRejectedValueOnce(+ new Error("snapshot template-snapshot not found"),+ );+ runtimeMocks.snapshotExists.mockResolvedValue(true);+ const options = resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE });++ await expect(+ createMicrosandboxHandle({+ backendName: "microsandbox",+ createInput: {+ runtimeContext: { appRoot: "/tmp/eve-app" },+ sessionKey: "session-key",+ templateKey: "template-key",+ },+ options,+ optionsHash: "options-hash",+ }),+ ).rejects.toBeInstanceOf(SandboxTemplateNotProvisionedError);+ });});describe("prewarmMicrosandboxTemplate", () => {diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.tsindex 63ee437e4..1ece7163d 100644--- a/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.ts+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.ts@@ -21,6 +21,7 @@ import {createPreparedMicrosandbox,createProviderName,doesPathExist,+ isMicrosandboxNotFoundError,loadMicrosandboxModule,type MicrosandboxVm,removeSnapshotIfExists,@@ -204,12 +205,14 @@ export async function createMicrosandboxHandle(input: {sessionKey: input.createInput.sessionKey,tags: sessionTags,});- return cacheHandle(- activeSessionKey,- createHandle(sandbox, input.backendName, input.optionsHash, () => {- activeMicrosandboxSessionHandles.delete(activeSessionKey);- }),- );+ if (sandbox !== null) {+ return cacheHandle(+ activeSessionKey,+ createHandle(sandbox, input.backendName, input.optionsHash, () => {+ activeMicrosandboxSessionHandles.delete(activeSessionKey);+ }),+ );+ }}let snapshotName: string | null = null;@@ -240,16 +243,31 @@ export async function createMicrosandboxHandle(input: {"eve-sbx-ses",`${input.createInput.sessionKey}:${randomUUID()}`,);- const sandbox = await createPreparedMicrosandbox({- fromSnapshot: snapshotName ?? undefined,- module,- name: sandboxName,- networkPolicy: input.options.networkPolicy,- options: input.options,- sessionKey: input.createInput.sessionKey,- setupBaseRuntime: snapshotName === null,- tags: sessionTags,- });+ let sandbox: MicrosandboxVm;+ try {+ sandbox = await createPreparedMicrosandbox({+ fromSnapshot: snapshotName ?? undefined,+ module,+ name: sandboxName,+ networkPolicy: input.options.networkPolicy,+ options: input.options,+ sessionKey: input.createInput.sessionKey,+ setupBaseRuntime: snapshotName === null,+ tags: sessionTags,+ });+ } catch (error) {+ if (+ snapshotName !== null &&+ input.createInput.templateKey !== null &&+ isMicrosandboxNotFoundError(error)+ ) {+ throw new SandboxTemplateNotProvisionedError({+ backendName: input.backendName,+ templateKey: input.createInput.templateKey,+ });+ }+ throw error;+ }await sandbox.writeMetadata(metadataPath, input.optionsHash);return cacheHandle(diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.test.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.test.tsindex 9398e6525..77f7e04af 100644--- a/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.test.ts+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.test.ts@@ -38,13 +38,15 @@ describe.skipIf(process.platform === "win32")("connectMicrosandbox", () => {version: 2,};- const vm = await connectMicrosandbox({- metadata,- metadataPath: "/tmp/eve-microsandbox-session/metadata.json",- module: runtime.module,- options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),- sessionKey: "session-key",- });+ const vm = expectConnected(+ await connectMicrosandbox({+ metadata,+ metadataPath: "/tmp/eve-microsandbox-session/metadata.json",+ module: runtime.module,+ options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),+ sessionKey: "session-key",+ }),+ );await vm.setNetworkPolicy("deny-all");@@ -70,13 +72,15 @@ describe.skipIf(process.platform === "win32")("connectMicrosandbox", () => {version: 2,};- const vm = await connectMicrosandbox({- metadata,- metadataPath: "/tmp/eve-microsandbox-session/metadata.json",- module: runtime.module,- options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),- sessionKey: "session-key",- });+ const vm = expectConnected(+ await connectMicrosandbox({+ metadata,+ metadataPath: "/tmp/eve-microsandbox-session/metadata.json",+ module: runtime.module,+ options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),+ sessionKey: "session-key",+ }),+ );const captured = await vm.captureState("options-hash");@@ -103,13 +107,15 @@ describe.skipIf(process.platform === "win32")("connectMicrosandbox", () => {};try {- const vm = await connectMicrosandbox({- metadata,- metadataPath: "/tmp/eve-microsandbox-session/metadata.json",- module: runtime.module,- options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),- sessionKey: "session-key",- });+ const vm = expectConnected(+ await connectMicrosandbox({+ metadata,+ metadataPath: "/tmp/eve-microsandbox-session/metadata.json",+ module: runtime.module,+ options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),+ sessionKey: "session-key",+ }),+ );const captured = await vm.captureState("options-hash");@@ -161,6 +167,28 @@ describe.skipIf(process.platform === "win32")("connectMicrosandbox", () => {}),);});++ it("returns null when a stopped session snapshot disappeared", async () => {+ const sandbox = createMockMicrosandbox();+ const runtime = createMockMicrosandboxModule(sandbox, { status: "stopped" });+ const metadata: MicrosandboxSessionMetadata = {+ networkPolicy: "allow-all",+ optionsHash: "options-hash",+ sandboxName: "eve-sbx-ses-old",+ stateSnapshotName: "eve-sbx-state-missing",+ version: 2,+ };++ await expect(+ connectMicrosandbox({+ metadata,+ metadataPath: "/tmp/eve-microsandbox-session/metadata.json",+ module: runtime.module,+ options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),+ sessionKey: "session-key",+ }),+ ).resolves.toBeNull();+ });});describe.skipIf(process.platform === "win32")("MicrosandboxVm", () => {@@ -266,6 +294,13 @@ async function withTimeout<T>(promise: Promise<T>, timeoutMs = 100): Promise<T>}}+function expectConnected(vm: MicrosandboxVm | null): MicrosandboxVm {+ if (vm === null) {+ throw new Error("Expected microsandbox reconnect to return a VM.");+ }+ return vm;+}+function createMockMicrosandboxModule(sandbox: ReturnType<typeof createMockMicrosandbox>,options: { readonly snapshots?: readonly string[]; readonly status?: string } = {},@@ -301,7 +336,7 @@ function createMockMicrosandboxModule(Snapshot: {async get(snapshotName: string) {if (!snapshots.has(snapshotName)) {- throw new Error(`snapshot ${snapshotName} not found`);+ throw new Error(`snapshot store unavailable for ${snapshotName}`);}return {};},diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.tsindex 2ed89f84b..b3dc247b5 100644--- a/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.ts+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.ts@@ -333,14 +333,17 @@ export async function connectMicrosandbox(input: {readonly options: ResolvedMicrosandboxOptions;readonly sessionKey: string;readonly tags?: SandboxBackendTags;-}): Promise<MicrosandboxVm> {+}): Promise<MicrosandboxVm | null> {let handle;try {handle = await input.module.Sandbox.get(input.metadata.sandboxName);} catch (error) {- if (!isMicrosandboxNotFoundError(error) || input.metadata.stateSnapshotName === undefined) {+ if (!isMicrosandboxNotFoundError(error)) {throw error;}+ if (input.metadata.stateSnapshotName === undefined) {+ return null;+ }return await restoreMicrosandboxSessionSnapshot(input);}@@ -380,14 +383,12 @@ async function restoreMicrosandboxSessionSnapshot(input: {readonly options: ResolvedMicrosandboxOptions;readonly sessionKey: string;readonly tags?: SandboxBackendTags;-}): Promise<MicrosandboxVm> {+}): Promise<MicrosandboxVm | null> {if (input.metadata.stateSnapshotName === undefined ||!(await snapshotExists(input.module, input.metadata.stateSnapshotName))) {- throw new Error(- `Microsandbox session snapshot is missing for sandbox "${input.metadata.sandboxName}".`,- );+ return null;}const sandboxName = createProviderName("eve-sbx-ses", `${input.sessionKey}:${randomUUID()}`);@@ -536,11 +537,8 @@ export async function snapshotExists(try {await module.Snapshot.get(snapshotName);return true;- } catch (error) {- if (isMicrosandboxNotFoundError(error)) {- return false;- }- throw error;+ } catch {+ return false;}}@@ -652,7 +650,7 @@ async function removeSandboxIfExists(}}-function isMicrosandboxNotFoundError(error: unknown): boolean {+export function isMicrosandboxNotFoundError(error: unknown): boolean {if (!(error instanceof Error)) {return false;}diff --git a/packages/eve/src/execution/sandbox/bindings/vercel-errors.ts b/packages/eve/src/execution/sandbox/bindings/vercel-errors.tsindex a54401bde..14cc7f7e0 100644--- a/packages/eve/src/execution/sandbox/bindings/vercel-errors.ts+++ b/packages/eve/src/execution/sandbox/bindings/vercel-errors.ts@@ -4,21 +4,21 @@ export function isVercelSnapshotUnavailableError(error: unknown): boolean {(candidate as { response?: { status?: number } }).response?.status ??(candidate as { status?: number }).status ??(candidate as { statusCode?: number }).statusCode;- if (status !== 410) {- continue;+ if (status === 410) {+ return true;}+ }- const message = [- (candidate as { message?: unknown }).message,- (candidate as { text?: unknown }).text,- (candidate as { json?: { error?: { message?: unknown } } }).json?.error?.message,- ]- .filter((part): part is string => typeof part === "string")- .join("\n");+ return false;+}- if (- /snapshot .*expired|expired .*snapshot|snapshot .*deleted|deleted .*snapshot/i.test(message)- ) {+export function isVercelSandboxMissingError(error: unknown): boolean {+ for (const candidate of walkErrorChain(error)) {+ const status =+ (candidate as { response?: { status?: number } }).response?.status ??+ (candidate as { status?: number }).status ??+ (candidate as { statusCode?: number }).statusCode;+ if (status === 404) {return true;}}diff --git a/packages/eve/src/execution/sandbox/bindings/vercel.test.ts b/packages/eve/src/execution/sandbox/bindings/vercel.test.tsindex 62dc4e01b..bc7632fb6 100644--- a/packages/eve/src/execution/sandbox/bindings/vercel.test.ts+++ b/packages/eve/src/execution/sandbox/bindings/vercel.test.ts@@ -777,7 +777,7 @@ describe("createVercelSandbox", () => {json: {error: {code: "bad_request",- message: "Snapshot expired or deleted.",+ message: "Resource is gone.",},},response: { status: 410 },@@ -840,6 +840,45 @@ describe("createVercelSandbox", () => {});});+ it("rebuilds a Vercel template when the named sandbox disappears during prewarm", async () => {+ const staleTemplate = createMockSandbox({ name: "template-key" });+ const freshTemplate = createMockSandbox({ name: "template-key" });+ const missingTemplateError = Object.assign(new Error("Status code 404 is not ok"), {+ response: { status: 404 },+ });+ vi.mocked(staleTemplate.snapshot).mockRejectedValueOnce(missingTemplateError);++ const create = vi+ .fn()+ .mockResolvedValueOnce(staleTemplate)+ .mockResolvedValueOnce(freshTemplate);+ const sandboxModule = {+ Sandbox: {+ create,+ get: vi.fn().mockResolvedValue(null),+ },+ };+ const log = vi.fn();++ const backend = createTestVercelSandbox({+ loadSandboxModule: async () => sandboxModule as never,+ });++ await expect(+ backend.prewarm({+ log,+ runtimeContext: { appRoot: "/tmp/test-app-root" },+ seedFiles: [],+ templateKey: "template-key",+ }),+ ).resolves.toEqual({ reused: false });++ expect(create).toHaveBeenCalledTimes(2);+ expect(staleTemplate.snapshot).toHaveBeenCalledTimes(1);+ expect(freshTemplate.snapshot).toHaveBeenCalledTimes(1);+ expect(log).toHaveBeenCalledWith("cached template disappeared; rebuilding sandbox template");+ });+it("resumes a stopped session sandbox via Sandbox.get instead of creating a new one", async () => {const templateSandbox = createMockSandbox({name: "template-key",diff --git a/packages/eve/src/execution/sandbox/bindings/vercel.ts b/packages/eve/src/execution/sandbox/bindings/vercel.tsindex 826589074..bcf51f291 100644--- a/packages/eve/src/execution/sandbox/bindings/vercel.ts+++ b/packages/eve/src/execution/sandbox/bindings/vercel.ts@@ -42,20 +42,17 @@ import {type VercelSandboxCreateParams,type VercelSandboxModule,} from "#execution/sandbox/bindings/vercel-create-sdk.js";-import { isVercelSnapshotUnavailableError } from "#execution/sandbox/bindings/vercel-errors.js";+import {+ isVercelSandboxMissingError,+ isVercelSnapshotUnavailableError,+} from "#execution/sandbox/bindings/vercel-errors.js";import { getNamedVercelSandbox } from "#execution/sandbox/bindings/vercel-lookup.js";-/**- * Construction input for {@link createVercelSandbox}. Internal —- * the public surface is the `vercel()` factory under- * `eve/sandbox`.- */export interface CreateVercelSandboxInput {readonly createSandbox?: CreateVercelSandbox;readonly createOptions?: SandboxCreateOptions;readonly loadSandboxModule?: () => Promise<VercelSandboxModule>;}-/*** Creates the Vercel-backed sandbox backend.*@@ -108,7 +105,10 @@ export function createVercelSandbox(tags,});} catch (error) {- if (template !== null && isVercelSnapshotUnavailableError(error)) {+ if (+ template !== null &&+ (isVercelSnapshotUnavailableError(error) || isVercelSandboxMissingError(error))+ ) {prewarmedTemplates.delete(template.templateKey);const staleTemplate = await getNamedVercelSandbox({createOptions,@@ -139,7 +139,7 @@ export function createVercelSandbox(): Promise<SandboxBackendPrewarmResult> {let outcome: EnsureTemplateOutcome;try {- outcome = await ensureTemplate({+ outcome = await ensureTemplateWithUnavailableRetry({bootstrap: prewarmInput.bootstrap,createOptions,createSandbox,@@ -184,6 +184,20 @@ interface EnsureTemplateInput {readonly templateKey: string;}+async function ensureTemplateWithUnavailableRetry(+ input: EnsureTemplateInput,+): Promise<EnsureTemplateOutcome> {+ try {+ return await ensureTemplate(input);+ } catch (error) {+ if (!isVercelSnapshotUnavailableError(error) && !isVercelSandboxMissingError(error)) {+ throw error;+ }+ input.log?.("cached template disappeared; rebuilding sandbox template");+ return await ensureTemplate(input);+ }+}+async function readTemplate(input: {readonly createOptions: SandboxCreateOptions;readonly loadSandboxModule: () => Promise<VercelSandboxModule>;