Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

Handle missing sandbox backend state (#20)

JJ Kasper16f5a52f1000
diff --git a/.changeset/vercel-sandbox-missing-template.md b/.changeset/vercel-sandbox-missing-template.md
new file mode 100644
index 000000000..0dc135898
--- /dev/null
+++ b/.changeset/vercel-sandbox-missing-template.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Handle missing sandbox template and session state more gracefully across Vercel, Microsandbox, and Docker backends. Eve now treats stale Vercel template references, missing Microsandbox session/template snapshots, and Docker template image races as recoverable provisioning misses so the runtime can rebuild or create a fresh sandbox automatically.
diff --git a/packages/eve/src/execution/sandbox/bindings/docker.integration.test.ts b/packages/eve/src/execution/sandbox/bindings/docker.integration.test.ts
index 82cb2af99..a297259e4 100644
--- a/packages/eve/src/execution/sandbox/bindings/docker.integration.test.ts
+++ b/packages/eve/src/execution/sandbox/bindings/docker.integration.test.ts
@@ -262,6 +262,30 @@ describe("createDockerSandboxBackend create", () => {
).rejects.toThrow(SandboxTemplateNotProvisionedError);
});
+ it("throws SandboxTemplateNotProvisionedError when a template-backed container fails to start", async () => {
+ const appRoot = await createScratchDirectory("eve-docker-sandbox-");
+ const { cli } = createFakeDockerCli((args) => {
+ if (isContainerInspect(args)) {
+ return { exitCode: 1, stderr: "No such container" };
+ }
+ if (isImageInspect(args, TEMPLATE_IMAGE)) {
+ return { exitCode: 0, stdout: "sha256:abc\n" };
+ }
+ if (args[0] === "run") {
+ return { exitCode: 1, stderr: "template-backed container failed to start" };
+ }
+ return undefined;
+ });
+
+ await expect(
+ createEngine({ cli }).create({
+ runtimeContext: { appRoot },
+ sessionKey: SESSION_KEY,
+ templateKey: TEMPLATE_KEY,
+ }),
+ ).rejects.toThrow(SandboxTemplateNotProvisionedError);
+ });
+
it("creates a session container from the template image with labels and tags", async () => {
const appRoot = await createScratchDirectory("eve-docker-sandbox-");
const previousRunId = process.env[EVE_DEVELOPMENT_SANDBOX_RUN_ID_ENV];
diff --git a/packages/eve/src/execution/sandbox/bindings/docker.ts b/packages/eve/src/execution/sandbox/bindings/docker.ts
index 02028f82b..0c11220d5 100644
--- a/packages/eve/src/execution/sandbox/bindings/docker.ts
+++ b/packages/eve/src/execution/sandbox/bindings/docker.ts
@@ -233,16 +233,26 @@ export function createDockerSandboxBackend(
);
}
- await startDockerContainer({
- cli,
- containerName,
- image,
- initialNetworkPolicy:
- createInput.templateKey === null ? "allow-all" : options.networkPolicy,
- options,
- role: "session",
- tags: createInput.tags,
- });
+ try {
+ await startDockerContainer({
+ cli,
+ containerName,
+ image,
+ initialNetworkPolicy:
+ createInput.templateKey === null ? "allow-all" : options.networkPolicy,
+ options,
+ role: "session",
+ tags: createInput.tags,
+ });
+ } catch (error) {
+ if (createInput.templateKey !== null) {
+ throw new SandboxTemplateNotProvisionedError({
+ backendName: DOCKER_BACKEND_NAME,
+ templateKey: createInput.templateKey,
+ });
+ }
+ throw error;
+ }
if (createInput.templateKey === null) {
await runDockerBaseSetup(cli, containerName);
diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.test.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.test.ts
index 7f7075f5e..cd9be1c97 100644
--- a/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.test.ts
+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.test.ts
@@ -5,12 +5,14 @@ import {
createMicrosandboxHandle,
prewarmMicrosandboxTemplate,
} from "#execution/sandbox/bindings/microsandbox-lifecycle.js";
+import { SandboxTemplateNotProvisionedError } from "#public/definitions/sandbox-backend.js";
import {
MICROSANDBOX_DEFAULT_IMAGE,
resolveMicrosandboxOptions,
} from "#execution/sandbox/bindings/microsandbox-options.js";
const runtimeMocks = vi.hoisted(() => ({
+ connectMicrosandbox: vi.fn(),
createPreparedMicrosandbox: vi.fn(),
createProviderName: vi.fn((prefix: string, key: string) => `${prefix}-${key}`),
doesPathExist: vi.fn(async () => false),
@@ -57,6 +59,7 @@ describe("createMicrosandboxHandle", () => {
clearActiveMicrosandboxSessionHandlesForTest();
vi.clearAllMocks();
runtimeMocks.loadMicrosandboxModule.mockResolvedValue({} as never);
+ runtimeMocks.connectMicrosandbox.mockReset();
runtimeMocks.sandboxExists.mockResolvedValue(false);
runtimeMocks.snapshotExists.mockResolvedValue(true);
metadataMocks.readSessionMetadata.mockResolvedValue(null);
@@ -106,6 +109,65 @@ describe("createMicrosandboxHandle", () => {
expect(secondHandle).toBe(firstHandle);
expect(runtimeMocks.createPreparedMicrosandbox).toHaveBeenCalledTimes(1);
});
+
+ it("creates fresh from the template when persisted session state disappeared", async () => {
+ const vm = createFakeMicrosandboxVm("session-key");
+ runtimeMocks.connectMicrosandbox.mockResolvedValueOnce(null);
+ runtimeMocks.createPreparedMicrosandbox.mockResolvedValue(vm);
+ runtimeMocks.snapshotExists.mockResolvedValue(true);
+ const options = resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE });
+
+ const handle = await createMicrosandboxHandle({
+ backendName: "microsandbox",
+ createInput: {
+ existingMetadata: {
+ optionsHash: "options-hash",
+ sandboxName: "deleted-sandbox",
+ stateSnapshotName: "deleted-session-snapshot",
+ version: 2,
+ },
+ runtimeContext: { appRoot: "/tmp/eve-app" },
+ sessionKey: "session-key",
+ templateKey: "template-key",
+ },
+ options,
+ optionsHash: "options-hash",
+ });
+
+ expect(runtimeMocks.connectMicrosandbox).toHaveBeenCalledTimes(1);
+ expect(runtimeMocks.createPreparedMicrosandbox).toHaveBeenCalledWith(
+ expect.objectContaining({
+ fromSnapshot: "template-snapshot",
+ sessionKey: "session-key",
+ setupBaseRuntime: false,
+ }),
+ );
+ await expect(handle.captureState()).resolves.toMatchObject({
+ backendName: "microsandbox",
+ sessionKey: "session-key",
+ });
+ });
+
+ it("reports a missing template snapshot race as not provisioned", async () => {
+ runtimeMocks.createPreparedMicrosandbox.mockRejectedValueOnce(
+ new Error("snapshot template-snapshot not found"),
+ );
+ runtimeMocks.snapshotExists.mockResolvedValue(true);
+ const options = resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE });
+
+ await expect(
+ createMicrosandboxHandle({
+ backendName: "microsandbox",
+ createInput: {
+ runtimeContext: { appRoot: "/tmp/eve-app" },
+ sessionKey: "session-key",
+ templateKey: "template-key",
+ },
+ options,
+ optionsHash: "options-hash",
+ }),
+ ).rejects.toBeInstanceOf(SandboxTemplateNotProvisionedError);
+ });
});
describe("prewarmMicrosandboxTemplate", () => {
diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.ts
index 63ee437e4..1ece7163d 100644
--- a/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.ts
+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox-lifecycle.ts
@@ -21,6 +21,7 @@ import {
createPreparedMicrosandbox,
createProviderName,
doesPathExist,
+ isMicrosandboxNotFoundError,
loadMicrosandboxModule,
type MicrosandboxVm,
removeSnapshotIfExists,
@@ -204,12 +205,14 @@ export async function createMicrosandboxHandle(input: {
sessionKey: input.createInput.sessionKey,
tags: sessionTags,
});
- return cacheHandle(
- activeSessionKey,
- createHandle(sandbox, input.backendName, input.optionsHash, () => {
- activeMicrosandboxSessionHandles.delete(activeSessionKey);
- }),
- );
+ if (sandbox !== null) {
+ return cacheHandle(
+ activeSessionKey,
+ createHandle(sandbox, input.backendName, input.optionsHash, () => {
+ activeMicrosandboxSessionHandles.delete(activeSessionKey);
+ }),
+ );
+ }
}
let snapshotName: string | null = null;
@@ -240,16 +243,31 @@ export async function createMicrosandboxHandle(input: {
"eve-sbx-ses",
`${input.createInput.sessionKey}:${randomUUID()}`,
);
- const sandbox = await createPreparedMicrosandbox({
- fromSnapshot: snapshotName ?? undefined,
- module,
- name: sandboxName,
- networkPolicy: input.options.networkPolicy,
- options: input.options,
- sessionKey: input.createInput.sessionKey,
- setupBaseRuntime: snapshotName === null,
- tags: sessionTags,
- });
+ let sandbox: MicrosandboxVm;
+ try {
+ sandbox = await createPreparedMicrosandbox({
+ fromSnapshot: snapshotName ?? undefined,
+ module,
+ name: sandboxName,
+ networkPolicy: input.options.networkPolicy,
+ options: input.options,
+ sessionKey: input.createInput.sessionKey,
+ setupBaseRuntime: snapshotName === null,
+ tags: sessionTags,
+ });
+ } catch (error) {
+ if (
+ snapshotName !== null &&
+ input.createInput.templateKey !== null &&
+ isMicrosandboxNotFoundError(error)
+ ) {
+ throw new SandboxTemplateNotProvisionedError({
+ backendName: input.backendName,
+ templateKey: input.createInput.templateKey,
+ });
+ }
+ throw error;
+ }
await sandbox.writeMetadata(metadataPath, input.optionsHash);
return cacheHandle(
diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.test.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.test.ts
index 9398e6525..77f7e04af 100644
--- a/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.test.ts
+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.test.ts
@@ -38,13 +38,15 @@ describe.skipIf(process.platform === "win32")("connectMicrosandbox", () => {
version: 2,
};
- const vm = await connectMicrosandbox({
- metadata,
- metadataPath: "/tmp/eve-microsandbox-session/metadata.json",
- module: runtime.module,
- options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),
- sessionKey: "session-key",
- });
+ const vm = expectConnected(
+ await connectMicrosandbox({
+ metadata,
+ metadataPath: "/tmp/eve-microsandbox-session/metadata.json",
+ module: runtime.module,
+ options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),
+ sessionKey: "session-key",
+ }),
+ );
await vm.setNetworkPolicy("deny-all");
@@ -70,13 +72,15 @@ describe.skipIf(process.platform === "win32")("connectMicrosandbox", () => {
version: 2,
};
- const vm = await connectMicrosandbox({
- metadata,
- metadataPath: "/tmp/eve-microsandbox-session/metadata.json",
- module: runtime.module,
- options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),
- sessionKey: "session-key",
- });
+ const vm = expectConnected(
+ await connectMicrosandbox({
+ metadata,
+ metadataPath: "/tmp/eve-microsandbox-session/metadata.json",
+ module: runtime.module,
+ options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),
+ sessionKey: "session-key",
+ }),
+ );
const captured = await vm.captureState("options-hash");
@@ -103,13 +107,15 @@ describe.skipIf(process.platform === "win32")("connectMicrosandbox", () => {
};
try {
- const vm = await connectMicrosandbox({
- metadata,
- metadataPath: "/tmp/eve-microsandbox-session/metadata.json",
- module: runtime.module,
- options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),
- sessionKey: "session-key",
- });
+ const vm = expectConnected(
+ await connectMicrosandbox({
+ metadata,
+ metadataPath: "/tmp/eve-microsandbox-session/metadata.json",
+ module: runtime.module,
+ options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),
+ sessionKey: "session-key",
+ }),
+ );
const captured = await vm.captureState("options-hash");
@@ -161,6 +167,28 @@ describe.skipIf(process.platform === "win32")("connectMicrosandbox", () => {
}),
);
});
+
+ it("returns null when a stopped session snapshot disappeared", async () => {
+ const sandbox = createMockMicrosandbox();
+ const runtime = createMockMicrosandboxModule(sandbox, { status: "stopped" });
+ const metadata: MicrosandboxSessionMetadata = {
+ networkPolicy: "allow-all",
+ optionsHash: "options-hash",
+ sandboxName: "eve-sbx-ses-old",
+ stateSnapshotName: "eve-sbx-state-missing",
+ version: 2,
+ };
+
+ await expect(
+ connectMicrosandbox({
+ metadata,
+ metadataPath: "/tmp/eve-microsandbox-session/metadata.json",
+ module: runtime.module,
+ options: resolveMicrosandboxOptions({ image: MICROSANDBOX_DEFAULT_IMAGE }),
+ sessionKey: "session-key",
+ }),
+ ).resolves.toBeNull();
+ });
});
describe.skipIf(process.platform === "win32")("MicrosandboxVm", () => {
@@ -266,6 +294,13 @@ async function withTimeout<T>(promise: Promise<T>, timeoutMs = 100): Promise<T>
}
}
+function expectConnected(vm: MicrosandboxVm | null): MicrosandboxVm {
+ if (vm === null) {
+ throw new Error("Expected microsandbox reconnect to return a VM.");
+ }
+ return vm;
+}
+
function createMockMicrosandboxModule(
sandbox: ReturnType<typeof createMockMicrosandbox>,
options: { readonly snapshots?: readonly string[]; readonly status?: string } = {},
@@ -301,7 +336,7 @@ function createMockMicrosandboxModule(
Snapshot: {
async get(snapshotName: string) {
if (!snapshots.has(snapshotName)) {
- throw new Error(`snapshot ${snapshotName} not found`);
+ throw new Error(`snapshot store unavailable for ${snapshotName}`);
}
return {};
},
diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.ts
index 2ed89f84b..b3dc247b5 100644
--- a/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.ts
+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.ts
@@ -333,14 +333,17 @@ export async function connectMicrosandbox(input: {
readonly options: ResolvedMicrosandboxOptions;
readonly sessionKey: string;
readonly tags?: SandboxBackendTags;
-}): Promise<MicrosandboxVm> {
+}): Promise<MicrosandboxVm | null> {
let handle;
try {
handle = await input.module.Sandbox.get(input.metadata.sandboxName);
} catch (error) {
- if (!isMicrosandboxNotFoundError(error) || input.metadata.stateSnapshotName === undefined) {
+ if (!isMicrosandboxNotFoundError(error)) {
throw error;
}
+ if (input.metadata.stateSnapshotName === undefined) {
+ return null;
+ }
return await restoreMicrosandboxSessionSnapshot(input);
}
@@ -380,14 +383,12 @@ async function restoreMicrosandboxSessionSnapshot(input: {
readonly options: ResolvedMicrosandboxOptions;
readonly sessionKey: string;
readonly tags?: SandboxBackendTags;
-}): Promise<MicrosandboxVm> {
+}): Promise<MicrosandboxVm | null> {
if (
input.metadata.stateSnapshotName === undefined ||
!(await snapshotExists(input.module, input.metadata.stateSnapshotName))
) {
- throw new Error(
- `Microsandbox session snapshot is missing for sandbox "${input.metadata.sandboxName}".`,
- );
+ return null;
}
const sandboxName = createProviderName("eve-sbx-ses", `${input.sessionKey}:${randomUUID()}`);
@@ -536,11 +537,8 @@ export async function snapshotExists(
try {
await module.Snapshot.get(snapshotName);
return true;
- } catch (error) {
- if (isMicrosandboxNotFoundError(error)) {
- return false;
- }
- throw error;
+ } catch {
+ return false;
}
}
@@ -652,7 +650,7 @@ async function removeSandboxIfExists(
}
}
-function isMicrosandboxNotFoundError(error: unknown): boolean {
+export function isMicrosandboxNotFoundError(error: unknown): boolean {
if (!(error instanceof Error)) {
return false;
}
diff --git a/packages/eve/src/execution/sandbox/bindings/vercel-errors.ts b/packages/eve/src/execution/sandbox/bindings/vercel-errors.ts
index a54401bde..14cc7f7e0 100644
--- a/packages/eve/src/execution/sandbox/bindings/vercel-errors.ts
+++ b/packages/eve/src/execution/sandbox/bindings/vercel-errors.ts
@@ -4,21 +4,21 @@ export function isVercelSnapshotUnavailableError(error: unknown): boolean {
(candidate as { response?: { status?: number } }).response?.status ??
(candidate as { status?: number }).status ??
(candidate as { statusCode?: number }).statusCode;
- if (status !== 410) {
- continue;
+ if (status === 410) {
+ return true;
}
+ }
- const message = [
- (candidate as { message?: unknown }).message,
- (candidate as { text?: unknown }).text,
- (candidate as { json?: { error?: { message?: unknown } } }).json?.error?.message,
- ]
- .filter((part): part is string => typeof part === "string")
- .join("\n");
+ return false;
+}
- if (
- /snapshot .*expired|expired .*snapshot|snapshot .*deleted|deleted .*snapshot/i.test(message)
- ) {
+export function isVercelSandboxMissingError(error: unknown): boolean {
+ for (const candidate of walkErrorChain(error)) {
+ const status =
+ (candidate as { response?: { status?: number } }).response?.status ??
+ (candidate as { status?: number }).status ??
+ (candidate as { statusCode?: number }).statusCode;
+ if (status === 404) {
return true;
}
}
diff --git a/packages/eve/src/execution/sandbox/bindings/vercel.test.ts b/packages/eve/src/execution/sandbox/bindings/vercel.test.ts
index 62dc4e01b..bc7632fb6 100644
--- a/packages/eve/src/execution/sandbox/bindings/vercel.test.ts
+++ b/packages/eve/src/execution/sandbox/bindings/vercel.test.ts
@@ -777,7 +777,7 @@ describe("createVercelSandbox", () => {
json: {
error: {
code: "bad_request",
- message: "Snapshot expired or deleted.",
+ message: "Resource is gone.",
},
},
response: { status: 410 },
@@ -840,6 +840,45 @@ describe("createVercelSandbox", () => {
});
});
+ it("rebuilds a Vercel template when the named sandbox disappears during prewarm", async () => {
+ const staleTemplate = createMockSandbox({ name: "template-key" });
+ const freshTemplate = createMockSandbox({ name: "template-key" });
+ const missingTemplateError = Object.assign(new Error("Status code 404 is not ok"), {
+ response: { status: 404 },
+ });
+ vi.mocked(staleTemplate.snapshot).mockRejectedValueOnce(missingTemplateError);
+
+ const create = vi
+ .fn()
+ .mockResolvedValueOnce(staleTemplate)
+ .mockResolvedValueOnce(freshTemplate);
+ const sandboxModule = {
+ Sandbox: {
+ create,
+ get: vi.fn().mockResolvedValue(null),
+ },
+ };
+ const log = vi.fn();
+
+ const backend = createTestVercelSandbox({
+ loadSandboxModule: async () => sandboxModule as never,
+ });
+
+ await expect(
+ backend.prewarm({
+ log,
+ runtimeContext: { appRoot: "/tmp/test-app-root" },
+ seedFiles: [],
+ templateKey: "template-key",
+ }),
+ ).resolves.toEqual({ reused: false });
+
+ expect(create).toHaveBeenCalledTimes(2);
+ expect(staleTemplate.snapshot).toHaveBeenCalledTimes(1);
+ expect(freshTemplate.snapshot).toHaveBeenCalledTimes(1);
+ expect(log).toHaveBeenCalledWith("cached template disappeared; rebuilding sandbox template");
+ });
+
it("resumes a stopped session sandbox via Sandbox.get instead of creating a new one", async () => {
const templateSandbox = createMockSandbox({
name: "template-key",
diff --git a/packages/eve/src/execution/sandbox/bindings/vercel.ts b/packages/eve/src/execution/sandbox/bindings/vercel.ts
index 826589074..bcf51f291 100644
--- a/packages/eve/src/execution/sandbox/bindings/vercel.ts
+++ b/packages/eve/src/execution/sandbox/bindings/vercel.ts
@@ -42,20 +42,17 @@ import {
type VercelSandboxCreateParams,
type VercelSandboxModule,
} from "#execution/sandbox/bindings/vercel-create-sdk.js";
-import { isVercelSnapshotUnavailableError } from "#execution/sandbox/bindings/vercel-errors.js";
+import {
+ isVercelSandboxMissingError,
+ isVercelSnapshotUnavailableError,
+} from "#execution/sandbox/bindings/vercel-errors.js";
import { getNamedVercelSandbox } from "#execution/sandbox/bindings/vercel-lookup.js";
-/**
- * Construction input for {@link createVercelSandbox}. Internal —
- * the public surface is the `vercel()` factory under
- * `eve/sandbox`.
- */
export interface CreateVercelSandboxInput {
readonly createSandbox?: CreateVercelSandbox;
readonly createOptions?: SandboxCreateOptions;
readonly loadSandboxModule?: () => Promise<VercelSandboxModule>;
}
-
/**
* Creates the Vercel-backed sandbox backend.
*
@@ -108,7 +105,10 @@ export function createVercelSandbox(
tags,
});
} catch (error) {
- if (template !== null && isVercelSnapshotUnavailableError(error)) {
+ if (
+ template !== null &&
+ (isVercelSnapshotUnavailableError(error) || isVercelSandboxMissingError(error))
+ ) {
prewarmedTemplates.delete(template.templateKey);
const staleTemplate = await getNamedVercelSandbox({
createOptions,
@@ -139,7 +139,7 @@ export function createVercelSandbox(
): Promise<SandboxBackendPrewarmResult> {
let outcome: EnsureTemplateOutcome;
try {
- outcome = await ensureTemplate({
+ outcome = await ensureTemplateWithUnavailableRetry({
bootstrap: prewarmInput.bootstrap,
createOptions,
createSandbox,
@@ -184,6 +184,20 @@ interface EnsureTemplateInput {
readonly templateKey: string;
}
+async function ensureTemplateWithUnavailableRetry(
+ input: EnsureTemplateInput,
+): Promise<EnsureTemplateOutcome> {
+ try {
+ return await ensureTemplate(input);
+ } catch (error) {
+ if (!isVercelSnapshotUnavailableError(error) && !isVercelSandboxMissingError(error)) {
+ throw error;
+ }
+ input.log?.("cached template disappeared; rebuilding sandbox template");
+ return await ensureTemplate(input);
+ }
+}
+
async function readTemplate(input: {
readonly createOptions: SandboxCreateOptions;
readonly loadSandboxModule: () => Promise<VercelSandboxModule>;