Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

perf(sandbox): skip redundant setup for prepared Vercel sandboxes (#5276)

Chad Hietalab848a517a731
diff --git a/.changeset/fresh-sandbox-base-runtime.md b/.changeset/fresh-sandbox-base-runtime.md
new file mode 100644
index 000000000..983714847
--- /dev/null
+++ b/.changeset/fresh-sandbox-base-runtime.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Avoid an extra setup command when creating a Vercel sandbox from a prepared snapshot of the standard eve image without Drive mounts. eve now checks the base runtime again after authored preparation, so a `prepare` callback that breaks it fails the build; existing sandboxes, resumes, custom images or sources, Drive mounts, and sandboxes without a prepared snapshot keep their setup checks. Vercel templates rebuild once so every snapshot has passed the new check.
diff --git a/docs/sandbox/vercel.mdx b/docs/sandbox/vercel.mdx
index 45f696d18..750799b5b 100644
--- a/docs/sandbox/vercel.mdx
+++ b/docs/sandbox/vercel.mdx
@@ -53,12 +53,14 @@ Outside Vercel and without a linked project, such as in CI or on a self-hosted s
## Snapshot preparation
-eve creates a temporary, non-persistent Vercel Sandbox during build, installs the base runtime, writes managed workspace and skill files, runs the authored `prepare` callback, and captures a snapshot. The build artifact records the snapshot ID.
+eve creates a temporary, non-persistent Vercel Sandbox during build, installs the base runtime, writes managed workspace and skill files, and runs the authored `prepare` callback. It then checks the base runtime again before capturing the snapshot, so a `prepare` callback that breaks it fails the build. The build artifact records the snapshot ID.
When a live session sandbox does not exist, eve creates it directly from that snapshot ID. Runtime does not need to look up the temporary template Sandbox.
When the environment has no `prepare` callback, no managed workspace or skill files, and no `source`, a snapshot would contain only the base runtime. eve skips it: the build records no snapshot ID, and each new session sandbox starts from the image and installs the base runtime itself. A `source` always produces a snapshot, so sessions start from what the build captured instead of fetching the source again.
+A new session sandbox created from a prepared snapshot of the standard eve image, without Drive mounts, inherits the base runtime the build checked and runs no setup command. Existing or resumed sandboxes, custom images or snapshot sources, sandboxes with Drive mounts, and sandboxes created without a prepared snapshot still run base-runtime setup when eve attaches them.
+
## Initialize after open
Initialize session-specific state after `open()` and before returning the sandbox. This selector writes a configuration file into the persistent `/workspace` filesystem:
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/preparation.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/preparation.eval.ts
index dbc9bf94a..4eca99a5e 100644
--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/preparation.eval.ts
+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/preparation.eval.ts
@@ -2,16 +2,14 @@ import { defineEval } from "eve/evals";
import { PREPARATION_MARKER_PATH, PREPARATION_MARKER_TOKEN } from "./shared";
-// The prompt directs the model to run the backticked `bash` command; a
-// non-error result containing the marker token proves the preparation-written
-// file is visible inside the sandbox.
export default defineEval({
tags: ["real-model"],
- description:
- "Sandbox smoke: `VercelSandbox.environment({ prepare })` runs before the first bash call.",
+ description: "Sandbox preparation and Bash process substitution work in a fresh session.",
async test(t) {
await t.send(
- `Run the bash command \`cat ${PREPARATION_MARKER_PATH}\` and reply with the file contents verbatim.`,
+ "Alice is checking her newly prepared workspace. Run the bash command " +
+ `\`cmp <(printf workspace-ready) <(printf workspace-ready) && cat ${PREPARATION_MARKER_PATH}\` ` +
+ "and reply with the file contents verbatim.",
);
t.succeeded();
diff --git a/packages/eve/src/execution/sandbox/bindings/vercel.test.ts b/packages/eve/src/execution/sandbox/bindings/vercel.test.ts
index c93433c6f..cba6d70cd 100644
--- a/packages/eve/src/execution/sandbox/bindings/vercel.test.ts
+++ b/packages/eve/src/execution/sandbox/bindings/vercel.test.ts
@@ -351,6 +351,7 @@ describe("createVercelSandbox", () => {
source: { snapshotId: "prepared-snapshot", type: "snapshot" },
}),
);
+ expect(sessionSandbox.runCommand).not.toHaveBeenCalled();
});
it("creates sandboxes with Vercel API credentials from the environment", async () => {
@@ -729,7 +730,7 @@ describe("createVercelSandbox", () => {
expect(sessionArgs?.[0]).toMatchObject({ timeout: 30 * 60 * 1_000 });
});
- it("mounts a Drive read-write by name and passes Drive references through", async () => {
+ it("mounts a Drive read-write by name, passes Drive references through, and checks the overlaid base runtime", async () => {
// Like the SDK's `Drive`: a `name` getter, with its metadata in a `drive` field.
class Drive {
readonly drive = { id: "drv_team" };
@@ -753,6 +754,7 @@ describe("createVercelSandbox", () => {
);
await provider.openSession({
appRoot: "/tmp/test-app-root",
+ prepared: { snapshotId: "prepared-snapshot" },
sandboxName: "session-key",
});
expect(create).toHaveBeenCalledWith(
@@ -763,6 +765,7 @@ describe("createVercelSandbox", () => {
},
}),
);
+ expect(sessionSandbox.runCommand).toHaveBeenCalledTimes(1);
});
/*
@@ -856,7 +859,7 @@ describe("createVercelSandbox", () => {
expect(fake.sandboxModule.Sandbox.create).not.toHaveBeenCalled();
});
- it("derives template keys from create options, image, resources, and source revision as before", async () => {
+ it("derives template keys from create options, image, resources, and source revision", async () => {
const managedFiles = {
resourcesKey: "resources-golden",
skills: [{ content: "# Skill", relativePath: "golden/SKILL.md" }],
@@ -902,13 +905,13 @@ describe("createVercelSandbox", () => {
};
expect(keys).toEqual({
- defaults: "eve-sbx-tpl-vercel-1f6d699611395741504fc8a4b10df805",
- managedFiles: "eve-sbx-tpl-vercel-82b4e1f8627add9d53d329878f5bb047",
- image: "eve-sbx-tpl-vercel-1f48759fa1e48bb0d5af6910a423593a",
- source: "eve-sbx-tpl-vercel-2f6b04f764532fa85e252b7c80bbfe56",
- options: "eve-sbx-tpl-vercel-e7a2ead2a2bdd5be4fbbc9025a7cd05b",
- credentials: "eve-sbx-tpl-vercel-e7a2ead2a2bdd5be4fbbc9025a7cd05b",
- sourceRevision: "eve-sbx-tpl-vercel-d995fa0bea286a7b26343f99a4c5e0a1",
+ defaults: "eve-sbx-tpl-vercel-554ce4314fb7a0503d92d30c416064d8",
+ managedFiles: "eve-sbx-tpl-vercel-d9abc1ab182ef14873f57fc00a0246c3",
+ image: "eve-sbx-tpl-vercel-64eba2454d7a47c0c897d5e82b365f4f",
+ source: "eve-sbx-tpl-vercel-11097efe04a48959d9420dd8ab306e12",
+ options: "eve-sbx-tpl-vercel-61b688a6c98003848d40c5a13046c703",
+ credentials: "eve-sbx-tpl-vercel-61b688a6c98003848d40c5a13046c703",
+ sourceRevision: "eve-sbx-tpl-vercel-f638671cfec9d0a12db226863fa3ed9f",
});
});
});
@@ -1060,7 +1063,7 @@ describe("createVercelSandbox", () => {
source: { snapshotId: "prepared-snapshot", type: "snapshot" },
}),
);
- expect(sandboxes.get(base)?.runCommand).toHaveBeenCalled();
+ expect(sandboxes.get(base)?.runCommand).not.toHaveBeenCalled();
});
it("adopts the fresh sandbox another instance made first after the same expiry", async () => {
@@ -1105,7 +1108,7 @@ describe("createVercelSandbox", () => {
expect(sandboxModule.Sandbox.create).toHaveBeenCalledWith(
expect.objectContaining({ name: base }),
);
- expect(sandboxes.get(base)?.runCommand).toHaveBeenCalled();
+ expect(sandboxes.get(base)?.runCommand).not.toHaveBeenCalled();
});
it("surfaces any other 410 instead of replacing the sandbox", async () => {
@@ -1886,7 +1889,7 @@ describe("createVercelSandbox", () => {
]);
});
- it("prepares the base runtime during sandbox init", async () => {
+ it("prepares the base runtime in the template rather than its fresh snapshot-backed session", async () => {
const templateSandbox = createMockSandbox({ name: "template" });
const sessionSandbox = createMockSandbox({ name: "session" });
const sandboxModule = {
@@ -1909,31 +1912,130 @@ describe("createVercelSandbox", () => {
sandboxName: "session-key",
});
- for (const sandbox of [templateSandbox, sessionSandbox]) {
- const calls = vi.mocked(sandbox.runCommand).mock.calls;
- expect(calls).toHaveLength(1);
+ const calls = templateSandbox.runCommand.mock.calls;
+ expect(calls).toHaveLength(2);
- const setupCall = calls[0]?.[0] as {
- args?: string[];
- cmd?: string;
- sudo?: boolean;
- };
- expect(setupCall).toMatchObject({ cmd: "bash" });
- expect(setupCall.sudo).toBeUndefined();
- const setupScript = setupCall.args?.[1] ?? "";
- expect(setupScript).toContain("mkdir -p /workspace");
- expect(setupScript).toContain("command -v bash");
- expect(setupScript).toContain("ln -s /proc/self/fd /dev/fd");
- expect(setupScript).toContain("test /dev/fd -ef /proc/self/fd");
- expect(setupScript).not.toContain("apt-get");
- expect(setupScript).not.toContain("gpgv");
- expect(setupScript).not.toContain("node --version");
- expect(setupScript).not.toContain("npm");
- expect(setupScript).not.toContain("python3");
- expect(setupScript).not.toContain("ripgrep");
- expect(setupScript).not.toContain("sudo mkdir");
- expect(setupScript).not.toContain("chown");
- }
+ const setupCall = calls[0]?.[0] as {
+ args?: string[];
+ cmd?: string;
+ sudo?: boolean;
+ };
+ expect(setupCall).toMatchObject({ cmd: "bash" });
+ expect(setupCall.sudo).toBeUndefined();
+ const setupScript = setupCall.args?.[1] ?? "";
+ expect(setupScript).toContain("mkdir -p /workspace");
+ expect(setupScript).toContain("command -v bash");
+ expect(setupScript).toContain("ln -s /proc/self/fd /dev/fd");
+ expect(setupScript).toContain("test /dev/fd -ef /proc/self/fd");
+ expect(setupScript).not.toContain("apt-get");
+ expect(setupScript).not.toContain("gpgv");
+ expect(setupScript).not.toContain("node --version");
+ expect(setupScript).not.toContain("npm");
+ expect(setupScript).not.toContain("python3");
+ expect(setupScript).not.toContain("ripgrep");
+ expect(setupScript).not.toContain("sudo mkdir");
+ expect(setupScript).not.toContain("chown");
+ expect(sessionSandbox.runCommand).not.toHaveBeenCalled();
+ });
+
+ it("checks the base runtime again after authored preparation and before capturing the snapshot", async () => {
+ const template = createMockSandbox({ name: "template" });
+ const prepare = vi.fn(async () => {});
+ const provider = createTestVercelSandbox({
+ loadSandboxModule: async () =>
+ ({
+ Sandbox: {
+ create: createSandboxesInOrder(template),
+ get: vi.fn().mockResolvedValue(null),
+ },
+ }) as never,
+ prepare,
+ });
+
+ await provider.prepare({ appRoot: "/tmp/test-app-root" });
+
+ expect(template.runCommand).toHaveBeenCalledTimes(2);
+ const [initial, final] = template.runCommand.mock.invocationCallOrder;
+ expect(initial).toBeLessThan(prepare.mock.invocationCallOrder[0]!);
+ expect(final).toBeGreaterThan(prepare.mock.invocationCallOrder[0]!);
+ expect(final).toBeLessThan(template.snapshot.mock.invocationCallOrder[0]!);
+ });
+
+ it("does not snapshot authored preparation that leaves the base runtime unusable", async () => {
+ const template = createMockSandbox({ name: "template" });
+ template.runCommand.mockResolvedValueOnce(createMockCommandResult()).mockResolvedValue({
+ ...createMockCommandResult(),
+ exitCode: 70,
+ });
+ const prepare = vi.fn(async () => {});
+ const provider = createTestVercelSandbox({
+ loadSandboxModule: async () =>
+ ({
+ Sandbox: {
+ create: createSandboxesInOrder(template),
+ get: vi.fn().mockResolvedValue(null),
+ },
+ }) as never,
+ prepare,
+ });
+
+ await expect(provider.prepare({ appRoot: "/tmp/test-app-root" })).rejects.toThrow(
+ "Failed to initialize Vercel sandbox base runtime",
+ );
+ expect(prepare).toHaveBeenCalledTimes(1);
+ expect(template.snapshot).not.toHaveBeenCalled();
+ expect(template.delete).toHaveBeenCalledTimes(1);
+ });
+
+ it.each([undefined, "custom-image"])(
+ "initializes a fresh image session without a prepared snapshot (%s)",
+ async (image) => {
+ const session = createMockSandbox({ name: "session" });
+ const provider = createTestVercelSandbox({
+ createOptions: { image },
+ loadSandboxModule: async () =>
+ ({
+ Sandbox: {
+ create: vi.fn().mockResolvedValue(session),
+ get: vi.fn().mockResolvedValue(null),
+ },
+ }) as never,
+ });
+
+ await provider.openSession({
+ appRoot: "/tmp/test-app-root",
+ prepared: {},
+ sandboxName: "session-key",
+ });
+
+ expect(session.runCommand).toHaveBeenCalledTimes(1);
+ expect(session.runCommand).toHaveBeenCalledWith(expect.objectContaining({ cmd: "bash" }));
+ },
+ );
+
+ it.each([
+ { image: "custom-image" },
+ { source: { snapshotId: "authored-snapshot", type: "snapshot" as const } },
+ ])("keeps base checks for prepared snapshots with custom sources (%j)", async (createOptions) => {
+ const session = createMockSandbox({ name: "session" });
+ const provider = createTestVercelSandbox({
+ createOptions,
+ loadSandboxModule: async () =>
+ ({
+ Sandbox: {
+ create: vi.fn().mockResolvedValue(session),
+ get: vi.fn().mockResolvedValue(null),
+ },
+ }) as never,
+ });
+
+ await provider.openSession({
+ appRoot: "/tmp/test-app-root",
+ prepared: { snapshotId: "prepared-snapshot" },
+ sandboxName: "session-key",
+ });
+
+ expect(session.runCommand).toHaveBeenCalledTimes(1);
});
it("retries base runtime setup through sudo when the default user fails", async () => {
@@ -1961,7 +2063,7 @@ describe("createVercelSandbox", () => {
appRoot: "/tmp/test-app-root",
});
- expect(templateSandbox.runCommand).toHaveBeenCalledTimes(2);
+ expect(templateSandbox.runCommand).toHaveBeenCalledTimes(3);
const firstCall = vi.mocked(templateSandbox.runCommand).mock.calls[0]?.[0] as {
args?: string[];
cmd?: string;
diff --git a/packages/eve/src/execution/sandbox/bindings/vercel.ts b/packages/eve/src/execution/sandbox/bindings/vercel.ts
index 2f6d6bdb2..24ce77f4f 100644
--- a/packages/eve/src/execution/sandbox/bindings/vercel.ts
+++ b/packages/eve/src/execution/sandbox/bindings/vercel.ts
@@ -102,6 +102,9 @@ export function createVercelSandbox(
timeout: input.createOptions?.timeout ?? DEFAULT_SANDBOX_TIMEOUT_MS,
};
const createSandbox = input.createSandbox ?? createVercelEveImageSandbox;
+ const usesEveBase =
+ createOptions.source === undefined &&
+ (createOptions.image ?? VERCEL_EVE_SANDBOX_IMAGE) === VERCEL_EVE_SANDBOX_IMAGE;
return {
async onSessionEnd(context, state, { abortSignal }) {
@@ -113,9 +116,8 @@ export function createVercelSandbox(
});
},
async prepare(context) {
- // Without inputs a snapshot only repeats the base setup every session
- // runs anyway. An authored `source` keeps one, so sessions start from
- // the build's capture instead of fetching the source again.
+ // Without build-time inputs, sessions can initialize directly from the image.
+ // An authored `source` keeps a snapshot instead of being fetched again per session.
if (context.isEmpty && createOptions.source === undefined) return {};
try {
const templateKey = `eve-sbx-tpl-vercel-${context
@@ -125,7 +127,7 @@ export function createVercelSandbox(
createOptions.source === undefined
? (createOptions.image ?? VERCEL_EVE_SANDBOX_IMAGE)
: undefined,
- version: 3,
+ version: 4,
})
.slice(0, 32)}`;
const snapshotId = await ensureTemplate({
@@ -202,6 +204,8 @@ export function createVercelSandbox(
const session = await ensureUsableSession({
// Only an expired snapshot gives a tool session a fresh sandbox.
expired: toolSession ? isVercelSnapshotNotFoundError : isVercelSnapshotUnavailableError,
+ inheritsBaseRuntime:
+ artifact.snapshotId !== undefined && mounts === undefined && usesEveBase,
input: ensureSessionInput,
session: await ensureSession(ensureSessionInput),
});
@@ -371,6 +375,8 @@ async function prepareTemplateSnapshot(
await ensureVercelSandboxBaseRuntime(sandbox);
await applyInitialVercelNetworkPolicy(sandbox, input.createOptions.networkPolicy);
await input.context.populate(createVercelConnection(sandbox));
+ // Authored preparation can change the base paths inherited by every new session.
+ await ensureVercelSandboxBaseRuntime(sandbox);
// The returned snapshot, not the handle's cached `currentSnapshotId`, which the SDK may not refresh.
return (await sandbox.snapshot()).snapshotId;
} finally {
@@ -446,11 +452,17 @@ interface VercelSandboxSessionCreateResult {
async function ensureUsableSession(input: {
readonly expired: (error: unknown) => boolean;
+ readonly inheritsBaseRuntime: boolean;
readonly input: EnsureSessionInput;
readonly session: VercelSandboxSessionCreateResult;
}): Promise<VercelSandboxSessionCreateResult> {
+ const ensureBaseRuntime = async (session: VercelSandboxSessionCreateResult) => {
+ // A fresh restore inherits the base runtime checked before the snapshot was captured.
+ if (session.created && input.inheritsBaseRuntime) return;
+ await ensureVercelSandboxBaseRuntime(session.sandbox);
+ };
try {
- await ensureVercelSandboxBaseRuntime(input.session.sandbox);
+ await ensureBaseRuntime(input.session);
return input.session;
} catch (error) {
if (input.session.created || !input.expired(error)) {
@@ -469,7 +481,7 @@ async function ensureUsableSession(input: {
if (!isVercelSandboxMissingError(error)) throw error;
}
const replacement = await ensureSession(input.input);
- await ensureVercelSandboxBaseRuntime(replacement.sandbox);
+ await ensureBaseRuntime(replacement);
return replacement;
}