The Open Framework for Building Agents
feat(eve): report microsandbox create progress and enrich failures (#90)
Rui11a9a3ed30f7
diff --git a/.changeset/clear-microsandbox-prewarm.md b/.changeset/clear-microsandbox-prewarm.mdnew file mode 100644index 000000000..315741639--- /dev/null+++ b/.changeset/clear-microsandbox-prewarm.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Report image-pull and VM-boot progress during microsandbox creation, and include phase and provider-specific recovery guidance when prewarm fails.diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox-create.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox-create.tsnew file mode 100644index 000000000..72ea38689--- /dev/null+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox-create.ts@@ -0,0 +1,166 @@+import { toErrorMessage } from "#shared/errors.js";+import type {+ MicrosandboxError,+ MicrosandboxErrorCode,+ PullProgressEvent,+ Sandbox as MicrosandboxSandbox,+ SandboxBuilder as MicrosandboxSandboxBuilder,+} from "microsandbox";++type MicrosandboxCreatePhase =+ | "resolving the image"+ | "downloading and materializing image layers"+ | "assembling the image filesystem"+ | "image ready; booting microsandbox VM";++export async function createMicrosandboxWithProgress(input: {+ readonly builder: MicrosandboxSandboxBuilder;+ readonly errorType: typeof MicrosandboxError;+ readonly log?: (message: string) => void;+ readonly source: string;+}): Promise<MicrosandboxSandbox> {+ const progress = { phase: "resolving the image" as MicrosandboxCreatePhase };++ try {+ const creation = await input.builder.createWithPullProgress();+ const progressReporter = reportMicrosandboxCreateProgress(creation, progress, input.log);+ try {+ return await withHeartbeat(progress, input.log, async () => await creation.awaitSandbox());+ } finally {+ await progressReporter.catch(() => {});+ }+ } catch (error) {+ throw enrichMicrosandboxError({+ context: `Failed to create microsandbox VM from ${input.source} while ${progress.phase}`,+ error,+ errorType: input.errorType,+ });+ }+}++export function enrichMicrosandboxError(input: {+ readonly context: string;+ readonly error: unknown;+ readonly errorType?: typeof MicrosandboxError;+}): Error {+ if (input.error instanceof MicrosandboxDiagnosticError) {+ return input.error;+ }++ const code = readMicrosandboxErrorCode(input.error, input.errorType);+ const codeLabel = code === undefined ? "" : ` [${code}]`;+ const detail = toErrorMessage(input.error);+ const detailSentence = /[.!?]$/u.test(detail) ? detail : `${detail}.`;+ const hint = microsandboxErrorHint(code);+ const hintSuffix = hint === undefined ? "" : ` ${hint}`;+ return new MicrosandboxDiagnosticError(+ `${input.context}${codeLabel}: ${detailSentence}${hintSuffix}`,+ { cause: input.error },+ );+}++class MicrosandboxDiagnosticError extends Error {}++async function reportMicrosandboxCreateProgress(+ events: AsyncIterable<PullProgressEvent>,+ progress: { phase: MicrosandboxCreatePhase },+ log: ((message: string) => void) | undefined,+): Promise<void> {+ for await (const event of events) {+ const nextPhase = microsandboxCreatePhase(event, progress.phase);+ if (nextPhase === progress.phase) {+ continue;+ }+ progress.phase = nextPhase;+ log?.(nextPhase);+ }+}++function microsandboxCreatePhase(+ event: PullProgressEvent,+ currentPhase: MicrosandboxCreatePhase,+): MicrosandboxCreatePhase {+ switch (event.kind) {+ case "resolving":+ case "resolved":+ return "resolving the image";+ case "layerDownloadProgress":+ case "layerDownloadComplete":+ case "layerDownloadVerifying":+ case "layerMaterializeStarted":+ case "layerMaterializeProgress":+ case "layerMaterializeWriting":+ case "layerMaterializeComplete":+ return "downloading and materializing image layers";+ case "stitchMergingTrees":+ case "stitchWritingFsmeta":+ case "stitchWritingVmdk":+ case "stitchComplete":+ return "assembling the image filesystem";+ case "complete":+ return "image ready; booting microsandbox VM";+ default:+ return currentPhase;+ }+}++async function withHeartbeat<T>(+ progress: { readonly phase: MicrosandboxCreatePhase },+ log: ((message: string) => void) | undefined,+ callback: () => Promise<T>,+): Promise<T> {+ log?.(progress.phase);+ if (log === undefined) {+ return await callback();+ }++ const startedAt = Date.now();+ const timer = setInterval(() => {+ const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);+ log(`${progress.phase} (${elapsedSeconds}s elapsed)`);+ }, 10_000);+ timer.unref?.();++ try {+ return await callback();+ } finally {+ clearInterval(timer);+ }+}++// Recovery guidance keyed by error code, and the single source of truth+// for which microsandbox codes we recognize: only codes with guidance+// are trusted when duck-typing (see `readMicrosandboxErrorCode`).+const MICROSANDBOX_ERROR_HINTS: Partial<Record<MicrosandboxErrorCode, string>> = {+ database:+ "Check that the microsandbox npm package and installed VM runtime use the same version. If versions changed, use a clean MSB_HOME or migrate the existing database.",+ imageNotFound: "Check that the image exists and that the registry credentials can pull it.",+ libkrunfwNotFound:+ "Run `npx microsandbox install` and verify that MSB_PATH points to the installed runtime.",+};++function readMicrosandboxErrorCode(+ error: unknown,+ errorType: typeof MicrosandboxError | undefined,+): MicrosandboxErrorCode | undefined {+ if (errorType !== undefined && error instanceof errorType) {+ return error.code;+ }+ // Without the module's error class we can only duck-type. Trust a+ // `code` string only when it names a code we have guidance for, so a+ // generic Node error (e.g. ENOENT) surfacing during prewarm is not+ // mislabeled as a microsandbox code.+ if (+ error instanceof Error &&+ "code" in error &&+ typeof error.code === "string" &&+ Object.hasOwn(MICROSANDBOX_ERROR_HINTS, error.code)+ ) {+ return error.code as MicrosandboxErrorCode;+ }+ return undefined;+}++function microsandboxErrorHint(code: MicrosandboxErrorCode | undefined): string | undefined {+ return code === undefined ? undefined : MICROSANDBOX_ERROR_HINTS[code];+}diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.test.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.test.tsindex 77f7e04af..07819c9ee 100644--- a/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.test.ts+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.test.ts@@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest";import type { MicrosandboxSessionMetadata } from "#execution/sandbox/bindings/microsandbox-metadata.js";import {connectMicrosandbox,+ createPreparedMicrosandbox,MicrosandboxVm,} from "#execution/sandbox/bindings/microsandbox-runtime.js";import {@@ -253,12 +254,167 @@ describe.skipIf(process.platform === "win32")("MicrosandboxVm", () => {});});+describe.skipIf(process.platform === "win32")("createPreparedMicrosandbox", () => {+ it("reports the current creation phase while the VM is still starting", async () => {+ vi.useFakeTimers();+ const sandbox = createMockMicrosandbox();+ const created = Promise.withResolvers<typeof sandbox>();+ const log = vi.fn();+ const module = createCreationModule({+ create: () => created.promise,+ progress: [+ { kind: "resolving", reference: MICROSANDBOX_DEFAULT_IMAGE },+ {+ kind: "complete",+ layerCount: 12,+ reference: MICROSANDBOX_DEFAULT_IMAGE,+ },+ ],+ });++ try {+ const pending = createPreparedMicrosandbox({+ log,+ module: module as never,+ name: "template-vm",+ networkPolicy: "deny-all",+ options: resolveMicrosandboxOptions(undefined),+ sessionKey: "template-key",+ setupBaseRuntime: false,+ });++ await vi.advanceTimersByTimeAsync(10_000);++ expect(log).toHaveBeenCalledWith("image ready; booting microsandbox VM (10s elapsed)");++ created.resolve(sandbox);+ await pending;+ } finally {+ vi.useRealTimers();+ }+ });++ it("adds image and provider context when VM creation rejects", async () => {+ class TestMicrosandboxError extends Error {+ readonly code = "imageNotFound";+ }++ const cause = new TestMicrosandboxError("manifest was not found");+ const module = {+ ...createCreationModule({+ create: async () => {+ throw cause;+ },+ progress: [],+ }),+ MicrosandboxError: TestMicrosandboxError,+ };++ const creation = createPreparedMicrosandbox({+ module: module as never,+ name: "template-vm",+ networkPolicy: "deny-all",+ options: resolveMicrosandboxOptions(undefined),+ sessionKey: "template-key",+ setupBaseRuntime: false,+ });++ await expect(creation).rejects.toMatchObject({+ cause,+ message: expect.stringContaining(+ `Failed to create microsandbox VM from image "${MICROSANDBOX_DEFAULT_IMAGE}" ` ++ "while resolving the image [imageNotFound]: manifest was not found. " ++ "Check that the image exists and that the registry credentials can pull it.",+ ),+ });+ });++ it("reports interleaved layer work as one stable phase", async () => {+ const log = vi.fn();+ const module = createCreationModule({+ create: async () => createMockMicrosandbox(),+ progress: [+ { kind: "resolving", reference: MICROSANDBOX_DEFAULT_IMAGE },+ { kind: "layerDownloadProgress" },+ { kind: "layerMaterializeStarted" },+ { kind: "layerDownloadProgress" },+ { kind: "layerMaterializeComplete" },+ { kind: "stitchWritingVmdk" },+ {+ kind: "complete",+ layerCount: 2,+ reference: MICROSANDBOX_DEFAULT_IMAGE,+ },+ ],+ });++ await createPreparedMicrosandbox({+ log,+ module: module as never,+ name: "template-vm",+ networkPolicy: "deny-all",+ options: resolveMicrosandboxOptions(undefined),+ sessionKey: "template-key",+ setupBaseRuntime: false,+ });++ expect(log.mock.calls.map(([message]) => message)).toEqual([+ "resolving the image",+ "downloading and materializing image layers",+ "assembling the image filesystem",+ "image ready; booting microsandbox VM",+ ]);+ });+});+function createMockMicrosandbox() {return {async stop() {},};}+function createCreationModule(input: {+ readonly create: () => Promise<ReturnType<typeof createMockMicrosandbox>>;+ readonly progress: readonly Record<string, unknown>[];+}) {+ const builder = {+ cpus: returnBuilder,+ async create() {+ return await input.create();+ },+ async createWithPullProgress() {+ return {+ async *[Symbol.asyncIterator]() {+ yield* input.progress;+ },+ awaitSandbox: input.create,+ };+ },+ detached: returnBuilder,+ disableNetwork: returnBuilder,+ envs: returnBuilder,+ image: returnBuilder,+ labels: returnBuilder,+ memory: returnBuilder,+ pullPolicy: returnBuilder,+ replace: returnBuilder,+ user: returnBuilder,+ workdir: returnBuilder,+ };++ function returnBuilder() {+ return builder;+ }++ return {+ Sandbox: {+ builder() {+ return builder;+ },+ },+ };+}+function createHangingExecHandleAfterExit(events: unknown[]) {return {async kill() {},@@ -322,6 +478,7 @@ function createMockMicrosandboxModule(const snapshots = new Set(options.snapshots ?? []);const module = {+ MicrosandboxError: class extends Error {},Sandbox: {async get() {return oldHandle;@@ -399,6 +556,14 @@ function createMockSandboxBuilder(create: (fromSnapshot: string) => unknown) {async create() {return create(fromSnapshot);},+ async createWithPullProgress() {+ return {+ async *[Symbol.asyncIterator]() {},+ async awaitSandbox() {+ return create(fromSnapshot);+ },+ };+ },};return builder;}diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.tsindex dfd32e28c..549eb283b 100644--- a/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.ts+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox-runtime.ts@@ -3,6 +3,7 @@ import { access } from "node:fs/promises";import { posix } from "node:path";import { shellQuote } from "#execution/sandbox/shell-quote.js";+import { createMicrosandboxWithProgress } from "#execution/sandbox/bindings/microsandbox-create.js";import {applyMicrosandboxNetwork,createMicrosandboxNetworkPlan,@@ -292,6 +293,7 @@ export async function createPreparedMicrosandbox(input: {const initialNetworkPolicy = input.setupBaseRuntime ? "allow-all" : input.networkPolicy;const sandbox = await createMicrosandbox({fromSnapshot: input.fromSnapshot,+ log: input.log,module: input.module,name: input.name,networkPolicy: initialNetworkPolicy,@@ -590,6 +592,7 @@ export async function doesPathExist(path: string): Promise<boolean> {async function createMicrosandbox(input: {readonly fromSnapshot?: string;+ readonly log?: (message: string) => void;readonly module: MicrosandboxModule;readonly name: string;readonly networkPolicy?: SandboxNetworkPolicy;@@ -618,7 +621,16 @@ async function createMicrosandbox(input: {builder = builder.user(input.user);}- return await applyMicrosandboxNetwork(builder, input.networkPolicy).create();+ const source =+ input.fromSnapshot === undefined+ ? `image "${input.options.image}"`+ : `snapshot "${input.fromSnapshot}"`;+ return await createMicrosandboxWithProgress({+ builder: applyMicrosandboxNetwork(builder, input.networkPolicy),+ errorType: input.module.MicrosandboxError,+ log: input.log,+ source,+ });}async function removeSandboxIfExists(diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox.test.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox.test.tsindex d55548daa..21a425bf3 100644--- a/packages/eve/src/execution/sandbox/bindings/microsandbox.test.ts+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox.test.ts@@ -1,4 +1,4 @@-import { describe, expect, it } from "vitest";+import { beforeEach, describe, expect, it, vi } from "vitest";import { createMicrosandboxSandboxBackend } from "#execution/sandbox/bindings/microsandbox.js";import {@@ -10,11 +10,22 @@ import {resolveMicrosandboxOptions,} from "#execution/sandbox/bindings/microsandbox-options.js";+const lifecycleMocks = vi.hoisted(() => ({+ createMicrosandboxHandle: vi.fn(),+ prewarmMicrosandboxTemplate: vi.fn(),+}));++vi.mock("#execution/sandbox/bindings/microsandbox-lifecycle.js", () => lifecycleMocks);+// The microsandbox native bindings ship for macOS (Apple Silicon) and// glibc Linux only; keep every microsandbox suite off Windows.const onWindows = process.platform === "win32";describe.skipIf(onWindows)("createMicrosandboxSandboxBackend", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });+it("exposes the stable backend name without loading microsandbox", () => {expect(createMicrosandboxSandboxBackend().name).toBe("microsandbox");});@@ -32,6 +43,30 @@ describe.skipIf(onWindows)("createMicrosandboxSandboxBackend", () => {expect(base.setup.autoInstall).toBe(true);expect(noInstall.setup.autoInstall).toBe(false);});++ it("adds version-skew guidance to database failures during prewarm", async () => {+ const cause = Object.assign(+ new Error("Migration file of version 'm20260606_000001_named_volume_kinds' is missing"),+ { code: "database" },+ );+ lifecycleMocks.prewarmMicrosandboxTemplate.mockRejectedValueOnce(cause);++ const prewarm = createMicrosandboxSandboxBackend().prewarm?.({+ runtimeContext: { appRoot: "/tmp/eve-app" },+ seedFiles: [],+ templateKey: "template-key",+ });++ await expect(prewarm).rejects.toMatchObject({+ cause,+ message: expect.stringContaining(+ 'Failed to prewarm microsandbox template "template-key" [database]: ' ++ "Migration file of version 'm20260606_000001_named_volume_kinds' is missing. " ++ "Check that the microsandbox npm package and installed VM runtime use the same version. " ++ "If versions changed, use a clean MSB_HOME or migrate the existing database.",+ ),+ });+ });});describe.skipIf(onWindows)("createMicrosandboxNetworkPlan", () => {diff --git a/packages/eve/src/execution/sandbox/bindings/microsandbox.ts b/packages/eve/src/execution/sandbox/bindings/microsandbox.tsindex a918e064f..a0f1df9e8 100644--- a/packages/eve/src/execution/sandbox/bindings/microsandbox.ts+++ b/packages/eve/src/execution/sandbox/bindings/microsandbox.ts@@ -2,6 +2,7 @@ import {createMicrosandboxHandle,prewarmMicrosandboxTemplate,} from "#execution/sandbox/bindings/microsandbox-lifecycle.js";+import { enrichMicrosandboxError } from "#execution/sandbox/bindings/microsandbox-create.js";import {microsandboxOptionsForHash,resolveMicrosandboxOptions,@@ -56,12 +57,19 @@ export function createMicrosandboxSandboxBackend(async prewarm(prewarmInput: SandboxBackendPrewarmInput<MicrosandboxBootstrapUseOptions>,): Promise<SandboxBackendPrewarmResult> {- return await prewarmMicrosandboxTemplate({- backendName: MICROSANDBOX_BACKEND_NAME,- options,- optionsHash,- prewarmInput,- });+ try {+ return await prewarmMicrosandboxTemplate({+ backendName: MICROSANDBOX_BACKEND_NAME,+ options,+ optionsHash,+ prewarmInput,+ });+ } catch (error) {+ throw enrichMicrosandboxError({+ context: `Failed to prewarm microsandbox template "${prewarmInput.templateKey}"`,+ error,+ });+ }},async create(createInput: SandboxBackendCreateInput,