The Open Framework for Building Agents
fix(computer-use): restart the desktop after sandbox resume (#4006)
e0-gh-vercel-connect[bot]db0cee2aade0
diff --git a/packages/eve-computer-use/extension/lib/computer-use-sandbox.ts b/packages/eve-computer-use/extension/lib/computer-use-sandbox.tsindex 79ec6b9da..ed1ff0f13 100644--- a/packages/eve-computer-use/extension/lib/computer-use-sandbox.ts+++ b/packages/eve-computer-use/extension/lib/computer-use-sandbox.ts@@ -6,6 +6,7 @@ import {computerUseDisplayCommand,computerUseDriverStartCommand,computerUsePaths,+ computerUseReadyCommand,} from "./computer-use.ts";import { shellQuote } from "./shell.ts";@@ -69,6 +70,22 @@ export async function startComputerUse(}}+/**+ * Restart the desktop when a resumed sandbox lost its processes.+ *+ * Sandbox resume does not rerun consumer startup, so the display and driver+ * started there are gone after a stop. The check is one cheap command when+ * everything is already live.+ */+export async function ensureComputerUse(+ sandbox: Pick<SandboxSession, "resolvePath" | "run">,+ abortSignal?: AbortSignal,+): Promise<void> {+ const ready = await sandbox.run({ command: computerUseReadyCommand(sandbox), abortSignal });+ if (ready.exitCode === 0) return;+ await startComputerUse(sandbox);+}+function computerUseInstallScript(sandbox: Pick<SandboxSession, "resolvePath">): string {const paths = computerUsePaths(sandbox);return `#!/usr/bin/env bashdiff --git a/packages/eve-computer-use/extension/lib/computer-use.ts b/packages/eve-computer-use/extension/lib/computer-use.tsindex 030b1e6a6..8a75a6136 100644--- a/packages/eve-computer-use/extension/lib/computer-use.ts+++ b/packages/eve-computer-use/extension/lib/computer-use.ts@@ -197,6 +197,17 @@ export function computerUseDriverStartCommand(sandbox: PathSandbox): string {].join("\n");}+/** Exit 0 only when the display, window manager, and driver are all live. */+export function computerUseReadyCommand(sandbox: PathSandbox): string {+ const paths = computerUsePaths(sandbox);+ return [+ "set -euo pipefail",+ `xdpyinfo -display ${DISPLAY} >/dev/null 2>&1`,+ `DISPLAY=${DISPLAY} xprop -root _NET_SUPPORTING_WM_CHECK 2>/dev/null | grep -Eq 'window id # 0x[1-9a-f]'`,+ `COMPUTER_USE_REQUEST='{"action":"health"}' COMPUTER_USE_SOCKET_PATH=${shellQuote(paths.driverSocket)} node ${shellQuote(`${paths.driverRoot}/client.mjs`)} >/dev/null 2>&1`,+ ].join("\n");+}+export function computerUseDisplayCommand(sandbox: PathSandbox): string {const { root } = computerUsePaths(sandbox);const xfconf = [diff --git a/packages/eve-computer-use/extension/tools/computer_use.ts b/packages/eve-computer-use/extension/tools/computer_use.tsindex 13b28a73a..1034c2bdd 100644--- a/packages/eve-computer-use/extension/tools/computer_use.ts+++ b/packages/eve-computer-use/extension/tools/computer_use.ts@@ -14,6 +14,7 @@ import {recordingPath,screenshotPath,} from "../lib/computer-use.ts";+import { ensureComputerUse } from "../lib/computer-use-sandbox.ts";const SCREEN_WIDTH = 1920;const SCREEN_HEIGHT = 1080;@@ -272,6 +273,8 @@ export default defineTool({if (input.action === "sequence") validateSequenceBudget(input.actions);+ await ensureComputerUse(sandbox, ctx.abortSignal);+const action =input.action === "arrange_pair"? arrangePairAction(input.pair, input.waitForMs)diff --git a/packages/eve-computer-use/test/scenario/computer-use.test.ts b/packages/eve-computer-use/test/scenario/computer-use.test.tsindex b2a05cecf..93ba0eb47 100644--- a/packages/eve-computer-use/test/scenario/computer-use.test.ts+++ b/packages/eve-computer-use/test/scenario/computer-use.test.ts@@ -15,6 +15,7 @@ import {computerUseDriverStartCommand,computerUseFocusCommand,computerUseLaunchCommand,+ computerUseReadyCommand,computerUseWindowLayoutCommand,recordingPath,screenshotPath,@@ -85,6 +86,7 @@ test("computer-use derives every artifact and driver path from the sandbox", ()test("computer-use startup commands remain valid and use resolved paths", () => {const display = computerUseDisplayCommand(sandbox);const driver = computerUseDriverStartCommand(sandbox);+ const ready = computerUseReadyCommand(sandbox);assert.match(display, /if ! xdpyinfo -display :99/u);assert.match(display, /-screen 0 1920x1080x24/u);assert.match(display, /xfwm4 --replace --compositor=on/u);@@ -92,8 +94,12 @@ test("computer-use startup commands remain valid and use resolved paths", () =>assert.match(driver, /CUA_DRIVER_RS_TELEMETRY_ENABLED=false/u);assert.match(driver, /COMPUTER_USE_REQUEST='\{"action":"health"\}'/u);assert.match(driver, /custom\/workspace\/.eve-code\/computer-use-driver\/server\.mjs/u);+ assert.match(ready, /xdpyinfo -display :99/u);+ assert.match(ready, /_NET_SUPPORTING_WM_CHECK/u);+ assert.match(ready, /custom\/workspace\/computer-use\/cua-driver\.sock/u);assertValidShell(display);assertValidShell(driver);+ assertValidShell(ready);});test("computer-use launch, focus, sizing, and layout commands remain valid", () => {@@ -238,17 +244,55 @@ test("computer-use sends a recording sequence through one resolved driver reques}),},);- assert.equal(commands.length, 1);+ assert.deepEqual(+ commands.map(({ command }) => command),+ [computerUseReadyCommand(sandbox), computerUseDriverCommand(sandbox)],+ );assert.equal(- commands[0]?.env?.COMPUTER_USE_SOCKET_PATH,+ commands[1]?.env?.COMPUTER_USE_SOCKET_PATH,"/custom/workspace/computer-use/cua-driver.sock",);- const request = JSON.parse(commands[0]?.env?.COMPUTER_USE_REQUEST ?? "");+ const request = JSON.parse(commands[1]?.env?.COMPUTER_USE_REQUEST ?? "");assert.equal(request.action.actions[0].path, "/custom/workspace/computer-use/take.mp4");assert.equal(output.path, "/custom/workspace/computer-use/take.mp4");assert.deepEqual(output.timings, [{ action: "record_start", durationMs: 12, result: null }]);});+test("computer-use restarts a desktop lost to sandbox resume before acting", async () => {+ const commands: string[] = [];+ const tool = computerUseTool as {+ execute(input: unknown, ctx: unknown): Promise<{ screenshot: string }>;+ };+ const output = await tool.execute(+ { request: { action: "screenshot" } },+ {+ abortSignal: undefined,+ getSandbox: async () => ({+ ...sandbox,+ run: async ({ command }: { command: string }) => {+ commands.push(command);+ // A resumed sandbox has no display or driver processes.+ if (command === computerUseReadyCommand(sandbox)) {+ return { exitCode: 1, stderr: "", stdout: "" };+ }+ return {+ exitCode: 0,+ stderr: "",+ stdout: driverResponse("/custom/workspace/computer-use/latest.png", "desktop"),+ };+ },+ }),+ },+ );+ assert.deepEqual(commands, [+ computerUseReadyCommand(sandbox),+ computerUseDisplayCommand(sandbox),+ computerUseDriverStartCommand(sandbox),+ computerUseDriverCommand(sandbox),+ ]);+ assert.equal(output.screenshot, "/custom/workspace/computer-use/latest.png");+});+test("computer-use advertises an object root while preserving action constraints", async () => {const { z } = await import("zod");const schema = z.toJSONSchema(computerUseTool.inputSchema as import("zod").z.ZodType, {