Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(computer-use): restart the desktop after sandbox resume (#4006)

e0-gh-vercel-connect[bot]db0cee2aade0
diff --git a/packages/eve-computer-use/extension/lib/computer-use-sandbox.ts b/packages/eve-computer-use/extension/lib/computer-use-sandbox.ts
index 79ec6b9da..ed1ff0f13 100644
--- a/packages/eve-computer-use/extension/lib/computer-use-sandbox.ts
+++ b/packages/eve-computer-use/extension/lib/computer-use-sandbox.ts
@@ -6,6 +6,7 @@ import {
computerUseDisplayCommand,
computerUseDriverStartCommand,
computerUsePaths,
+ computerUseReadyCommand,
} from "./computer-use.ts";
import { shellQuote } from "./shell.ts";
@@ -69,6 +70,22 @@ export async function startComputerUse(
}
}
+/**
+ * Restart the desktop when a resumed sandbox lost its processes.
+ *
+ * Sandbox resume does not rerun consumer startup, so the display and driver
+ * started there are gone after a stop. The check is one cheap command when
+ * everything is already live.
+ */
+export async function ensureComputerUse(
+ sandbox: Pick<SandboxSession, "resolvePath" | "run">,
+ abortSignal?: AbortSignal,
+): Promise<void> {
+ const ready = await sandbox.run({ command: computerUseReadyCommand(sandbox), abortSignal });
+ if (ready.exitCode === 0) return;
+ await startComputerUse(sandbox);
+}
+
function computerUseInstallScript(sandbox: Pick<SandboxSession, "resolvePath">): string {
const paths = computerUsePaths(sandbox);
return `#!/usr/bin/env bash
diff --git a/packages/eve-computer-use/extension/lib/computer-use.ts b/packages/eve-computer-use/extension/lib/computer-use.ts
index 030b1e6a6..8a75a6136 100644
--- a/packages/eve-computer-use/extension/lib/computer-use.ts
+++ b/packages/eve-computer-use/extension/lib/computer-use.ts
@@ -197,6 +197,17 @@ export function computerUseDriverStartCommand(sandbox: PathSandbox): string {
].join("\n");
}
+/** Exit 0 only when the display, window manager, and driver are all live. */
+export function computerUseReadyCommand(sandbox: PathSandbox): string {
+ const paths = computerUsePaths(sandbox);
+ return [
+ "set -euo pipefail",
+ `xdpyinfo -display ${DISPLAY} >/dev/null 2>&1`,
+ `DISPLAY=${DISPLAY} xprop -root _NET_SUPPORTING_WM_CHECK 2>/dev/null | grep -Eq 'window id # 0x[1-9a-f]'`,
+ `COMPUTER_USE_REQUEST='{"action":"health"}' COMPUTER_USE_SOCKET_PATH=${shellQuote(paths.driverSocket)} node ${shellQuote(`${paths.driverRoot}/client.mjs`)} >/dev/null 2>&1`,
+ ].join("\n");
+}
+
export function computerUseDisplayCommand(sandbox: PathSandbox): string {
const { root } = computerUsePaths(sandbox);
const xfconf = [
diff --git a/packages/eve-computer-use/extension/tools/computer_use.ts b/packages/eve-computer-use/extension/tools/computer_use.ts
index 13b28a73a..1034c2bdd 100644
--- a/packages/eve-computer-use/extension/tools/computer_use.ts
+++ b/packages/eve-computer-use/extension/tools/computer_use.ts
@@ -14,6 +14,7 @@ import {
recordingPath,
screenshotPath,
} from "../lib/computer-use.ts";
+import { ensureComputerUse } from "../lib/computer-use-sandbox.ts";
const SCREEN_WIDTH = 1920;
const SCREEN_HEIGHT = 1080;
@@ -272,6 +273,8 @@ export default defineTool({
if (input.action === "sequence") validateSequenceBudget(input.actions);
+ await ensureComputerUse(sandbox, ctx.abortSignal);
+
const action =
input.action === "arrange_pair"
? arrangePairAction(input.pair, input.waitForMs)
diff --git a/packages/eve-computer-use/test/scenario/computer-use.test.ts b/packages/eve-computer-use/test/scenario/computer-use.test.ts
index b2a05cecf..93ba0eb47 100644
--- a/packages/eve-computer-use/test/scenario/computer-use.test.ts
+++ b/packages/eve-computer-use/test/scenario/computer-use.test.ts
@@ -15,6 +15,7 @@ import {
computerUseDriverStartCommand,
computerUseFocusCommand,
computerUseLaunchCommand,
+ computerUseReadyCommand,
computerUseWindowLayoutCommand,
recordingPath,
screenshotPath,
@@ -85,6 +86,7 @@ test("computer-use derives every artifact and driver path from the sandbox", ()
test("computer-use startup commands remain valid and use resolved paths", () => {
const display = computerUseDisplayCommand(sandbox);
const driver = computerUseDriverStartCommand(sandbox);
+ const ready = computerUseReadyCommand(sandbox);
assert.match(display, /if ! xdpyinfo -display :99/u);
assert.match(display, /-screen 0 1920x1080x24/u);
assert.match(display, /xfwm4 --replace --compositor=on/u);
@@ -92,8 +94,12 @@ test("computer-use startup commands remain valid and use resolved paths", () =>
assert.match(driver, /CUA_DRIVER_RS_TELEMETRY_ENABLED=false/u);
assert.match(driver, /COMPUTER_USE_REQUEST='\{"action":"health"\}'/u);
assert.match(driver, /custom\/workspace\/.eve-code\/computer-use-driver\/server\.mjs/u);
+ assert.match(ready, /xdpyinfo -display :99/u);
+ assert.match(ready, /_NET_SUPPORTING_WM_CHECK/u);
+ assert.match(ready, /custom\/workspace\/computer-use\/cua-driver\.sock/u);
assertValidShell(display);
assertValidShell(driver);
+ assertValidShell(ready);
});
test("computer-use launch, focus, sizing, and layout commands remain valid", () => {
@@ -238,17 +244,55 @@ test("computer-use sends a recording sequence through one resolved driver reques
}),
},
);
- assert.equal(commands.length, 1);
+ assert.deepEqual(
+ commands.map(({ command }) => command),
+ [computerUseReadyCommand(sandbox), computerUseDriverCommand(sandbox)],
+ );
assert.equal(
- commands[0]?.env?.COMPUTER_USE_SOCKET_PATH,
+ commands[1]?.env?.COMPUTER_USE_SOCKET_PATH,
"/custom/workspace/computer-use/cua-driver.sock",
);
- const request = JSON.parse(commands[0]?.env?.COMPUTER_USE_REQUEST ?? "");
+ const request = JSON.parse(commands[1]?.env?.COMPUTER_USE_REQUEST ?? "");
assert.equal(request.action.actions[0].path, "/custom/workspace/computer-use/take.mp4");
assert.equal(output.path, "/custom/workspace/computer-use/take.mp4");
assert.deepEqual(output.timings, [{ action: "record_start", durationMs: 12, result: null }]);
});
+test("computer-use restarts a desktop lost to sandbox resume before acting", async () => {
+ const commands: string[] = [];
+ const tool = computerUseTool as {
+ execute(input: unknown, ctx: unknown): Promise<{ screenshot: string }>;
+ };
+ const output = await tool.execute(
+ { request: { action: "screenshot" } },
+ {
+ abortSignal: undefined,
+ getSandbox: async () => ({
+ ...sandbox,
+ run: async ({ command }: { command: string }) => {
+ commands.push(command);
+ // A resumed sandbox has no display or driver processes.
+ if (command === computerUseReadyCommand(sandbox)) {
+ return { exitCode: 1, stderr: "", stdout: "" };
+ }
+ return {
+ exitCode: 0,
+ stderr: "",
+ stdout: driverResponse("/custom/workspace/computer-use/latest.png", "desktop"),
+ };
+ },
+ }),
+ },
+ );
+ assert.deepEqual(commands, [
+ computerUseReadyCommand(sandbox),
+ computerUseDisplayCommand(sandbox),
+ computerUseDriverStartCommand(sandbox),
+ computerUseDriverCommand(sandbox),
+ ]);
+ assert.equal(output.screenshot, "/custom/workspace/computer-use/latest.png");
+});
+
test("computer-use advertises an object root while preserving action constraints", async () => {
const { z } = await import("zod");
const schema = z.toJSONSchema(computerUseTool.inputSchema as import("zod").z.ZodType, {