Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

test: prove native drag scroll and double-click on current desktop

Task Builder314c795fde87
diff --git a/evals/packages/hosts/src/local.ts b/evals/packages/hosts/src/local.ts
index 0f9b1a9c..a8e54f29 100644
--- a/evals/packages/hosts/src/local.ts
+++ b/evals/packages/hosts/src/local.ts
@@ -886,6 +886,7 @@ async function ensureDisplay(repoRoot: string, env: NodeJS.ProcessEnv, log: (mes
if (process.platform === "win32") {
// Workspace dev scripts use POSIX environment assignments.
// Keep the emulator scoped to this isolated eval process.
+ env.pnpm_config_shell_emulator = "true";
env.npm_config_shell_emulator = "true";
if (!pnpmEntry || !/pnpm\.(?:c|m)?js$/i.test(pnpmEntry) || !existsSync(pnpmEntry)) {
throw new Error("Run Windows Electron journeys through pnpm so npm_execpath names its JS entry.");
diff --git a/evals/packages/labs/fixtures/portable-computer-use-app.mjs b/evals/packages/labs/fixtures/portable-computer-use-app.mjs
index 041423a8..b9a19e43 100644
--- a/evals/packages/labs/fixtures/portable-computer-use-app.mjs
+++ b/evals/packages/labs/fixtures/portable-computer-use-app.mjs
@@ -11,7 +11,7 @@ app.setPath("userData", process.argv.at(-1));
const ready = app.whenReady().then(async () => {
for (const [index, title] of ["Workspace window", "Other window"].entries()) {
const window = new BrowserWindow({ width: 620, height: 450, x: 50 + index * 700, y: 120, title, webPreferences: { nodeIntegration: false, contextIsolation: true } });
- const html = '<title>' + title + '</title><style>body{margin:0;background:white;color:black;font:16px sans-serif}button{position:absolute;left:70px;top:70px;width:180px;height:55px}input{position:absolute;left:70px;top:160px;width:300px;height:40px;caret-color:transparent}#count{position:absolute;left:70px;top:230px}</style><button id="increment">Increment</button><input id="draft" aria-label="Fixture draft" value="Initial draft"><p id="count">0</p><script>window.counter=0;document.getElementById("increment").onclick=()=>{document.getElementById("count").textContent=String(++window.counter)}</script>';
+ const html = "<title>" + title + "</title>" + "<style>body{margin:0;background:white;color:black;font:16px sans-serif}button{position:absolute;left:70px;top:70px;width:180px;height:55px}input{position:absolute;left:70px;top:160px;width:300px;height:40px;caret-color:transparent}#count{position:absolute;left:70px;top:230px}#doubles{left:420px;width:150px}#scroll{position:absolute;left:420px;top:160px;width:150px;height:230px;overflow:auto;background:#ddddff}#drag{position:absolute;left:70px;top:290px;width:120px;height:55px;background:#008080;touch-action:none;user-select:none}</style><button id=\"increment\">Increment</button><input id=\"draft\" aria-label=\"Fixture draft\" value=\"Initial draft\"><p id=\"count\">0</p><button id=\"doubles\">Double click</button><div id=\"scroll\"><div style=\"height:1000px\">Scroll me</div></div><div id=\"drag\">Drag me</div><script>window.counter=0;window.pointer={doubles:0,held:false};document.getElementById(\"increment\").onclick=()=>{document.getElementById(\"count\").textContent=String(++window.counter)};document.getElementById(\"doubles\").ondblclick=()=>{document.getElementById(\"doubles\").textContent=String(++window.pointer.doubles)};const handle=document.getElementById(\"drag\");let start;handle.onpointerdown=e=>{window.pointer.held=true;start={x:e.clientX,y:e.clientY,left:handle.offsetLeft,top:handle.offsetTop};handle.setPointerCapture(e.pointerId)};handle.onpointermove=e=>{if(window.pointer.held&&e.buttons===1){handle.style.left=(start.left+e.clientX-start.x)+\"px\";handle.style.top=(start.top+e.clientY-start.y)+\"px\"}};handle.onpointerup=()=>{window.pointer.held=false};</script>";
await window.loadURL("data:text/html," + encodeURIComponent(html)); windows.push(window);
}
});
@@ -20,6 +20,7 @@ createInterface({ input: channel }).on("line", async (line) => {
try {
await ready; let result;
if (request.method === "state") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript('({ count: window.counter, draft: document.getElementById("draft").value })')));
+ else if (request.method === "pointer_state") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript('({ doubles: window.pointer.doubles, scrolled: document.getElementById("scroll").scrollTop > 0, dragged: document.getElementById("drag").offsetLeft > 170, released: !window.pointer.held })')));
else if (request.method === "bounds") result = windows.map((w) => ({ bounds: w.getBounds(), contentBounds: w.getContentBounds(), scale: screen.getDisplayMatching(w.getBounds()).scaleFactor }));
else if (request.method === "focus_other") { windows[1].focus(); result = {}; }
else if (request.method === "outside_input") {
diff --git a/evals/specs/computer-use-portable-window.e2e.test.ts b/evals/specs/computer-use-portable-window.e2e.test.ts
index d506e60a..447724aa 100644
--- a/evals/specs/computer-use-portable-window.e2e.test.ts
+++ b/evals/specs/computer-use-portable-window.e2e.test.ts
@@ -42,6 +42,7 @@ test("Windows and Linux X11 Computer Use keeps native input within the approved
expect(acted, JSON.stringify(acted)).toMatchObject({ ok: true, dispatched: true });
await expect.poll(() => world.state()).toEqual([{ count: 1, draft: "Initial draft" }, { count: 0, draft: "Initial draft" }]);
expect(toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "repeat-observation", action: { type: "click", ...point } })).code).toBe("stale_observation");
+ expect(toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "first-click", action: { type: "click", ...point } })).code).toBe("duplicate_request");
});
await step("Native keyboard input edits only the selected window", async () => {
const act = async (request: string, action: object) => {
@@ -57,6 +58,24 @@ test("Windows and Linux X11 Computer Use keeps native input within the approved
await act("write-draft", { type: "type", text: "OmniRush 123!" });
await expect.poll(() => world.state()).toEqual([{ count: 1, draft: "OmniRush 123!" }, { count: 0, draft: "Initial draft" }]);
});
+ await step("Native double-click, scroll and drag affect only the selected window and release input", async () => {
+ const act = async (request: string, actionFor: (observation: Record<string, unknown>) => Promise<object>) => {
+ const observed = toolState(await world.call("computer_observe", { session_id: session }));
+ expect(observed.ok, JSON.stringify(observed)).toBe(true);
+ const action = await actionFor(observed);
+ const result = toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: request, action }));
+ expect(result, JSON.stringify(result)).toMatchObject({ ok: true, dispatched: true });
+ };
+ await act("double-click", async (observed) => ({ type: "double_click", ...await world.point(observed, 480, 95) }));
+ await act("scroll", async (observed) => ({ type: "scroll", axis: "vertical", delta: 3, ...await world.point(observed, 480, 240) }));
+ await act("drag", async (observed) => ({ type: "drag", path: await Promise.all([{ x: 120, y: 310 }, { x: 140, y: 315 }, { x: 180, y: 325 }, { x: 240, y: 330 }].map(({ x, y }) => world.point(observed, x, y))) }));
+ await expect.poll(() => world.pointerState()).toEqual([
+ { doubles: 1, scrolled: true, dragged: true, released: true },
+ { doubles: 0, scrolled: false, dragged: false, released: true },
+ ]);
+ await act("restore-draft-focus", async (observed) => ({ type: "click", ...await world.point(observed, 150, 180) }));
+ expect(await world.state()).toEqual([{ count: 1, draft: "OmniRush 123!" }, { count: 0, draft: "Initial draft" }]);
+ });
await step("Changed pixels and out-of-window coordinates cannot dispatch input", async () => {
const observed = toolState(await world.call("computer_observe", { session_id: session }));
await world.change();
@@ -65,6 +84,14 @@ test("Windows and Linux X11 Computer Use keeps native input within the approved
expect(toolState(await world.call("computer_act", { session_id: session, observation_id: fresh.observation_id, request_id: "outside-click", action: { type: "click", x: -1, y: 20 } }))).toMatchObject({ ok: false, code: "invalid_action", may_have_acted: false });
expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);
});
+ await step("System-wide shortcuts are rejected without closing or changing either window", async () => {
+ const observed = toolState(await world.call("computer_observe", { session_id: session }));
+ const result = toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "system-shortcut", action: { type: "key", key: "F4", modifiers: ["alt"] } }));
+ expect(result, JSON.stringify(result)).toMatchObject({ ok: false, code: "unsupported_action" });
+ expect(toolState(await world.call("computer_session_status", { session_id: session })).state).toBe("paused");
+ expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);
+ await world.approve("resume");
+ });
await step("Focus moving to another window stops keyboard input", async () => {
const observed = toolState(await world.call("computer_observe", { session_id: session }));
await world.focusOther();
@@ -114,10 +141,14 @@ test("Windows and Linux X11 Computer Use keeps native input within the approved
const observed = toolState(await world.call("computer_observe", { session_id: opened.session_id, include_image: false }));
expect(observed.ok).toBe(true);
expect(toolState(await world.call("computer_act", { session_id: opened.session_id, observation_id: observed.observation_id, request_id: "read-only-type", action: { type: "type", text: "Forbidden" } })).code).toBe("scope_denied");
+ const baseline = await world.imagePixel(await world.call("computer_observe", { session_id: opened.session_id }));
+ if (!baseline || typeof baseline !== "object" || !("rgb" in baseline) || !Array.isArray(baseline.rgb)) throw new Error("Missing selected-window image sample.");
+ // Windows capture can round colours during its native pixel conversion.
+ for (const channel of baseline.rgb) { expect(channel).toBeGreaterThanOrEqual(234); expect(channel).toBeLessThanOrEqual(242); }
await world.cover();
const covered = await world.call("computer_observe", { session_id: opened.session_id });
expect(toolState(covered).ok, JSON.stringify(toolState(covered))).toBe(true);
- expect(await world.imagePixel(covered)).toMatchObject({ rgb: [238, 238, 238] });
+ expect(await world.imagePixel(covered)).toEqual(baseline);
await world.closeClient();
await expect.poll(() => world.hostState()).toEqual([]);
expect(await world.state()).toEqual(stoppedState);
diff --git a/evals/worlds/computer-use.ts b/evals/worlds/computer-use.ts
index b463b6f6..0d26bcbe 100644
--- a/evals/worlds/computer-use.ts
+++ b/evals/worlds/computer-use.ts
@@ -360,6 +360,7 @@ export async function portableComputerUseWorld(_seed: Seed, { place }: { place:
focusOther: () => fixture.request("focus_other"),
cover: () => fixture.request("cover"),
outsideInput: () => fixture.request("outside_input"),
+ pointerState: () => fixture.request("pointer_state"),
imagePixel(reply: unknown) {
if (!record(reply) || !Array.isArray(reply.content)) throw new Error("Missing image content.");
const image = reply.content.find((item: unknown) => record(item) && item.type === "image");