A desktop coding agent with free access to frontier models.
TypeScript
test: prove native drag scroll and double-click on current desktop
Task Builder314c795fde87
diff --git a/evals/packages/hosts/src/local.ts b/evals/packages/hosts/src/local.tsindex 0f9b1a9c..a8e54f29 100644--- a/evals/packages/hosts/src/local.ts+++ b/evals/packages/hosts/src/local.ts@@ -886,6 +886,7 @@ async function ensureDisplay(repoRoot: string, env: NodeJS.ProcessEnv, log: (mesif (process.platform === "win32") {// Workspace dev scripts use POSIX environment assignments.// Keep the emulator scoped to this isolated eval process.+ env.pnpm_config_shell_emulator = "true";env.npm_config_shell_emulator = "true";if (!pnpmEntry || !/pnpm\.(?:c|m)?js$/i.test(pnpmEntry) || !existsSync(pnpmEntry)) {throw new Error("Run Windows Electron journeys through pnpm so npm_execpath names its JS entry.");diff --git a/evals/packages/labs/fixtures/portable-computer-use-app.mjs b/evals/packages/labs/fixtures/portable-computer-use-app.mjsindex 041423a8..b9a19e43 100644--- a/evals/packages/labs/fixtures/portable-computer-use-app.mjs+++ b/evals/packages/labs/fixtures/portable-computer-use-app.mjs@@ -11,7 +11,7 @@ app.setPath("userData", process.argv.at(-1));const ready = app.whenReady().then(async () => {for (const [index, title] of ["Workspace window", "Other window"].entries()) {const window = new BrowserWindow({ width: 620, height: 450, x: 50 + index * 700, y: 120, title, webPreferences: { nodeIntegration: false, contextIsolation: true } });- const html = '<title>' + title + '</title><style>body{margin:0;background:white;color:black;font:16px sans-serif}button{position:absolute;left:70px;top:70px;width:180px;height:55px}input{position:absolute;left:70px;top:160px;width:300px;height:40px;caret-color:transparent}#count{position:absolute;left:70px;top:230px}</style><button id="increment">Increment</button><input id="draft" aria-label="Fixture draft" value="Initial draft"><p id="count">0</p><script>window.counter=0;document.getElementById("increment").onclick=()=>{document.getElementById("count").textContent=String(++window.counter)}</script>';+ const html = "<title>" + title + "</title>" + "<style>body{margin:0;background:white;color:black;font:16px sans-serif}button{position:absolute;left:70px;top:70px;width:180px;height:55px}input{position:absolute;left:70px;top:160px;width:300px;height:40px;caret-color:transparent}#count{position:absolute;left:70px;top:230px}#doubles{left:420px;width:150px}#scroll{position:absolute;left:420px;top:160px;width:150px;height:230px;overflow:auto;background:#ddddff}#drag{position:absolute;left:70px;top:290px;width:120px;height:55px;background:#008080;touch-action:none;user-select:none}</style><button id=\"increment\">Increment</button><input id=\"draft\" aria-label=\"Fixture draft\" value=\"Initial draft\"><p id=\"count\">0</p><button id=\"doubles\">Double click</button><div id=\"scroll\"><div style=\"height:1000px\">Scroll me</div></div><div id=\"drag\">Drag me</div><script>window.counter=0;window.pointer={doubles:0,held:false};document.getElementById(\"increment\").onclick=()=>{document.getElementById(\"count\").textContent=String(++window.counter)};document.getElementById(\"doubles\").ondblclick=()=>{document.getElementById(\"doubles\").textContent=String(++window.pointer.doubles)};const handle=document.getElementById(\"drag\");let start;handle.onpointerdown=e=>{window.pointer.held=true;start={x:e.clientX,y:e.clientY,left:handle.offsetLeft,top:handle.offsetTop};handle.setPointerCapture(e.pointerId)};handle.onpointermove=e=>{if(window.pointer.held&&e.buttons===1){handle.style.left=(start.left+e.clientX-start.x)+\"px\";handle.style.top=(start.top+e.clientY-start.y)+\"px\"}};handle.onpointerup=()=>{window.pointer.held=false};</script>";await window.loadURL("data:text/html," + encodeURIComponent(html)); windows.push(window);}});@@ -20,6 +20,7 @@ createInterface({ input: channel }).on("line", async (line) => {try {await ready; let result;if (request.method === "state") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript('({ count: window.counter, draft: document.getElementById("draft").value })')));+ else if (request.method === "pointer_state") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript('({ doubles: window.pointer.doubles, scrolled: document.getElementById("scroll").scrollTop > 0, dragged: document.getElementById("drag").offsetLeft > 170, released: !window.pointer.held })')));else if (request.method === "bounds") result = windows.map((w) => ({ bounds: w.getBounds(), contentBounds: w.getContentBounds(), scale: screen.getDisplayMatching(w.getBounds()).scaleFactor }));else if (request.method === "focus_other") { windows[1].focus(); result = {}; }else if (request.method === "outside_input") {diff --git a/evals/specs/computer-use-portable-window.e2e.test.ts b/evals/specs/computer-use-portable-window.e2e.test.tsindex d506e60a..447724aa 100644--- a/evals/specs/computer-use-portable-window.e2e.test.ts+++ b/evals/specs/computer-use-portable-window.e2e.test.ts@@ -42,6 +42,7 @@ test("Windows and Linux X11 Computer Use keeps native input within the approvedexpect(acted, JSON.stringify(acted)).toMatchObject({ ok: true, dispatched: true });await expect.poll(() => world.state()).toEqual([{ count: 1, draft: "Initial draft" }, { count: 0, draft: "Initial draft" }]);expect(toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "repeat-observation", action: { type: "click", ...point } })).code).toBe("stale_observation");+ expect(toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "first-click", action: { type: "click", ...point } })).code).toBe("duplicate_request");});await step("Native keyboard input edits only the selected window", async () => {const act = async (request: string, action: object) => {@@ -57,6 +58,24 @@ test("Windows and Linux X11 Computer Use keeps native input within the approvedawait act("write-draft", { type: "type", text: "OmniRush 123!" });await expect.poll(() => world.state()).toEqual([{ count: 1, draft: "OmniRush 123!" }, { count: 0, draft: "Initial draft" }]);});+ await step("Native double-click, scroll and drag affect only the selected window and release input", async () => {+ const act = async (request: string, actionFor: (observation: Record<string, unknown>) => Promise<object>) => {+ const observed = toolState(await world.call("computer_observe", { session_id: session }));+ expect(observed.ok, JSON.stringify(observed)).toBe(true);+ const action = await actionFor(observed);+ const result = toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: request, action }));+ expect(result, JSON.stringify(result)).toMatchObject({ ok: true, dispatched: true });+ };+ await act("double-click", async (observed) => ({ type: "double_click", ...await world.point(observed, 480, 95) }));+ await act("scroll", async (observed) => ({ type: "scroll", axis: "vertical", delta: 3, ...await world.point(observed, 480, 240) }));+ await act("drag", async (observed) => ({ type: "drag", path: await Promise.all([{ x: 120, y: 310 }, { x: 140, y: 315 }, { x: 180, y: 325 }, { x: 240, y: 330 }].map(({ x, y }) => world.point(observed, x, y))) }));+ await expect.poll(() => world.pointerState()).toEqual([+ { doubles: 1, scrolled: true, dragged: true, released: true },+ { doubles: 0, scrolled: false, dragged: false, released: true },+ ]);+ await act("restore-draft-focus", async (observed) => ({ type: "click", ...await world.point(observed, 150, 180) }));+ expect(await world.state()).toEqual([{ count: 1, draft: "OmniRush 123!" }, { count: 0, draft: "Initial draft" }]);+ });await step("Changed pixels and out-of-window coordinates cannot dispatch input", async () => {const observed = toolState(await world.call("computer_observe", { session_id: session }));await world.change();@@ -65,6 +84,14 @@ test("Windows and Linux X11 Computer Use keeps native input within the approvedexpect(toolState(await world.call("computer_act", { session_id: session, observation_id: fresh.observation_id, request_id: "outside-click", action: { type: "click", x: -1, y: 20 } }))).toMatchObject({ ok: false, code: "invalid_action", may_have_acted: false });expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);});+ await step("System-wide shortcuts are rejected without closing or changing either window", async () => {+ const observed = toolState(await world.call("computer_observe", { session_id: session }));+ const result = toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "system-shortcut", action: { type: "key", key: "F4", modifiers: ["alt"] } }));+ expect(result, JSON.stringify(result)).toMatchObject({ ok: false, code: "unsupported_action" });+ expect(toolState(await world.call("computer_session_status", { session_id: session })).state).toBe("paused");+ expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);+ await world.approve("resume");+ });await step("Focus moving to another window stops keyboard input", async () => {const observed = toolState(await world.call("computer_observe", { session_id: session }));await world.focusOther();@@ -114,10 +141,14 @@ test("Windows and Linux X11 Computer Use keeps native input within the approvedconst observed = toolState(await world.call("computer_observe", { session_id: opened.session_id, include_image: false }));expect(observed.ok).toBe(true);expect(toolState(await world.call("computer_act", { session_id: opened.session_id, observation_id: observed.observation_id, request_id: "read-only-type", action: { type: "type", text: "Forbidden" } })).code).toBe("scope_denied");+ const baseline = await world.imagePixel(await world.call("computer_observe", { session_id: opened.session_id }));+ if (!baseline || typeof baseline !== "object" || !("rgb" in baseline) || !Array.isArray(baseline.rgb)) throw new Error("Missing selected-window image sample.");+ // Windows capture can round colours during its native pixel conversion.+ for (const channel of baseline.rgb) { expect(channel).toBeGreaterThanOrEqual(234); expect(channel).toBeLessThanOrEqual(242); }await world.cover();const covered = await world.call("computer_observe", { session_id: opened.session_id });expect(toolState(covered).ok, JSON.stringify(toolState(covered))).toBe(true);- expect(await world.imagePixel(covered)).toMatchObject({ rgb: [238, 238, 238] });+ expect(await world.imagePixel(covered)).toEqual(baseline);await world.closeClient();await expect.poll(() => world.hostState()).toEqual([]);expect(await world.state()).toEqual(stoppedState);diff --git a/evals/worlds/computer-use.ts b/evals/worlds/computer-use.tsindex b463b6f6..0d26bcbe 100644--- a/evals/worlds/computer-use.ts+++ b/evals/worlds/computer-use.ts@@ -360,6 +360,7 @@ export async function portableComputerUseWorld(_seed: Seed, { place }: { place:focusOther: () => fixture.request("focus_other"),cover: () => fixture.request("cover"),outsideInput: () => fixture.request("outside_input"),+ pointerState: () => fixture.request("pointer_state"),imagePixel(reply: unknown) {if (!record(reply) || !Array.isArray(reply.content)) throw new Error("Missing image content.");const image = reply.content.find((item: unknown) => record(item) && item.type === "image");