Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
test: display and pointer integration tests use the computer_* tools (LPT-934)
alexshpunt-clanker[bot]f361574eb937
diff --git a/test/display-observation.integration.test.ts b/test/display-observation.integration.test.tsindex 86de7d2..3a51400 100644--- a/test/display-observation.integration.test.ts+++ b/test/display-observation.integration.test.ts@@ -9,7 +9,7 @@ import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";const yoloRead = `${SCRIPT_HELPERS}const view = await tools.computer_read({path:"computer:machine/local/display/primary"});text(body(view)); image(view);-const input = await insert(observation(view),{action:"click",x:0,y:0});+const input = await tools.computer_click({window:observation(view),x:0,y:0});text("observation input: " + input.outcome + " / " + input.effect);`;for (const mode of ["yolo", "safe"] as const) {@@ -25,13 +25,13 @@ for (const mode of ["yolo", "safe"] as const) {const result = await new PiIntegrationTest({testName: `primary-display-${mode}`, artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve(`test/fixtures/${mode === "safe" ? "safe" : "display"}-extension.ts`)], isolateUserResources: true,- tools: ["codemode", "computer_read", "computer_insert"],+ tools: ["codemode", "computer_read", "computer_click", "computer_delete"],conversation: [assistantMessage([toolCall({ id: "read", name: mode === "safe" ? "computer_read" : "codemode", arguments: mode === "safe"? { path: "computer:machine/local/display/primary" } : { code: yoloRead } })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "mixed", name: "computer_read", arguments: { path: "computer:machine/local/display/primary", views: ["ui"] } })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "promote", name: "computer_read", arguments: { path: "computer:machine/local/display/primary", mode: "yolo" } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "display-input", name: "computer_insert", arguments: { path: "computer:machine/local/display/primary", text: '{"action":"click","x":0,"y":0}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "display-input", name: "computer_click", arguments: { window: "computer:machine/local/display/primary", x: 0, y: 0 } })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Read-only display source checked.")]),],diff --git a/test/pointer-hover.integration.test.ts b/test/pointer-hover.integration.test.tsindex fa7fa1d..a750e74 100644--- a/test/pointer-hover.integration.test.ts+++ b/test/pointer-hover.integration.test.ts@@ -5,9 +5,8 @@ import { expect, test } from "vitest";import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";const fixture = (name: string) => path.resolve("test/fixtures", name);-// Any refused input spends its observation, so each check that must reach the session uses a fresh overview.const prelude = `${SCRIPT_HELPERS}-const act=async(view,command)=>(await insert(typeof view==="string"?view:observation(view),command)).outcome;+const hover=async(window,command)=>(await tools.computer_hover({window,...command})).outcome;const observe=async window=>tools.computer_read({path:window,views:['capture:{"hints":false}']});const control=async operation=>tools.fixture_pointer_control({operation});const open=async()=>(await openApp("Notepad")).window;@@ -25,7 +24,7 @@ async function scenario(testName: string, code: string) {extensions: [fixture("pointer-hover-extension.ts")],isolateUserResources: true,piCommand: process.env.PI_HOVER_PI_COMMAND,- tools: ["codemode", "computer_insert", "fixture_pointer_control"],+ tools: ["codemode", "computer_delete", "fixture_pointer_control"],conversation: [assistantMessage([toolCall({ id: "pointer", name: "codemode", arguments: { code: prelude + "\n" + code } })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")])],@@ -37,69 +36,52 @@ async function scenario(testName: string, code: string) {}// Protects the production Pi schema/Codemode/session/MCP boundary, not native hover UI effects.-test("real Pi grounds move and bounded hover once and preserves refusal and uncertain receipts without fallback", async () => {+test("real Pi grounds bounded hover in the latest Read and preserves uncertain receipts without fallback", async () => {const { calls, nested } = await scenario("native-pointer-receipts", `let view=await observe(first);const other=await observe(second);-// The observation address binds its window; identity fields cannot redirect it.-if(await act(view,{action:"move",x:100,y:80,window_ref:second})!=="refused")throw Error("Wrong-window observation admitted");-if(await act(view,{action:"hover",hint:hints(other)[0],dwell_ms:100})!=="refused")throw Error("Foreign hint admitted");+// A ref belongs to its window's latest Read; another window's ref cannot redirect input.+if(await hover(first,{target:"9:z",dwell_ms:100})!=="refused")throw Error("Unknown ref admitted");view=await observe(first);-const background=await insert(observation(view),{action:"move",x:100,y:80});-if(background.outcome!=="refused"||background.code!=="background_unavailable"||background.effect!=="not-applied")throw Error("Background refusal lost");-if(await act(view,{action:"move",x:100,y:80,delivery_mode:"foreground"})!=="refused")throw Error("Refused observation replayed");-view=await observe(first);-const hover={action:"hover",hint:hints(view)[0],dwell_ms:100,delivery_mode:"foreground"};-if(await act(view,hover)!=="delivered")throw Error("Truthful hover completion lost");-if(await act(view,hover)!=="refused")throw Error("Hover replayed");-view=await tools.computer_read({path:first,views:["image"]});-if(await act(view,{action:"move",x:120,y:90,delivery_mode:"foreground"})!=="delivered")throw Error("Image-only move refused");+const target=hints(view)[0];+if(await hover(first,{target,dwell_ms:100})!=="delivered")throw Error("Truthful hover completion lost");+if(await hover(first,{target,dwell_ms:100})!=="refused")throw Error("Spent ref replayed");+await tools.computer_read({path:first,views:["image"]});+if(await hover(first,{x:120,y:90,dwell_ms:50})!=="delivered")throw Error("Image-only point refused");await control("partial");view=await observe(first);-const partial={action:"hover",hint:hints(view)[0],dwell_ms:100,delivery_mode:"foreground"};-if(await act(view,partial)!=="unknown")throw Error("Interrupted native prefix lost");-if(await act(view,partial)!=="refused")throw Error("Unknown pointer replayed");-await control("malformed");view=await observe(first);-const malformed={action:"move",x:130,y:100,delivery_mode:"foreground"};-if(await act(view,malformed)!=="unknown")throw Error("Malformed successful receipt called delivered");-if(await act(view,malformed)!=="refused")throw Error("Malformed outcome replayed");-for(const command of [- {action:"hover",x:10,y:10}, {action:"hover",x:10,y:10,dwell_ms:0}, {action:"hover",x:10,y:10,dwell_ms:10001},- {action:"move",x:10,y:10,dwell_ms:1}, {action:"move",x:10,y:10,button:"left"},- {action:"sequence",sequence:[{action:"move",x:10,y:10}]}-]){view=await observe(first);if(await act(view,command)!=="refused")throw Error("Unsupported pointer request admitted: "+JSON.stringify(command));}-text("Pointer contract passed: admission, receipts and no replay");`);- const outcomes = nested.filter(call => call.name === "computer_insert").map(call => call.computer);+if(await hover(first,{target:hints(view)[0],dwell_ms:100})!=="unknown")throw Error("Interrupted native prefix lost");+await control("malformed");await tools.computer_read({path:first,views:["image"]});+if(await hover(first,{x:130,y:100,dwell_ms:50})!=="unknown")throw Error("Malformed successful receipt called delivered");+text("Pointer contract passed: admission and receipts");`);+ const outcomes = nested.filter(call => call.name === "computer_hover").map(call => call.computer);const hovered = outcomes.find(item => item.status === "delivered" && item.state?.dwell_elapsed_ms !== undefined)!;expect(hovered.state).toMatchObject({ dwell_elapsed_ms: 100, foreground_changed: null });expect(hovered.action_capability.actual_delivery).toBe("foreground");const interrupted = outcomes.find(item => item.status === "unknown" && item.state?.attempted === true)!;expect(interrupted.state).toMatchObject({ attempted: true, delivered: true, dwell_elapsed_ms: 20 });- const refusedBackground = outcomes.find(item => item.state?.code === "background_unavailable")!;- expect(refusedBackground.state.attempted).toBe(false);const actions = calls.filter(c => !["list_apps", "list_windows", "launch_app", "fixture_identity", "get_window_state", "fixture_control"].includes(c.name));- expect(actions.map(c => c.name)).toEqual(Array(5).fill("move_pointer"));+ expect(actions.map(c => c.name)).toEqual(Array(4).fill("move_pointer"));expect(actions.map(c => c.arguments)).toEqual([- { pid: 42, window_id: 91, capture_id: expect.any(String), x: 100, y: 80, dwell_ms: 0, delivery_mode: "background" },{ pid: 42, window_id: 91, element_token: expect.any(String), dwell_ms: 100, delivery_mode: "foreground" },- { pid: 42, window_id: 91, capture_id: expect.any(String), x: 120, y: 90, dwell_ms: 0, delivery_mode: "foreground" },+ { pid: 42, window_id: 91, capture_id: expect.any(String), x: 120, y: 90, dwell_ms: 50, delivery_mode: "foreground" },{ pid: 42, window_id: 91, element_token: expect.any(String), dwell_ms: 100, delivery_mode: "foreground" },- { pid: 42, window_id: 91, capture_id: expect.any(String), x: 130, y: 100, dwell_ms: 0, delivery_mode: "foreground" },+ { pid: 42, window_id: 91, capture_id: expect.any(String), x: 130, y: 100, dwell_ms: 50, delivery_mode: "foreground" },]);- expect(new Set(actions.filter(c => c.arguments.capture_id).map(c => c.arguments.capture_id)).size).toBe(3);+ expect(new Set(actions.filter(c => c.arguments.capture_id).map(c => c.arguments.capture_id)).size).toBe(2);});test("real Pi never sends pointer input through Safe-disallowed, closed or reused window references", async () => {const { calls } = await scenario("native-pointer-access", `const listed=await windows();const external=listed.split("\\n").find(line=>line.includes("| Unowned fixture |")).match(/computer:[\\w-]+\\/w\\d+[a-z]{2}/)[0];-const externalView=await observe(external);+await observe(external);await control("safe");-if(await act(externalView,{action:"move",x:100,y:80,delivery_mode:"foreground"})!=="refused")throw Error("Safe sent input to external window");+if(await hover(external,{x:100,y:80,dwell_ms:100})!=="refused")throw Error("Safe sent input to external window");let view=await observe(first);-if(await act(view,{action:"hover",hint:hints(view)[0],dwell_ms:100,delivery_mode:"foreground"})!=="delivered")throw Error("Safe denied owned window");+if(await hover(first,{target:hints(view)[0],dwell_ms:100})!=="delivered")throw Error("Safe denied owned window");view=await observe(first);await control("close_first");-if(await act(view,{action:"move",x:100,y:80,delivery_mode:"foreground"})!=="refused")throw Error("Closed window admitted");+if(await hover(first,{x:100,y:80,dwell_ms:100})!=="refused")throw Error("Closed window admitted");view=await observe(second);await control("recycle_second");-if(await act(view,{action:"hover",hint:hints(view)[0],dwell_ms:100,delivery_mode:"foreground"})!=="refused")throw Error("Reused handle admitted");+if(await hover(second,{target:hints(view)[0],dwell_ms:100})!=="refused")throw Error("Reused handle admitted");text("Pointer contract passed: Safe and native lifetime guards");`);const actions = calls.filter(c => !["list_apps", "list_windows", "launch_app", "fixture_identity", "get_window_state", "fixture_control"].includes(c.name));expect(actions).toEqual([{ name: "move_pointer", arguments: {diff --git a/test/selected-displays.integration.test.ts b/test/selected-displays.integration.test.tsindex 2ed6233..81e6c6a 100644--- a/test/selected-displays.integration.test.ts+++ b/test/selected-displays.integration.test.ts@@ -17,13 +17,13 @@ const rows=r.split("\\n").filter(line=>/^computer:local\\/d\\d+[a-z]{2} /.test(lif(rows.length!==2)throw Error("Native discovery failed");const row=rows.find(line=>JSON.parse(line.slice(line.lastIndexOf(" | ")+3)).bounds.x<0);store("monitor",row.split(" | ")[0]);text(row);`;- // Display observations are readonly: input into one refuses before dispatch and keeps it for detail.+ // Display observations are readonly: input tools take only window addresses, so they refuse before dispatch and keep it for detail.const observe = `${SCRIPT_HELPERS}const r=await tools.computer_read({path:load("monitor")});if(typeof r==="string")throw Error("Invalid read grant");store("overview",observation(r));-const input=await insert(load("overview"),{action:"click",x:1,y:1});-if(input.outcome!=="refused"||input.effect!=="not-applied"||!/readonly/.test(input.reason))throw Error("Display pixels grounded input");+const input=await tools.computer_click({window:load("overview"),x:1,y:1});+if(input.outcome!=="refused"||input.effect!=="not-applied"||!/window address/.test(input.reason))throw Error("Display pixels grounded input");text("overview read; input refused");`;const crop = `const r=await tools.computer_read({path:load("overview"),views:["region:100,80,100,80"]});if(typeof r==="string")throw Error("Selected crop failed");@@ -31,13 +31,13 @@ image(r);text("cropped");`;try {const result = await new PiIntegrationTest({ testName: `selected-displays-${mode}`, artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve(`test/fixtures/selected-display-${mode === "safe" ? "safe-" : ""}extension.ts`)], isolateUserResources: true,- tools: ["codemode", "computer_read", "computer_insert"], conversation: [+ tools: ["codemode", "computer_read", "computer_click", "computer_delete"], conversation: [assistantMessage([toolCall({ id: "discovery", name: mode === "safe" ? "computer_read" : "codemode", arguments: mode === "safe" ? { path: "computer:machine/local/displays" } : { code: discover } })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "overview", name: mode === "safe" ? "computer_read" : "codemode", arguments: mode === "safe" ? { path: "computer:machine/local/display/d-unowned" } : { code: observe } })], { stopReason: "toolUse" }),...(mode === "yolo" ? [assistantMessage([toolCall({ id: "crop", name: "codemode", arguments: { code: crop } })], { stopReason: "toolUse" })] : []),- assistantMessage([toolCall({ id: "input", name: mode === "safe" ? "computer_insert" : "codemode", arguments: mode === "safe"- ? { path: "computer:machine/local/display/d-unowned", text: '{"action":"click","x":1,"y":1}' }- : { code: 'const r=await tools.computer_insert({path:load("monitor"),text:JSON.stringify({action:"click",x:1,y:1})});text(r.outcome+": "+r.reason);' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "input", name: mode === "safe" ? "computer_click" : "codemode", arguments: mode === "safe"+ ? { window: "computer:machine/local/display/d-unowned", x: 1, y: 1 }+ : { code: 'const r=await tools.computer_click({window:load("monitor"),x:1,y:1});text(r.outcome+": "+r.reason);' } })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Read-only monitor selection checked.")]),],@@ -49,7 +49,7 @@ image(r);text("cropped");`;expect(getToolExecution(result, "input").isError).toBe(true);} else {for (const id of ["discovery", "overview", "crop", "input"]) expect(getToolExecution(result, id).isError, getToolResultText(result, id)).toBe(false);- expect(getToolResultText(result, "input")).toContain("refused: A display address accepts only resize");+ expect(getToolResultText(result, "input")).toContain("refused: Give the window address");expect(images("crop")).toHaveLength(1);const executions = getToolExecutions(result);const [listed] = nestedCalls(executions, "discovery"), [overview] = nestedCalls(executions, "overview"), [crop] = nestedCalls(executions, "crop");