Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

test: display and pointer integration tests use the computer_* tools (LPT-934)

alexshpunt-clanker[bot]f361574eb937
diff --git a/test/display-observation.integration.test.ts b/test/display-observation.integration.test.ts
index 86de7d2..3a51400 100644
--- a/test/display-observation.integration.test.ts
+++ b/test/display-observation.integration.test.ts
@@ -9,7 +9,7 @@ import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";
const yoloRead = `${SCRIPT_HELPERS}
const view = await tools.computer_read({path:"computer:machine/local/display/primary"});
text(body(view)); image(view);
-const input = await insert(observation(view),{action:"click",x:0,y:0});
+const input = await tools.computer_click({window:observation(view),x:0,y:0});
text("observation input: " + input.outcome + " / " + input.effect);`;
for (const mode of ["yolo", "safe"] as const) {
@@ -25,13 +25,13 @@ for (const mode of ["yolo", "safe"] as const) {
const result = await new PiIntegrationTest({
testName: `primary-display-${mode}`, artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve(`test/fixtures/${mode === "safe" ? "safe" : "display"}-extension.ts`)], isolateUserResources: true,
- tools: ["codemode", "computer_read", "computer_insert"],
+ tools: ["codemode", "computer_read", "computer_click", "computer_delete"],
conversation: [
assistantMessage([toolCall({ id: "read", name: mode === "safe" ? "computer_read" : "codemode", arguments: mode === "safe"
? { path: "computer:machine/local/display/primary" } : { code: yoloRead } })], { stopReason: "toolUse" }),
assistantMessage([toolCall({ id: "mixed", name: "computer_read", arguments: { path: "computer:machine/local/display/primary", views: ["ui"] } })], { stopReason: "toolUse" }),
assistantMessage([toolCall({ id: "promote", name: "computer_read", arguments: { path: "computer:machine/local/display/primary", mode: "yolo" } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "display-input", name: "computer_insert", arguments: { path: "computer:machine/local/display/primary", text: '{"action":"click","x":0,"y":0}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "display-input", name: "computer_click", arguments: { window: "computer:machine/local/display/primary", x: 0, y: 0 } })], { stopReason: "toolUse" }),
assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Read-only display source checked.")]),
],
diff --git a/test/pointer-hover.integration.test.ts b/test/pointer-hover.integration.test.ts
index fa7fa1d..a750e74 100644
--- a/test/pointer-hover.integration.test.ts
+++ b/test/pointer-hover.integration.test.ts
@@ -5,9 +5,8 @@ import { expect, test } from "vitest";
import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";
const fixture = (name: string) => path.resolve("test/fixtures", name);
-// Any refused input spends its observation, so each check that must reach the session uses a fresh overview.
const prelude = `${SCRIPT_HELPERS}
-const act=async(view,command)=>(await insert(typeof view==="string"?view:observation(view),command)).outcome;
+const hover=async(window,command)=>(await tools.computer_hover({window,...command})).outcome;
const observe=async window=>tools.computer_read({path:window,views:['capture:{"hints":false}']});
const control=async operation=>tools.fixture_pointer_control({operation});
const open=async()=>(await openApp("Notepad")).window;
@@ -25,7 +24,7 @@ async function scenario(testName: string, code: string) {
extensions: [fixture("pointer-hover-extension.ts")],
isolateUserResources: true,
piCommand: process.env.PI_HOVER_PI_COMMAND,
- tools: ["codemode", "computer_insert", "fixture_pointer_control"],
+ tools: ["codemode", "computer_delete", "fixture_pointer_control"],
conversation: [assistantMessage([toolCall({ id: "pointer", name: "codemode", arguments: { code: prelude + "\n" + code } })], { stopReason: "toolUse" }),
assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")])],
@@ -37,69 +36,52 @@ async function scenario(testName: string, code: string) {
}
// Protects the production Pi schema/Codemode/session/MCP boundary, not native hover UI effects.
-test("real Pi grounds move and bounded hover once and preserves refusal and uncertain receipts without fallback", async () => {
+test("real Pi grounds bounded hover in the latest Read and preserves uncertain receipts without fallback", async () => {
const { calls, nested } = await scenario("native-pointer-receipts", `
let view=await observe(first);const other=await observe(second);
-// The observation address binds its window; identity fields cannot redirect it.
-if(await act(view,{action:"move",x:100,y:80,window_ref:second})!=="refused")throw Error("Wrong-window observation admitted");
-if(await act(view,{action:"hover",hint:hints(other)[0],dwell_ms:100})!=="refused")throw Error("Foreign hint admitted");
+// A ref belongs to its window's latest Read; another window's ref cannot redirect input.
+if(await hover(first,{target:"9:z",dwell_ms:100})!=="refused")throw Error("Unknown ref admitted");
view=await observe(first);
-const background=await insert(observation(view),{action:"move",x:100,y:80});
-if(background.outcome!=="refused"||background.code!=="background_unavailable"||background.effect!=="not-applied")throw Error("Background refusal lost");
-if(await act(view,{action:"move",x:100,y:80,delivery_mode:"foreground"})!=="refused")throw Error("Refused observation replayed");
-view=await observe(first);
-const hover={action:"hover",hint:hints(view)[0],dwell_ms:100,delivery_mode:"foreground"};
-if(await act(view,hover)!=="delivered")throw Error("Truthful hover completion lost");
-if(await act(view,hover)!=="refused")throw Error("Hover replayed");
-view=await tools.computer_read({path:first,views:["image"]});
-if(await act(view,{action:"move",x:120,y:90,delivery_mode:"foreground"})!=="delivered")throw Error("Image-only move refused");
+const target=hints(view)[0];
+if(await hover(first,{target,dwell_ms:100})!=="delivered")throw Error("Truthful hover completion lost");
+if(await hover(first,{target,dwell_ms:100})!=="refused")throw Error("Spent ref replayed");
+await tools.computer_read({path:first,views:["image"]});
+if(await hover(first,{x:120,y:90,dwell_ms:50})!=="delivered")throw Error("Image-only point refused");
await control("partial");view=await observe(first);
-const partial={action:"hover",hint:hints(view)[0],dwell_ms:100,delivery_mode:"foreground"};
-if(await act(view,partial)!=="unknown")throw Error("Interrupted native prefix lost");
-if(await act(view,partial)!=="refused")throw Error("Unknown pointer replayed");
-await control("malformed");view=await observe(first);
-const malformed={action:"move",x:130,y:100,delivery_mode:"foreground"};
-if(await act(view,malformed)!=="unknown")throw Error("Malformed successful receipt called delivered");
-if(await act(view,malformed)!=="refused")throw Error("Malformed outcome replayed");
-for(const command of [
- {action:"hover",x:10,y:10}, {action:"hover",x:10,y:10,dwell_ms:0}, {action:"hover",x:10,y:10,dwell_ms:10001},
- {action:"move",x:10,y:10,dwell_ms:1}, {action:"move",x:10,y:10,button:"left"},
- {action:"sequence",sequence:[{action:"move",x:10,y:10}]}
-]){view=await observe(first);if(await act(view,command)!=="refused")throw Error("Unsupported pointer request admitted: "+JSON.stringify(command));}
-text("Pointer contract passed: admission, receipts and no replay");`);
- const outcomes = nested.filter(call => call.name === "computer_insert").map(call => call.computer);
+if(await hover(first,{target:hints(view)[0],dwell_ms:100})!=="unknown")throw Error("Interrupted native prefix lost");
+await control("malformed");await tools.computer_read({path:first,views:["image"]});
+if(await hover(first,{x:130,y:100,dwell_ms:50})!=="unknown")throw Error("Malformed successful receipt called delivered");
+text("Pointer contract passed: admission and receipts");`);
+ const outcomes = nested.filter(call => call.name === "computer_hover").map(call => call.computer);
const hovered = outcomes.find(item => item.status === "delivered" && item.state?.dwell_elapsed_ms !== undefined)!;
expect(hovered.state).toMatchObject({ dwell_elapsed_ms: 100, foreground_changed: null });
expect(hovered.action_capability.actual_delivery).toBe("foreground");
const interrupted = outcomes.find(item => item.status === "unknown" && item.state?.attempted === true)!;
expect(interrupted.state).toMatchObject({ attempted: true, delivered: true, dwell_elapsed_ms: 20 });
- const refusedBackground = outcomes.find(item => item.state?.code === "background_unavailable")!;
- expect(refusedBackground.state.attempted).toBe(false);
const actions = calls.filter(c => !["list_apps", "list_windows", "launch_app", "fixture_identity", "get_window_state", "fixture_control"].includes(c.name));
- expect(actions.map(c => c.name)).toEqual(Array(5).fill("move_pointer"));
+ expect(actions.map(c => c.name)).toEqual(Array(4).fill("move_pointer"));
expect(actions.map(c => c.arguments)).toEqual([
- { pid: 42, window_id: 91, capture_id: expect.any(String), x: 100, y: 80, dwell_ms: 0, delivery_mode: "background" },
{ pid: 42, window_id: 91, element_token: expect.any(String), dwell_ms: 100, delivery_mode: "foreground" },
- { pid: 42, window_id: 91, capture_id: expect.any(String), x: 120, y: 90, dwell_ms: 0, delivery_mode: "foreground" },
+ { pid: 42, window_id: 91, capture_id: expect.any(String), x: 120, y: 90, dwell_ms: 50, delivery_mode: "foreground" },
{ pid: 42, window_id: 91, element_token: expect.any(String), dwell_ms: 100, delivery_mode: "foreground" },
- { pid: 42, window_id: 91, capture_id: expect.any(String), x: 130, y: 100, dwell_ms: 0, delivery_mode: "foreground" },
+ { pid: 42, window_id: 91, capture_id: expect.any(String), x: 130, y: 100, dwell_ms: 50, delivery_mode: "foreground" },
]);
- expect(new Set(actions.filter(c => c.arguments.capture_id).map(c => c.arguments.capture_id)).size).toBe(3);
+ expect(new Set(actions.filter(c => c.arguments.capture_id).map(c => c.arguments.capture_id)).size).toBe(2);
});
test("real Pi never sends pointer input through Safe-disallowed, closed or reused window references", async () => {
const { calls } = await scenario("native-pointer-access", `
const listed=await windows();
const external=listed.split("\\n").find(line=>line.includes("| Unowned fixture |")).match(/computer:[\\w-]+\\/w\\d+[a-z]{2}/)[0];
-const externalView=await observe(external);
+await observe(external);
await control("safe");
-if(await act(externalView,{action:"move",x:100,y:80,delivery_mode:"foreground"})!=="refused")throw Error("Safe sent input to external window");
+if(await hover(external,{x:100,y:80,dwell_ms:100})!=="refused")throw Error("Safe sent input to external window");
let view=await observe(first);
-if(await act(view,{action:"hover",hint:hints(view)[0],dwell_ms:100,delivery_mode:"foreground"})!=="delivered")throw Error("Safe denied owned window");
+if(await hover(first,{target:hints(view)[0],dwell_ms:100})!=="delivered")throw Error("Safe denied owned window");
view=await observe(first);await control("close_first");
-if(await act(view,{action:"move",x:100,y:80,delivery_mode:"foreground"})!=="refused")throw Error("Closed window admitted");
+if(await hover(first,{x:100,y:80,dwell_ms:100})!=="refused")throw Error("Closed window admitted");
view=await observe(second);await control("recycle_second");
-if(await act(view,{action:"hover",hint:hints(view)[0],dwell_ms:100,delivery_mode:"foreground"})!=="refused")throw Error("Reused handle admitted");
+if(await hover(second,{target:hints(view)[0],dwell_ms:100})!=="refused")throw Error("Reused handle admitted");
text("Pointer contract passed: Safe and native lifetime guards");`);
const actions = calls.filter(c => !["list_apps", "list_windows", "launch_app", "fixture_identity", "get_window_state", "fixture_control"].includes(c.name));
expect(actions).toEqual([{ name: "move_pointer", arguments: {
diff --git a/test/selected-displays.integration.test.ts b/test/selected-displays.integration.test.ts
index 2ed6233..81e6c6a 100644
--- a/test/selected-displays.integration.test.ts
+++ b/test/selected-displays.integration.test.ts
@@ -17,13 +17,13 @@ const rows=r.split("\\n").filter(line=>/^computer:local\\/d\\d+[a-z]{2} /.test(l
if(rows.length!==2)throw Error("Native discovery failed");
const row=rows.find(line=>JSON.parse(line.slice(line.lastIndexOf(" | ")+3)).bounds.x<0);
store("monitor",row.split(" | ")[0]);text(row);`;
- // Display observations are readonly: input into one refuses before dispatch and keeps it for detail.
+ // Display observations are readonly: input tools take only window addresses, so they refuse before dispatch and keep it for detail.
const observe = `${SCRIPT_HELPERS}
const r=await tools.computer_read({path:load("monitor")});
if(typeof r==="string")throw Error("Invalid read grant");
store("overview",observation(r));
-const input=await insert(load("overview"),{action:"click",x:1,y:1});
-if(input.outcome!=="refused"||input.effect!=="not-applied"||!/readonly/.test(input.reason))throw Error("Display pixels grounded input");
+const input=await tools.computer_click({window:load("overview"),x:1,y:1});
+if(input.outcome!=="refused"||input.effect!=="not-applied"||!/window address/.test(input.reason))throw Error("Display pixels grounded input");
text("overview read; input refused");`;
const crop = `const r=await tools.computer_read({path:load("overview"),views:["region:100,80,100,80"]});
if(typeof r==="string")throw Error("Selected crop failed");
@@ -31,13 +31,13 @@ image(r);text("cropped");`;
try {
const result = await new PiIntegrationTest({ testName: `selected-displays-${mode}`, artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve(`test/fixtures/selected-display-${mode === "safe" ? "safe-" : ""}extension.ts`)], isolateUserResources: true,
- tools: ["codemode", "computer_read", "computer_insert"], conversation: [
+ tools: ["codemode", "computer_read", "computer_click", "computer_delete"], conversation: [
assistantMessage([toolCall({ id: "discovery", name: mode === "safe" ? "computer_read" : "codemode", arguments: mode === "safe" ? { path: "computer:machine/local/displays" } : { code: discover } })], { stopReason: "toolUse" }),
assistantMessage([toolCall({ id: "overview", name: mode === "safe" ? "computer_read" : "codemode", arguments: mode === "safe" ? { path: "computer:machine/local/display/d-unowned" } : { code: observe } })], { stopReason: "toolUse" }),
...(mode === "yolo" ? [assistantMessage([toolCall({ id: "crop", name: "codemode", arguments: { code: crop } })], { stopReason: "toolUse" })] : []),
- assistantMessage([toolCall({ id: "input", name: mode === "safe" ? "computer_insert" : "codemode", arguments: mode === "safe"
- ? { path: "computer:machine/local/display/d-unowned", text: '{"action":"click","x":1,"y":1}' }
- : { code: 'const r=await tools.computer_insert({path:load("monitor"),text:JSON.stringify({action:"click",x:1,y:1})});text(r.outcome+": "+r.reason);' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "input", name: mode === "safe" ? "computer_click" : "codemode", arguments: mode === "safe"
+ ? { window: "computer:machine/local/display/d-unowned", x: 1, y: 1 }
+ : { code: 'const r=await tools.computer_click({window:load("monitor"),x:1,y:1});text(r.outcome+": "+r.reason);' } })], { stopReason: "toolUse" }),
assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Read-only monitor selection checked.")]),
],
@@ -49,7 +49,7 @@ image(r);text("cropped");`;
expect(getToolExecution(result, "input").isError).toBe(true);
} else {
for (const id of ["discovery", "overview", "crop", "input"]) expect(getToolExecution(result, id).isError, getToolResultText(result, id)).toBe(false);
- expect(getToolResultText(result, "input")).toContain("refused: A display address accepts only resize");
+ expect(getToolResultText(result, "input")).toContain("refused: Give the window address");
expect(images("crop")).toHaveLength(1);
const executions = getToolExecutions(result);
const [listed] = nestedCalls(executions, "discovery"), [overview] = nestedCalls(executions, "overview"), [crop] = nestedCalls(executions, "crop");