Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

test: sequence, semantic, detail, hold, held-path and capture-drag integration tests use the computer_* tools (LPT-934)

alexshpunt-clanker[bot]577f7d04aeb3
diff --git a/test/capture-drag.integration.test.ts b/test/capture-drag.integration.test.ts
index e6e7559..546032f 100644
--- a/test/capture-drag.integration.test.ts
+++ b/test/capture-drag.integration.test.ts
@@ -1,11 +1,11 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import path from "node:path";
-import { assistantMessage, getToolExecution, getToolExecutions, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";
+import { assistantMessage, getToolExecution, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";
import { expect, test } from "vitest";
-import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";
+import { SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";
// This checks the real Pi/MCP boundary, not native Windows application behavior.
-test("real Pi keeps drag capture binding and stops on resize refusal until reobserved", async () => {
+test("real Pi binds a drag to the latest capture and refuses it after the window changed size until it is read again", async () => {
await mkdir(".tmp/lpt-619", { recursive: true });
const cwd = await mkdtemp(path.resolve(".tmp/lpt-619/pi-capture-")); const log = path.join(cwd, "calls.jsonl");
await mkdir(path.join(cwd, ".pi"));
@@ -13,42 +13,34 @@ test("real Pi keeps drag capture binding and stops on resize refusal until reobs
args: [path.resolve("test/fixtures/complete-input-driver.mjs"), log, "capture-drag"] }));
const code = `${SCRIPT_HELPERS}
const opened = await openApp("Notepad");
-// These checks keep one observation for pixel input, so they turn the default successor off.
-const act = (view, sequence) => insert(view, {action:"sequence",sequence,successor:false});
-const drag={action:"drag",from_x:30,from_y:40,to_x:100,to_y:80,duration_ms:100,steps:4};
-const stopped=await act(opened.observation,[{action:"key",key:"r"},drag,{action:"key",key:"b"}]);
-if(stopped.outcome!=="refused" || stopped.code!=="capture_frame_mismatch" || stopped.effect!=="unknown") throw Error("Resize refusal or completed prefix lost");
-text(JSON.stringify(stopped.steps));
-if((await act(opened.observation,[drag])).outcome!=="refused") throw Error("Consumed overview replayed");
-const view=observation(await tools.computer_read({path:opened.window}));
-const fresh=await act(view,[drag]);
-if(fresh.outcome!=="delivered" || fresh.steps?.length!==1) throw Error("Fresh capture drag refused");
-if((await act(view,[drag])).outcome!=="refused") throw Error("Fresh overview remained armed");
+const drag = () => tools.computer_drag({window:opened.window,from:{x:30,y:40},to:{x:100,y:80},duration_ms:100});
+if((await drag()).outcome!=="delivered") throw Error("Drag on the first capture refused");
+if((await tools.computer_press({window:opened.window,key:"r"})).outcome!=="delivered") throw Error("Key refused");
+await windows();
+const stale=await drag();
+if(stale.outcome!=="refused" || stale.code!=="capture_frame_mismatch" || stale.effect!=="not-applied") throw Error("Resize refusal lost: "+stale.text);
+await tools.computer_read({path:opened.window});
+const fresh=await drag();
+if(fresh.outcome!=="delivered") throw Error("Fresh capture drag refused: "+fresh.text);
text("Capture-bound drag refusal and fresh recovery passed");`;
try {
const result = await new PiIntegrationTest({ testName: "capture-bound-drag", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve("test/fixtures/capture-drag-extension.ts")], isolateUserResources: true,
- tools: ["codemode", "computer_insert"],
+ tools: ["codemode", "computer_delete"],
conversation: [assistantMessage([toolCall({ id: "drag", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")])],
}).run("Use the synthetic MCP capture fixture; stop after resize refusal and observe before trying fresh input.");
expect(getToolExecution(result, "drag").isError, getToolResultText(result, "drag")).toBe(false);
expect(getToolResultText(result, "drag")).toContain("Capture-bound drag refusal and fresh recovery passed");
- const nested = nestedCalls(getToolExecutions(result), "drag");
- const opened = nested[0].computer, reread = nested.find(call => call.name === "computer_read")!.computer;
- expect(reread.source_identity).toEqual(opened.source_identity);
- const sequences = nested.filter(call => call.computer.state?.sequence).map(call => call.computer);
- expect(sequences.map(item => [item.status, item.state.sequence.completed, item.state.sequence.attempted])).toEqual([["refused", 1, 2], ["delivered", 1, 1]]);
const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
const input = calls.filter(c => ["press_key", "drag"].includes(c.name));
- expect(input.map(c => c.name)).toEqual(["press_key", "drag", "drag"]);
- expect(input[0].arguments.key).toBe("r");
- expect(input.slice(1).map(c => c.arguments)).toEqual(["capture-1", "capture-2"].map(capture_id => ({
- target: { kind: "window", pid: 42, window_id: 91 }, delivery_mode: "background", capture_id,
- from_x: 30, from_y: 40, to_x: 100, to_y: 80, duration_ms: 100, steps: 4,
- })));
- expect(calls.filter(c => c.name === "get_window_state")).toHaveLength(2);
+ expect(input.map(c => c.name)).toEqual(["drag", "press_key", "drag", "drag"]);
+ // Each drag names the latest capture before it; the stale one carries the capture taken before the window grew.
+ const latest = (index: number) => calls.slice(0, index).filter(c => c.name === "get_window_state").length;
+ const drags = calls.flatMap((c, index) => c.name === "drag" ? [{ args: c.arguments, capture: `capture-${latest(index)}` }] : []);
+ for (const drag of drags) expect(drag.args).toEqual({ target: { kind: "window", pid: 42, window_id: 91 }, delivery_mode: "background",
+ capture_id: drag.capture, from_x: 30, from_y: 40, to_x: 100, to_y: 80, duration_ms: 100 });
expect(calls.some(c => c.name === "bring_to_front")).toBe(false);
} finally { await rm(cwd, { recursive: true, force: true }); }
});
diff --git a/test/complete-input.integration.test.ts b/test/complete-input.integration.test.ts
index a905053..812d810 100644
--- a/test/complete-input.integration.test.ts
+++ b/test/complete-input.integration.test.ts
@@ -41,10 +41,10 @@ text("Complete sequence guards passed");`;
const result = await new PiIntegrationTest({
testName: "complete-input", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,
- tools: ["codemode", "computer_insert"],
+ tools: ["codemode", "computer_delete"],
conversation: [
assistantMessage([toolCall({ id: "sequence", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")]),
],
}).run("Use complete known-target sequences. Never replay a completed prefix or uncertain input.");
diff --git a/test/display-observation.integration.test.ts b/test/display-observation.integration.test.ts
index 746c7b4..86de7d2 100644
--- a/test/display-observation.integration.test.ts
+++ b/test/display-observation.integration.test.ts
@@ -32,7 +32,7 @@ for (const mode of ["yolo", "safe"] as const) {
assistantMessage([toolCall({ id: "mixed", name: "computer_read", arguments: { path: "computer:machine/local/display/primary", views: ["ui"] } })], { stopReason: "toolUse" }),
assistantMessage([toolCall({ id: "promote", name: "computer_read", arguments: { path: "computer:machine/local/display/primary", mode: "yolo" } })], { stopReason: "toolUse" }),
assistantMessage([toolCall({ id: "display-input", name: "computer_insert", arguments: { path: "computer:machine/local/display/primary", text: '{"action":"click","x":0,"y":0}' } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Read-only display source checked.")]),
],
}).run("Observe the primary display only when permitted. Never change mode or use display pixels as window input.");
diff --git a/test/extension-modes.integration.test.ts b/test/extension-modes.integration.test.ts
index 0f83783..29eff21 100644
--- a/test/extension-modes.integration.test.ts
+++ b/test/extension-modes.integration.test.ts
@@ -33,7 +33,7 @@ test("without Pi Agent IDE, tool_search finds the deferred computer tools and th
{ id: "keys", name: "tool_search", arguments: { query: "computer type press select timeline" } },
{ id: "guide", name: "computer_read", arguments: { path: "docs:computer" } },
{ id: "session", name: "computer_read", arguments: { path: "computer:session" } },
- { id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } },
+ { id: "close", name: "computer_delete", arguments: { path: "computer:session" } },
{ id: "open", name: "tool_search", arguments: { query: "computer open window" } },
{ id: "more", name: "tool_search", arguments: { query: "computer wait browser" } },
{ id: "display", name: "computer_display", arguments: { display: "computer:display/primary", width: 1920, height: 1080 } },
@@ -92,7 +92,7 @@ test("with Pi Agent IDE, computer use extends Read, Search, Insert and Delete an
{ id: "input", name: "tool_search", arguments: { query: "computer click hover drag scroll" } },
{ id: "keys", name: "tool_search", arguments: { query: "computer type press select timeline" } },
{ id: "session", name: "read", arguments: { path: "computer:session" } },
- { id: "close", name: "insert", arguments: { path: "computer:session", text: '{"action":"close"}' } },
+ { id: "close", name: "delete", arguments: { path: "computer:session" } },
{ id: "force", name: "delete", arguments: { path: "computer:machine/local?force=true" } },
{ id: "delete", name: "delete", arguments: { path: "computer:session" } },
]);
diff --git a/test/extension.integration.test.ts b/test/extension.integration.test.ts
index bea7b1f..05bba9b 100644
--- a/test/extension.integration.test.ts
+++ b/test/extension.integration.test.ts
@@ -29,7 +29,7 @@ test("standalone Pi opens two windows with first observations, rejects invented
assistantMessage([toolCall({ id: "stale", name: "computer_write", arguments: {
path: "computer:observation/not-an-observation", content: "Do not dispatch",
} })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("The standalone contract was exercised.")]),
],
}).run("Open two new test windows with first observations, refuse an invented write reference, and release runtime.");
diff --git a/test/fixtures/complete-input-driver.mjs b/test/fixtures/complete-input-driver.mjs
index cc186cd..836b77b 100644
--- a/test/fixtures/complete-input-driver.mjs
+++ b/test/fixtures/complete-input-driver.mjs
@@ -8,6 +8,7 @@ import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprot
// Deterministic MCP input receipts, not evidence of native application behavior.
const log = process.argv[2];
const captureDrag = process.argv[3] === "capture-drag";
+let resize = false;
const semantic = process.argv[3] === "semantic";
let capture; let captureCount = 0;
const image = await sharp({ create: { width: 500, height: 350, channels: 3, background: "white" } }).png().toBuffer();
@@ -21,7 +22,10 @@ server.setRequestHandler(CallToolRequestSchema, async ({ params: { name, argumen
if (name === "fixture_identity") state = { identity: "owned-lifetime" };
else if (name === "list_apps") state = { apps: [{ name: "Notepad", launch_path: "fixture-notepad" }] };
else if (name === "launch_app") { windows.push({ pid: 42, window_id: 91, app_name: "Notepad", bounds: { x: 10, y: 20, width: 1000, height: 700 } }); state = { pid: 42 }; }
- else if (name === "list_windows") state = { windows };
+ else if (name === "list_windows") {
+ if (resize) { windows[0].bounds.width = 1200; resize = false; }
+ state = { windows };
+ }
else if (name === "get_window_state") {
state = { window_bounds: windows[0].bounds, screenshot_width: 500, screenshot_height: 350, capture_id: "once",
elements: [{ element_token: "field", role: "textbox", label: "Field", actions: ["type_text"] }] }; pixels = args.include_screenshot === false ? undefined : image;
@@ -54,7 +58,8 @@ server.setRequestHandler(CallToolRequestSchema, async ({ params: { name, argumen
: { route: "synthetic_events", delivery: { mode: args.delivery_mode } }),
};
else throw Error("Unexpected dispatch " + name);
- if (captureDrag && name === "press_key" && args.key === "r") windows[0].bounds.width = 1200;
+ // The window grows outside the agent's input: the next window listing after "r" applies it, after every capture of the press.
+ if (captureDrag && name === "press_key" && args.key === "r") resize = true;
return { structuredContent: state, content: [{ type: "text", text: JSON.stringify(state) },
...(pixels ? [{ type: "image", data: pixels.toString("base64"), mimeType: "image/png" }] : [])] };
});
diff --git a/test/fixtures/standalone-computer.ts b/test/fixtures/standalone-computer.ts
index 69ce038..18b9ff7 100644
--- a/test/fixtures/standalone-computer.ts
+++ b/test/fixtures/standalone-computer.ts
@@ -2,8 +2,11 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
import { ComputerResources } from "../../src/computer-resources.ts";
import type { ComputerMode, ComputerResult, ComputerSession } from "../../src/session.ts";
import { registerStandaloneTools } from "../../src/standalone-tools.ts";
+import { registerComputerTools } from "../../src/computer-tools.ts";
-export const STANDALONE_TOOLS = ["computer_read", "computer_search", "computer_write", "computer_insert"];
+export const STANDALONE_TOOLS = ["computer_read", "computer_search", "computer_delete", "computer_open", "computer_window", "computer_display", "computer_wait",
+ "computer_browser", "computer_files", "computer_clipboard", "computer_record", "computer_click", "computer_hover", "computer_drag", "computer_scroll",
+ "computer_type", "computer_press", "computer_select", "computer_timeline"];
/**
* Register the production standalone tools over a fixture session and declare them at once. tool_search
@@ -13,6 +16,7 @@ export function registerFixtureComputerTools(pi: ExtensionAPI, getSession: (cwd:
closeSession: () => Promise<ComputerResult>, getMode?: () => ComputerMode): ComputerResources {
const resources = new ComputerResources(getSession, closeSession, undefined, undefined, getMode);
pi.on("session_start", () => {
+ registerComputerTools(pi, resources);
registerStandaloneTools(pi, resources, "Fixture computer guide.");
pi.setActiveTools([...new Set([...pi.getActiveTools(), ...STANDALONE_TOOLS])]);
});
@@ -35,9 +39,9 @@ const address = (value, kind, label) => {
};
const observation = value => address(value, "observation", "Observation");
const hints = value => [...body(value).matchAll(/^([0-9a-z]+:[a-z]+) \| /gm)].map(match => match[1]);
-const insert = (path, command) => tools.computer_insert({ path, text: JSON.stringify(command) });
const windows = async query => tools.computer_search({ path: "computer:machine/local/windows", query: query ?? "" });
-const openApp = app => insert("computer:machine/local", { action: "open", app });
+const openApp = app => tools.computer_open({ at: "computer:machine/local", app });
+const closeAll = () => tools.computer_delete({ path: "computer:session" });
`;
/** The computer tool calls a Codemode script made, in order, with their real results. */
diff --git a/test/held-path.integration.test.ts b/test/held-path.integration.test.ts
index 6381cc5..f46bee2 100644
--- a/test/held-path.integration.test.ts
+++ b/test/held-path.integration.test.ts
@@ -5,33 +5,27 @@ import { assistantMessage, getToolExecution, getToolExecutions, getToolResultTex
import { expect, test } from "vitest";
import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";
-test("real Pi Codemode rejects an invalid path batch and preserves a partial stroke without replay", async () => {
+test("real Pi draws a held path through its points and preserves a partial stroke as unknown", async () => {
await mkdir(".tmp/cua-drag-fork", { recursive: true });
const cwd = await mkdtemp(path.resolve(".tmp/cua-drag-fork/pi-path-")); const log = path.join(cwd, "calls.jsonl");
await mkdir(path.join(cwd, ".pi"));
await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({
command: process.execPath, args: [path.resolve("test/fixtures/complete-input-driver.mjs"), log],
}));
- // A refusal spends its observation, so the valid path acts on a fresh overview.
const code = `${SCRIPT_HELPERS}
const opened = await openApp("Notepad");
-const send = (view, sequence) => insert(view, {action:"sequence",delivery_mode:"foreground",sequence});
-const invalid = await send(opened.observation,[{action:"key",key:"a"},{action:"drag",from_x:20,from_y:30,via:[{x:501,y:40}],to_x:100,to_y:30}]);
-if(invalid.outcome !== "refused" || invalid.effect !== "not-applied") throw Error("Invalid tail admitted");
-const view = observation(await tools.computer_read({path:opened.window}));
-const stopped = await send(view,[{action:"drag",from_x:20,from_y:30,via:[{x:40,y:80},{x:90,y:70}],to_x:100,to_y:30,duration_ms:800},{action:"key",key:"a"}]);
-if(stopped.outcome !== "unknown" || stopped.effect !== "unknown") throw Error("Lost partial path");
+const stopped = await tools.computer_drag({window:opened.window,foreground:true,from:{x:20,y:30},via:[{x:40,y:80},{x:90,y:70}],to:{x:100,y:30},duration_ms:800});
+if(stopped.outcome !== "unknown" || stopped.effect !== "unknown") throw Error("Lost partial path: " + stopped.text);
text(stopped.text);
-if((await send(view,[{action:"key",key:"a"}])).outcome !== "refused") throw Error("Replay admitted");
text("Held path guards passed");`;
try {
const result = await new PiIntegrationTest({
testName: "held-path", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve("test/fixtures/held-path-extension.ts")], isolateUserResources: true,
- tools: ["codemode", "computer_insert"],
+ tools: ["codemode", "computer_delete"],
conversation: [
assistantMessage([toolCall({ id: "path", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")]),
],
}).run("Draw only a bounded known-target path. Never replay partial input.");
@@ -39,7 +33,6 @@ text("Held path guards passed");`;
expect(getToolResultText(result, "path")).toContain("Held path guards passed");
const stopped = nestedCalls(getToolExecutions(result), "path").find(call => call.computer.status === "unknown")!.computer;
expect(stopped.state.delivery.delivered_count).toBe(2);
- expect(stopped.state.sequence).toMatchObject({ attempted: 1, completed: 0 });
const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
const inputs = calls.filter(call => ["press_key","drag","fixture_held_path"].includes(call.name));
expect(inputs).toEqual([{ name: "fixture_held_path", arguments: {
diff --git a/test/key-hold.integration.test.ts b/test/key-hold.integration.test.ts
index 97bd885..bae8c18 100644
--- a/test/key-hold.integration.test.ts
+++ b/test/key-hold.integration.test.ts
@@ -5,35 +5,30 @@ import { assistantMessage, getToolExecution, getToolResultText, PiIntegrationTes
import { expect, test } from "vitest";
import { SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";
-test("real Pi admits one bounded hold, refuses an invalid pointer tail and preserves release uncertainty without replay", async () => {
+test("real Pi holds keys for a bounded time and preserves release uncertainty", async () => {
await mkdir(".tmp/key-holds", { recursive: true });
const cwd = await mkdtemp(path.resolve(".tmp/key-holds/pi-")); const log = path.join(cwd, "calls.jsonl");
await mkdir(path.join(cwd, ".pi"));
await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({
command: process.execPath, args: [path.resolve("test/fixtures/complete-input-driver.mjs"), log],
}));
- // A refusal spends its observation, so each later hold acts on a fresh overview.
const code = `${SCRIPT_HELPERS}
const opened = await openApp("Notepad");
-const fresh = async () => observation(await tools.computer_read({path:opened.window}));
-const send = (view, command) => insert(view, {action:"hold",delivery_mode:"foreground",...command});
-if((await send(opened.observation,{keys:["shift"],duration_ms:500,actions:[{action:"click",x:20,y:30},{action:"drag",from_x:20,from_y:30,via:[{x:501,y:40}],to_x:100,to_y:30}]})).outcome !== "refused") throw Error("Invalid pointer tail admitted");
-const complete = await send(await fresh(),{keys:["shift","w"],duration_ms:500,actions:[{action:"click",x:20,y:30}]});
-if(complete.outcome !== "delivered" || !complete.text.includes("keys_released=true")) throw Error("Complete hold receipt lost");
-const view = await fresh();
-const partial = await send(view,{keys:["escape"],duration_ms:500});
-if(partial.outcome !== "unknown" || !partial.text.includes("keys_released=false")) throw Error("Release uncertainty lost");
+const press = command => tools.computer_press({window:opened.window,foreground:true,hold_ms:500,...command});
+const complete = await press({key:"shift+w"});
+if(complete.outcome !== "delivered" || !complete.text.includes("keys_released=true")) throw Error("Complete hold receipt lost: " + complete.text);
+const partial = await press({key:"escape"});
+if(partial.outcome !== "unknown" || !partial.text.includes("keys_released=false")) throw Error("Release uncertainty lost: " + partial.text);
text(partial.text);
-if((await send(view,{keys:["w"],duration_ms:500})).outcome !== "refused") throw Error("Uncertain hold replay admitted");
text("Bounded key hold guards passed");`;
try {
const result = await new PiIntegrationTest({
testName: "key-hold", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve("test/fixtures/held-path-extension.ts")], isolateUserResources: true,
- tools: ["codemode", "computer_insert"],
+ tools: ["codemode", "computer_delete"],
conversation: [
assistantMessage([toolCall({ id: "hold", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")]),
],
}).run("Use one bounded hold. Never replay uncertain or partially completed input.");
@@ -42,7 +37,7 @@ text("Bounded key hold guards passed");`;
expect(getToolResultText(result, "hold")).toContain("Bounded key hold guards passed");
const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
expect(calls.filter(call => ["fixture_key_hold", "press_key", "drag", "click"].includes(call.name))).toEqual([
- { name: "fixture_key_hold", arguments: {pid:42,window_id:91,delivery_mode:"foreground",keys:["shift","w"],duration_ms:500,actions:[{action:"click",x:20,y:30}]} },
+ { name: "fixture_key_hold", arguments: {pid:42,window_id:91,delivery_mode:"foreground",keys:["shift","w"],duration_ms:500} },
{ name: "fixture_key_hold", arguments: {pid:42,window_id:91,delivery_mode:"foreground",keys:["escape"],duration_ms:500} },
]);
} finally { await rm(cwd, { recursive: true, force: true }); }
diff --git a/test/native-detail.integration.test.ts b/test/native-detail.integration.test.ts
index 5dffaa2..a0cd71c 100644
--- a/test/native-detail.integration.test.ts
+++ b/test/native-detail.integration.test.ts
@@ -19,7 +19,9 @@ if (opened.outcome !== "observed" || !opened.observation) throw Error("Open retu
const region = "region:100,80,100,80";
const detail = await tools.computer_read({path:opened.observation,views:[region]});
text(body(detail)); image(detail);
-const stale = await insert(opened.observation,{action:"click",element_token:"close"});
+const ref = hints(opened)[0];
+if (!ref) throw Error("Open listed no refs: " + body(opened));
+const stale = await tools.computer_click({window:opened.window,target:ref});
if (stale.outcome !== "refused" || stale.effect !== "not-applied") throw Error("Old overview still armed");
const overview = await tools.computer_read({path:"computer:machine/local/display/primary"});
const crop = await tools.computer_read({path:observation(overview),views:[region]});
@@ -32,10 +34,10 @@ text("Read-only detail guards passed");`;
const result = await new PiIntegrationTest({
testName: "native-detail", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,
- tools: ["codemode", "computer_insert"],
+ tools: ["codemode", "computer_delete"],
conversation: [
assistantMessage([toolCall({ id: "detail", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")]),
],
}).run("Read details only. Never dispatch input from the old overview or publish intermediate full-display pixels.");
diff --git a/test/observation-granularity.integration.test.ts b/test/observation-granularity.integration.test.ts
index d64b78a..b623ede 100644
--- a/test/observation-granularity.integration.test.ts
+++ b/test/observation-granularity.integration.test.ts
@@ -44,10 +44,10 @@ text("Three mode contracts passed; exactly two native clicks.");
const result = await new PiIntegrationTest({
testName: "observation-granularity", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,
- tools: ["codemode", "computer_insert"],
+ tools: ["codemode", "computer_delete"],
conversation: [
assistantMessage([toolCall({ id: "parts", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")]),
],
}).run("Use three observation granularities without inheriting omitted authority.");
diff --git a/test/pointer-hover.integration.test.ts b/test/pointer-hover.integration.test.ts
index 2bbf5aa..fa7fa1d 100644
--- a/test/pointer-hover.integration.test.ts
+++ b/test/pointer-hover.integration.test.ts
@@ -27,7 +27,7 @@ async function scenario(testName: string, code: string) {
piCommand: process.env.PI_HOVER_PI_COMMAND,
tools: ["codemode", "computer_insert", "fixture_pointer_control"],
conversation: [assistantMessage([toolCall({ id: "pointer", name: "codemode", arguments: { code: prelude + "\n" + code } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")])],
}).run("Exercise native pointer admission against synthetic MCP windows only. Never operate a real desktop.");
expect(getToolExecution(result, "pointer").isError, getToolResultText(result, "pointer")).toBe(false);
diff --git a/test/portable-session.integration.test.ts b/test/portable-session.integration.test.ts
index 2552b46..d90b8fb 100644
--- a/test/portable-session.integration.test.ts
+++ b/test/portable-session.integration.test.ts
@@ -23,9 +23,9 @@ text(JSON.stringify({platform:"linux",metadata:rows.length,lifetime:"deferred",l
try {
const result = await new PiIntegrationTest({ testName: "portable-lifetime-boundary", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve("test/fixtures/portable-extension.ts")], isolateUserResources: true,
- tools: ["codemode", "computer_insert"], conversation: [
+ tools: ["codemode", "computer_delete"], conversation: [
assistantMessage([toolCall({ id: "portable", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Portable admission checked without app launch or input.")]),
] }).run("Inspect only synthetic metadata and verify deferred lifetime refusal. Do not launch apps or send input.");
expect(getToolExecution(result, "portable").isError, getToolResultText(result, "portable")).toBe(false);
diff --git a/test/safe-extension.integration.test.ts b/test/safe-extension.integration.test.ts
index 74e9995..419cd5b 100644
--- a/test/safe-extension.integration.test.ts
+++ b/test/safe-extension.integration.test.ts
@@ -25,7 +25,7 @@ test("real Pi restores Safe, filters foreign windows, rejects policy arguments,
assistantMessage([toolCall({ id: "foreign", name: "computer_search", arguments: { path: "computer:machine/local/windows", query: "" } })], { stopReason: "toolUse" }),
assistantMessage([toolCall({ id: "promote", name: "computer_insert", arguments: { ...open, mode: "yolo" } })], { stopReason: "toolUse" }),
assistantMessage([toolCall({ id: "owned", name: "computer_insert", arguments: open })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Safe remained selected.")]),
],
}).run("Exercise Safe window policy without changing the saved user choice.");
diff --git a/test/selected-displays.integration.test.ts b/test/selected-displays.integration.test.ts
index 627d6b5..2ed6233 100644
--- a/test/selected-displays.integration.test.ts
+++ b/test/selected-displays.integration.test.ts
@@ -38,7 +38,7 @@ image(r);text("cropped");`;
assistantMessage([toolCall({ id: "input", name: mode === "safe" ? "computer_insert" : "codemode", arguments: mode === "safe"
? { path: "computer:machine/local/display/d-unowned", text: '{"action":"click","x":1,"y":1}' }
: { code: 'const r=await tools.computer_insert({path:load("monitor"),text:JSON.stringify({action:"click",x:1,y:1})});text(r.outcome+": "+r.reason);' } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Read-only monitor selection checked.")]),
],
}).run("Discover and read a selected monitor only when permitted. Do not promote display pixels to input.");
diff --git a/test/semantic-actions.integration.test.ts b/test/semantic-actions.integration.test.ts
index 729c253..430694f 100644
--- a/test/semantic-actions.integration.test.ts
+++ b/test/semantic-actions.integration.test.ts
@@ -14,38 +14,39 @@ test("real Pi dispatches explicit UIA intents once and keeps provider uncertaint
// The peer names its semantic control with the fixed native token "field".
const code = `${SCRIPT_HELPERS}
const opened=await openApp("Notepad");
-const uiView=async()=>{const view=await tools.computer_read({path:opened.window,views:["ui"]});if(typeof view!=="string")throw Error("UI-only returned pixels");return observation(view);};
-for(const action of ["invoke","select","set_value"]) {
- const view=await uiView();
- const command={action,element_token:"field",...(action==="set_value"?{text:"new value"}:{})};
- if((await insert(view,command)).outcome!=="delivered")throw Error("Semantic completion lost");
- if((await insert(view,command)).outcome!=="refused")throw Error("Consumed token replayed");
+const ref=async()=>{const view=await tools.computer_read({path:opened.window,views:["ui"]});if(typeof view!=="string")throw Error("UI-only returned pixels");return hints(view)[0];};
+// computer_select without values selects the element; computer_type with replace sets its value.
+const steps=[["computer_select",{}],["computer_type",{text:"new value",replace:true}]];
+for(const [tool,args] of steps) {
+ const target=await ref();
+ const done=await tools[tool]({window:opened.window,target,...args});
+ if(done.outcome!=="delivered")throw Error("Semantic completion lost: "+done.text);
+ if((await tools[tool]({window:opened.window,target,...args})).outcome!=="refused")throw Error("Spent ref replayed");
}
-const view=await uiView();
-const command={action:"set_value",element_token:"field",text:"provider-failure"};
-const failed=await insert(view,command);
-if(failed.outcome!=="unknown"||(await insert(view,command)).outcome!=="refused")throw Error("Provider uncertainty was replayed");
+const target=await ref();
+const failed=await tools.computer_type({window:opened.window,target,text:"provider-failure",replace:true});
+if(failed.outcome!=="unknown"||(await tools.computer_type({window:opened.window,target,text:"provider-failure",replace:true})).outcome!=="refused")throw Error("Provider uncertainty was replayed");
text("Strict semantic intents and unknown provider outcome passed");`;
try {
const result = await new PiIntegrationTest({ testName: "strict-semantic-actions", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve("test/fixtures/semantic-actions-extension.ts")], isolateUserResources: true,
- tools: ["codemode", "computer_insert"],
+ tools: ["codemode", "computer_delete"],
conversation: [assistantMessage([toolCall({ id: "semantic", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")])],
}).run("Exercise strict semantics on synthetic MCP controls only. Never use physical or foreground input.");
expect(getToolExecution(result, "semantic").isError, getToolResultText(result, "semantic")).toBe(false);
expect(getToolResultText(result, "semantic")).toContain("Strict semantic intents and unknown provider outcome passed");
const dispatched = nestedCalls(getToolExecutions(result), "semantic").filter(call => call.computer.action_capability?.route === "accessibility").map(call => call.computer);
- expect(dispatched.map(item => item.status)).toEqual(["delivered", "delivered", "delivered", "unknown"]);
- for (const item of dispatched.slice(0, 3)) expect(item.action_capability.semantic.app_effect).toBe("unverified");
- expect(dispatched[3].state.attempted).toBe(true);
+ expect(dispatched.map(item => item.status)).toEqual(["delivered", "delivered", "unknown"]);
+ for (const item of dispatched.slice(0, 2)) expect(item.action_capability.semantic.app_effect).toBe("unverified");
+ expect(dispatched[2].state.attempted).toBe(true);
const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
const actions = calls.filter(c => !["list_apps", "list_windows", "launch_app", "fixture_identity", "get_window_state"].includes(c.name));
- expect(actions.map(c => c.name)).toEqual(Array(4).fill("semantic_action"));
- expect(actions.map(c => c.arguments)).toEqual(["invoke", "select", "set_value", "set_value"].map((operation, i) => ({
+ expect(actions.map(c => c.name)).toEqual(Array(3).fill("semantic_action"));
+ expect(actions.map(c => c.arguments)).toEqual(["select", "set_value", "set_value"].map((operation, i) => ({
pid: 42, window_id: 91, element_token: "field", operation,
- ...(operation === "set_value" ? { value: i === 3 ? "provider-failure" : "new value" } : {}),
+ ...(operation === "set_value" ? { value: i === 2 ? "provider-failure" : "new value" } : {}),
})));
} finally { await rm(cwd, { recursive: true, force: true }); }
});
diff --git a/test/sequence.integration.test.ts b/test/sequence.integration.test.ts
index 9211190..e00c464 100644
--- a/test/sequence.integration.test.ts
+++ b/test/sequence.integration.test.ts
@@ -12,9 +12,9 @@ test("real Pi stores a sequence without images and forwards only explicitly sele
await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({ command: process.execPath,
args: [path.resolve("test/fixtures/detail-driver.mjs"), log] }));
const capture = `${SCRIPT_HELPERS}
-const result=await tools.computer_read({path:"computer:machine/local/display/primary",views:['sequence:{"duration_ms":3000,"fps":2,"max_frames":3}']});
-if(typeof result!=="string")throw Error("Capture unexpectedly forwarded frames");
-store("temporal",address(result,"sequence","Sequence"));text(result.match(/^Frames: .*$/m)[0]);`;
+const result=await tools.computer_record({at:"computer:machine/local/display/primary",action:"sequence",duration_ms:3000,fps:2,max_frames:3});
+if(result.outcome!=="observed")throw Error("Capture failed: "+result.text);
+store("temporal",address(result,"sequence","Sequence"));text(result.text.match(/^Frames: .*$/m)[0]);`;
const select = `const result=await tools.computer_read({path:load("temporal"),views:['frames:{"indexes":[0,2]}']});
if(result.type!=="images"||result.images.length!==2)throw Error("Selection failed");
if(!/Selected frames: 0, 2; readonly, input=none/.test(result.note))throw Error("Frames ground input");
@@ -23,10 +23,10 @@ text("Only selected temporal frames forwarded");`;
try {
const result = await new PiIntegrationTest({ testName: "temporal-sequence", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,
- tools: ["codemode", "computer_insert"], conversation: [
+ tools: ["codemode", "computer_delete"], conversation: [
assistantMessage([toolCall({ id: "capture", name: "codemode", arguments: { code: capture } })], { stopReason: "toolUse" }),
assistantMessage([toolCall({ id: "select", name: "codemode", arguments: { code: select } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")]),
],
}).run("Capture a bounded temporal resource, then forward exactly frames zero and two, without input.");
diff --git a/test/shell-desktop.integration.test.ts b/test/shell-desktop.integration.test.ts
index 10b6253..fec774f 100644
--- a/test/shell-desktop.integration.test.ts
+++ b/test/shell-desktop.integration.test.ts
@@ -30,7 +30,7 @@ text("observed");`;
assistantMessage([toolCall({ id: "discover", name: mode === "safe" ? "computer_search" : "codemode",
arguments: mode === "safe" ? { path: "computer:machine/local/windows", query: "" } : { code: discover } })], { stopReason: "toolUse" }),
...(mode === "yolo" ? [assistantMessage([toolCall({ id: "observe", name: "codemode", arguments: { code: observe } })], { stopReason: "toolUse" })] : []),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Read-only membership contract checked.")]),
] }).run("Read only permitted window membership. Do not switch desktops, activate windows or send input.");
expect(getToolExecution(result, "discover").isError, getToolResultText(result, "discover")).toBe(false);
diff --git a/test/ui-resources.integration.test.ts b/test/ui-resources.integration.test.ts
index 5042f12..c40eef8 100644
--- a/test/ui-resources.integration.test.ts
+++ b/test/ui-resources.integration.test.ts
@@ -44,10 +44,10 @@ text("UI resource guards passed: 200 collected, 199 search matches, one native c
const result = await new PiIntegrationTest({
testName: "ui-resources", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,
- tools: ["codemode", "computer_insert"],
+ tools: ["codemode", "computer_delete"],
conversation: [
assistantMessage([toolCall({ id: "ui", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")]),
],
}).run("Refine observed UI resources locally and act only on the exact current enabled target.");
diff --git a/test/window-capture-failure.integration.test.ts b/test/window-capture-failure.integration.test.ts
index 4695cc6..65a046f 100644
--- a/test/window-capture-failure.integration.test.ts
+++ b/test/window-capture-failure.integration.test.ts
@@ -29,10 +29,10 @@ text("Minimized screenshot refused with native cause; old input refused; UI-only
const result = await new PiIntegrationTest({
testName: "window-capture-failure", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,
- tools: ["codemode", "computer_insert"],
+ tools: ["codemode", "computer_delete"],
conversation: [
assistantMessage([toolCall({ id: "capture", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),
- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")]),
],
}).run("Observe the capture failure without activating or restoring any window.");