Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
test: sequence, semantic, detail, hold, held-path and capture-drag integration tests use the computer_* tools (LPT-934)
alexshpunt-clanker[bot]577f7d04aeb3
diff --git a/test/capture-drag.integration.test.ts b/test/capture-drag.integration.test.tsindex e6e7559..546032f 100644--- a/test/capture-drag.integration.test.ts+++ b/test/capture-drag.integration.test.ts@@ -1,11 +1,11 @@import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";import path from "node:path";-import { assistantMessage, getToolExecution, getToolExecutions, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";+import { assistantMessage, getToolExecution, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";import { expect, test } from "vitest";-import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";+import { SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";// This checks the real Pi/MCP boundary, not native Windows application behavior.-test("real Pi keeps drag capture binding and stops on resize refusal until reobserved", async () => {+test("real Pi binds a drag to the latest capture and refuses it after the window changed size until it is read again", async () => {await mkdir(".tmp/lpt-619", { recursive: true });const cwd = await mkdtemp(path.resolve(".tmp/lpt-619/pi-capture-")); const log = path.join(cwd, "calls.jsonl");await mkdir(path.join(cwd, ".pi"));@@ -13,42 +13,34 @@ test("real Pi keeps drag capture binding and stops on resize refusal until reobsargs: [path.resolve("test/fixtures/complete-input-driver.mjs"), log, "capture-drag"] }));const code = `${SCRIPT_HELPERS}const opened = await openApp("Notepad");-// These checks keep one observation for pixel input, so they turn the default successor off.-const act = (view, sequence) => insert(view, {action:"sequence",sequence,successor:false});-const drag={action:"drag",from_x:30,from_y:40,to_x:100,to_y:80,duration_ms:100,steps:4};-const stopped=await act(opened.observation,[{action:"key",key:"r"},drag,{action:"key",key:"b"}]);-if(stopped.outcome!=="refused" || stopped.code!=="capture_frame_mismatch" || stopped.effect!=="unknown") throw Error("Resize refusal or completed prefix lost");-text(JSON.stringify(stopped.steps));-if((await act(opened.observation,[drag])).outcome!=="refused") throw Error("Consumed overview replayed");-const view=observation(await tools.computer_read({path:opened.window}));-const fresh=await act(view,[drag]);-if(fresh.outcome!=="delivered" || fresh.steps?.length!==1) throw Error("Fresh capture drag refused");-if((await act(view,[drag])).outcome!=="refused") throw Error("Fresh overview remained armed");+const drag = () => tools.computer_drag({window:opened.window,from:{x:30,y:40},to:{x:100,y:80},duration_ms:100});+if((await drag()).outcome!=="delivered") throw Error("Drag on the first capture refused");+if((await tools.computer_press({window:opened.window,key:"r"})).outcome!=="delivered") throw Error("Key refused");+await windows();+const stale=await drag();+if(stale.outcome!=="refused" || stale.code!=="capture_frame_mismatch" || stale.effect!=="not-applied") throw Error("Resize refusal lost: "+stale.text);+await tools.computer_read({path:opened.window});+const fresh=await drag();+if(fresh.outcome!=="delivered") throw Error("Fresh capture drag refused: "+fresh.text);text("Capture-bound drag refusal and fresh recovery passed");`;try {const result = await new PiIntegrationTest({ testName: "capture-bound-drag", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/capture-drag-extension.ts")], isolateUserResources: true,- tools: ["codemode", "computer_insert"],+ tools: ["codemode", "computer_delete"],conversation: [assistantMessage([toolCall({ id: "drag", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")])],}).run("Use the synthetic MCP capture fixture; stop after resize refusal and observe before trying fresh input.");expect(getToolExecution(result, "drag").isError, getToolResultText(result, "drag")).toBe(false);expect(getToolResultText(result, "drag")).toContain("Capture-bound drag refusal and fresh recovery passed");- const nested = nestedCalls(getToolExecutions(result), "drag");- const opened = nested[0].computer, reread = nested.find(call => call.name === "computer_read")!.computer;- expect(reread.source_identity).toEqual(opened.source_identity);- const sequences = nested.filter(call => call.computer.state?.sequence).map(call => call.computer);- expect(sequences.map(item => [item.status, item.state.sequence.completed, item.state.sequence.attempted])).toEqual([["refused", 1, 2], ["delivered", 1, 1]]);const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));const input = calls.filter(c => ["press_key", "drag"].includes(c.name));- expect(input.map(c => c.name)).toEqual(["press_key", "drag", "drag"]);- expect(input[0].arguments.key).toBe("r");- expect(input.slice(1).map(c => c.arguments)).toEqual(["capture-1", "capture-2"].map(capture_id => ({- target: { kind: "window", pid: 42, window_id: 91 }, delivery_mode: "background", capture_id,- from_x: 30, from_y: 40, to_x: 100, to_y: 80, duration_ms: 100, steps: 4,- })));- expect(calls.filter(c => c.name === "get_window_state")).toHaveLength(2);+ expect(input.map(c => c.name)).toEqual(["drag", "press_key", "drag", "drag"]);+ // Each drag names the latest capture before it; the stale one carries the capture taken before the window grew.+ const latest = (index: number) => calls.slice(0, index).filter(c => c.name === "get_window_state").length;+ const drags = calls.flatMap((c, index) => c.name === "drag" ? [{ args: c.arguments, capture: `capture-${latest(index)}` }] : []);+ for (const drag of drags) expect(drag.args).toEqual({ target: { kind: "window", pid: 42, window_id: 91 }, delivery_mode: "background",+ capture_id: drag.capture, from_x: 30, from_y: 40, to_x: 100, to_y: 80, duration_ms: 100 });expect(calls.some(c => c.name === "bring_to_front")).toBe(false);} finally { await rm(cwd, { recursive: true, force: true }); }});diff --git a/test/complete-input.integration.test.ts b/test/complete-input.integration.test.tsindex a905053..812d810 100644--- a/test/complete-input.integration.test.ts+++ b/test/complete-input.integration.test.ts@@ -41,10 +41,10 @@ text("Complete sequence guards passed");`;const result = await new PiIntegrationTest({testName: "complete-input", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,- tools: ["codemode", "computer_insert"],+ tools: ["codemode", "computer_delete"],conversation: [assistantMessage([toolCall({ id: "sequence", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")]),],}).run("Use complete known-target sequences. Never replay a completed prefix or uncertain input.");diff --git a/test/display-observation.integration.test.ts b/test/display-observation.integration.test.tsindex 746c7b4..86de7d2 100644--- a/test/display-observation.integration.test.ts+++ b/test/display-observation.integration.test.ts@@ -32,7 +32,7 @@ for (const mode of ["yolo", "safe"] as const) {assistantMessage([toolCall({ id: "mixed", name: "computer_read", arguments: { path: "computer:machine/local/display/primary", views: ["ui"] } })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "promote", name: "computer_read", arguments: { path: "computer:machine/local/display/primary", mode: "yolo" } })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "display-input", name: "computer_insert", arguments: { path: "computer:machine/local/display/primary", text: '{"action":"click","x":0,"y":0}' } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Read-only display source checked.")]),],}).run("Observe the primary display only when permitted. Never change mode or use display pixels as window input.");diff --git a/test/extension-modes.integration.test.ts b/test/extension-modes.integration.test.tsindex 0f83783..29eff21 100644--- a/test/extension-modes.integration.test.ts+++ b/test/extension-modes.integration.test.ts@@ -33,7 +33,7 @@ test("without Pi Agent IDE, tool_search finds the deferred computer tools and th{ id: "keys", name: "tool_search", arguments: { query: "computer type press select timeline" } },{ id: "guide", name: "computer_read", arguments: { path: "docs:computer" } },{ id: "session", name: "computer_read", arguments: { path: "computer:session" } },- { id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } },+ { id: "close", name: "computer_delete", arguments: { path: "computer:session" } },{ id: "open", name: "tool_search", arguments: { query: "computer open window" } },{ id: "more", name: "tool_search", arguments: { query: "computer wait browser" } },{ id: "display", name: "computer_display", arguments: { display: "computer:display/primary", width: 1920, height: 1080 } },@@ -92,7 +92,7 @@ test("with Pi Agent IDE, computer use extends Read, Search, Insert and Delete an{ id: "input", name: "tool_search", arguments: { query: "computer click hover drag scroll" } },{ id: "keys", name: "tool_search", arguments: { query: "computer type press select timeline" } },{ id: "session", name: "read", arguments: { path: "computer:session" } },- { id: "close", name: "insert", arguments: { path: "computer:session", text: '{"action":"close"}' } },+ { id: "close", name: "delete", arguments: { path: "computer:session" } },{ id: "force", name: "delete", arguments: { path: "computer:machine/local?force=true" } },{ id: "delete", name: "delete", arguments: { path: "computer:session" } },]);diff --git a/test/extension.integration.test.ts b/test/extension.integration.test.tsindex bea7b1f..05bba9b 100644--- a/test/extension.integration.test.ts+++ b/test/extension.integration.test.ts@@ -29,7 +29,7 @@ test("standalone Pi opens two windows with first observations, rejects inventedassistantMessage([toolCall({ id: "stale", name: "computer_write", arguments: {path: "computer:observation/not-an-observation", content: "Do not dispatch",} })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("The standalone contract was exercised.")]),],}).run("Open two new test windows with first observations, refuse an invented write reference, and release runtime.");diff --git a/test/fixtures/complete-input-driver.mjs b/test/fixtures/complete-input-driver.mjsindex cc186cd..836b77b 100644--- a/test/fixtures/complete-input-driver.mjs+++ b/test/fixtures/complete-input-driver.mjs@@ -8,6 +8,7 @@ import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprot// Deterministic MCP input receipts, not evidence of native application behavior.const log = process.argv[2];const captureDrag = process.argv[3] === "capture-drag";+let resize = false;const semantic = process.argv[3] === "semantic";let capture; let captureCount = 0;const image = await sharp({ create: { width: 500, height: 350, channels: 3, background: "white" } }).png().toBuffer();@@ -21,7 +22,10 @@ server.setRequestHandler(CallToolRequestSchema, async ({ params: { name, argumenif (name === "fixture_identity") state = { identity: "owned-lifetime" };else if (name === "list_apps") state = { apps: [{ name: "Notepad", launch_path: "fixture-notepad" }] };else if (name === "launch_app") { windows.push({ pid: 42, window_id: 91, app_name: "Notepad", bounds: { x: 10, y: 20, width: 1000, height: 700 } }); state = { pid: 42 }; }- else if (name === "list_windows") state = { windows };+ else if (name === "list_windows") {+ if (resize) { windows[0].bounds.width = 1200; resize = false; }+ state = { windows };+ }else if (name === "get_window_state") {state = { window_bounds: windows[0].bounds, screenshot_width: 500, screenshot_height: 350, capture_id: "once",elements: [{ element_token: "field", role: "textbox", label: "Field", actions: ["type_text"] }] }; pixels = args.include_screenshot === false ? undefined : image;@@ -54,7 +58,8 @@ server.setRequestHandler(CallToolRequestSchema, async ({ params: { name, argumen: { route: "synthetic_events", delivery: { mode: args.delivery_mode } }),};else throw Error("Unexpected dispatch " + name);- if (captureDrag && name === "press_key" && args.key === "r") windows[0].bounds.width = 1200;+ // The window grows outside the agent's input: the next window listing after "r" applies it, after every capture of the press.+ if (captureDrag && name === "press_key" && args.key === "r") resize = true;return { structuredContent: state, content: [{ type: "text", text: JSON.stringify(state) },...(pixels ? [{ type: "image", data: pixels.toString("base64"), mimeType: "image/png" }] : [])] };});diff --git a/test/fixtures/standalone-computer.ts b/test/fixtures/standalone-computer.tsindex 69ce038..18b9ff7 100644--- a/test/fixtures/standalone-computer.ts+++ b/test/fixtures/standalone-computer.ts@@ -2,8 +2,11 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";import { ComputerResources } from "../../src/computer-resources.ts";import type { ComputerMode, ComputerResult, ComputerSession } from "../../src/session.ts";import { registerStandaloneTools } from "../../src/standalone-tools.ts";+import { registerComputerTools } from "../../src/computer-tools.ts";-export const STANDALONE_TOOLS = ["computer_read", "computer_search", "computer_write", "computer_insert"];+export const STANDALONE_TOOLS = ["computer_read", "computer_search", "computer_delete", "computer_open", "computer_window", "computer_display", "computer_wait",+ "computer_browser", "computer_files", "computer_clipboard", "computer_record", "computer_click", "computer_hover", "computer_drag", "computer_scroll",+ "computer_type", "computer_press", "computer_select", "computer_timeline"];/*** Register the production standalone tools over a fixture session and declare them at once. tool_search@@ -13,6 +16,7 @@ export function registerFixtureComputerTools(pi: ExtensionAPI, getSession: (cwd:closeSession: () => Promise<ComputerResult>, getMode?: () => ComputerMode): ComputerResources {const resources = new ComputerResources(getSession, closeSession, undefined, undefined, getMode);pi.on("session_start", () => {+ registerComputerTools(pi, resources);registerStandaloneTools(pi, resources, "Fixture computer guide.");pi.setActiveTools([...new Set([...pi.getActiveTools(), ...STANDALONE_TOOLS])]);});@@ -35,9 +39,9 @@ const address = (value, kind, label) => {};const observation = value => address(value, "observation", "Observation");const hints = value => [...body(value).matchAll(/^([0-9a-z]+:[a-z]+) \| /gm)].map(match => match[1]);-const insert = (path, command) => tools.computer_insert({ path, text: JSON.stringify(command) });const windows = async query => tools.computer_search({ path: "computer:machine/local/windows", query: query ?? "" });-const openApp = app => insert("computer:machine/local", { action: "open", app });+const openApp = app => tools.computer_open({ at: "computer:machine/local", app });+const closeAll = () => tools.computer_delete({ path: "computer:session" });`;/** The computer tool calls a Codemode script made, in order, with their real results. */diff --git a/test/held-path.integration.test.ts b/test/held-path.integration.test.tsindex 6381cc5..f46bee2 100644--- a/test/held-path.integration.test.ts+++ b/test/held-path.integration.test.ts@@ -5,33 +5,27 @@ import { assistantMessage, getToolExecution, getToolExecutions, getToolResultTeximport { expect, test } from "vitest";import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";-test("real Pi Codemode rejects an invalid path batch and preserves a partial stroke without replay", async () => {+test("real Pi draws a held path through its points and preserves a partial stroke as unknown", async () => {await mkdir(".tmp/cua-drag-fork", { recursive: true });const cwd = await mkdtemp(path.resolve(".tmp/cua-drag-fork/pi-path-")); const log = path.join(cwd, "calls.jsonl");await mkdir(path.join(cwd, ".pi"));await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({command: process.execPath, args: [path.resolve("test/fixtures/complete-input-driver.mjs"), log],}));- // A refusal spends its observation, so the valid path acts on a fresh overview.const code = `${SCRIPT_HELPERS}const opened = await openApp("Notepad");-const send = (view, sequence) => insert(view, {action:"sequence",delivery_mode:"foreground",sequence});-const invalid = await send(opened.observation,[{action:"key",key:"a"},{action:"drag",from_x:20,from_y:30,via:[{x:501,y:40}],to_x:100,to_y:30}]);-if(invalid.outcome !== "refused" || invalid.effect !== "not-applied") throw Error("Invalid tail admitted");-const view = observation(await tools.computer_read({path:opened.window}));-const stopped = await send(view,[{action:"drag",from_x:20,from_y:30,via:[{x:40,y:80},{x:90,y:70}],to_x:100,to_y:30,duration_ms:800},{action:"key",key:"a"}]);-if(stopped.outcome !== "unknown" || stopped.effect !== "unknown") throw Error("Lost partial path");+const stopped = await tools.computer_drag({window:opened.window,foreground:true,from:{x:20,y:30},via:[{x:40,y:80},{x:90,y:70}],to:{x:100,y:30},duration_ms:800});+if(stopped.outcome !== "unknown" || stopped.effect !== "unknown") throw Error("Lost partial path: " + stopped.text);text(stopped.text);-if((await send(view,[{action:"key",key:"a"}])).outcome !== "refused") throw Error("Replay admitted");text("Held path guards passed");`;try {const result = await new PiIntegrationTest({testName: "held-path", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/held-path-extension.ts")], isolateUserResources: true,- tools: ["codemode", "computer_insert"],+ tools: ["codemode", "computer_delete"],conversation: [assistantMessage([toolCall({ id: "path", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")]),],}).run("Draw only a bounded known-target path. Never replay partial input.");@@ -39,7 +33,6 @@ text("Held path guards passed");`;expect(getToolResultText(result, "path")).toContain("Held path guards passed");const stopped = nestedCalls(getToolExecutions(result), "path").find(call => call.computer.status === "unknown")!.computer;expect(stopped.state.delivery.delivered_count).toBe(2);- expect(stopped.state.sequence).toMatchObject({ attempted: 1, completed: 0 });const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));const inputs = calls.filter(call => ["press_key","drag","fixture_held_path"].includes(call.name));expect(inputs).toEqual([{ name: "fixture_held_path", arguments: {diff --git a/test/key-hold.integration.test.ts b/test/key-hold.integration.test.tsindex 97bd885..bae8c18 100644--- a/test/key-hold.integration.test.ts+++ b/test/key-hold.integration.test.ts@@ -5,35 +5,30 @@ import { assistantMessage, getToolExecution, getToolResultText, PiIntegrationTesimport { expect, test } from "vitest";import { SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";-test("real Pi admits one bounded hold, refuses an invalid pointer tail and preserves release uncertainty without replay", async () => {+test("real Pi holds keys for a bounded time and preserves release uncertainty", async () => {await mkdir(".tmp/key-holds", { recursive: true });const cwd = await mkdtemp(path.resolve(".tmp/key-holds/pi-")); const log = path.join(cwd, "calls.jsonl");await mkdir(path.join(cwd, ".pi"));await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({command: process.execPath, args: [path.resolve("test/fixtures/complete-input-driver.mjs"), log],}));- // A refusal spends its observation, so each later hold acts on a fresh overview.const code = `${SCRIPT_HELPERS}const opened = await openApp("Notepad");-const fresh = async () => observation(await tools.computer_read({path:opened.window}));-const send = (view, command) => insert(view, {action:"hold",delivery_mode:"foreground",...command});-if((await send(opened.observation,{keys:["shift"],duration_ms:500,actions:[{action:"click",x:20,y:30},{action:"drag",from_x:20,from_y:30,via:[{x:501,y:40}],to_x:100,to_y:30}]})).outcome !== "refused") throw Error("Invalid pointer tail admitted");-const complete = await send(await fresh(),{keys:["shift","w"],duration_ms:500,actions:[{action:"click",x:20,y:30}]});-if(complete.outcome !== "delivered" || !complete.text.includes("keys_released=true")) throw Error("Complete hold receipt lost");-const view = await fresh();-const partial = await send(view,{keys:["escape"],duration_ms:500});-if(partial.outcome !== "unknown" || !partial.text.includes("keys_released=false")) throw Error("Release uncertainty lost");+const press = command => tools.computer_press({window:opened.window,foreground:true,hold_ms:500,...command});+const complete = await press({key:"shift+w"});+if(complete.outcome !== "delivered" || !complete.text.includes("keys_released=true")) throw Error("Complete hold receipt lost: " + complete.text);+const partial = await press({key:"escape"});+if(partial.outcome !== "unknown" || !partial.text.includes("keys_released=false")) throw Error("Release uncertainty lost: " + partial.text);text(partial.text);-if((await send(view,{keys:["w"],duration_ms:500})).outcome !== "refused") throw Error("Uncertain hold replay admitted");text("Bounded key hold guards passed");`;try {const result = await new PiIntegrationTest({testName: "key-hold", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/held-path-extension.ts")], isolateUserResources: true,- tools: ["codemode", "computer_insert"],+ tools: ["codemode", "computer_delete"],conversation: [assistantMessage([toolCall({ id: "hold", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")]),],}).run("Use one bounded hold. Never replay uncertain or partially completed input.");@@ -42,7 +37,7 @@ text("Bounded key hold guards passed");`;expect(getToolResultText(result, "hold")).toContain("Bounded key hold guards passed");const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));expect(calls.filter(call => ["fixture_key_hold", "press_key", "drag", "click"].includes(call.name))).toEqual([- { name: "fixture_key_hold", arguments: {pid:42,window_id:91,delivery_mode:"foreground",keys:["shift","w"],duration_ms:500,actions:[{action:"click",x:20,y:30}]} },+ { name: "fixture_key_hold", arguments: {pid:42,window_id:91,delivery_mode:"foreground",keys:["shift","w"],duration_ms:500} },{ name: "fixture_key_hold", arguments: {pid:42,window_id:91,delivery_mode:"foreground",keys:["escape"],duration_ms:500} },]);} finally { await rm(cwd, { recursive: true, force: true }); }diff --git a/test/native-detail.integration.test.ts b/test/native-detail.integration.test.tsindex 5dffaa2..a0cd71c 100644--- a/test/native-detail.integration.test.ts+++ b/test/native-detail.integration.test.ts@@ -19,7 +19,9 @@ if (opened.outcome !== "observed" || !opened.observation) throw Error("Open retuconst region = "region:100,80,100,80";const detail = await tools.computer_read({path:opened.observation,views:[region]});text(body(detail)); image(detail);-const stale = await insert(opened.observation,{action:"click",element_token:"close"});+const ref = hints(opened)[0];+if (!ref) throw Error("Open listed no refs: " + body(opened));+const stale = await tools.computer_click({window:opened.window,target:ref});if (stale.outcome !== "refused" || stale.effect !== "not-applied") throw Error("Old overview still armed");const overview = await tools.computer_read({path:"computer:machine/local/display/primary"});const crop = await tools.computer_read({path:observation(overview),views:[region]});@@ -32,10 +34,10 @@ text("Read-only detail guards passed");`;const result = await new PiIntegrationTest({testName: "native-detail", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,- tools: ["codemode", "computer_insert"],+ tools: ["codemode", "computer_delete"],conversation: [assistantMessage([toolCall({ id: "detail", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")]),],}).run("Read details only. Never dispatch input from the old overview or publish intermediate full-display pixels.");diff --git a/test/observation-granularity.integration.test.ts b/test/observation-granularity.integration.test.tsindex d64b78a..b623ede 100644--- a/test/observation-granularity.integration.test.ts+++ b/test/observation-granularity.integration.test.ts@@ -44,10 +44,10 @@ text("Three mode contracts passed; exactly two native clicks.");const result = await new PiIntegrationTest({testName: "observation-granularity", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,- tools: ["codemode", "computer_insert"],+ tools: ["codemode", "computer_delete"],conversation: [assistantMessage([toolCall({ id: "parts", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")]),],}).run("Use three observation granularities without inheriting omitted authority.");diff --git a/test/pointer-hover.integration.test.ts b/test/pointer-hover.integration.test.tsindex 2bbf5aa..fa7fa1d 100644--- a/test/pointer-hover.integration.test.ts+++ b/test/pointer-hover.integration.test.ts@@ -27,7 +27,7 @@ async function scenario(testName: string, code: string) {piCommand: process.env.PI_HOVER_PI_COMMAND,tools: ["codemode", "computer_insert", "fixture_pointer_control"],conversation: [assistantMessage([toolCall({ id: "pointer", name: "codemode", arguments: { code: prelude + "\n" + code } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")])],}).run("Exercise native pointer admission against synthetic MCP windows only. Never operate a real desktop.");expect(getToolExecution(result, "pointer").isError, getToolResultText(result, "pointer")).toBe(false);diff --git a/test/portable-session.integration.test.ts b/test/portable-session.integration.test.tsindex 2552b46..d90b8fb 100644--- a/test/portable-session.integration.test.ts+++ b/test/portable-session.integration.test.ts@@ -23,9 +23,9 @@ text(JSON.stringify({platform:"linux",metadata:rows.length,lifetime:"deferred",ltry {const result = await new PiIntegrationTest({ testName: "portable-lifetime-boundary", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/portable-extension.ts")], isolateUserResources: true,- tools: ["codemode", "computer_insert"], conversation: [+ tools: ["codemode", "computer_delete"], conversation: [assistantMessage([toolCall({ id: "portable", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Portable admission checked without app launch or input.")]),] }).run("Inspect only synthetic metadata and verify deferred lifetime refusal. Do not launch apps or send input.");expect(getToolExecution(result, "portable").isError, getToolResultText(result, "portable")).toBe(false);diff --git a/test/safe-extension.integration.test.ts b/test/safe-extension.integration.test.tsindex 74e9995..419cd5b 100644--- a/test/safe-extension.integration.test.ts+++ b/test/safe-extension.integration.test.ts@@ -25,7 +25,7 @@ test("real Pi restores Safe, filters foreign windows, rejects policy arguments,assistantMessage([toolCall({ id: "foreign", name: "computer_search", arguments: { path: "computer:machine/local/windows", query: "" } })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "promote", name: "computer_insert", arguments: { ...open, mode: "yolo" } })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "owned", name: "computer_insert", arguments: open })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Safe remained selected.")]),],}).run("Exercise Safe window policy without changing the saved user choice.");diff --git a/test/selected-displays.integration.test.ts b/test/selected-displays.integration.test.tsindex 627d6b5..2ed6233 100644--- a/test/selected-displays.integration.test.ts+++ b/test/selected-displays.integration.test.ts@@ -38,7 +38,7 @@ image(r);text("cropped");`;assistantMessage([toolCall({ id: "input", name: mode === "safe" ? "computer_insert" : "codemode", arguments: mode === "safe"? { path: "computer:machine/local/display/d-unowned", text: '{"action":"click","x":1,"y":1}' }: { code: 'const r=await tools.computer_insert({path:load("monitor"),text:JSON.stringify({action:"click",x:1,y:1})});text(r.outcome+": "+r.reason);' } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Read-only monitor selection checked.")]),],}).run("Discover and read a selected monitor only when permitted. Do not promote display pixels to input.");diff --git a/test/semantic-actions.integration.test.ts b/test/semantic-actions.integration.test.tsindex 729c253..430694f 100644--- a/test/semantic-actions.integration.test.ts+++ b/test/semantic-actions.integration.test.ts@@ -14,38 +14,39 @@ test("real Pi dispatches explicit UIA intents once and keeps provider uncertaint// The peer names its semantic control with the fixed native token "field".const code = `${SCRIPT_HELPERS}const opened=await openApp("Notepad");-const uiView=async()=>{const view=await tools.computer_read({path:opened.window,views:["ui"]});if(typeof view!=="string")throw Error("UI-only returned pixels");return observation(view);};-for(const action of ["invoke","select","set_value"]) {- const view=await uiView();- const command={action,element_token:"field",...(action==="set_value"?{text:"new value"}:{})};- if((await insert(view,command)).outcome!=="delivered")throw Error("Semantic completion lost");- if((await insert(view,command)).outcome!=="refused")throw Error("Consumed token replayed");+const ref=async()=>{const view=await tools.computer_read({path:opened.window,views:["ui"]});if(typeof view!=="string")throw Error("UI-only returned pixels");return hints(view)[0];};+// computer_select without values selects the element; computer_type with replace sets its value.+const steps=[["computer_select",{}],["computer_type",{text:"new value",replace:true}]];+for(const [tool,args] of steps) {+ const target=await ref();+ const done=await tools[tool]({window:opened.window,target,...args});+ if(done.outcome!=="delivered")throw Error("Semantic completion lost: "+done.text);+ if((await tools[tool]({window:opened.window,target,...args})).outcome!=="refused")throw Error("Spent ref replayed");}-const view=await uiView();-const command={action:"set_value",element_token:"field",text:"provider-failure"};-const failed=await insert(view,command);-if(failed.outcome!=="unknown"||(await insert(view,command)).outcome!=="refused")throw Error("Provider uncertainty was replayed");+const target=await ref();+const failed=await tools.computer_type({window:opened.window,target,text:"provider-failure",replace:true});+if(failed.outcome!=="unknown"||(await tools.computer_type({window:opened.window,target,text:"provider-failure",replace:true})).outcome!=="refused")throw Error("Provider uncertainty was replayed");text("Strict semantic intents and unknown provider outcome passed");`;try {const result = await new PiIntegrationTest({ testName: "strict-semantic-actions", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/semantic-actions-extension.ts")], isolateUserResources: true,- tools: ["codemode", "computer_insert"],+ tools: ["codemode", "computer_delete"],conversation: [assistantMessage([toolCall({ id: "semantic", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")])],}).run("Exercise strict semantics on synthetic MCP controls only. Never use physical or foreground input.");expect(getToolExecution(result, "semantic").isError, getToolResultText(result, "semantic")).toBe(false);expect(getToolResultText(result, "semantic")).toContain("Strict semantic intents and unknown provider outcome passed");const dispatched = nestedCalls(getToolExecutions(result), "semantic").filter(call => call.computer.action_capability?.route === "accessibility").map(call => call.computer);- expect(dispatched.map(item => item.status)).toEqual(["delivered", "delivered", "delivered", "unknown"]);- for (const item of dispatched.slice(0, 3)) expect(item.action_capability.semantic.app_effect).toBe("unverified");- expect(dispatched[3].state.attempted).toBe(true);+ expect(dispatched.map(item => item.status)).toEqual(["delivered", "delivered", "unknown"]);+ for (const item of dispatched.slice(0, 2)) expect(item.action_capability.semantic.app_effect).toBe("unverified");+ expect(dispatched[2].state.attempted).toBe(true);const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));const actions = calls.filter(c => !["list_apps", "list_windows", "launch_app", "fixture_identity", "get_window_state"].includes(c.name));- expect(actions.map(c => c.name)).toEqual(Array(4).fill("semantic_action"));- expect(actions.map(c => c.arguments)).toEqual(["invoke", "select", "set_value", "set_value"].map((operation, i) => ({+ expect(actions.map(c => c.name)).toEqual(Array(3).fill("semantic_action"));+ expect(actions.map(c => c.arguments)).toEqual(["select", "set_value", "set_value"].map((operation, i) => ({pid: 42, window_id: 91, element_token: "field", operation,- ...(operation === "set_value" ? { value: i === 3 ? "provider-failure" : "new value" } : {}),+ ...(operation === "set_value" ? { value: i === 2 ? "provider-failure" : "new value" } : {}),})));} finally { await rm(cwd, { recursive: true, force: true }); }});diff --git a/test/sequence.integration.test.ts b/test/sequence.integration.test.tsindex 9211190..e00c464 100644--- a/test/sequence.integration.test.ts+++ b/test/sequence.integration.test.ts@@ -12,9 +12,9 @@ test("real Pi stores a sequence without images and forwards only explicitly seleawait writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({ command: process.execPath,args: [path.resolve("test/fixtures/detail-driver.mjs"), log] }));const capture = `${SCRIPT_HELPERS}-const result=await tools.computer_read({path:"computer:machine/local/display/primary",views:['sequence:{"duration_ms":3000,"fps":2,"max_frames":3}']});-if(typeof result!=="string")throw Error("Capture unexpectedly forwarded frames");-store("temporal",address(result,"sequence","Sequence"));text(result.match(/^Frames: .*$/m)[0]);`;+const result=await tools.computer_record({at:"computer:machine/local/display/primary",action:"sequence",duration_ms:3000,fps:2,max_frames:3});+if(result.outcome!=="observed")throw Error("Capture failed: "+result.text);+store("temporal",address(result,"sequence","Sequence"));text(result.text.match(/^Frames: .*$/m)[0]);`;const select = `const result=await tools.computer_read({path:load("temporal"),views:['frames:{"indexes":[0,2]}']});if(result.type!=="images"||result.images.length!==2)throw Error("Selection failed");if(!/Selected frames: 0, 2; readonly, input=none/.test(result.note))throw Error("Frames ground input");@@ -23,10 +23,10 @@ text("Only selected temporal frames forwarded");`;try {const result = await new PiIntegrationTest({ testName: "temporal-sequence", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,- tools: ["codemode", "computer_insert"], conversation: [+ tools: ["codemode", "computer_delete"], conversation: [assistantMessage([toolCall({ id: "capture", name: "codemode", arguments: { code: capture } })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "select", name: "codemode", arguments: { code: select } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")]),],}).run("Capture a bounded temporal resource, then forward exactly frames zero and two, without input.");diff --git a/test/shell-desktop.integration.test.ts b/test/shell-desktop.integration.test.tsindex 10b6253..fec774f 100644--- a/test/shell-desktop.integration.test.ts+++ b/test/shell-desktop.integration.test.ts@@ -30,7 +30,7 @@ text("observed");`;assistantMessage([toolCall({ id: "discover", name: mode === "safe" ? "computer_search" : "codemode",arguments: mode === "safe" ? { path: "computer:machine/local/windows", query: "" } : { code: discover } })], { stopReason: "toolUse" }),...(mode === "yolo" ? [assistantMessage([toolCall({ id: "observe", name: "codemode", arguments: { code: observe } })], { stopReason: "toolUse" })] : []),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Read-only membership contract checked.")]),] }).run("Read only permitted window membership. Do not switch desktops, activate windows or send input.");expect(getToolExecution(result, "discover").isError, getToolResultText(result, "discover")).toBe(false);diff --git a/test/ui-resources.integration.test.ts b/test/ui-resources.integration.test.tsindex 5042f12..c40eef8 100644--- a/test/ui-resources.integration.test.ts+++ b/test/ui-resources.integration.test.ts@@ -44,10 +44,10 @@ text("UI resource guards passed: 200 collected, 199 search matches, one native cconst result = await new PiIntegrationTest({testName: "ui-resources", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,- tools: ["codemode", "computer_insert"],+ tools: ["codemode", "computer_delete"],conversation: [assistantMessage([toolCall({ id: "ui", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")]),],}).run("Refine observed UI resources locally and act only on the exact current enabled target.");diff --git a/test/window-capture-failure.integration.test.ts b/test/window-capture-failure.integration.test.tsindex 4695cc6..65a046f 100644--- a/test/window-capture-failure.integration.test.ts+++ b/test/window-capture-failure.integration.test.ts@@ -29,10 +29,10 @@ text("Minimized screenshot refused with native cause; old input refused; UI-onlyconst result = await new PiIntegrationTest({testName: "window-capture-failure", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,- tools: ["codemode", "computer_insert"],+ tools: ["codemode", "computer_delete"],conversation: [assistantMessage([toolCall({ id: "capture", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "close", name: "computer_insert", arguments: { path: "computer:session", text: '{"action":"close"}' } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")]),],}).run("Observe the capture failure without activating or restoring any window.");