Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
test: integration tests expect real input for points, keys and drags (LPT-942)
alexshpunt-clanker[bot]b19975b48ff3
diff --git a/test/complete-input.integration.test.ts b/test/complete-input.integration.test.tsindex cfb66be..aef9ed4 100644--- a/test/complete-input.integration.test.ts+++ b/test/complete-input.integration.test.ts@@ -5,7 +5,7 @@ import { assistantMessage, getToolExecution, getToolExecutions, getToolResultTeximport { expect, test } from "vitest";import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";-test("real Pi sends keys, chords and drags in the background and keeps partial and refused outcomes apart", async () => {+test("real Pi sends keys, chords and drags as real input and keeps partial and refused outcomes apart", async () => {await mkdir(".tmp/cua-gestures", { recursive: true });const cwd = await mkdtemp(path.resolve(".tmp/cua-gestures/pi-contract-")); const log = path.join(cwd, "calls.jsonl");await mkdir(path.join(cwd, ".pi"));@@ -23,8 +23,8 @@ if (drag.outcome !== "delivered") throw Error("Drag refused: " + drag.text);const partial = await press("escape");if (partial.outcome !== "unknown" || partial.effect !== "unknown") throw Error("Partial key reported as " + partial.outcome);text(partial.text);-const refused = await press("delete", {foreground:false});-if (refused.outcome !== "refused" || refused.effect !== "not-applied") throw Error("Background refusal reported as " + refused.outcome);+const refused = await press("delete");+if (refused.outcome !== "refused" || refused.effect !== "not-applied") throw Error("Native refusal reported as " + refused.outcome);text(refused.text);text("Complete sequence guards passed");`;try {@@ -53,7 +53,8 @@ text("Complete sequence guards passed");`;expect(input.map(c => c.name)).toEqual(["press_key", "hotkey", "drag", "press_key", "press_key"]);expect(input.map(c => c.arguments.key ?? c.arguments.keys)).toEqual(["a", ["shift", "b"], undefined, "escape", "delete"]);expect(input.map(c => c.arguments.target)).toEqual(Array(5).fill({ kind: "window", pid: 42, window_id: 91 }));- expect(input.every(c => c.arguments.delivery_mode === "background")).toBe(true);+ // Keys and drags are real input: every one goes in the foreground, and a refused key is not retried.+ expect(input.every(c => c.arguments.delivery_mode === "foreground")).toBe(true);expect(calls.some(c => c.name === "bring_to_front")).toBe(false);} finally { await rm(cwd, { recursive: true, force: true }); }});diff --git a/test/coordinate-scroll.integration.test.ts b/test/coordinate-scroll.integration.test.tsindex 5761780..3f57628 100644--- a/test/coordinate-scroll.integration.test.ts+++ b/test/coordinate-scroll.integration.test.ts@@ -8,7 +8,7 @@ const root = path.resolve(import.meta.dirname, "..");const fixture = (name: string) => path.join(root, "test/fixtures", name);// Pattern: real-Pi extensions example and the ordinary IDE computer-provider scenarios.-// Contract: computer_scroll preserves capture authority and structured uncertain prefixes; IDE Insert refuses.+// Contract: computer_scroll at a point is real input that preserves capture authority and structured uncertain prefixes; IDE Insert refuses.async function scenario(testName: string, scenario: "receipts" | "authority", providerFirst: boolean) {const consumer = IDE_EXTENSION, provider = fixture("coordinate-scroll-extension.ts");await mkdir(path.join(root, ".tmp/lpt-730"), { recursive: true });@@ -45,12 +45,12 @@ const native = (x: number, y: number, delta_x: number, delta_y: number, deliverytest.each([false, true])("ordinary IDE scroll preserves capture-bound deltas and uncertain prefixes (providerFirst=%s)", async providerFirst => {const actions = inputCalls(await scenario("coordinate-scroll-receipts", "receipts", providerFirst));- expect(actions.map(call => call.name)).toEqual(Array(10).fill("scroll_at"));+ // Every scroll at a point is real input in the foreground.+ expect(actions.map(call => call.name)).toEqual(Array(7).fill("scroll_at"));expect(actions.map(call => call.arguments)).toEqual([- native(100, 80, 0, 2, "background"), native(100, 80, -2, 3, "background"), native(100, 80, -2, 3), native(120, 90, 1, 0),- ...Array.from({ length: 5 }, () => native(130, 100, 1, 1)), native(130, 100, 1, 1, "background"),+ native(100, 80, -2, 3), native(120, 90, 1, 0), ...Array.from({ length: 5 }, () => native(130, 100, 1, 1)),]);- // The refusal spent its capture, so the foreground retry captures the window again: every scroll has its own capture.+ // Every scroll binds to its own capture.expect(new Set(actions.map(call => call.arguments.capture_id)).size).toBe(actions.length);}, 120_000);diff --git a/test/fixtures/coordinate-scroll-extension.ts b/test/fixtures/coordinate-scroll-extension.tsindex 8e387d4..7b319b4 100644--- a/test/fixtures/coordinate-scroll-extension.ts+++ b/test/fixtures/coordinate-scroll-extension.ts@@ -111,24 +111,17 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {const image = view.content.find(block => block.type === "image");assert(image?.mimeType === "image/png" && image.data, "Native overview image did not reach actual Pi");if (scenario === "receipts") {- // foreground: false keeps a background refusal as it is.- const refused = await rejected(first, { foreground: false }), refusal = semantic(refused), refusalState = record(refusal.state);- assert(refusal.status === "refused" && refusal.effect === "not-applied" && refusalState.code === "background_unavailable" && refusalState.pointer_attempted === false,- "Structured background refusal lost its no-input evidence");- // A refusal spends the observation; by default the same refusal sends the scroll again in the foreground, and the- // window keeps foreground delivery.- await observe(first);+ // A scroll at a point is real input: the wheel goes in the foreground at once.const delivered = await scroll(first, { delta_x: -2, delta_y: 3 }), receipt = semantic(delivered), state = record(receipt.state);assert(!delivered.isError && receipt.status === "delivered" && receipt.effect === "applied" && state.unit === "wheel_detents" &&JSON.stringify(state.delivered_axes) === '["y","x"]' && state.pointer_changed === null && record(receipt.action_capability).actual_delivery === "foreground",`Truthful two-axis structured receipt lost: ${textOf(delivered)}`);- assert(textOf(delivered).includes("refused background input"), "The foreground retry was not reported");await observe(first, "image");const imageOnly = await scroll(first, { x: 120, y: 90, delta_x: 1, delta_y: 0 });assert(!imageOnly.isError && semantic(imageOnly).status === "delivered", "Image-only scroll refused");- for (const operation of ["partial", "last_check", "wrong_unit", "wrong_delta", "wrong_axes", "background_success"]) {+ for (const operation of ["partial", "last_check", "wrong_unit", "wrong_delta", "wrong_axes"]) {await control(operation); view = await observe(first);- const result = await rejected(first, { x: 130, y: 100, delta_x: 1, delta_y: 1, ...(operation === "background_success" ? { foreground: false } : {}) });+ const result = await rejected(first, { x: 130, y: 100, delta_x: 1, delta_y: 1 });const action = semantic(result), native = record(action.state);assert(action.status === "unknown" && action.effect === "unknown", `${operation} lost structured uncertainty`);if (operation === "partial") assert(native.pointer_delivered === true && JSON.stringify(native.attempted_axes) === '["y","x"]' &&@@ -147,18 +140,18 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {assert(cropped.content.some(block => block.type === "image" && block.mimeType === "image/jpeg") && field(cropped.details, "input") === "none" &&field(cropped.details, "observation_id") === undefined, "Detail granted input authority");await control("capability_off"); await observe(first);- await rejected(first, { foreground: true });+ await rejected(first);await control("capability_on");const external = reference(await ok("search", { path: "computer:machine/local/windows", query: "Unowned fixture" }), "window");await observe(external); await control("safe");- await rejected(external, { foreground: true });+ await rejected(external);await observe(first, "image");- const owned = await scroll(first, { foreground: true });+ const owned = await scroll(first);assert(!owned.isError && semantic(owned).status === "delivered", "Safe denied owned image scroll");await observe(first); await control("close_first");- await rejected(first, { foreground: true });+ await rejected(first);await observe(second); await control("recycle_second");- await rejected(second, { foreground: true });+ await rejected(second);// Insert never acts on a window; it names the tool that does.const insert = await denied("insert", { path: first, text: JSON.stringify({ action: "scroll", x: 100, y: 80, delta_y: 2, unit: "wheel_detents" }) });assert(textOf(insert).includes("computer_scroll"), "Insert refusal named no tool");diff --git a/test/fixtures/ide-provider-extension.ts b/test/fixtures/ide-provider-extension.tsindex d51c6aa..e2aed0a 100644--- a/test/fixtures/ide-provider-extension.ts+++ b/test/fixtures/ide-provider-extension.ts@@ -121,7 +121,7 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {const hint = hintOf(view), literal = ' {"looks":"like JSON"}\n"quoted text"\n ';await refused("write", { path: reference(view, "observation"), content: literal }, "computer_type");const nativeBeforeType = (await journal()).length;- await ok("computer_type", { window: first, target: hint, text: literal, delay_ms: 0, foreground: true });+ await ok("computer_type", { window: first, target: hint, text: literal, delay_ms: 0 });assert(!await capturedBeforeInput(nativeBeforeType), "computer_type refreshed native capture");const written = (await inputs()).at(-1);assert(written?.name === "type_text" && written.arguments.text === literal && written.arguments.delay_ms === 0, "Literal typing normalized text, added a command or lost delay");diff --git a/test/held-path.integration.test.ts b/test/held-path.integration.test.tsindex f46bee2..c6ddad2 100644--- a/test/held-path.integration.test.ts+++ b/test/held-path.integration.test.ts@@ -14,7 +14,7 @@ test("real Pi draws a held path through its points and preserves a partial strok}));const code = `${SCRIPT_HELPERS}const opened = await openApp("Notepad");-const stopped = await tools.computer_drag({window:opened.window,foreground:true,from:{x:20,y:30},via:[{x:40,y:80},{x:90,y:70}],to:{x:100,y:30},duration_ms:800});+const stopped = await tools.computer_drag({window:opened.window,from:{x:20,y:30},via:[{x:40,y:80},{x:90,y:70}],to:{x:100,y:30},duration_ms:800});if(stopped.outcome !== "unknown" || stopped.effect !== "unknown") throw Error("Lost partial path: " + stopped.text);text(stopped.text);text("Held path guards passed");`;diff --git a/test/key-hold.integration.test.ts b/test/key-hold.integration.test.tsindex bae8c18..551260e 100644--- a/test/key-hold.integration.test.ts+++ b/test/key-hold.integration.test.ts@@ -14,7 +14,7 @@ test("real Pi holds keys for a bounded time and preserves release uncertainty",}));const code = `${SCRIPT_HELPERS}const opened = await openApp("Notepad");-const press = command => tools.computer_press({window:opened.window,foreground:true,hold_ms:500,...command});+const press = command => tools.computer_press({window:opened.window,hold_ms:500,...command});const complete = await press({key:"shift+w"});if(complete.outcome !== "delivered" || !complete.text.includes("keys_released=true")) throw Error("Complete hold receipt lost: " + complete.text);const partial = await press({key:"escape"});