Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
test: window management, restore and mode integration tests use computer_window and check the Write/Insert refusal (LPT-934)
alexshpunt-clanker[bot]bfc7febbd01c
diff --git a/test/extension-modes.integration.test.ts b/test/extension-modes.integration.test.tsindex 29eff21..b5ec8e1 100644--- a/test/extension-modes.integration.test.ts+++ b/test/extension-modes.integration.test.ts@@ -43,7 +43,7 @@ test("without Pi Agent IDE, tool_search finds the deferred computer tools and th{ id: "click", name: "computer_click", arguments: { window: "computer:machine/local", target: "1:a" } },{ id: "delete", name: "computer_delete", arguments: { path: "computer:session" } },]);- expect(getToolCallNames(result)).toEqual(["tool_search", "tool_search", "tool_search", "computer_read", "computer_read", "computer_insert", "tool_search", "tool_search", "computer_display", "tool_search", "computer_files", "computer_record", "computer_click", "computer_delete"]);+ expect(getToolCallNames(result)).toEqual(["tool_search", "tool_search", "tool_search", "computer_read", "computer_read", "computer_delete", "tool_search", "tool_search", "computer_display", "tool_search", "computer_files", "computer_record", "computer_click", "computer_delete"]);for (const tool of ["computer_click", "computer_hover", "computer_drag", "computer_scroll", "computer_type", "computer_press", "computer_select", "computer_timeline"])expect(["find", "input", "keys"].map(id => getToolResultText(result, id)).join("\n")).toContain(`- ${tool}:`);for (const id of ["find", "guide", "session", "close", "delete"])@@ -52,7 +52,7 @@ test("without Pi Agent IDE, tool_search finds the deferred computer tools and th// tool_search loads the best matches of each query, at most 8 by default, so the guide names one query per cluster.// A query lists only tools it newly loads; together the guide's cluster queries load every computer tool.const loaded = ["find", "open", "more", "files", "input", "keys"].map(id => getToolResultText(result, id)).join("\n");- for (const tool of ["computer_search", "computer_insert", "computer_write", "computer_delete", "computer_open", "computer_window", "computer_display",+ for (const tool of ["computer_search", "computer_delete", "computer_open", "computer_window", "computer_display","computer_wait", "computer_browser", "computer_files", "computer_clipboard", "computer_record", "computer_click", "computer_hover", "computer_drag","computer_scroll", "computer_type", "computer_press", "computer_select", "computer_timeline"]) expect(loaded).toContain(`- ${tool}:`);// Input goes to a window address and refuses anything else before native work.@@ -82,9 +82,9 @@ test("an IDE that refuses this plugin API version leaves computer use standaloneexpect(getToolResultText(result, "session")).toContain("computer:machine/local");}, 120_000);-test("with Pi Agent IDE, computer use extends Read, Search, Insert and Delete and adds only the typed computer tools", async () => {+test("with Pi Agent IDE, computer use extends Read, Search and Delete, refuses Write and Insert, and adds only the typed computer tools", async () => {for (const extensions of [[ide, computerUse, "builtin:tool-search"], [computerUse, ide, "builtin:tool-search"]]) {- const result = await run("with-ide", extensions, ["read", "insert", "delete", "tool_search"], [+ const result = await run("with-ide", extensions, ["read", "write", "insert", "delete", "tool_search"], [{ id: "find", name: "tool_search", arguments: { query: "computer read windows displays" } },{ id: "open", name: "tool_search", arguments: { query: "computer open window" } },{ id: "more", name: "tool_search", arguments: { query: "computer wait browser" } },@@ -92,6 +92,8 @@ test("with Pi Agent IDE, computer use extends Read, Search, Insert and Delete an{ id: "input", name: "tool_search", arguments: { query: "computer click hover drag scroll" } },{ id: "keys", name: "tool_search", arguments: { query: "computer type press select timeline" } },{ id: "session", name: "read", arguments: { path: "computer:session" } },+ { id: "insert", name: "insert", arguments: { path: "computer:machine/local", text: '{"action":"open","app":"Notepad"}' } },+ { id: "write", name: "write", arguments: { path: "computer:machine/local", content: "hello" } },{ id: "close", name: "delete", arguments: { path: "computer:session" } },{ id: "force", name: "delete", arguments: { path: "computer:machine/local?force=true" } },{ id: "delete", name: "delete", arguments: { path: "computer:session" } },@@ -107,6 +109,10 @@ test("with Pi Agent IDE, computer use extends Read, Search, Insert and Delete anfor (const id of ["session", "close", "delete"])expect(getToolExecution(result, id).isError, getToolResultText(result, id)).toBe(false);expect(getToolResultText(result, "session")).toContain("computer:session");+ // Write and Insert only name the typed tool that does what they were sent; nothing reaches a machine.+ for (const id of ["insert", "write"]) expect(getToolExecution(result, id).isError).toBe(true);+ expect(getToolResultText(result, "insert")).toContain("Insert does not act on computer resources; nothing was sent. Use computer_open.");+ expect(getToolResultText(result, "write")).toContain("Write does not act on computer resources; nothing was sent. Use computer_type.");expect(getToolResultText(result, "close")).toContain("Computer closed");expect(getToolResultText(result, "delete")).toContain("Computer closed");}diff --git a/test/fixtures/window-management-extension.ts b/test/fixtures/window-management-extension.tsindex 06fef7e..910e7e6 100644--- a/test/fixtures/window-management-extension.ts+++ b/test/fixtures/window-management-extension.ts@@ -16,7 +16,7 @@ function find(value: unknown, key: string): unknown {for (const child of Object.values(value)) { const found = find(child, key); if (found !== undefined) return found; }return undefined;}-function reference(result: Result, kind: "window" | "window-state" | "observation"): string {+function reference(result: Result, kind: "window" | "observation"): string {const found = body(result).match(addressPattern(kind));assert(found, `No ${kind} resource: ${body(result)}`); return found[0];}@@ -78,104 +78,87 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {const journal = async (): Promise<Row[]> => (await readFile(log!, "utf8")).trim().split("\n").filter(Boolean).map(line => JSON.parse(line));const count = async (name: string) => (await journal()).filter(row => row.name === name).length;const control = (operation: string, window_id = 91) => driver!.call("fixture_control", { operation, window_id }, signal);+ // A state Read only looks: no pixels, no address that grants anything, and the window's observations stay.const stateRead = async (window: string) => {const result = await ok("read", { path: window, views: ["state"] });assert(result.content.every(block => block.type !== "image"), "Management state granted pixels");+ assert(!/computer:window-state\//.test(body(result)), `State Read issued a window-state address: ${body(result)}`);if (management) {assert(find(result, "coordinate_space") === "windows_desktop_pixels" && find(result, "bounds_source") !== undefined,`State lost explicit desktop bounds source: ${body(result)}`);assert(find(result, "virtual_desktop") !== undefined, `State dropped native desktop membership: ${body(result)}`);}- return reference(result, "window-state");+ return result;};- const insert = (state: string, command: Record<string, unknown>) => call("insert", { path: state, text: JSON.stringify(command) });- const move = { action: "move", x: -20, y: 120, unit: "windows_desktop_pixels" };- const open = async () => reference(await ok("insert", { path: "computer:machine/local", text: '{"action":"open","app":"Notepad"}' }), "window");- assert(!pi.getAllTools().some(tool => /^computer_(read|search|write|insert)$/.test(tool.name)), "Standalone computer tools leaked into the IDE provider");+ // computer_window samples the window's state itself and sends one command against that sample.+ const manage = (window: string, command: Record<string, unknown>, callSignal = signal) => call("computer_window", { window, ...command }, callSignal);+ const move = { action: "move", x: -20, y: 120 };+ const open = async () => reference(await ok("computer_open", { at: "computer:machine/local", app: "Notepad" }), "window");+ assert(!pi.getAllTools().some(tool => /^computer_(read|search|delete)$/.test(tool.name)), "Standalone computer tools leaked into the IDE provider");const first = await open(), second = await open();const listed = await ok("search", { path: "computer:machine/local/windows", query: "Owned management fixture" });assert(body(listed).includes(first) && body(listed).includes(second), "Search lost exact-source references");- let state = await stateRead(first);+ await stateRead(first);if (scenario === "workflow") {- const statusBefore = await count("get_window_control_state"), captureBefore = await count("get_window_state");- const cached = await ok("read", { path: state });- assert(reference(cached, "window-state") === state, "Cached Read renewed one-use admission");- assert(await count("get_window_control_state") === statusBefore && await count("get_window_state") === captureBefore,- "Cached Read acquired hidden metadata or pixels");const invalid = [- { action: "move", x: 1, y: 2 }, { ...move, unit: "css_pixels" }, { ...move, x: 1.5 }, { ...move, x: 100001 },- { ...move, y: -100001 }, { action: "move", x: 1, unit: "windows_desktop_pixels" },- { action: "resize", width: 0, height: 80, unit: "windows_desktop_pixels" },- { action: "resize", width: 32768, height: 80, unit: "windows_desktop_pixels" },- { action: "resize", width: 100, height: 2.5, unit: "windows_desktop_pixels" },- { ...move, width: 10 }, { ...move, pid: 42 }, { ...move, window_id: 91 }, { ...move, state_id: "foreign" },- { ...move, delivery_mode: "foreground" }, { ...move, hint: "1" }, { ...move, animation: {} },- { action: "sequence", sequence: [move] }, { action: "focus", unit: "windows_desktop_pixels" },+ { action: "move", x: 1 }, { ...move, x: 100001 }, { ...move, y: -100001 },+ { action: "resize", width: 0, height: 80 }, { action: "resize", width: 32768, height: 80 },+ { ...move, width: 10 }, { ...move, unit: "css_pixels" }, { ...move, pid: 42 }, { ...move, state_id: "foreign" }, { ...move, delivery_mode: "foreground" },{ action: "minimize", x: 1 }, { action: "maximize", height: 5 }, { action: "click", x: 1, y: 1 },];- await denied("write", { path: state, content: "never type" });- for (const command of invalid) await denied("insert", { path: state, text: JSON.stringify(command) });- assert(await count("manage_window") === 0, "Invalid geometry or borrowed input authority reached native mutation");- const result = await insert(state, move);+ // Write and Insert never act on a window; they name the tool that does.+ assert(body(await denied("write", { path: first, content: "never type" })).includes("computer_type"), "Write refusal named no tool");+ assert(body(await denied("insert", { path: first, text: JSON.stringify(move) })).includes("computer_window"), "Insert refusal named no tool");+ for (const command of invalid) {+ const result = await call("computer_window", { window: first, ...command });+ assert(result.isError, `computer_window accepted ${JSON.stringify(command)}: ${body(result).split("\n")[0]}`);+ }+ assert(await count("manage_window") === 0, "Invalid geometry reached native mutation");+ const result = await manage(first, move);assert(!result.isError && semantic(result).effect === "applied", `Valid desktop move failed: ${body(result)}`);assert(find(result, "mutation_attempted") === true && find(result, "postcondition_confirmed") === true,"Delivered management discarded native completion prefix");- await denied("insert", { path: state, text: JSON.stringify(move) }); await denied("read", { path: state });- assert(await count("manage_window") === 1 && await count("get_window_control_state") === statusBefore,- "Consumed admission replayed or editor resolution refreshed native state");- const operations = [{ action: "resize", width: 180, height: 120, unit: "windows_desktop_pixels" },- { action: "focus" }, { action: "maximize" }, { action: "minimize" }];- for (const command of operations) {- await control("reset"); state = await stateRead(first);- const result = await insert(state, command);+ for (const command of [{ action: "resize", width: 180, height: 120 }, { action: "focus" }, { action: "maximize" }, { action: "minimize" }]) {+ await control("reset");+ const result = await manage(first, command);assert(!result.isError && semantic(result).effect === "applied", `Valid ${command.action} failed: ${body(result)}`);}- await control("reset"); await control("outer_bounds"); state = await stateRead(first);- const outer = await insert(state, move);+ await control("reset"); await control("outer_bounds");+ const outer = await manage(first, move);assert(!outer.isError && find(outer, "bounds_source") === "window_rect", "Explicit outer bounds became visible-frame coordinates");- await control("reset"); const retired = await stateRead(first); await stateRead(first);- await denied("insert", { path: retired, text: JSON.stringify(move) });- state = await stateRead(first); const ui = await ok("read", { path: first, views: ["ui"] });- await denied("insert", { path: state, text: JSON.stringify(move) });- const observation = reference(ui, "observation");- await denied("insert", { path: observation, text: JSON.stringify(move) });- await stateRead(first); await denied("write", { path: observation, content: "retired UI" });- state = await stateRead(first);- await ok("read", { path: first, views: ['sequence:{"duration_ms":200,"fps":10,"max_frames":2}'] });- await denied("insert", { path: state, text: JSON.stringify(move) });- assert(await count("manage_window") === 6, "Read-only authority retirement sent a mutation");+ // A state Read between a UI Read and input keeps the UI observation's refs current.+ const ui = await ok("read", { path: first, views: ["ui"] });+ await stateRead(first);+ await ok("read", { path: reference(ui, "observation") });+ assert(await count("manage_window") === 6, "Read-only state sent a mutation");} else if (scenario === "admission") {const external = reference(await ok("search", { path: "computer:machine/local/windows", query: "Unowned management fixture" }), "window");- const externalState = await stateRead(external); mode = "safe";- const refused = await denied("insert", { path: externalState, text: JSON.stringify(move) });+ await stateRead(external); mode = "safe";+ const refused = await denied("computer_window", { window: external, ...move });assert(semantic(refused).effect === "not-applied", "Safe denial lost no-dispatch effect");assert(await count("manage_window") === 0, "Safe denial dispatched management");- state = await stateRead(first); const prior = await count("get_window_control_state");- await ok("read", { path: state }); assert(await count("get_window_control_state") === prior, "Safe cached Read renewed state");const aborted = new AbortController(); aborted.abort();- const cancelled = await call("insert", { path: state, text: JSON.stringify(move) }, aborted.signal);+ const cancelled = await manage(first, move, aborted.signal);assert(cancelled.isError && await count("manage_window") === 0, "Pre-dispatch cancellation mutated the source");- state = await stateRead(first); await control("close_first");- await denied("read", { path: state }); await denied("insert", { path: state, text: JSON.stringify(move) });- state = await stateRead(second); await control("reuse_second");- await denied("insert", { path: state, text: JSON.stringify(move) });+ await control("close_first");+ await denied("computer_window", { window: first, ...move });+ await control("reuse_second");+ await denied("computer_window", { window: second, ...move });assert(await count("manage_window") === 0, "Closed/reused source sent native mutation");mode = "yolo";const replacement = reference(await ok("search", { path: "computer:machine/local/windows", query: "Owned management fixture 92" }), "window");- state = await stateRead(replacement); await control("reused_after_dispatch", 92);- const reused = await denied("insert", { path: state, text: JSON.stringify(move) });+ await control("reused_after_dispatch", 92);+ const reused = await denied("computer_window", { window: replacement, ...move });assert(semantic(reused).effect === "unknown" && find(reused, "postcondition_confirmed") === true,"Post-dispatch reuse became success or discarded completed prefix");- await denied("insert", { path: state, text: JSON.stringify(move) });- assert(await count("manage_window") === 1, "Post-dispatch reuse was replayed");+ assert(await count("manage_window") === 1, "Post-dispatch reuse sent an extra mutation");} else if (scenario === "typed") {- // computer_window samples its own one-use state: no window-state address, and invalid fields never reach native code.const window = (args: Record<string, unknown>, target = first) => call("computer_window", { window: target, ...args });for (const bad of [{ action: "move", x: 1 }, { action: "maximize", x: 1 }, { action: "resize", width: 10 }, { action: "restore", y: 2 }]) {const result = await window(bad);assert(result.isError, `computer_window accepted ${JSON.stringify(bad)}: ${body(result)}`);}- await denied("computer_window", { window: state, action: "focus" });+ await denied("computer_window", { window: "computer:machine/local", action: "focus" });assert(await count("manage_window") === 0, "Invalid computer_window calls reached native management");const moved = await window({ action: "move", x: -20, y: 120 });assert(!moved.isError && semantic(moved).effect === "applied" && body(moved).includes("on local"), `computer_window move failed: ${body(moved)}`);@@ -217,47 +200,38 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {} else if (scenario === "receipts") {for (const operation of ["refused", "partial", "final_cancel", "wrong_state", "wrong_window", "wrong_operation", "wrong_route","missing_focus", "missing_prefix", "missing_receipt", "wrong_request", "wrong_before", "constrained", "changed_source", "contradictory_foreground"]) {- await control("reset"); state = await stateRead(first); await control(operation);- const result = await denied("insert", { path: state, text: JSON.stringify(move) }), action = semantic(result);+ await control("reset"); await control(operation);+ const result = await denied("computer_window", { window: first, ...move }), action = semantic(result);assert(action.effect === (operation === "refused" ? "not-applied" : "unknown"), `Wrong effect for ${operation}: ${body(result)}`);if (["partial", "final_cancel"].includes(operation)) assert(find(result, "mutation_attempted") === true, `${operation} lost attempted prefix`);if (operation === "final_cancel") assert(find(result, "postcondition_confirmed") === true, "Final cancellation lost completed prefix");- const before = await count("manage_window");- await denied("insert", { path: state, text: JSON.stringify(move) }); await denied("read", { path: state });- assert(await count("manage_window") === before, `${operation} was replayed`);}- await control("reset"); state = await stateRead(first);+ await control("reset");const abortAfter = new AbortController(); cancelAfterDispatch = abortAfter;- const cancelled = await call("insert", { path: state, text: JSON.stringify(move) }, abortAfter.signal);+ const cancelled = await manage(first, move, abortAfter.signal);assert(cancelled.isError && semantic(cancelled).effect === "unknown" && find(cancelled, "postcondition_confirmed") === true,`Cancellation after native completion discarded the delivered prefix: ${body(cancelled)}`);- const countAfter = await count("manage_window");- await denied("insert", { path: state, text: JSON.stringify(move) });- assert(await count("manage_window") === countAfter, "Post-dispatch cancellation was replayed");} else {- await control("minimized"); state = await stateRead(first);- const focus = await denied("insert", { path: state, text: '{"action":"focus"}' });+ await control("minimized");+ const focus = await denied("computer_window", { window: first, action: "focus" });assert(semantic(focus).effect === "not-applied" && find(focus, "mutation_attempted") === false && await count("restore_window") === 0,"Focus implicitly restored or activated a minimized target");- state = await stateRead(first);- const restored = await insert(state, { action: "restore" });+ const restored = await manage(first, { action: "restore" });assert(!restored.isError && semantic(restored).effect === "applied" && find(restored, "restore_requested") === true,`Control admission lost existing restore: ${body(restored)}`);- await denied("insert", { path: state, text: JSON.stringify(move) });for (const situation of ["other_desktop", "unknown_desktop"]) {- await control("reset"); await control(situation); state = await stateRead(first);- const result = await denied("insert", { path: state, text: '{"action":"focus"}' });+ await control("reset"); await control(situation);+ const result = await denied("computer_window", { window: first, action: "focus" });assert(semantic(result).effect === "not-applied" && find(result, "mutation_attempted") === false,`${situation} activation implicitly switched desktops`);}- management = false; await control("reset"); await control("minimized"); state = await stateRead(first);- const legacyReads = await count("get_window_status"), managementBefore = await count("manage_window");- await ok("read", { path: state });- await denied("insert", { path: state, text: JSON.stringify(move) });- assert(await count("manage_window") === managementBefore && await count("get_window_status") === legacyReads,- "Restore-only state borrowed management authority or silently refreshed state");- state = await stateRead(first);- const legacy = await insert(state, { action: "restore" });+ management = false; await control("reset"); await control("minimized");+ const managementBefore = await count("manage_window");+ await stateRead(first);+ const refused = await denied("computer_window", { window: first, ...move });+ assert(/only restore and activate/.test(body(refused)), `Restore-only runtime refusal changed: ${body(refused)}`);+ assert(await count("manage_window") === managementBefore, "Restore-only state borrowed management authority");+ const legacy = await manage(first, { action: "restore" });assert(!legacy.isError && find(legacy, "restore_requested") === true, `Legacy flat restore changed: ${body(legacy)}`);restoration = false;const before = await count("get_window_status");@@ -268,7 +242,7 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {assert(rows.every(row => ["fixture_identity", "fixture_control", "list_apps", "list_windows", "launch_app", "get_window_state","get_window_status", "get_window_control_state", "restore_window", "manage_window"].includes(row.name)),"Management used hidden input, desktop capture or activation fallback");- await ok("insert", { path: "computer:session", text: '{"action":"close"}' });+ await ok("delete", { path: "computer:session" });return { content: [{ type: "text" as const, text: `IDE window management ${scenario} contract passed` }],details: { scenario, mutations: rows.filter(row => row.name === "manage_window").length } };},diff --git a/test/fixtures/window-restore-extension.ts b/test/fixtures/window-restore-extension.tsindex bf10d45..9b8d16e 100644--- a/test/fixtures/window-restore-extension.ts+++ b/test/fixtures/window-restore-extension.ts@@ -15,7 +15,7 @@ function find(value: unknown, key: string): unknown {for (const child of Object.values(value)) { const found = find(child, key); if (found !== undefined) return found; }return undefined;}-function reference(result: Result, kind: "window" | "window-state" | "observation"): string {+function reference(result: Result, kind: "window" | "observation"): string {const found = body(result).match(addressPattern(kind));assert(found, `No ${kind} resource in native tool receipt: ${body(result)}`);return found[0];@@ -30,8 +30,8 @@ function semantic(result: Result): Record<string, unknown> {export default async function fixture(pi: ExtensionAPI): Promise<void> {const source = process.env.PI_RESTORE_PROVIDER_SOURCE ?? path.resolve(import.meta.dirname, "../..");const load = (name: string) => import(pathToFileURL(path.join(source, "src", name)).href);- const [{ registerComputerProvider }, { loadComputerConfig }, { McpDriver }, { ComputerSession }] =- await Promise.all([load("provider.ts"), load("config.ts"), load("driver.ts"), load("session.ts")]);+ const [{ registerComputerProvider }, { loadComputerConfig }, { McpDriver }, { ComputerSession }, { registerComputerTools }] =+ await Promise.all([load("provider.ts"), load("config.ts"), load("driver.ts"), load("session.ts"), load("computer-tools.ts")]);const Base = McpDriver as typeof import("../../src/driver.ts").McpDriver;let capability = true, mode: ComputerMode = "yolo", session: ComputerSession | undefined, driver: FixtureDriver | undefined, log: string | undefined;class FixtureDriver extends Base {@@ -49,7 +49,8 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {const close = async (): Promise<ComputerResult> => session ? session.close() : {content: [{ type: "text", text: "No fixture runtime is running." }], details: { status: "closed", mode },};- await registerComputerProvider(pi, getSession, close);+ const provider = await registerComputerProvider(pi, getSession, close);+ registerComputerTools(pi, provider.resources);pi.registerTool({name: "verify_window_restore", label: "Verify metadata-only window restore",description: "Exercise real IDE resource tools against synthetic window restore receipts, never a desktop.",@@ -68,35 +69,31 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {const journal = async () => (await readFile(log!, "utf8")).trim().split("\n").filter(Boolean).map(line => JSON.parse(line));const count = async (name: string) => (await journal()).filter(row => row.name === name).length;const control = (operation: string) => driver!.call("fixture_control", { operation }, signal);+ // A state Read only looks: no pixels and no address that grants anything.const stateRead = async (window: string) => {const value = await ok("read", { path: window, views: ["state"] });assert(value.content.every(block => block.type !== "image"), "State Read granted pixel authority");+ assert(!/computer:window-state\//.test(body(value)), `State Read issued a window-state address: ${body(value)}`);assert(find(value, "minimized") === true && find(value, "foreground_at_target") === false && find(value, "keyboard_focus_at_target") === null,`State lost separate native window/focus measurements: ${body(value)}`);- return reference(value, "window-state");+ return value;};- const restore = (state: string) => call("insert", { path: state, text: '{"action":"restore"}' });- const open = async () => reference(await ok("insert", { path: "computer:machine/local", text: '{"action":"open","app":"Notepad"}' }), "window");- assert(!pi.getAllTools().some(tool => /^computer_/.test(tool.name)), "Duplicate computer tools escaped primary IDE provider");+ // computer_window samples the window's state itself and restores against that sample.+ const restore = (window: string) => call("computer_window", { window, action: "restore" });+ const open = async () => reference(await ok("computer_open", { at: "computer:machine/local", app: "Notepad" }), "window");+ assert(!pi.getAllTools().some(tool => /^computer_(read|search|delete)$/.test(tool.name)), "Standalone computer tools escaped primary IDE provider");const first = await open(), second = await open();const listed = await ok("search", { path: "computer:machine/local/windows", query: "Owned restore fixture" });assert(body(listed).includes(first) && body(listed).includes(second), "Search lost exact native window references");- let state = await stateRead(first);+ await stateRead(first);if (scenario === "workflow") {- const before = await count("get_window_status"), cached = await ok("read", { path: state });- assert(reference(cached, "window-state") === state, "Cached state Read issued fresh authority");- assert(await count("get_window_status") === before && await count("get_window_state") === 2,- "Cached state Read or editor resolution silently acquired native state/pixels");- for (const [name, args] of [- ["write", { path: state, content: "must not type" }],- ["insert", { path: state, text: '{"action":"click","x":1,"y":1}' }],- ["insert", { path: state, text: '{"action":"restore","window_id":92}' }],- ["insert", { path: state, text: '{"action":"restore","state_id":"other"}' }],- ["insert", { path: state, text: '{"action":"restore","delivery_mode":"foreground"}' }],- ["insert", { path: state, text: '{"action":"sequence","sequence":[]}' }],- ] as const) await denied(name, args);- assert(await count("restore_window") === 0, "Invalid state command dispatched restoration");- const result = await restore(state);+ // Write and Insert never act on a window; invalid restore fields never reach native code.+ await denied("write", { path: first, content: "must not type" });+ assert(body(await denied("insert", { path: first, text: '{"action":"restore"}' })).includes("computer_window"), "Insert refusal named no tool");+ for (const extra of [{ window_id: 92 }, { state_id: "other" }, { delivery_mode: "foreground" }, { x: 1 }])+ await denied("computer_window", { window: first, action: "restore", ...extra });+ assert(await count("restore_window") === 0, "Invalid restore dispatched restoration");+ const result = await restore(first);assert(!result.isError, `Admitted restore failed: ${body(result)}`);const action = semantic(result);assert(action.ok === true && action.effect === "applied" && action.status === "delivered", "Confirmed restoration lost semantic success");@@ -104,50 +101,32 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {"Restore collapsed foreground and keyboard focus or discarded prefix");assert(body(result).includes("restore_requested=true") && /keyboard_focus(?:_at_target)?=(?:null|unknown|unavailable)/.test(body(result)),`Human tool receipt hid restoration/focus prefix: ${body(result)}`);- await denied("insert", { path: state, text: '{"action":"restore"}' });- await denied("read", { path: state });- assert(await count("restore_window") === 1 && await count("get_window_status") === before, "Consumed state was replayed or silently refreshed");- const old = await stateRead(first); state = await stateRead(first);- await denied("insert", { path: old, text: '{"action":"restore"}' });- const ui = await ok("read", { path: first, views: ["ui"] });- await denied("insert", { path: state, text: '{"action":"restore"}' });- const observation = reference(ui, "observation");- await stateRead(first);- await denied("write", { path: observation, content: "retired UI" });- assert(await count("restore_window") === 1, "Retired metadata/UI authority dispatched restoration");- const beforeSequence = await stateRead(first);- await ok("read", { path: first, views: ['sequence:{"duration_ms":200,"fps":10,"max_frames":2}'] });- await denied("insert", { path: beforeSequence, text: '{"action":"restore"}' });- assert(await count("restore_window") === 1, "Temporal Read retained restore authority");+ assert(await count("restore_window") === 1, "Restore dispatched more than once");} else {const external = reference(await ok("search", { path: "computer:machine/local/windows", query: "Unowned restore fixture" }), "window");- const externalState = await stateRead(external); mode = "safe";- const refusal = await denied("insert", { path: externalState, text: '{"action":"restore"}' });+ await stateRead(external); mode = "safe";+ const refusal = await denied("computer_window", { window: external, action: "restore" });assert(semantic(refusal).effect === "not-applied" && body(refusal).includes("Safe mode"), "Safe denial hid no-dispatch effect");- const current = await stateRead(first);- const cachedBefore = await count("get_window_status");- await ok("read", { path: current });- assert(await count("get_window_status") === cachedBefore, "Owned cached state Read refreshed under Safe");+ await stateRead(first);for (const operation of ["refused", "partial", "final_cancel", "wrong_state", "wrong_window", "wrong_route", "missing_focus", "not_foreground"]) {- const address = await stateRead(first); await control(operation);- const result = await denied("insert", { path: address, text: '{"action":"restore"}' }), action = semantic(result);+ await control(operation);+ const before = await count("restore_window");+ const result = await denied("computer_window", { window: first, action: "restore" }), action = semantic(result);assert(action.effect === (operation === "refused" ? "not-applied" : "unknown"), `Wrong semantic effect for ${operation}: ${body(result)}`);assert(find(action, "restore_attempted") === (operation !== "refused"), `Native prefix discarded for ${operation}`);if (operation === "final_cancel") assert(find(action, "activation_succeeded") === true, "Final cancellation lost fully delivered prefix");- const before = await count("restore_window");- await denied("insert", { path: address, text: '{"action":"restore"}' });- assert(await count("restore_window") === before, `Unknown/refused ${operation} was replayed`);+ assert(await count("restore_window") === before + 1, `${operation} was sent more than once`);}- state = await stateRead(first); await control("close_first");- await denied("read", { path: state });- await denied("insert", { path: state, text: '{"action":"restore"}' });- state = await stateRead(second); await control("reuse_second");- await denied("insert", { path: state, text: '{"action":"restore"}' });+ await control("close_first");+ await denied("read", { path: first, views: ["state"] });+ await denied("computer_window", { window: first, action: "restore" });+ await control("reuse_second");+ await denied("computer_window", { window: second, action: "restore" });const beforeReuse = await count("restore_window");mode = "yolo";const replacement = reference(await ok("search", { path: "computer:machine/local/windows", query: "Owned restore fixture 92" }), "window");- state = await stateRead(replacement); await control("reused_after_restore");- const reused = await denied("insert", { path: state, text: '{"action":"restore"}' });+ await control("reused_after_restore");+ const reused = await denied("computer_window", { window: replacement, action: "restore" });assert(semantic(reused).effect === "unknown" && find(reused, "activation_succeeded") === true, "Post-dispatch reuse lost completed prefix or became success");assert(await count("restore_window") === beforeReuse + 1, "Closed/reused reference sent an extra restoration");capability = false;@@ -160,7 +139,7 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {const rows = await journal();assert(rows.every(row => ["fixture_identity", "fixture_control", "list_apps", "list_windows", "launch_app", "get_window_state", "get_window_status", "restore_window"].includes(row.name)),"Restore used hidden input, capture fallback or bring_to_front");- await ok("insert", { path: "computer:session", text: '{"action":"close"}' });+ await ok("delete", { path: "computer:session" });return { content: [{ type: "text" as const, text: `IDE window restore ${scenario} contract passed` }],details: { scenario, restores: rows.filter(row => row.name === "restore_window").length, statusReads: rows.filter(row => row.name === "get_window_status").length } };},diff --git a/test/window-management.integration.test.ts b/test/window-management.integration.test.tsindex 68dcd1e..988deb9 100644--- a/test/window-management.integration.test.ts+++ b/test/window-management.integration.test.ts@@ -9,7 +9,7 @@ const fixture = path.join(root, "test/fixtures/window-management-extension.ts");const driver = path.join(root, "test/fixtures/window-management-driver.mjs");// Pattern: executable real-Pi extension example and the ordinary-IDE restore suite.-// Contract: native desktop geometry is one-use management authority, never pixel/input authority.+// Contract: computer_window samples native desktop geometry for one command; it is never pixel/input authority.for (const scenario of ["workflow", "admission", "receipts", "restore", "typed"] as const) test.each([false, true])(`ordinary IDE window management protects ${scenario} (providerFirst=%s)`,async providerFirst => {@@ -24,7 +24,7 @@ for (const scenario of ["workflow", "admission", "receipts", "restore", "typed"]artifactsDir: testArtifactsDir(import.meta.filename), cwd, isolateUserResources: true, rawMode: false,piCommand: PI_COMMAND,extensions: providerFirst ? [fixture, extension] : [extension, fixture],- tools: ["read", "search", "write", "insert", "verify_window_management"], timeoutMs: 90_000,+ tools: ["read", "search", "write", "insert", "delete", "verify_window_management"], timeoutMs: 90_000,conversation: [assistantMessage([toolCall({ id: "management-proof", name: "verify_window_management", arguments: { scenario } })], { stopReason: "toolUse" }),assistantMessage([text("Finished the synthetic window management scenario.")])],}).run("Use ordinary IDE tools against the synthetic MCP window fixture only. Never operate a real desktop or use input fallback.");diff --git a/test/window-restore.integration.test.ts b/test/window-restore.integration.test.tsindex f25b145..6c27424 100644--- a/test/window-restore.integration.test.ts+++ b/test/window-restore.integration.test.ts@@ -9,7 +9,7 @@ const fixture = path.join(root, "test/fixtures/window-restore-extension.ts");const driver = path.join(root, "test/fixtures/window-restore-driver.mjs");// Pattern: real-Pi extension example + ordinary IDE provider cases.-// Contract: state resources grant one restore, no pixel/input authority, and preserve interrupted native prefixes.+// Contract: computer_window restores against its own state sample, grants no pixel/input authority, and preserves interrupted native prefixes.for (const scenario of ["workflow", "admission"] as const) test.each([false, true])(`ordinary IDE metadata-only restore protects ${scenario} (providerFirst=%s)`,async providerFirst => {@@ -24,7 +24,7 @@ for (const scenario of ["workflow", "admission"] as const) test.each([false, truartifactsDir: testArtifactsDir(import.meta.filename), cwd, isolateUserResources: true, rawMode: false,piCommand: PI_COMMAND,extensions: providerFirst ? [fixture, extension] : [extension, fixture],- tools: ["read", "search", "write", "insert", "verify_window_restore"], timeoutMs: 90_000,+ tools: ["read", "search", "write", "insert", "delete", "verify_window_restore"], timeoutMs: 90_000,conversation: [assistantMessage([toolCall({ id: "restore-proof", name: "verify_window_restore", arguments: { scenario } })], { stopReason: "toolUse" }),assistantMessage([text("Finished the synthetic window restore scenario.")])],}).run("Use ordinary IDE tools against the synthetic native window fixture. Never activate real windows or use keyboard/pointer fallback.");