Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Tests: a click that closes its dialog is delivered and names the app's window in front (LPT-954)
alexshpunt-clanker[bot]3acad28e305e
diff --git a/test/computer-presentation.test.ts b/test/computer-presentation.test.tsindex 1da282b..dbd4250 100644--- a/test/computer-presentation.test.ts+++ b/test/computer-presentation.test.ts@@ -139,3 +139,10 @@ test("a wait on a browser page shows the snapshot of its last sample, satisfiedexpect(text).toContain("computer:observation/o7");}});++test("a receipt without a successor gives its reason once", () => {+ const reason = "This window closed or is hidden; the app's frontmost window is \"Main\".";+ const text = presentComputerResult({ content: [], details: { status: "delivered", mode: "yolo", successor: { reason } } } as ComputerResult);+ expect(text).toContain(`Successor: none. ${reason}`);+ expect(text.split(reason)).toHaveLength(2);+});diff --git a/test/dialog-close.test.ts b/test/dialog-close.test.tsnew file mode 100644index 0000000..d5829ee--- /dev/null+++ b/test/dialog-close.test.ts@@ -0,0 +1,71 @@+import { expect, test, vi } from "vitest";+import { ComputerSession, type DriverResult, type Driver } from "../src/session.ts";++// LPT-954: a click that closes its window, such as OK in a dialog, hands the foreground to another window of the app.+// The driver reports that window as foreground_after; the receipt is delivered and names it.+const dialog = { pid: 42, window_id: 100 };+const main = { pid: 42, window_id: 200 };+function fixture(click: DriverResult, mode: "yolo" | "safe" = "yolo") {+ let windows: object[] = [];+ const driver: Driver = {+ identity: vi.fn(async window => `${window.pid}:${window.window_id}`),+ call: vi.fn<Driver["call"]>(async name => {+ if (name === "list_windows") return { content: [], structuredContent: { windows } };+ if (name === "list_apps") return { content: [], structuredContent: { apps: [{ name: "Editor", launch_path: "C:\\Editor\\editor.exe" }] } };+ if (name === "launch_app") windows = [{ ...dialog, title: "Create a New Image" }];+ if (name === "launch_app") return { content: [{ type: "text", text: JSON.stringify({ pid: 42, windows }) }] };+ if (name === "get_window_state") return { content: [{ type: "image", mimeType: "image/png", data: "cG5n" }],+ structuredContent: { snapshot_id: "s1", screenshot_width: 400, screenshot_height: 300, elements: [] } };+ return click;+ }),+ close: vi.fn(async () => {}),+ };+ return new ComputerSession(driver, () => mode);+}+async function clickOk(session: ComputerSession) {+ const opened = await session.open("Editor");+ return session.act({ window_ref: opened.details.window_ref!, observation_id: opened.details.observation_id!, delivery_mode: "foreground", action: "click", x: 350, y: 250 });+}+const closed = { content: [{ type: "text" as const, text: "✅ Sent click via SendInput to pid 42 at (350,340) (delivery_mode:foreground)." },+ { type: "text" as const, text: "The click closed its target; window 200 of the same process is now in front: \"Untitled - Editor\"." }],+ structuredContent: { path: "pixel", verified: false, effect: "unverifiable", foreground_after: { ...main, title: "Untitled - Editor" } } };+const message = (result: Awaited<ReturnType<typeof clickOk>>) => result.content.filter(block => block.type === "text").map(block => block.text).join("\n");++test("a click that closes its window is delivered and names the window of the app now in front", async () => {+ const session = fixture(closed);+ const result = await clickOk(session);+ expect(result.details.status, message(result)).toBe("delivered");+ expect(message(result)).toMatch(/^The click closed this window; "Untitled - Editor" \(computer:window\/(w-[0-9a-f-]+)\) of the same app is now in front\./);+ const ref = /computer:window\/(w-[0-9a-f-]+)/.exec(message(result))![1];+ expect(session.windowTarget(ref)).toEqual(main);+});++test("Safe mode names the window now in front without granting it an address", async () => {+ const result = await clickOk(fixture(closed, "safe"));+ expect(result.details.status).toBe("delivered");+ expect(message(result)).toMatch(/^The click closed this window; "Untitled - Editor" of the same app is now in front\./);+});++test("after input, a window that closed or hid names the app's frontmost remaining window", async () => {+ let windows: object[] = [];+ const driver: Driver = {+ identity: vi.fn(async window => `${window.pid}:${window.window_id}`),+ call: vi.fn<Driver["call"]>(async name => {+ if (name === "list_windows") return { content: [], structuredContent: { windows } };+ if (name === "list_apps") return { content: [], structuredContent: { apps: [{ name: "Editor", launch_path: "C:\\Editor\\editor.exe" }] } };+ if (name === "launch_app") windows = [{ ...dialog, title: "Create a New Image" }];+ if (name === "launch_app") return { content: [{ type: "text", text: JSON.stringify({ pid: 42, windows }) }] };+ return { content: [{ type: "image", mimeType: "image/png", data: "cG5n" }], structuredContent: { snapshot_id: "s1", elements: [] } };+ }),+ close: vi.fn(async () => {}),+ };+ const session = new ComputerSession(driver, () => "yolo");+ await session.open("Editor");+ // While the window is listed, nothing is said.+ expect(await session.closedWindowNote(dialog)).toBeUndefined();+ // Listing order is the desktop's z-order, front first.+ windows = [{ pid: 7, window_id: 300, title: "Other app" }, { ...main, title: "Untitled - Editor" }, { pid: 42, window_id: 201, title: "Layers" }];+ expect(await session.closedWindowNote(dialog)).toMatch(/^This window closed or is hidden; the app's frontmost window is "Untitled - Editor" \(computer:window\/w-[0-9a-f-]+\)\.$/);+ windows = [{ pid: 7, window_id: 300, title: "Other app" }];+ expect(await session.closedWindowNote(dialog)).toBe("This window closed or is hidden; the app has no other visible window.");+});diff --git a/test/live/dialog-close.live.ts b/test/live/dialog-close.live.tsnew file mode 100644index 0000000..d8a7163--- /dev/null+++ b/test/live/dialog-close.live.ts@@ -0,0 +1,80 @@+import { afterAll, expect, test } from "vitest";+import { requireComputerAddress } from "../../src/computer-resources.ts";+import { clickPlan, pressPlan } from "../../src/computer-tools.ts";+import { presentComputerResult } from "../../src/provider.ts";+import { addresses, connectLive, openWindow, WINDOWS_POWERSHELL_PATH } from "./remote.ts";++// LPT-954: OK in a dialog closes the dialog, and Windows hands the foreground to the app's main window. The click is+// delivered and its receipt names the main window as the one now in front; Escape on the dialog is delivered too.+const live = await connectLive();+afterAll(async () => { await live.close(); });++function mainForm(title: string): string[] {+ const script = `Add-Type -AssemblyName System.Windows.Forms+$a = [Windows.Forms.Screen]::PrimaryScreen.WorkingArea+$f = New-Object Windows.Forms.Form; $f.Text = '${title}'; $f.StartPosition = 'Manual'; $f.Left = $a.Left + [int]($a.Width / 2) - 210; $f.Top = $a.Top + [int]($a.Height / 2) - 150; $f.Width = 420; $f.Height = 300+$o = New-Object Windows.Forms.Button; $o.Text = 'Open'; $o.Left = 10; $o.Top = 10; $o.Width = 120+$l = New-Object Windows.Forms.Label; $l.Left = 10; $l.Top = 60; $l.Width = 380; $l.Text = 'result:'+$o.Add_Click({+ $d = New-Object Windows.Forms.Form; $d.Text = '${title} dialog'; $d.StartPosition = 'CenterParent'; $d.Width = 260; $d.Height = 160+ $d.MinimizeBox = $false; $d.MaximizeBox = $false; $d.ShowInTaskbar = $false+ $ok = New-Object Windows.Forms.Button; $ok.Text = 'OK'; $ok.Left = 20; $ok.Top = 60; $ok.DialogResult = 'OK'+ $cancel = New-Object Windows.Forms.Button; $cancel.Text = 'Cancel'; $cancel.Left = 120; $cancel.Top = 60; $cancel.DialogResult = 'Cancel'+ $d.AcceptButton = $ok; $d.CancelButton = $cancel; $d.Controls.AddRange(@($ok, $cancel))+ $l.Text += ' ' + $d.ShowDialog($f)+})+$f.Controls.AddRange(@($o, $l)); [void]$f.ShowDialog()`;+ return ["-NoProfile", "-WindowStyle", "Hidden", "-EncodedCommand", Buffer.from(script, "utf16le").toString("base64")];+}++type Element = { role: string; name?: string; bounds: { window_image?: { x: number; y: number; width: number; height: number } } };+async function centre(window: string, name: string) {+ const full = await live.read(window, ["full"]);+ const box = (full.details.ui as { elements: Element[] }).elements.find(item => item.role === "Button" && item.name === name)!.bounds.window_image!;+ return { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) };+}++test("a click on OK that closes a dialog is delivered and names the window now in front", async () => {+ const title = `pi-dialog-${Date.now()}`;+ const window = await openWindow(live, live.machine, WINDOWS_POWERSHELL_PATH, mainForm(title), `regex:^${title}$`);+ const act = async (path: string, plan: ReturnType<typeof clickPlan>) => {+ const result = await live.resources.input(requireComputerAddress(path), live.cwd, plan);+ const body = presentComputerResult(result);+ console.log(`${result.details.status}\n${body}`);+ return { status: result.details.status, body };+ };+ const dialog = async () => {+ for (let i = 0; i < 20; i++) {+ const found = addresses(await live.search(live.windows, `regex:^${title} dialog$`), "window")[0];+ if (found) return found;+ await new Promise(resolve => setTimeout(resolve, 500));+ }+ throw new Error("The dialog did not open.");+ };+ const result = async () => /result:[^\n|]*/.exec(presentComputerResult(await live.read(window, ["ui"])))![0].trim();+ try {+ // Right after an app starts, Windows can still be handing the foreground back to the window that had it, and the+ // first click's dialog then loses the foreground to that window (an honest unknown). Let the start settle first.+ await new Promise(resolve => setTimeout(resolve, 1500));+ // The Open button shows a modal dialog from its click handler, so it gets the real pointer.+ expect((await act(window, clickPlan({ window, ...await centre(window, "Open") }))).status).toBe("delivered");+ const first = await dialog();+ const ok = await act(first, clickPlan({ window: first, ...await centre(first, "OK") }));+ expect(ok.status, ok.body).toBe("delivered");+ // The driver names the window in front when the dialog was already gone at its check after the click; otherwise+ // the change list finds the dialog gone and names the app's frontmost window.+ expect(ok.body).toMatch(new RegExp(`(The click closed this window; |the app's frontmost window is )"${title}" \\(computer:`));+ expect(await result()).toBe("result: OK");++ expect((await act(window, clickPlan({ window, ...await centre(window, "Open") }))).status).toBe("delivered");+ const second = await dialog();+ await live.read(second, []);+ expect((await act(second, pressPlan({ window: second, key: "Escape" }))).status).toBe("delivered");+ await new Promise(resolve => setTimeout(resolve, 300));+ expect(await result()).toBe("result: OK Cancel");+ } finally {+ // Force-close refuses windows the session found by title rather than opened; the process is this check's own.+ const { pid } = (await live.read(window, ["state"])).details.target as { pid: number };+ await live.kill(pid);+ }+});