Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Tests: a click that closes its dialog is delivered and names the app's window in front (LPT-954)

alexshpunt-clanker[bot]3acad28e305e
diff --git a/test/computer-presentation.test.ts b/test/computer-presentation.test.ts
index 1da282b..dbd4250 100644
--- a/test/computer-presentation.test.ts
+++ b/test/computer-presentation.test.ts
@@ -139,3 +139,10 @@ test("a wait on a browser page shows the snapshot of its last sample, satisfied
expect(text).toContain("computer:observation/o7");
}
});
+
+test("a receipt without a successor gives its reason once", () => {
+ const reason = "This window closed or is hidden; the app's frontmost window is \"Main\".";
+ const text = presentComputerResult({ content: [], details: { status: "delivered", mode: "yolo", successor: { reason } } } as ComputerResult);
+ expect(text).toContain(`Successor: none. ${reason}`);
+ expect(text.split(reason)).toHaveLength(2);
+});
diff --git a/test/dialog-close.test.ts b/test/dialog-close.test.ts
new file mode 100644
index 0000000..d5829ee
--- /dev/null
+++ b/test/dialog-close.test.ts
@@ -0,0 +1,71 @@
+import { expect, test, vi } from "vitest";
+import { ComputerSession, type DriverResult, type Driver } from "../src/session.ts";
+
+// LPT-954: a click that closes its window, such as OK in a dialog, hands the foreground to another window of the app.
+// The driver reports that window as foreground_after; the receipt is delivered and names it.
+const dialog = { pid: 42, window_id: 100 };
+const main = { pid: 42, window_id: 200 };
+function fixture(click: DriverResult, mode: "yolo" | "safe" = "yolo") {
+ let windows: object[] = [];
+ const driver: Driver = {
+ identity: vi.fn(async window => `${window.pid}:${window.window_id}`),
+ call: vi.fn<Driver["call"]>(async name => {
+ if (name === "list_windows") return { content: [], structuredContent: { windows } };
+ if (name === "list_apps") return { content: [], structuredContent: { apps: [{ name: "Editor", launch_path: "C:\\Editor\\editor.exe" }] } };
+ if (name === "launch_app") windows = [{ ...dialog, title: "Create a New Image" }];
+ if (name === "launch_app") return { content: [{ type: "text", text: JSON.stringify({ pid: 42, windows }) }] };
+ if (name === "get_window_state") return { content: [{ type: "image", mimeType: "image/png", data: "cG5n" }],
+ structuredContent: { snapshot_id: "s1", screenshot_width: 400, screenshot_height: 300, elements: [] } };
+ return click;
+ }),
+ close: vi.fn(async () => {}),
+ };
+ return new ComputerSession(driver, () => mode);
+}
+async function clickOk(session: ComputerSession) {
+ const opened = await session.open("Editor");
+ return session.act({ window_ref: opened.details.window_ref!, observation_id: opened.details.observation_id!, delivery_mode: "foreground", action: "click", x: 350, y: 250 });
+}
+const closed = { content: [{ type: "text" as const, text: "✅ Sent click via SendInput to pid 42 at (350,340) (delivery_mode:foreground)." },
+ { type: "text" as const, text: "The click closed its target; window 200 of the same process is now in front: \"Untitled - Editor\"." }],
+ structuredContent: { path: "pixel", verified: false, effect: "unverifiable", foreground_after: { ...main, title: "Untitled - Editor" } } };
+const message = (result: Awaited<ReturnType<typeof clickOk>>) => result.content.filter(block => block.type === "text").map(block => block.text).join("\n");
+
+test("a click that closes its window is delivered and names the window of the app now in front", async () => {
+ const session = fixture(closed);
+ const result = await clickOk(session);
+ expect(result.details.status, message(result)).toBe("delivered");
+ expect(message(result)).toMatch(/^The click closed this window; "Untitled - Editor" \(computer:window\/(w-[0-9a-f-]+)\) of the same app is now in front\./);
+ const ref = /computer:window\/(w-[0-9a-f-]+)/.exec(message(result))![1];
+ expect(session.windowTarget(ref)).toEqual(main);
+});
+
+test("Safe mode names the window now in front without granting it an address", async () => {
+ const result = await clickOk(fixture(closed, "safe"));
+ expect(result.details.status).toBe("delivered");
+ expect(message(result)).toMatch(/^The click closed this window; "Untitled - Editor" of the same app is now in front\./);
+});
+
+test("after input, a window that closed or hid names the app's frontmost remaining window", async () => {
+ let windows: object[] = [];
+ const driver: Driver = {
+ identity: vi.fn(async window => `${window.pid}:${window.window_id}`),
+ call: vi.fn<Driver["call"]>(async name => {
+ if (name === "list_windows") return { content: [], structuredContent: { windows } };
+ if (name === "list_apps") return { content: [], structuredContent: { apps: [{ name: "Editor", launch_path: "C:\\Editor\\editor.exe" }] } };
+ if (name === "launch_app") windows = [{ ...dialog, title: "Create a New Image" }];
+ if (name === "launch_app") return { content: [{ type: "text", text: JSON.stringify({ pid: 42, windows }) }] };
+ return { content: [{ type: "image", mimeType: "image/png", data: "cG5n" }], structuredContent: { snapshot_id: "s1", elements: [] } };
+ }),
+ close: vi.fn(async () => {}),
+ };
+ const session = new ComputerSession(driver, () => "yolo");
+ await session.open("Editor");
+ // While the window is listed, nothing is said.
+ expect(await session.closedWindowNote(dialog)).toBeUndefined();
+ // Listing order is the desktop's z-order, front first.
+ windows = [{ pid: 7, window_id: 300, title: "Other app" }, { ...main, title: "Untitled - Editor" }, { pid: 42, window_id: 201, title: "Layers" }];
+ expect(await session.closedWindowNote(dialog)).toMatch(/^This window closed or is hidden; the app's frontmost window is "Untitled - Editor" \(computer:window\/w-[0-9a-f-]+\)\.$/);
+ windows = [{ pid: 7, window_id: 300, title: "Other app" }];
+ expect(await session.closedWindowNote(dialog)).toBe("This window closed or is hidden; the app has no other visible window.");
+});
diff --git a/test/live/dialog-close.live.ts b/test/live/dialog-close.live.ts
new file mode 100644
index 0000000..d8a7163
--- /dev/null
+++ b/test/live/dialog-close.live.ts
@@ -0,0 +1,80 @@
+import { afterAll, expect, test } from "vitest";
+import { requireComputerAddress } from "../../src/computer-resources.ts";
+import { clickPlan, pressPlan } from "../../src/computer-tools.ts";
+import { presentComputerResult } from "../../src/provider.ts";
+import { addresses, connectLive, openWindow, WINDOWS_POWERSHELL_PATH } from "./remote.ts";
+
+// LPT-954: OK in a dialog closes the dialog, and Windows hands the foreground to the app's main window. The click is
+// delivered and its receipt names the main window as the one now in front; Escape on the dialog is delivered too.
+const live = await connectLive();
+afterAll(async () => { await live.close(); });
+
+function mainForm(title: string): string[] {
+ const script = `Add-Type -AssemblyName System.Windows.Forms
+$a = [Windows.Forms.Screen]::PrimaryScreen.WorkingArea
+$f = New-Object Windows.Forms.Form; $f.Text = '${title}'; $f.StartPosition = 'Manual'; $f.Left = $a.Left + [int]($a.Width / 2) - 210; $f.Top = $a.Top + [int]($a.Height / 2) - 150; $f.Width = 420; $f.Height = 300
+$o = New-Object Windows.Forms.Button; $o.Text = 'Open'; $o.Left = 10; $o.Top = 10; $o.Width = 120
+$l = New-Object Windows.Forms.Label; $l.Left = 10; $l.Top = 60; $l.Width = 380; $l.Text = 'result:'
+$o.Add_Click({
+ $d = New-Object Windows.Forms.Form; $d.Text = '${title} dialog'; $d.StartPosition = 'CenterParent'; $d.Width = 260; $d.Height = 160
+ $d.MinimizeBox = $false; $d.MaximizeBox = $false; $d.ShowInTaskbar = $false
+ $ok = New-Object Windows.Forms.Button; $ok.Text = 'OK'; $ok.Left = 20; $ok.Top = 60; $ok.DialogResult = 'OK'
+ $cancel = New-Object Windows.Forms.Button; $cancel.Text = 'Cancel'; $cancel.Left = 120; $cancel.Top = 60; $cancel.DialogResult = 'Cancel'
+ $d.AcceptButton = $ok; $d.CancelButton = $cancel; $d.Controls.AddRange(@($ok, $cancel))
+ $l.Text += ' ' + $d.ShowDialog($f)
+})
+$f.Controls.AddRange(@($o, $l)); [void]$f.ShowDialog()`;
+ return ["-NoProfile", "-WindowStyle", "Hidden", "-EncodedCommand", Buffer.from(script, "utf16le").toString("base64")];
+}
+
+type Element = { role: string; name?: string; bounds: { window_image?: { x: number; y: number; width: number; height: number } } };
+async function centre(window: string, name: string) {
+ const full = await live.read(window, ["full"]);
+ const box = (full.details.ui as { elements: Element[] }).elements.find(item => item.role === "Button" && item.name === name)!.bounds.window_image!;
+ return { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) };
+}
+
+test("a click on OK that closes a dialog is delivered and names the window now in front", async () => {
+ const title = `pi-dialog-${Date.now()}`;
+ const window = await openWindow(live, live.machine, WINDOWS_POWERSHELL_PATH, mainForm(title), `regex:^${title}$`);
+ const act = async (path: string, plan: ReturnType<typeof clickPlan>) => {
+ const result = await live.resources.input(requireComputerAddress(path), live.cwd, plan);
+ const body = presentComputerResult(result);
+ console.log(`${result.details.status}\n${body}`);
+ return { status: result.details.status, body };
+ };
+ const dialog = async () => {
+ for (let i = 0; i < 20; i++) {
+ const found = addresses(await live.search(live.windows, `regex:^${title} dialog$`), "window")[0];
+ if (found) return found;
+ await new Promise(resolve => setTimeout(resolve, 500));
+ }
+ throw new Error("The dialog did not open.");
+ };
+ const result = async () => /result:[^\n|]*/.exec(presentComputerResult(await live.read(window, ["ui"])))![0].trim();
+ try {
+ // Right after an app starts, Windows can still be handing the foreground back to the window that had it, and the
+ // first click's dialog then loses the foreground to that window (an honest unknown). Let the start settle first.
+ await new Promise(resolve => setTimeout(resolve, 1500));
+ // The Open button shows a modal dialog from its click handler, so it gets the real pointer.
+ expect((await act(window, clickPlan({ window, ...await centre(window, "Open") }))).status).toBe("delivered");
+ const first = await dialog();
+ const ok = await act(first, clickPlan({ window: first, ...await centre(first, "OK") }));
+ expect(ok.status, ok.body).toBe("delivered");
+ // The driver names the window in front when the dialog was already gone at its check after the click; otherwise
+ // the change list finds the dialog gone and names the app's frontmost window.
+ expect(ok.body).toMatch(new RegExp(`(The click closed this window; |the app's frontmost window is )"${title}" \\(computer:`));
+ expect(await result()).toBe("result: OK");
+
+ expect((await act(window, clickPlan({ window, ...await centre(window, "Open") }))).status).toBe("delivered");
+ const second = await dialog();
+ await live.read(second, []);
+ expect((await act(second, pressPlan({ window: second, key: "Escape" }))).status).toBe("delivered");
+ await new Promise(resolve => setTimeout(resolve, 300));
+ expect(await result()).toBe("result: OK Cancel");
+ } finally {
+ // Force-close refuses windows the session found by title rather than opened; the process is this check's own.
+ const { pid } = (await live.read(window, ["state"])).details.target as { pid: number };
+ await live.kill(pid);
+ }
+});