Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Tests: native multi-clicks, unit and live (LPT-948)
alexshpunt-clanker[bot]2f23d09fb758
diff --git a/test/complete-input.test.ts b/test/complete-input.test.tsindex 8f0c1bd..62f2898 100644--- a/test/complete-input.test.ts+++ b/test/complete-input.test.ts@@ -30,6 +30,8 @@ test.each([[{ action: "right_click", x: 100, y: 80, modifiers: ["shift"] }, "right_click", { x: 100, y: 80, modifier: ["shift"] }],[{ action: "double_click", element_token: "field" }, "double_click", { element_token: "field" }],[{ action: "click", x: 100, y: 80, button: "middle" }, "click", { x: 100, y: 80, button: "middle", capture_id: "capture-one" }],+ [{ action: "click", x: 100, y: 80, count: 3 }, "click", { x: 100, y: 80, count: 3, capture_id: "capture-one" }],+ [{ action: "click", element_token: "field", button: "right", count: 2 }, "click", { element_token: "field", button: "right", count: 2 }],[{ action: "scroll", element_token: "field", direction: "down", amount: 2, by: "page" }, "scroll", { element_token: "field", direction: "down", amount: 2, by: "page" }],[{ action: "drag", from_x: 30, from_y: 40, to_x: 200, to_y: 80, duration_ms: 250, steps: 12, button: "left", modifiers: ["ctrl"] }, "drag",{ from_x: 30, from_y: 40, to_x: 200, to_y: 80, duration_ms: 250, steps: 12, button: "left", modifier: ["ctrl"] }],@@ -53,6 +55,7 @@ test.each([{ action: "scroll", x: 100, y: 80, direction: "down" },{ action: "scroll", direction: "down", amount: 2 },{ action: "key", key: "a", duration_ms: 1000 }, { action: "click", x: 10, y: 10, keys: ["ctrl", "a"] },+ { action: "click", x: 10, y: 10, count: 4 }, { action: "click", x: 10, y: 10, count: 0 }, { action: "double_click", x: 10, y: 10, count: 3 },{ action: "sequence", sequence: [] },{ action: "sequence", sequence: [{ action: "key", key: "a" }, { action: "drag", from_x: 1, from_y: 1, to_x: NaN, to_y: 2 }] },{ action: "sequence", sequence: [{ action: "hover", x: 10, y: 10 }] },diff --git a/test/computer-resources.test.ts b/test/computer-resources.test.tsindex bb0e046..eb16adc 100644--- a/test/computer-resources.test.ts+++ b/test/computer-resources.test.ts@@ -299,6 +299,22 @@ test("a click on a ref goes through UI Automation only, and no pattern means theexpect(sent[1]).not.toHaveProperty("uia_only");});+// LPT-948: a triple click selects a field's value; no UI Automation pattern carries a multi-click, so it is real input.+test("triple clicks and right or middle double clicks reach a native window as real clicks", async () => {+ const f = await fixture();+ let view = await f.read();+ const triple = await f.resources.input(f.window, cwd, clickPlan({ window: f.window.source, target: view.hint!, count: 3 }));+ expect(triple.details.status).toBe("delivered");+ view = await f.read();+ await f.resources.input(f.window, cwd, clickPlan({ window: f.window.source, x: 100, y: 70, button: "right", count: 2 }));+ await f.resources.input(f.window, cwd, clickPlan({ window: f.window.source, x: 100, y: 70, button: "middle", count: 2 }));+ expect(f.inputs()).toEqual([+ ["click", { target: { kind: "window", pid: 42, window_id: 91 }, delivery_mode: "foreground", element_token: expect.any(String), count: 3 }, undefined],+ ["click", expect.objectContaining({ delivery_mode: "foreground", x: expect.any(Number), y: expect.any(Number), button: "right", count: 2 }), undefined],+ ["click", expect.objectContaining({ delivery_mode: "foreground", button: "middle", count: 2 }), undefined]]);+ expect(f.inputs().some(([, args]) => "uia_only" in (args as object))).toBe(false);+});+test("points and keys are real input; nothing posts window messages", async () => {const f = await fixture();(f.driver as Driver & { coordinateScroll: boolean }).coordinateScroll = true;diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.tsindex 73061fa..abe82b9 100644--- a/test/computer-tools.test.ts+++ b/test/computer-tools.test.ts@@ -99,7 +99,10 @@ test("input tools build one plan for native windows and one for browser pages",native: { action: "double_click", modifiers: ["ctrl"] }, web: { action: "timeline" } });expect(clickPlan({ window: w, target: "e1", count: 2, modifiers: ["ctrl"] }).web).toEqual({ action: "click", click_count: 2, modifiers: ["Control"] });expect(clickPlan({ window: w, target: "e1", button: "right" }).native).toEqual({ action: "right_click" });- expect(clickPlan({ window: w, target: "e1", count: 3 }).native).toBe("A triple click works on browser pages only.");+ // LPT-948: every multi-click goes to a native window as the driver's click with its button and count.+ expect(clickPlan({ window: w, target: "1:a", count: 3 }).native).toEqual({ action: "click", count: 3 });+ expect(clickPlan({ window: w, x: 1, y: 2, button: "right", count: 2, modifiers: ["shift"] }).native).toEqual({ action: "click", button: "right", count: 2, modifiers: ["shift"] });+ expect(clickPlan({ window: w, x: 1, y: 2, button: "middle", count: 3 }).native).toEqual({ action: "click", button: "middle", count: 3 });expect(() => clickPlan({ window: w })).toThrow("click needs a target or a point (x and y).");expect(() => clickPlan({ window: w, x: 1 })).toThrow("click needs both x and y for a point.");expect(() => clickPlan({ window: w, x: 1, y: 2, target: "1:a" })).toThrow("click takes a target or a point, not both.");diff --git a/test/live/multi-click.live.ts b/test/live/multi-click.live.tsnew file mode 100644index 0000000..54982df--- /dev/null+++ b/test/live/multi-click.live.ts@@ -0,0 +1,60 @@+import { afterAll, expect, test } from "vitest";+import { requireComputerAddress } from "../../src/computer-resources.ts";+import { clickPlan } from "../../src/computer-tools.ts";+import { deleteComputer, presentComputerResult } from "../../src/provider.ts";+import { connectLive, openWindow, WINDOWS_POWERSHELL_PATH } from "./remote.ts";++// LPT-948: triple clicks and right or middle double clicks reach a native window as real clicks. A WinForms pad logs+// every press it gets as its button and Windows' click count, so a double click shows as "R1 R2": Windows made the+// second press a double click because it came within the system double-click time and distance. Windows counts no+// further, so a triple click is "L1 L2 L1"; apps that select a line on a triple click time the third press themselves.+const live = await connectLive();+afterAll(async () => { await live.close(); });++function padForm(title: string): string[] {+ const script = `Add-Type -AssemblyName System.Windows.Forms+$f = New-Object Windows.Forms.Form; $f.Text = '${title}'; $f.Width = 420; $f.Height = 300+# Open at the middle of the screen, away from the top-left corner where other windows of the desktop often sit and+# would cover the pad: a click on a covered point lands on the covering window.+$a = [Windows.Forms.Screen]::PrimaryScreen.WorkingArea; $f.StartPosition = 'Manual'; $f.Left = $a.Left + $a.Width / 2; $f.Top = $a.Top + $a.Height / 2+$p = New-Object Windows.Forms.Panel; $p.Left = 10; $p.Top = 10; $p.Width = 380; $p.Height = 160; $p.BackColor = 'LightSteelBlue'; $p.AccessibleName = 'Pad'+$l = New-Object Windows.Forms.Label; $l.Left = 10; $l.Top = 190; $l.Width = 380; $l.Text = 'log:'+$p.Add_MouseDown({ param($s, $e) $l.Text += ' ' + $e.Button.ToString().Substring(0, 1) + $e.Clicks })+$f.Controls.AddRange(@($p, $l)); [void]$f.ShowDialog()`;+ return ["-NoProfile", "-WindowStyle", "Hidden", "-EncodedCommand", Buffer.from(script, "utf16le").toString("base64")];+}++test("a triple click and right or middle double and triple clicks arrive as one multi-click each", async () => {+ const title = `pi-multi-click-${Date.now()}`;+ const window = await openWindow(live, live.machine, WINDOWS_POWERSHELL_PATH, padForm(title), title);+ try {+ const full = await live.read(window, ["full"]);+ type Element = { hint: string; role: string; name?: string; bounds: { window_image?: { x: number; y: number; width: number; height: number } } };+ const elements = (full.details.ui as { elements: Element[] }).elements;+ const pad = elements.find(item => item.name === "Pad")!;+ const box = pad.bounds.window_image!;+ const point = { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) };+ const log = async () => /log:[^\n|]*/.exec(presentComputerResult(await live.read(window, ["ui"])))![0].trim();+ let expected = "log:";+ const click = async (plan: Parameters<typeof clickPlan>[0], presses: string) => {+ const result = await live.resources.input(requireComputerAddress(window), live.cwd, clickPlan(plan));+ const body = presentComputerResult(result);+ console.log(`${JSON.stringify({ ...plan, window: undefined })}: ${result.details.status}\n${body}`);+ expect(result.details.status, body).toBe("delivered");+ expected += ` ${presses}`;+ // The label repaints after the presses; give the form a moment before reading it.+ await new Promise(resolve => setTimeout(resolve, 300));+ expect(await log()).toBe(expected);+ };+ await click({ window, ...point, count: 3 }, "L1 L2 L1");+ await click({ window, ...point, button: "right", count: 2 }, "R1 R2");+ await click({ window, ...point, button: "middle", count: 2 }, "M1 M2");+ await click({ window, ...point, button: "middle", count: 3 }, "M1 M2 M1");+ await click({ window, ...point, button: "right", count: 3 }, "R1 R2 R1");+ // A ref of the latest Read: no UI Automation pattern carries a multi-click, so it goes straight to the real pointer.+ const ref = /^([0-9a-z]+:[a-z]+) \| Pane \| Pad/m.exec(presentComputerResult(await live.read(window, ["ui"])))![1];+ await click({ window, target: ref, count: 3 }, "L1 L2 L1");+ } finally {+ await deleteComputer(live.resources, `${window}?force=true`, { path: `${window}?force=true` }, live.cwd);+ }+});