Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Tests: native multi-clicks, unit and live (LPT-948)

alexshpunt-clanker[bot]2f23d09fb758
diff --git a/test/complete-input.test.ts b/test/complete-input.test.ts
index 8f0c1bd..62f2898 100644
--- a/test/complete-input.test.ts
+++ b/test/complete-input.test.ts
@@ -30,6 +30,8 @@ test.each([
[{ action: "right_click", x: 100, y: 80, modifiers: ["shift"] }, "right_click", { x: 100, y: 80, modifier: ["shift"] }],
[{ action: "double_click", element_token: "field" }, "double_click", { element_token: "field" }],
[{ action: "click", x: 100, y: 80, button: "middle" }, "click", { x: 100, y: 80, button: "middle", capture_id: "capture-one" }],
+ [{ action: "click", x: 100, y: 80, count: 3 }, "click", { x: 100, y: 80, count: 3, capture_id: "capture-one" }],
+ [{ action: "click", element_token: "field", button: "right", count: 2 }, "click", { element_token: "field", button: "right", count: 2 }],
[{ action: "scroll", element_token: "field", direction: "down", amount: 2, by: "page" }, "scroll", { element_token: "field", direction: "down", amount: 2, by: "page" }],
[{ action: "drag", from_x: 30, from_y: 40, to_x: 200, to_y: 80, duration_ms: 250, steps: 12, button: "left", modifiers: ["ctrl"] }, "drag",
{ from_x: 30, from_y: 40, to_x: 200, to_y: 80, duration_ms: 250, steps: 12, button: "left", modifier: ["ctrl"] }],
@@ -53,6 +55,7 @@ test.each([
{ action: "scroll", x: 100, y: 80, direction: "down" },
{ action: "scroll", direction: "down", amount: 2 },
{ action: "key", key: "a", duration_ms: 1000 }, { action: "click", x: 10, y: 10, keys: ["ctrl", "a"] },
+ { action: "click", x: 10, y: 10, count: 4 }, { action: "click", x: 10, y: 10, count: 0 }, { action: "double_click", x: 10, y: 10, count: 3 },
{ action: "sequence", sequence: [] },
{ action: "sequence", sequence: [{ action: "key", key: "a" }, { action: "drag", from_x: 1, from_y: 1, to_x: NaN, to_y: 2 }] },
{ action: "sequence", sequence: [{ action: "hover", x: 10, y: 10 }] },
diff --git a/test/computer-resources.test.ts b/test/computer-resources.test.ts
index bb0e046..eb16adc 100644
--- a/test/computer-resources.test.ts
+++ b/test/computer-resources.test.ts
@@ -299,6 +299,22 @@ test("a click on a ref goes through UI Automation only, and no pattern means the
expect(sent[1]).not.toHaveProperty("uia_only");
});
+// LPT-948: a triple click selects a field's value; no UI Automation pattern carries a multi-click, so it is real input.
+test("triple clicks and right or middle double clicks reach a native window as real clicks", async () => {
+ const f = await fixture();
+ let view = await f.read();
+ const triple = await f.resources.input(f.window, cwd, clickPlan({ window: f.window.source, target: view.hint!, count: 3 }));
+ expect(triple.details.status).toBe("delivered");
+ view = await f.read();
+ await f.resources.input(f.window, cwd, clickPlan({ window: f.window.source, x: 100, y: 70, button: "right", count: 2 }));
+ await f.resources.input(f.window, cwd, clickPlan({ window: f.window.source, x: 100, y: 70, button: "middle", count: 2 }));
+ expect(f.inputs()).toEqual([
+ ["click", { target: { kind: "window", pid: 42, window_id: 91 }, delivery_mode: "foreground", element_token: expect.any(String), count: 3 }, undefined],
+ ["click", expect.objectContaining({ delivery_mode: "foreground", x: expect.any(Number), y: expect.any(Number), button: "right", count: 2 }), undefined],
+ ["click", expect.objectContaining({ delivery_mode: "foreground", button: "middle", count: 2 }), undefined]]);
+ expect(f.inputs().some(([, args]) => "uia_only" in (args as object))).toBe(false);
+});
+
test("points and keys are real input; nothing posts window messages", async () => {
const f = await fixture();
(f.driver as Driver & { coordinateScroll: boolean }).coordinateScroll = true;
diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.ts
index 73061fa..abe82b9 100644
--- a/test/computer-tools.test.ts
+++ b/test/computer-tools.test.ts
@@ -99,7 +99,10 @@ test("input tools build one plan for native windows and one for browser pages",
native: { action: "double_click", modifiers: ["ctrl"] }, web: { action: "timeline" } });
expect(clickPlan({ window: w, target: "e1", count: 2, modifiers: ["ctrl"] }).web).toEqual({ action: "click", click_count: 2, modifiers: ["Control"] });
expect(clickPlan({ window: w, target: "e1", button: "right" }).native).toEqual({ action: "right_click" });
- expect(clickPlan({ window: w, target: "e1", count: 3 }).native).toBe("A triple click works on browser pages only.");
+ // LPT-948: every multi-click goes to a native window as the driver's click with its button and count.
+ expect(clickPlan({ window: w, target: "1:a", count: 3 }).native).toEqual({ action: "click", count: 3 });
+ expect(clickPlan({ window: w, x: 1, y: 2, button: "right", count: 2, modifiers: ["shift"] }).native).toEqual({ action: "click", button: "right", count: 2, modifiers: ["shift"] });
+ expect(clickPlan({ window: w, x: 1, y: 2, button: "middle", count: 3 }).native).toEqual({ action: "click", button: "middle", count: 3 });
expect(() => clickPlan({ window: w })).toThrow("click needs a target or a point (x and y).");
expect(() => clickPlan({ window: w, x: 1 })).toThrow("click needs both x and y for a point.");
expect(() => clickPlan({ window: w, x: 1, y: 2, target: "1:a" })).toThrow("click takes a target or a point, not both.");
diff --git a/test/live/multi-click.live.ts b/test/live/multi-click.live.ts
new file mode 100644
index 0000000..54982df
--- /dev/null
+++ b/test/live/multi-click.live.ts
@@ -0,0 +1,60 @@
+import { afterAll, expect, test } from "vitest";
+import { requireComputerAddress } from "../../src/computer-resources.ts";
+import { clickPlan } from "../../src/computer-tools.ts";
+import { deleteComputer, presentComputerResult } from "../../src/provider.ts";
+import { connectLive, openWindow, WINDOWS_POWERSHELL_PATH } from "./remote.ts";
+
+// LPT-948: triple clicks and right or middle double clicks reach a native window as real clicks. A WinForms pad logs
+// every press it gets as its button and Windows' click count, so a double click shows as "R1 R2": Windows made the
+// second press a double click because it came within the system double-click time and distance. Windows counts no
+// further, so a triple click is "L1 L2 L1"; apps that select a line on a triple click time the third press themselves.
+const live = await connectLive();
+afterAll(async () => { await live.close(); });
+
+function padForm(title: string): string[] {
+ const script = `Add-Type -AssemblyName System.Windows.Forms
+$f = New-Object Windows.Forms.Form; $f.Text = '${title}'; $f.Width = 420; $f.Height = 300
+# Open at the middle of the screen, away from the top-left corner where other windows of the desktop often sit and
+# would cover the pad: a click on a covered point lands on the covering window.
+$a = [Windows.Forms.Screen]::PrimaryScreen.WorkingArea; $f.StartPosition = 'Manual'; $f.Left = $a.Left + $a.Width / 2; $f.Top = $a.Top + $a.Height / 2
+$p = New-Object Windows.Forms.Panel; $p.Left = 10; $p.Top = 10; $p.Width = 380; $p.Height = 160; $p.BackColor = 'LightSteelBlue'; $p.AccessibleName = 'Pad'
+$l = New-Object Windows.Forms.Label; $l.Left = 10; $l.Top = 190; $l.Width = 380; $l.Text = 'log:'
+$p.Add_MouseDown({ param($s, $e) $l.Text += ' ' + $e.Button.ToString().Substring(0, 1) + $e.Clicks })
+$f.Controls.AddRange(@($p, $l)); [void]$f.ShowDialog()`;
+ return ["-NoProfile", "-WindowStyle", "Hidden", "-EncodedCommand", Buffer.from(script, "utf16le").toString("base64")];
+}
+
+test("a triple click and right or middle double and triple clicks arrive as one multi-click each", async () => {
+ const title = `pi-multi-click-${Date.now()}`;
+ const window = await openWindow(live, live.machine, WINDOWS_POWERSHELL_PATH, padForm(title), title);
+ try {
+ const full = await live.read(window, ["full"]);
+ type Element = { hint: string; role: string; name?: string; bounds: { window_image?: { x: number; y: number; width: number; height: number } } };
+ const elements = (full.details.ui as { elements: Element[] }).elements;
+ const pad = elements.find(item => item.name === "Pad")!;
+ const box = pad.bounds.window_image!;
+ const point = { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) };
+ const log = async () => /log:[^\n|]*/.exec(presentComputerResult(await live.read(window, ["ui"])))![0].trim();
+ let expected = "log:";
+ const click = async (plan: Parameters<typeof clickPlan>[0], presses: string) => {
+ const result = await live.resources.input(requireComputerAddress(window), live.cwd, clickPlan(plan));
+ const body = presentComputerResult(result);
+ console.log(`${JSON.stringify({ ...plan, window: undefined })}: ${result.details.status}\n${body}`);
+ expect(result.details.status, body).toBe("delivered");
+ expected += ` ${presses}`;
+ // The label repaints after the presses; give the form a moment before reading it.
+ await new Promise(resolve => setTimeout(resolve, 300));
+ expect(await log()).toBe(expected);
+ };
+ await click({ window, ...point, count: 3 }, "L1 L2 L1");
+ await click({ window, ...point, button: "right", count: 2 }, "R1 R2");
+ await click({ window, ...point, button: "middle", count: 2 }, "M1 M2");
+ await click({ window, ...point, button: "middle", count: 3 }, "M1 M2 M1");
+ await click({ window, ...point, button: "right", count: 3 }, "R1 R2 R1");
+ // A ref of the latest Read: no UI Automation pattern carries a multi-click, so it goes straight to the real pointer.
+ const ref = /^([0-9a-z]+:[a-z]+) \| Pane \| Pad/m.exec(presentComputerResult(await live.read(window, ["ui"])))![1];
+ await click({ window, target: ref, count: 3 }, "L1 L2 L1");
+ } finally {
+ await deleteComputer(live.resources, `${window}?force=true`, { path: `${window}?force=true` }, live.cwd);
+ }
+});