Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Triple clicks and right or middle double clicks on native windows (LPT-948)

alexshpunt-clanker[bot]e940bed2f500
diff --git a/src/complete-input.ts b/src/complete-input.ts
index 7e105b4..1fafaa7 100644
--- a/src/complete-input.ts
+++ b/src/complete-input.ts
@@ -13,6 +13,8 @@ export interface InputCommand {
dwell_ms?: number;
delay_ms?: number;
button?: "left" | "right" | "middle";
+ /** click only: 2 is a double click, 3 a triple click, at the system double-click timing. */
+ count?: number;
direction?: "up" | "down" | "left" | "right";
amount?: number; by?: "line" | "page";
/** Signed native wheel detents at a pixel point; positive values scroll content right/down. */
@@ -51,7 +53,7 @@ const modifiers = ["ctrl", "shift", "alt", "win"];
const fields: Record<InputCommand["action"], string[]> = {
move: [], hover: ["dwell_ms"],
key: ["key", "modifiers"], hotkey: ["keys"], type_text: ["text", "delay_ms"], set_value: ["text"],
- click: ["button", "modifiers"], right_click: ["modifiers"], double_click: ["modifiers"],
+ click: ["button", "count", "modifiers"], right_click: ["modifiers"], double_click: ["modifiers"],
scroll: ["direction", "amount", "by", "delta_x", "delta_y", "unit"], drag: ["from_x", "from_y", "to_x", "to_y", "duration_ms", "steps", "button", "modifiers", "via"],
select_all: [], invoke: [], select: [],
hold: ["keys", "duration_ms", "actions"],
@@ -114,6 +116,7 @@ export function validateInput(command: InputStep, observed: { tokens: Set<string
}
if (command.delay_ms !== undefined && !integer(command.delay_ms, 0, 200)) throw new Error("delay_ms must be 0..200.");
if (command.button !== undefined && !["left", "right", "middle"].includes(command.button)) throw new Error("Unsupported button.");
+ if (command.count !== undefined && !integer(command.count, 1, 3)) throw new Error("count must be 1, 2 or 3.");
if (command.action === "scroll") {
if (pixel) {
if (command.direction !== undefined || command.amount !== undefined || command.by !== undefined) throw new Error("Coordinate scroll cannot combine detents with direction, amount or by.");
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index 8c6c9d8..80af3e0 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -1084,8 +1084,8 @@ export class ComputerResources {
const action = String(command.action);
// What the agent sees structurally is pressed structurally: a click or scroll on a target goes through UI Automation
// alone, and the driver refuses before sending anything when no pattern carries it. Points, keys, drags and timelines
- // are real input; semantic actions and menus are UI Automation calls of their own.
- const structural = !plan.real && plan.target !== undefined && plan.point === undefined && STRUCTURAL.includes(action) && command.unit === undefined && command.modifiers === undefined;
+ // are real input; semantic actions and menus are UI Automation calls of their own. No pattern carries a multi-click.
+ const structural = !plan.real && plan.target !== undefined && plan.point === undefined && STRUCTURAL.includes(action) && command.unit === undefined && command.modifiers === undefined && command.count === undefined;
if (structural) command.uia_only = true;
else if (!isSemanticAction(action) && action !== "menu") command.delivery_mode = "foreground";
let target = plan.target;
diff --git a/src/computer-tools.ts b/src/computer-tools.ts
index 17d8e6b..30213ce 100644
--- a/src/computer-tools.ts
+++ b/src/computer-tools.ts
@@ -311,17 +311,17 @@ function holding(modifiers: string[] | undefined, start: number, end: number, ev
const clickParameters = Type.Object({
window: inputWindow, target: inputTarget("The element to click"), x: coordinate("Left"), y: coordinate("Top"), space: pointSpace,
button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")], { description: "Mouse button, default left." })),
- count: Type.Optional(Type.Integer({ minimum: 1, maximum: 3, description: "Clicks: 2 is a double click, 3 a triple click (browser pages). Default 1." })),
+ count: Type.Optional(Type.Integer({ minimum: 1, maximum: 3, description: "Clicks: 2 is a double click, 3 a triple click. Default 1." })),
modifiers: modifiersField, timeout_ms: locatorTimeout,
}, { additionalProperties: false });
/** What computer_click sends to a native window and to a browser page. */
export function clickPlan(params: Static<typeof clickParameters>): InputPlan {
const count = params.count ?? 1, button = params.button ?? "left", modifiers = params.modifiers?.length ? { modifiers: params.modifiers } : {};
- const native = count === 3 ? "A triple click works on browser pages only."
- : button === "right" ? count === 2 ? "A right double click works on browser pages only." : { action: "right_click", ...modifiers, ...timeout(params) }
- : count === 2 ? button === "middle" ? "A middle double click works on browser pages only." : { action: "double_click", ...modifiers, ...timeout(params) }
- : { action: "click", ...(button === "middle" ? { button } : {}), ...modifiers, ...timeout(params) };
+ // A native window gets the driver's own single, right and double clicks; every other multi-click is its click with a count.
+ const native = count === 1 ? button === "right" ? { action: "right_click", ...modifiers, ...timeout(params) } : { action: "click", ...(button === "middle" ? { button } : {}), ...modifiers, ...timeout(params) }
+ : count === 2 && button === "left" ? { action: "double_click", ...modifiers, ...timeout(params) }
+ : { action: "click", ...(button !== "left" ? { button } : {}), count, ...modifiers, ...timeout(params) };
const aimed = aim(params, "click", true);
// A point of a page gets the presses Playwright's mouse.click sends, numbered so the page sees double and triple clicks.
const presses = Array.from({ length: count }, (_, index) => [{ at_ms: 0, type: "button_down", button, click_count: index + 1 }, { at_ms: 0, type: "button_up", button, click_count: index + 1 }]).flat();