Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Triple clicks and right or middle double clicks on native windows (LPT-948)
alexshpunt-clanker[bot]e940bed2f500
diff --git a/src/complete-input.ts b/src/complete-input.tsindex 7e105b4..1fafaa7 100644--- a/src/complete-input.ts+++ b/src/complete-input.ts@@ -13,6 +13,8 @@ export interface InputCommand {dwell_ms?: number;delay_ms?: number;button?: "left" | "right" | "middle";+ /** click only: 2 is a double click, 3 a triple click, at the system double-click timing. */+ count?: number;direction?: "up" | "down" | "left" | "right";amount?: number; by?: "line" | "page";/** Signed native wheel detents at a pixel point; positive values scroll content right/down. */@@ -51,7 +53,7 @@ const modifiers = ["ctrl", "shift", "alt", "win"];const fields: Record<InputCommand["action"], string[]> = {move: [], hover: ["dwell_ms"],key: ["key", "modifiers"], hotkey: ["keys"], type_text: ["text", "delay_ms"], set_value: ["text"],- click: ["button", "modifiers"], right_click: ["modifiers"], double_click: ["modifiers"],+ click: ["button", "count", "modifiers"], right_click: ["modifiers"], double_click: ["modifiers"],scroll: ["direction", "amount", "by", "delta_x", "delta_y", "unit"], drag: ["from_x", "from_y", "to_x", "to_y", "duration_ms", "steps", "button", "modifiers", "via"],select_all: [], invoke: [], select: [],hold: ["keys", "duration_ms", "actions"],@@ -114,6 +116,7 @@ export function validateInput(command: InputStep, observed: { tokens: Set<string}if (command.delay_ms !== undefined && !integer(command.delay_ms, 0, 200)) throw new Error("delay_ms must be 0..200.");if (command.button !== undefined && !["left", "right", "middle"].includes(command.button)) throw new Error("Unsupported button.");+ if (command.count !== undefined && !integer(command.count, 1, 3)) throw new Error("count must be 1, 2 or 3.");if (command.action === "scroll") {if (pixel) {if (command.direction !== undefined || command.amount !== undefined || command.by !== undefined) throw new Error("Coordinate scroll cannot combine detents with direction, amount or by.");diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex 8c6c9d8..80af3e0 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -1084,8 +1084,8 @@ export class ComputerResources {const action = String(command.action);// What the agent sees structurally is pressed structurally: a click or scroll on a target goes through UI Automation// alone, and the driver refuses before sending anything when no pattern carries it. Points, keys, drags and timelines- // are real input; semantic actions and menus are UI Automation calls of their own.- const structural = !plan.real && plan.target !== undefined && plan.point === undefined && STRUCTURAL.includes(action) && command.unit === undefined && command.modifiers === undefined;+ // are real input; semantic actions and menus are UI Automation calls of their own. No pattern carries a multi-click.+ const structural = !plan.real && plan.target !== undefined && plan.point === undefined && STRUCTURAL.includes(action) && command.unit === undefined && command.modifiers === undefined && command.count === undefined;if (structural) command.uia_only = true;else if (!isSemanticAction(action) && action !== "menu") command.delivery_mode = "foreground";let target = plan.target;diff --git a/src/computer-tools.ts b/src/computer-tools.tsindex 17d8e6b..30213ce 100644--- a/src/computer-tools.ts+++ b/src/computer-tools.ts@@ -311,17 +311,17 @@ function holding(modifiers: string[] | undefined, start: number, end: number, evconst clickParameters = Type.Object({window: inputWindow, target: inputTarget("The element to click"), x: coordinate("Left"), y: coordinate("Top"), space: pointSpace,button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")], { description: "Mouse button, default left." })),- count: Type.Optional(Type.Integer({ minimum: 1, maximum: 3, description: "Clicks: 2 is a double click, 3 a triple click (browser pages). Default 1." })),+ count: Type.Optional(Type.Integer({ minimum: 1, maximum: 3, description: "Clicks: 2 is a double click, 3 a triple click. Default 1." })),modifiers: modifiersField, timeout_ms: locatorTimeout,}, { additionalProperties: false });/** What computer_click sends to a native window and to a browser page. */export function clickPlan(params: Static<typeof clickParameters>): InputPlan {const count = params.count ?? 1, button = params.button ?? "left", modifiers = params.modifiers?.length ? { modifiers: params.modifiers } : {};- const native = count === 3 ? "A triple click works on browser pages only."- : button === "right" ? count === 2 ? "A right double click works on browser pages only." : { action: "right_click", ...modifiers, ...timeout(params) }- : count === 2 ? button === "middle" ? "A middle double click works on browser pages only." : { action: "double_click", ...modifiers, ...timeout(params) }- : { action: "click", ...(button === "middle" ? { button } : {}), ...modifiers, ...timeout(params) };+ // A native window gets the driver's own single, right and double clicks; every other multi-click is its click with a count.+ const native = count === 1 ? button === "right" ? { action: "right_click", ...modifiers, ...timeout(params) } : { action: "click", ...(button === "middle" ? { button } : {}), ...modifiers, ...timeout(params) }+ : count === 2 && button === "left" ? { action: "double_click", ...modifiers, ...timeout(params) }+ : { action: "click", ...(button !== "left" ? { button } : {}), count, ...modifiers, ...timeout(params) };const aimed = aim(params, "click", true);// A point of a page gets the presses Playwright's mouse.click sends, numbered so the page sees double and triple clicks.const presses = Array.from({ length: count }, (_, index) => [{ at_ms: 0, type: "button_down", button, click_count: index + 1 }, { at_ms: 0, type: "button_up", button, click_count: index + 1 }]).flat();