Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
feat: typed input tools on native windows and browser elements (LPT-927 part a)
alexshpunt-clanker[bot]f6c72d670f2e
diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex 1944b5b..8078bfb 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -49,7 +49,38 @@ const BATCH_STEPS = 200;/** Native actions a batch may run; each later step re-resolves its target in a fresh UI walk. */const NATIVE_BATCH_ACTIONS = ["click", "double_click", "right_click", "type_text", "key", "hotkey", "scroll", "invoke", "select", "set_value"];/** Native actions that take a Playwright locator or JSON target. */-const NATIVE_LOCATOR_ACTIONS = ["click", "double_click", "right_click", "hover", "type_text", "scroll", "invoke", "select", "set_value", "check", "uncheck"];+const NATIVE_LOCATOR_ACTIONS = ["click", "double_click", "right_click", "hover", "type_text", "key", "scroll", "invoke", "select", "set_value", "check", "uncheck"];+/** A point in pixels of a window's latest image, or in CSS pixels of a browser page. */+export interface InputPoint { x: number; y: number }+/** One typed input on a window, built by a computer_* input tool; the window decides which backend runs it. */+export interface InputPlan {+ /** The tool's action for refusals, such as click. */+ label: string;+ /** A ref of the window's latest Read (a native ref such as 1:a, a page ref such as e12) or a Playwright locator. */+ target?: string;+ point?: InputPoint;+ /** drag: where it ends. */+ end?: { target?: string; point?: InputPoint };+ /** Browser windows: page (CSS pixels of the page, the default) or window (pixels of the window's latest image). */+ space?: "page" | "window";+ /** The native command without its target, or why native windows cannot run it. */+ native: Record<string, unknown> | string;+ /** The Playwright action without its target, or why browser pages cannot run it. */+ web: Record<string, unknown> | string;+ /** true sends native input in the foreground at once; false never leaves the background. */+ foreground?: boolean;+ /** Runs after this one is delivered, such as Enter after typing. */+ then?: InputPlan;+}+/** Native input that moves the real pointer, so it always goes in the foreground. */+const FOREGROUND_ONLY = ["hover", "move", "timeline"];+/** Native actions that only a locator target runs; a ref of the latest Read becomes the locator of its element. */+const LOCATOR_ONLY = ["check", "uncheck"];+/** What a target string is: a native ref of a window Read (1:a), a page ref of a DOM Read (e12, f1e3), or a Playwright locator. */+export function refKind(target: unknown): "hint" | "page" | "locator" | undefined {+ if (typeof target !== "string") return undefined;+ return /^[0-9a-z]+:[a-z]+$/.test(target) ? "hint" : /^(?:f\d+)?e\d+$/.test(target) ? "page" : "locator";+}/** Playwright verbs with a different native name. */const LOCATOR_VERBS: Record<string, string> = { fill: "set_value", dblclick: "double_click" };/**@@ -287,6 +318,8 @@ export class ComputerResources {private readonly starting = new Map<string, Promise<OpenedMachine>>();/** Sandbox launches and machine starts still in progress; a cancelled one finishes its own cleanup before it settles. */private readonly opening = new Set<Promise<unknown>>();+ /** Windows that refused background input once; typed input to them goes in the foreground. */+ private readonly foreground = new Set<string>();/** Short aliases of this run's long addresses (computer:zinpc/w3qk). */private readonly shorts = new ShortAddresses();/** Window names the agent gave, with the window's title when it was named. */@@ -877,6 +910,127 @@ export class ComputerResources {const result = await session.actPage(address.ref, action, signal);return this.stamp(place, result.details.dom ? this.remember({ window_ref: address.ref }, result) : result);}+ /**+ * One typed input on a window by its address. The window picks the backend: Playwright for page refs, locators+ * and CSS points of a browser page, the native driver and UI Automation otherwise. Native points are pixels of the+ * window's latest image; without a current one the window is captured first. The receipt names its machine.+ */+ async input(address: ComputerAddress, cwd: string, plan: InputPlan, signal?: AbortSignal): Promise<ComputerResult> {+ if (address.kind !== "window") throw new ComputerRequestError("Give the window address from Search, Read or a name, such as computer:zinpc/w3qk or computer:game.");+ const place = this.place(address);+ const session = await this.sessionFor(cwd, { window_ref: address.ref }, signal);+ let result = await this.inputStep(address.ref, session, plan, cwd, signal);+ // A follow-up such as Enter after typing runs only after the first part was delivered.+ if (plan.then && result.details.status === "delivered") {+ const next = await this.inputStep(address.ref, session, plan.then, cwd, signal);+ result = { ...next, content: [...result.content, ...next.content] };+ }+ return this.stamp(place, result);+ }+ private async inputStep(window: string, session: ComputerSession, plan: InputPlan, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {+ const ref = refKind(plan.target);+ const browser = session.isBrowserWindow(window);+ if (browser && plan.space !== "window" && ref !== "hint") return this.pageInput(window, session, plan, cwd, signal);+ if (plan.space !== undefined && !browser) throw new ComputerRequestError("space applies to browser windows only; points on other windows are pixels of the window's latest image.");+ if (ref === "page") throw new ComputerRequestError(`${plan.target} is a page ref from a DOM Read; ${browser ? "omit space: \"window\" to act on the page" : "this window is not a browser page, so use a ref from its latest Read or a Playwright locator"}.`);+ return this.nativeStep(window, session, plan, cwd, signal);+ }++ /** Typed input on the active page of a browser window through Playwright; a current DOM observation gives the change list. */+ private async pageInput(window: string, session: ComputerSession, plan: InputPlan, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {+ if (typeof plan.web === "string") throw new ComputerRequestError(plan.web);+ if (plan.point || plan.end) throw new ComputerRequestError(`${plan.label} at a point of a browser page is not available yet; give a target, or space: "window" for window pixels.`);+ const target = typeof plan.target === "string" ? refKind(plan.target) === "page" ? { ref: plan.target } : plan.target : undefined;+ const command = { ...plan.web, ...(target !== undefined ? { target } : {}) };+ let action: WebAction;+ try { action = validateWebAction(command); }+ catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }+ const dom = session.inputView(window).dom;+ if (dom && this.snapshots.has(dom)) return this.performAt({ kind: "observation", ref: dom, source: computerSource("observation", dom) }, cwd, JSON.stringify(command), "insert", signal);+ this.retireWindow(window);+ const result = await session.actPage(window, action, signal);+ return result.details.dom ? this.remember({ window_ref: window }, result) : result;+ }++ /**+ * Typed input on a native window: a locator resolves in fresh UI walks, a ref of the latest Read and points use the+ * window's current observation, captured first when there is none with the pixels a point needs. A background+ * refusal, which sends nothing, is retried at once in the foreground, and the window keeps foreground delivery.+ */+ private async nativeStep(window: string, session: ComputerSession, plan: InputPlan, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {+ if (typeof plan.native === "string") throw new ComputerRequestError(plan.native);+ const command: Record<string, unknown> = structuredClone(plan.native);+ const action = String(command.action);+ const pointerOnly = FOREGROUND_ONLY.includes(action);+ if (pointerOnly && plan.foreground === false) throw new ComputerRequestError(`${plan.label} on a native window moves the real pointer, so it needs the foreground.`);+ const foreground = plan.foreground ?? (pointerOnly || this.foreground.has(window));+ if (foreground && !isSemanticAction(action)) command.delivery_mode = "foreground";+ let target = plan.target;+ let view = session.inputView(window);+ // Actions that only exist on locators take a ref of the latest Read as the locator that names its element.+ if (refKind(target) === "hint" && (LOCATOR_ONLY.includes(action) || command.option !== undefined)) target = suggestLocator(this.hinted(view, target as string));+ if (command.option !== undefined) {+ if (refKind(target) !== "locator") throw new ComputerRequestError(`${plan.label} with values needs the list as target.`);+ target = `${target}.getByRole("option", { name: ${JSON.stringify(command.option)}, exact: true })`;+ delete command.option;+ }+ let result: ComputerResult;+ // What a foreground retry sends: the input observation is spent by then, so its refs become locators or points.+ let retry = (): InputPlan => plan;+ if (refKind(target) === "locator") {+ if (plan.end) throw new ComputerRequestError(`${plan.label} takes refs of the window's latest Read or points, not locators.`);+ result = await this.performAt({ kind: "window", ref: window, source: computerSource("window", window) }, cwd, JSON.stringify({ ...command, target }), "insert", signal);+ } else {+ const hint = refKind(target) === "hint" ? target as string : undefined;+ // Only a locator waits for its element; refs and points act on the current observation at once.+ delete command.timeout_ms;+ const pixels = plan.point !== undefined || plan.end !== undefined || command.unit === "wheel_detents" || action === "timeline";+ if (hint) this.hinted(view, hint);+ else if (!view.native || pixels && view.native.width === undefined) {+ const captured = await this.read({ kind: "window", ref: window, source: computerSource("window", window) }, cwd, ["image"], signal);+ if (!captured || captured.details.status !== "observed") return captured ?? { content: [{ type: "text", text: "The window could not be captured; nothing was sent." }], details: { status: "refused", mode: this.getMode(), window_ref: window }, isError: true };+ view = session.inputView(window);+ }+ const native = view.native!;+ if (pixels && native.width === undefined) throw new ComputerRequestError(`${plan.label} at a point needs the window's pixels: Read the window with an image, or give a locator.`);+ // Points are in the latest image the agent saw; the current observation can be a capture of another size.+ const sx = native.width && view.seen ? native.width / view.seen.width : 1, sy = native.height && view.seen ? native.height / view.seen.height : 1;+ const at = (point: InputPoint) => ({ x: Math.min(native.width! - 1, Math.round(point.x * sx)), y: Math.min(native.height! - 1, Math.round(point.y * sy)) });+ const seen = (point: InputPoint) => ({ x: point.x / sx, y: point.y / sy });+ const centre = (ref: string) => {+ const box = this.hinted(view, ref).bounds.window_image;+ if (!box || native.width === undefined) throw new ComputerRequestError(`${ref} has no place in the window's image; give a point or a locator.`);+ return { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) };+ };+ if (action === "drag") {+ const from = hint ? centre(hint) : plan.point ? at(plan.point) : undefined;+ const end = plan.end?.target !== undefined ? refKind(plan.end.target) === "hint" ? centre(plan.end.target as string) : undefined : plan.end?.point ? at(plan.end.point) : undefined;+ if (!from || !end) throw new ComputerRequestError("drag on a native window goes from and to refs of the window's latest Read or points.");+ Object.assign(command, { from_x: from.x, from_y: from.y, to_x: end.x, to_y: end.y });+ if (Array.isArray(command.via)) command.via = (command.via as InputPoint[]).map(at);+ retry = () => ({ ...plan, target: undefined, point: seen(from), end: { point: seen(end) } });+ } else if (action === "timeline") {+ command.events = (command.events as Record<string, unknown>[]).map(event => typeof event.x === "number" && typeof event.y === "number" ? { ...event, ...at(event as unknown as InputPoint) } : event);+ } else if (plan.point) Object.assign(command, at(plan.point));+ else if (command.unit === "wheel_detents") Object.assign(command, { x: Math.floor(native.width! / 2), y: Math.floor(native.height! / 2) });+ if (hint && action !== "drag") retry = () => NATIVE_LOCATOR_ACTIONS.includes(LOCATOR_VERBS[action] ?? action) ? { ...plan, target: suggestLocator(this.hinted(view, hint)) }+ : { ...plan, target: undefined, point: seen(centre(hint)) };+ const observation = { kind: "observation" as const, ref: native.id, source: computerSource("observation", native.id), ...(hint && action !== "drag" ? { hint } : {}) };+ result = await this.performAt(observation, cwd, JSON.stringify(command), "insert", signal);+ }+ const capability = result.details.action_capability;+ if (foreground || plan.foreground === false || result.details.status !== "refused" || capability?.background !== "unavailable") return result;+ this.foreground.add(window);+ const again = await this.nativeStep(window, session, { ...retry(), foreground: true }, cwd, signal);+ return { ...again, content: [{ type: "text", text: `The window refused background input (${capability.code}), so this call sent it in the foreground; later input to this window goes in the foreground.` }, ...again.content] };+ }++ /** The element of a ref in the window's current observation. */+ private hinted(view: ReturnType<ComputerSession["inputView"]>, hint: string): UiElement {+ const element = view.native?.ui?.elements.find(item => item.hint === hint);+ if (!element) throw new ComputerRequestError(`${hint} is not a ref of this window's latest Read, or input since then spent it: Read the window again, or give a Playwright locator.`);+ return element;+ }/** Dispatch exactly one literal Write or JSON Insert against its address-bound target; the receipt names its machine. */async perform(address: ComputerAddress, cwd: string, value: string, tool: "write" | "insert", signal?: AbortSignal): Promise<ComputerResult> {const place = this.place(address);diff --git a/src/computer-tools.ts b/src/computer-tools.tsindex cbae9fa..8c96c6b 100644--- a/src/computer-tools.ts+++ b/src/computer-tools.ts@@ -1,7 +1,7 @@import { Type, type JsonValue, type Static } from "@earendil-works/pi-ai";import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";import { Text } from "@earendil-works/pi-tui";-import { ComputerRequestError, type ComputerResources } from "./computer-resources.ts";+import { ComputerRequestError, type ComputerResources, type InputPlan } from "./computer-resources.ts";import type { ComputerAddress } from "./computer-address.ts";import { computerActionRenderer } from "./computer-action-renderer.ts";import { parseComputerViews } from "./computer-views.ts";@@ -242,6 +242,188 @@ export function recordCommand(params: Static<typeof recordParameters>): { commanreturn { command: { action: "audio_record", ...fields } };}+const inputWindow = Type.String({ description: "Window address from Search, Read or a name, such as computer:zinpc/w3qk or computer:game." });+const inputTarget = (what: string) => Type.Optional(Type.String({ description: `${what}: a ref from the window's latest Read (1:a in a window Read, e12 in a page snapshot) or a Playwright locator such as getByRole('button', { name: 'Save' }).` }));+const coordinate = (axis: string) => Type.Optional(Type.Number({ minimum: 0, description: `${axis} of a point instead of target: pixels of the window's latest image, or CSS pixels of a browser page.` }));+const pointSpace = Type.Optional(Type.Union([Type.Literal("page"), Type.Literal("window")], { description: "Browser windows: page (default) takes refs, locators and CSS points of the page; window takes refs and points of a window Read, such as the browser's own toolbar." }));+const foregroundField = Type.Optional(Type.Boolean({ description: "Native windows: true sends input in the foreground at once (it activates the window and moves the real pointer); false never leaves the background. By default input starts in the background and moves to the foreground when the window refuses it." }));+const modifiersField = Type.Optional(Type.Array(Type.Union([Type.Literal("ctrl"), Type.Literal("shift"), Type.Literal("alt"), Type.Literal("win")]), { maxItems: 4, description: "Keys held during the action." }));+const locatorTimeout = Type.Optional(Type.Integer({ minimum: 0, maximum: 60000, description: "How long a locator target may take to match one ready element, default 5000." }));+const PAGE_MODIFIERS: Record<string, string> = { ctrl: "Control", shift: "Shift", alt: "Alt", win: "Meta" };++type Endpoint = { target?: string; x?: number; y?: number };+/** The target or point of an action; required unless the action can go to the focused element. */+function aim(params: Endpoint, label: string, required: boolean): Pick<InputPlan, "target" | "point"> {+ if ((params.x === undefined) !== (params.y === undefined)) throw new ComputerRequestError(`${label} needs both x and y for a point.`);+ const point = params.x !== undefined ? { x: params.x, y: params.y! } : undefined;+ if (point && params.target !== undefined) throw new ComputerRequestError(`${label} takes a target or a point, not both.`);+ if (required && !point && params.target === undefined) throw new ComputerRequestError(`${label} needs a target or a point (x and y).`);+ return { ...(params.target !== undefined ? { target: params.target } : {}), ...(point ? { point } : {}) };+}+/** Fields every input plan shares. */+const shared = (params: { space?: "page" | "window"; foreground?: boolean }) =>+ ({ ...(params.space ? { space: params.space } : {}), ...(params.foreground !== undefined ? { foreground: params.foreground } : {}) });+const timeout = (params: { timeout_ms?: number }) => params.timeout_ms !== undefined ? { timeout_ms: params.timeout_ms } : {};+const pageModifiers = (modifiers?: string[]) => modifiers?.length ? { modifiers: modifiers.map(key => PAGE_MODIFIERS[key]) } : {};++const clickParameters = Type.Object({+ window: inputWindow, target: inputTarget("The element to click"), x: coordinate("Left"), y: coordinate("Top"), space: pointSpace,+ button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")], { description: "Mouse button, default left." })),+ count: Type.Optional(Type.Integer({ minimum: 1, maximum: 3, description: "Clicks: 2 is a double click, 3 a triple click (browser pages). Default 1." })),+ modifiers: modifiersField, timeout_ms: locatorTimeout, foreground: foregroundField,+}, { additionalProperties: false });++/** What computer_click sends to a native window and to a browser page. */+export function clickPlan(params: Static<typeof clickParameters>): InputPlan {+ const count = params.count ?? 1, button = params.button ?? "left", modifiers = params.modifiers?.length ? { modifiers: params.modifiers } : {};+ const native = count === 3 ? "A triple click works on browser pages only."+ : button === "right" ? count === 2 ? "A right double click works on browser pages only." : { action: "right_click", ...modifiers, ...timeout(params) }+ : count === 2 ? button === "middle" ? "A middle double click works on browser pages only." : { action: "double_click", ...modifiers, ...timeout(params) }+ : { action: "click", ...(button === "middle" ? { button } : {}), ...modifiers, ...timeout(params) };+ return { label: "click", ...aim(params, "click", true), ...shared(params), native,+ web: { action: "click", ...(button !== "left" ? { button } : {}), ...(count > 1 ? { click_count: count } : {}), ...pageModifiers(params.modifiers), ...timeout(params) } };+}++const hoverParameters = Type.Object({+ window: inputWindow, target: inputTarget("The element to point at"), x: coordinate("Left"), y: coordinate("Top"), space: pointSpace,+ dwell_ms: Type.Optional(Type.Integer({ minimum: 1, maximum: 10000, description: "Native windows: how long the pointer rests there before the result, default 300." })),+ timeout_ms: locatorTimeout,+}, { additionalProperties: false });++/** What computer_hover sends; on a native window it moves the real pointer, so it runs in the foreground. */+export function hoverPlan(params: Static<typeof hoverParameters>): InputPlan {+ return { label: "hover", ...aim(params, "hover", true), ...shared(params),+ native: { action: "hover", dwell_ms: params.dwell_ms ?? 300, ...timeout(params) }, web: { action: "hover", ...timeout(params) } };+}++const endpoint = (what: string) => Type.Object({+ target: Type.Optional(Type.String({ description: `${what}: a ref from the window's latest Read, or a Playwright locator on browser pages.` })),+ x: Type.Optional(Type.Number({ minimum: 0 })), y: Type.Optional(Type.Number({ minimum: 0 })),+}, { additionalProperties: false, description: `${what}: a target, or a point (x and y) in pixels of the window's latest image or CSS pixels of a browser page.` });+const dragParameters = Type.Object({+ window: inputWindow, from: endpoint("Where the drag starts"), to: endpoint("Where the drag ends"), space: pointSpace,+ via: Type.Optional(Type.Array(Type.Object({ x: Type.Number({ minimum: 0 }), y: Type.Number({ minimum: 0 }) }, { additionalProperties: false }),+ { minItems: 1, maxItems: 254, description: "Points the pointer passes through on the way, in order, for a path rather than a straight line; left button only." })),+ duration_ms: Type.Optional(Type.Integer({ minimum: 0, maximum: 10000, description: "How long the move takes, default 500." })),+ button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")], { description: "Button held during the drag, default left." })),+ modifiers: modifiersField, foreground: foregroundField,+}, { additionalProperties: false });++/** What computer_drag sends to a native window. */+export function dragPlan(params: Static<typeof dragParameters>): InputPlan {+ const end = aim(params.to, "to", true);+ return { label: "drag", ...aim(params.from, "from", true), end, ...shared(params),+ native: { action: "drag", ...(params.duration_ms !== undefined ? { duration_ms: params.duration_ms } : {}), ...(params.button && params.button !== "left" ? { button: params.button } : {}),+ ...(params.modifiers?.length ? { modifiers: params.modifiers } : {}), ...(params.via ? { via: params.via } : {}) },+ web: "drag on browser pages is not available yet; use space: \"window\" for window pixels." };+}++const scrollParameters = Type.Object({+ window: inputWindow, target: inputTarget("The element to scroll; omit for the middle of the window or page"), x: coordinate("Left"), y: coordinate("Top"), space: pointSpace,+ delta_x: Type.Optional(Type.Integer({ minimum: -50, maximum: 50, description: "Wheel notches to the right (negative: left)." })),+ delta_y: Type.Optional(Type.Integer({ minimum: -50, maximum: 50, description: "Wheel notches down (negative: up). One notch scrolls about 100 CSS pixels in a browser; apps decide their own distance." })),+ timeout_ms: locatorTimeout, foreground: foregroundField,+}, { additionalProperties: false });++/** What computer_scroll sends: wheel notches at a point, or an accessibility scroll of a native element target. */+export function scrollPlan(params: Static<typeof scrollParameters>): InputPlan {+ const dx = params.delta_x ?? 0, dy = params.delta_y ?? 0;+ if (!dx && !dy) throw new ComputerRequestError("scroll needs delta_x or delta_y.");+ const aimed = aim(params, "scroll", false);+ const native = aimed.target === undefined ? { action: "scroll", ...(dx ? { delta_x: dx } : {}), ...(dy ? { delta_y: dy } : {}), unit: "wheel_detents" }+ : dx && dy ? "Scroll a native element along one axis at a time, or give a point."+ : { action: "scroll", direction: dy > 0 ? "down" : dy < 0 ? "up" : dx > 0 ? "right" : "left", amount: Math.abs(dx || dy), ...timeout(params) };+ return { label: "scroll", ...aimed, ...shared(params), native,+ web: { action: "scroll", ...(dx ? { delta_x: dx * 100 } : {}), ...(dy ? { delta_y: dy * 100 } : {}), ...timeout(params) } };+}++const typeParameters = Type.Object({+ window: inputWindow, text: Type.String({ maxLength: 20000, description: "Text to type, as is: newlines and quotes stay text. Native windows take up to 2000 characters." }),+ target: inputTarget("The field to type into; omit for the focused element"), space: pointSpace,+ replace: Type.Optional(Type.Boolean({ description: "Replace the field's whole value instead of typing at the caret; needs target." })),+ submit: Type.Optional(Type.Boolean({ description: "Press Enter after the text." })),+ delay_ms: Type.Optional(Type.Integer({ minimum: 0, maximum: 200, description: "Native windows: pause between characters." })),+ timeout_ms: locatorTimeout, foreground: foregroundField,+}, { additionalProperties: false });++/** What computer_type sends, with Enter as a second step when submit is set. */+export function typePlan(params: Static<typeof typeParameters>): InputPlan {+ if (params.replace && params.target === undefined) throw new ComputerRequestError("replace needs the field as target.");+ // Enter goes to the same field: a semantic replace does not move the keyboard focus there.+ const enter: InputPlan = { label: "submit", ...(params.target !== undefined ? { target: params.target } : {}), ...shared(params), native: { action: "key", key: "return", ...timeout(params) }, web: { action: "press", key: "Enter", ...timeout(params) } };+ return { label: "type", ...aim(params, "type", false), ...shared(params),+ native: params.replace ? { action: "set_value", text: params.text, ...timeout(params) } : { action: "type_text", text: params.text, ...(params.delay_ms !== undefined ? { delay_ms: params.delay_ms } : {}), ...timeout(params) },+ web: params.delay_ms !== undefined ? "delay_ms applies to native windows." : { action: params.replace ? "fill" : "type", text: params.text, ...timeout(params) },+ ...(params.submit ? { then: enter } : {}) };+}++const pressParameters = Type.Object({+ window: inputWindow, key: Type.String({ maxLength: 64, description: "A key with optional modifiers joined by +, such as Enter, ctrl+s, shift+Tab, F5, ArrowDown or a." }),+ target: inputTarget("The element to focus first; omit for the focused element"), space: pointSpace,+ hold_ms: Type.Optional(Type.Integer({ minimum: 1, maximum: 10000, description: "Native windows: hold the keys down this long, for apps and games that read held keys." })),+ foreground: foregroundField,+}, { additionalProperties: false });+const PAGE_KEYS: Record<string, string> = { enter: "Enter", return: "Enter", esc: "Escape", escape: "Escape", tab: "Tab", space: "Space", backspace: "Backspace", delete: "Delete", del: "Delete",+ up: "ArrowUp", down: "ArrowDown", left: "ArrowLeft", right: "ArrowRight", home: "Home", end: "End", pageup: "PageUp", pagedown: "PageDown", pgup: "PageUp", pgdn: "PageDown" };++/** What computer_press sends: one key or chord, or keys held for hold_ms on a native window. */+export function pressPlan(params: Static<typeof pressParameters>): InputPlan {+ const parts = params.key.split("+").map(part => part.trim());+ if (parts.some(part => !part)) throw new ComputerRequestError("key joins a key and its modifiers with +, such as ctrl+s.");+ const key = parts.at(-1)!, held = parts.slice(0, -1).map(part => part.toLowerCase().replace(/^(control)$/, "ctrl").replace(/^(meta|windows|cmd)$/, "win"));+ if (held.some(part => !PAGE_MODIFIERS[part])) throw new ComputerRequestError("Modifiers are ctrl, shift, alt and win.");+ const native = params.hold_ms !== undefined ? { action: "hold", keys: [...held, key.toLowerCase()], duration_ms: params.hold_ms }+ : { action: "key", key: key.toLowerCase(), ...(held.length ? { modifiers: held } : {}) };+ const pageKey = PAGE_KEYS[key.toLowerCase()] ?? (/^f\d{1,2}$/i.test(key) ? key.toUpperCase() : key);+ return { label: "press", ...aim(params, "press", false), ...shared(params), native,+ web: params.hold_ms !== undefined ? "hold_ms on browser pages is not available yet." : { action: "press", key: [...held.map(part => PAGE_MODIFIERS[part]), pageKey].join("+") } };+}++const selectParameters = Type.Object({+ window: inputWindow, target: inputTarget("The list, select, checkbox or option; not used with menu"), space: pointSpace,+ values: Type.Optional(Type.Array(Type.String(), { minItems: 1, maxItems: 64, description: "Options to choose in the target list, by label or value. Native lists take one; omit values to select the target option itself." })),+ checked: Type.Optional(Type.Boolean({ description: "Set the target checkbox, radio or toggle to checked (true) or unchecked (false); nothing is sent when it already is." })),+ menu: Type.Optional(Type.Array(Type.String(), { minItems: 1, maxItems: 16, description: "Native windows: menu labels from the menu bar to the item, such as [\"File\", \"Save As...\"]." })),+ timeout_ms: locatorTimeout,+}, { additionalProperties: false });++/** What computer_select sends: options of a list, a checked state, or a menu item. */+export function selectPlan(params: Static<typeof selectParameters>): InputPlan {+ const chosen = [params.values && "values", params.checked !== undefined && "checked", params.menu && "menu"].filter(Boolean);+ if (chosen.length > 1) throw new ComputerRequestError(`select takes one of values, checked or menu, not ${chosen.join(" and ")}.`);+ if (params.menu) {+ if (params.target !== undefined) throw new ComputerRequestError("menu names its item by path; omit target.");+ return { label: "select", ...shared(params), native: { action: "menu", path: params.menu }, web: "menu works on native windows; use a target on browser pages." };+ }+ if (params.target === undefined) throw new ComputerRequestError("select needs a target.");+ const base = { label: "select", target: params.target, ...shared(params) };+ if (params.checked !== undefined) return { ...base, native: { action: params.checked ? "check" : "uncheck", ...timeout(params) }, web: { action: params.checked ? "check" : "uncheck", ...timeout(params) } };+ if (params.values) return { ...base, native: params.values.length > 1 ? "A native list takes one value at a time." : { action: "select", option: params.values[0], ...timeout(params) },+ web: { action: "select", values: params.values, ...timeout(params) } };+ return { ...base, native: { action: "select", ...timeout(params) }, web: "Give values: the options to choose in this select element." };+}++const timelineParameters = Type.Object({+ window: inputWindow, space: pointSpace,+ events: Type.Array(Type.Object({+ at_ms: Type.Integer({ minimum: 0, maximum: 120000, description: "Time from the start; never decreases." }),+ type: Type.Union([Type.Literal("key_down"), Type.Literal("key_up"), Type.Literal("button_down"), Type.Literal("button_up"), Type.Literal("move"), Type.Literal("move_relative"), Type.Literal("wheel"), Type.Literal("pen_down"), Type.Literal("pen_move"), Type.Literal("pen_up")]),+ key: Type.Optional(Type.String({ description: "key_down, key_up." })),+ button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle"), Type.Literal("x1"), Type.Literal("x2")], { description: "button_down, button_up; default left." })),+ x: Type.Optional(Type.Number({ minimum: 0, description: "move and pen events: pixels of the window's latest image." })),+ y: Type.Optional(Type.Number({ minimum: 0 })),+ dx: Type.Optional(Type.Integer({ description: "move_relative: raw mouse counts; wheel: notches." })),+ dy: Type.Optional(Type.Integer()),+ pressure: Type.Optional(Type.Number({ minimum: 0, maximum: 1, description: "Pen events." })),+ tilt_x: Type.Optional(Type.Integer({ minimum: -90, maximum: 90 })), tilt_y: Type.Optional(Type.Integer({ minimum: -90, maximum: 90 })),+ eraser: Type.Optional(Type.Boolean()), barrel: Type.Optional(Type.Boolean()),+ }, { additionalProperties: false }), { minItems: 1, maxItems: 5000, description: "Timed events. Every key and button pressed must be released; anything still held is released when the call ends." }),+}, { additionalProperties: false });++/** What computer_timeline sends to a native window, in the foreground. */+export function timelinePlan(params: Static<typeof timelineParameters>): InputPlan {+ return { label: "timeline", ...shared(params), native: { action: "timeline", events: params.events }, web: "timeline on browser pages is not available yet; use space: \"window\" for window pixels." };+}/*** Register the typed computer_* tools that replace Insert commands one cluster at a time. They are deferred: a* tool_search for computer loads them, so sessions that never use the computer pay nothing for them.@@ -359,6 +541,70 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou: resources.perform(address, ctx.cwd, JSON.stringify(run.command), "insert", signal)));},});+ /** Run one typed input on a window: the plan is built and checked before anything reaches the machine. */+ const input = (window: string, action: string, build: () => InputPlan, cwd: string, signal?: AbortSignal) => {+ let plan: InputPlan | undefined;+ return computerAction(resources, window, action, () => { plan = build(); return resources.address(window); },+ address => resources.input(address, cwd, plan!, signal)).then(actionResult);+ };+ const aimed = (args: Record<string, unknown>) => args.target !== undefined ? String(args.target) : args.x !== undefined ? `${args.x},${args.y}` : "";+ const inputAnnotations = { readOnlyHint: false, destructiveHint: true, openWorldHint: true };+ pi.registerTool({+ ...common, name: "computer_click", label: "Computer click",+ description: "Use computer_click to click an element or a point of a window: any button, double or triple clicks, with modifier keys. The result lists what changed.",+ parameters: clickParameters, annotations: inputAnnotations,+ ...rendered("computer_click", args => `${args.window} ${aimed(args)}${args.count ? ` x${args.count}` : ""}${args.button && args.button !== "left" ? ` ${args.button}` : ""}`),+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "click", () => clickPlan(params), ctx.cwd, signal); },+ });+ pi.registerTool({+ ...common, name: "computer_hover", label: "Computer hover",+ description: "Use computer_hover to rest the pointer on an element or a point without clicking, for tooltips, hover menus and hover states.",+ parameters: hoverParameters, annotations: inputAnnotations,+ ...rendered("computer_hover", args => `${args.window} ${aimed(args)}`),+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "hover", () => hoverPlan(params), ctx.cwd, signal); },+ });+ pi.registerTool({+ ...common, name: "computer_drag", label: "Computer drag",+ description: "Use computer_drag to press a button at one element or point, move to another, optionally through a path, and release there, for sliders, drag and drop and drawing.",+ parameters: dragParameters, annotations: inputAnnotations,+ ...rendered("computer_drag", args => `${args.window} ${aimed(args.from as Record<string, unknown>)} → ${aimed(args.to as Record<string, unknown>)}`),+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "drag", () => dragPlan(params), ctx.cwd, signal); },+ });+ pi.registerTool({+ ...common, name: "computer_scroll", label: "Computer scroll",+ description: "Use computer_scroll to turn the mouse wheel over an element, a point or the middle of a window or page.",+ parameters: scrollParameters, annotations: inputAnnotations,+ ...rendered("computer_scroll", args => `${args.window} ${aimed(args)} ${args.delta_x ?? 0},${args.delta_y ?? 0}`),+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "scroll", () => scrollPlan(params), ctx.cwd, signal); },+ });+ pi.registerTool({+ ...common, name: "computer_type", label: "Computer type",+ description: "Use computer_type to type text into a field or the focused element, or to replace a field's value, optionally pressing Enter after it.",+ parameters: typeParameters, annotations: inputAnnotations,+ ...rendered("computer_type", args => `${args.window} ${aimed(args)} ${String(args.text ?? "").length} chars${args.submit ? " + Enter" : ""}`),+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "type", () => typePlan(params), ctx.cwd, signal); },+ });+ pi.registerTool({+ ...common, name: "computer_press", label: "Computer press",+ description: "Use computer_press to press a key or shortcut such as Enter or ctrl+s, or to hold keys down for a time.",+ parameters: pressParameters, annotations: inputAnnotations,+ ...rendered("computer_press", args => `${args.window} ${args.key}${args.hold_ms ? ` for ${args.hold_ms} ms` : ""}`),+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "press", () => pressPlan(params), ctx.cwd, signal); },+ });+ pi.registerTool({+ ...common, name: "computer_select", label: "Computer select",+ description: "Use computer_select to choose options in a list or select element, set a checkbox, radio or toggle, or open an app's menu item.",+ parameters: selectParameters, annotations: inputAnnotations,+ ...rendered("computer_select", args => `${args.window} ${args.menu ? (args.menu as string[]).join(" > ") : `${args.target ?? ""} ${args.values ? (args.values as string[]).join(", ") : args.checked !== undefined ? (args.checked ? "checked" : "unchecked") : ""}`}`),+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "select", () => selectPlan(params), ctx.cwd, signal); },+ });+ pi.registerTool({+ ...common, name: "computer_timeline", label: "Computer timeline",+ description: "Use computer_timeline to play timed key, button, pointer, wheel and pen events in one call, for games, held keys across moves and pressure-sensitive strokes.",+ parameters: timelineParameters, annotations: inputAnnotations,+ ...rendered("computer_timeline", args => `${args.window} ${(args.events as unknown[] | undefined)?.length ?? 0} events`),+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "timeline", () => timelinePlan(params), ctx.cwd, signal); },+ });pi.registerTool({...common, name: "computer_display", label: "Computer display",description: "Use computer_display to change a display's resolution in YOLO mode, for example to test an app at another size; closing computer use restores it.",diff --git a/src/session.ts b/src/session.tsindex 14acf09..fb91dd9 100644--- a/src/session.ts+++ b/src/session.ts@@ -146,7 +146,10 @@ interface Observation { texts?: Map<string, string>; id: string; tokens: Set<strinterface WindowState { id: string; nativeId: string; kind: "restore" | "management" }/** The unannotated pixels of a window's latest capture with pixels, kept as the baseline of its next pixel comparison. */export interface WindowFrame { observation_id: string; png: Buffer; size: { maxLongEdge: number; scale: number } }-interface WindowEntry { ref: string; target: WindowTarget; identity: string; owned: boolean; observation?: Observation; windowState?: WindowState; dom?: { id: string; page: WebDom }; web?: WebBrowser; detail?: DetailContext; lastAction?: GuideMark; frame?: WindowFrame }+interface WindowEntry { ref: string; target: WindowTarget; identity: string; owned: boolean; observation?: Observation;+ /** Size of the window's latest image capture: points of typed input are in its pixels. Outlives the observation. */+ seen?: { width: number; height: number };+ windowState?: WindowState; dom?: { id: string; page: WebDom }; web?: WebBrowser; detail?: DetailContext; lastAction?: GuideMark; frame?: WindowFrame }interface App { name: string; launch_path?: string; bundle_id?: string; kind?: string }/** A discovery query: `regex:<pattern>` is a case-insensitive regular expression, anything else a case-insensitive substring. */export function textMatcher(query?: string): (text: string) => boolean {@@ -1029,6 +1032,7 @@ export class ComputerSession {const full = [element.label, element.value].filter((item): item is string => typeof item === "string" && item.length > 1024);if (full.length && ui.elements[index]) texts.set(ui.elements[index].hint, full.join("\n").slice(0, ELEMENT_TEXT_LIMIT));});+ if (hasImage && typeof state.screenshot_width === "number" && typeof state.screenshot_height === "number") entry.seen = { width: state.screenshot_width, height: state.screenshot_height };entry.observation = { id, ui, settings, ...(texts.size ? { texts } : {}),width: hasImage && typeof state.screenshot_width === "number" ? state.screenshot_width : undefined,height: hasImage && typeof state.screenshot_height === "number" ? state.screenshot_height : undefined,@@ -1790,6 +1794,17 @@ export class ComputerSession {}/** Whether a window still holds this observation for input. */holdsObservation(windowRef: string, id: string): boolean { return this.entries.get(windowRef)?.observation?.id === id; }+ /**+ * What typed input can use in a window now: its current observation (a DOM snapshot of a browser page, or a native+ * capture with its size and, until the first input spends them, its UI hints) and the size of its latest image,+ * which is the space of the agent's points.+ */+ inputView(ref: string): { dom?: string; native?: { id: string; width?: number; height?: number; ui?: UiResource }; seen?: { width: number; height: number } } {+ const entry = this.entries.get(ref), observed = entry?.observation;+ return { ...(entry?.dom ? { dom: entry.dom.id } : {}),+ ...(observed ? { native: { id: observed.id, width: observed.width, height: observed.height, ...(observed.ui && !observed.pixelsOnly ? { ui: observed.ui } : {}) } } : {}),+ ...(entry?.seen ? { seen: { ...entry.seen } } : {}) };+ }/** Whether this session issued a still-registered window reference. */hasWindow(ref: string): boolean { return this.entries.has(ref); }/** Whether this session issued a still-registered display reference. */diff --git a/src/web-browser.ts b/src/web-browser.tsindex c972950..0e856e1 100644--- a/src/web-browser.ts+++ b/src/web-browser.ts@@ -30,6 +30,8 @@ export interface WebAction {| "upload" | "wait_download" | "cancel_download" | "evaluate";target?: WebTarget;button?: "left" | "right" | "middle"; click_count?: number;+ /** Keys held during a click or hover. */+ modifiers?: ("Alt" | "Control" | "Meta" | "Shift")[];text?: string; key?: string; values?: string[];delta_x?: number; delta_y?: number;/** Element state for wait; load state for wait_load. */@@ -174,10 +176,10 @@ export function validateWebSnapshot(value: unknown = {}): WebSnapshotOptions {}const ACTIONS: Record<WebAction["action"], { fields: readonly string[]; target: "required" | "optional" | "none" }> = {- click: { fields: ["button", "click_count"], target: "required" },- hover: { fields: [], target: "required" },+ click: { fields: ["button", "click_count", "modifiers"], target: "required" },+ hover: { fields: ["modifiers"], target: "required" },fill: { fields: ["text"], target: "required" },- type: { fields: ["text"], target: "required" },+ type: { fields: ["text"], target: "optional" },press: { fields: ["key"], target: "optional" },check: { fields: [], target: "required" },uncheck: { fields: [], target: "required" },@@ -219,6 +221,11 @@ export function validateWebAction(value: unknown): WebAction {action.button = command.button as WebAction["button"];}if (command.click_count !== undefined) action.click_count = integer(command.click_count, 1, 3, "click_count");+ if (command.modifiers !== undefined) {+ if (!Array.isArray(command.modifiers) || command.modifiers.length > 4 || new Set(command.modifiers).size !== command.modifiers.length || command.modifiers.some(key => !["Alt", "Control", "Meta", "Shift"].includes(key as string)))+ throw Error("modifiers must be unique Alt, Control, Meta or Shift.");+ action.modifiers = command.modifiers as WebAction["modifiers"];+ }if (name === "fill" || name === "type") {if (typeof command.text !== "string" || command.text.length > 20_000) throw Error(`${name} needs text of at most 20000 characters.`);action.text = command.text;diff --git a/src/web-worker.ts b/src/web-worker.tsindex 7c4e161..20a5b69 100644--- a/src/web-worker.ts+++ b/src/web-worker.ts@@ -449,10 +449,10 @@ async function act(action: WebAction) {const target = action.target && locate(page, action.target);const run = async (): Promise<unknown> => {switch (action.action) {- case "click": return target!.click({ button: action.button, clickCount: action.click_count, timeout });- case "hover": return target!.hover({ timeout });+ case "click": return target!.click({ button: action.button, clickCount: action.click_count, modifiers: action.modifiers, timeout });+ case "hover": return target!.hover({ modifiers: action.modifiers, timeout });case "fill": return target!.fill(action.text!, { timeout });- case "type": return target!.pressSequentially(action.text!, { timeout });+ case "type": return target ? target.pressSequentially(action.text!, { timeout }) : page.keyboard.type(action.text!);case "press": return target ? target.press(action.key!, { timeout }) : page.keyboard.press(action.key!);case "check": return target!.check({ timeout });case "uncheck": return target!.uncheck({ timeout });diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.tsindex bc5578d..45f0fb6 100644--- a/test/computer-tools.test.ts+++ b/test/computer-tools.test.ts@@ -1,6 +1,6 @@import { expect, test, vi } from "vitest";-import { requireComputerAddress, ComputerResources } from "../src/computer-resources.ts";-import { clipboardCommand, filesCommand, openCommand, recordCommand, waitCommand, windowCommand } from "../src/computer-tools.ts";+import { requireComputerAddress, ComputerResources, refKind } from "../src/computer-resources.ts";+import { clickPlan, clipboardCommand, dragPlan, filesCommand, hoverPlan, openCommand, pressPlan, recordCommand, scrollPlan, selectPlan, timelinePlan, typePlan, waitCommand, windowCommand } from "../src/computer-tools.ts";import type { WebBrowser, WebLauncher } from "../src/web-browser.ts";import type { MachineHost } from "../src/machine-host.ts";import { mkdtemp, readFile, rm } from "node:fs/promises";@@ -80,6 +80,51 @@ test("computer_record maps video and audio to machine commands and sequence to texpect(() => recordCommand({ at: "m", action: "video_stop", seconds: 3 })).toThrow("video_stop takes no seconds.");});+test("input tools build one plan for native windows and one for browser pages", () => {+ const w = "computer:w";+ expect(refKind("1:a")).toBe("hint");+ expect(refKind("2b:ac")).toBe("hint");+ expect(refKind("e12")).toBe("page");+ expect(refKind("f1e3")).toBe("page");+ expect(refKind("getByRole('button', { name: 'Save' })")).toBe("locator");+ expect(clickPlan({ window: w, target: "1:a" })).toEqual({ label: "click", target: "1:a", native: { action: "click" }, web: { action: "click" } });+ expect(clickPlan({ window: w, x: 10, y: 20, count: 2, modifiers: ["ctrl"] })).toEqual({ label: "click", point: { x: 10, y: 20 },+ native: { action: "double_click", modifiers: ["ctrl"] }, web: { action: "click", click_count: 2, modifiers: ["Control"] } });+ expect(clickPlan({ window: w, target: "e1", button: "right" }).native).toEqual({ action: "right_click" });+ expect(clickPlan({ window: w, target: "e1", count: 3 }).native).toBe("A triple click works on browser pages only.");+ expect(() => clickPlan({ window: w })).toThrow("click needs a target or a point (x and y).");+ expect(() => clickPlan({ window: w, x: 1 })).toThrow("click needs both x and y for a point.");+ expect(() => clickPlan({ window: w, x: 1, y: 2, target: "1:a" })).toThrow("click takes a target or a point, not both.");+ expect(hoverPlan({ window: w, target: "1:a" }).native).toEqual({ action: "hover", dwell_ms: 300 });+ expect(dragPlan({ window: w, from: { target: "1:a" }, to: { x: 5, y: 6 }, via: [{ x: 3, y: 4 }] })).toMatchObject({ target: "1:a", end: { point: { x: 5, y: 6 } }, native: { action: "drag", via: [{ x: 3, y: 4 }] } });+ expect(scrollPlan({ window: w, delta_y: -2 })).toMatchObject({ native: { action: "scroll", delta_y: -2, unit: "wheel_detents" }, web: { action: "scroll", delta_y: -200 } });+ expect(scrollPlan({ window: w, target: "1:a", delta_x: 3 }).native).toEqual({ action: "scroll", direction: "right", amount: 3 });+ expect(() => scrollPlan({ window: w })).toThrow("scroll needs delta_x or delta_y.");+ const typed = typePlan({ window: w, target: "1:a", text: "Ada", replace: true, submit: true });+ expect(typed).toMatchObject({ native: { action: "set_value", text: "Ada" }, web: { action: "fill", text: "Ada" },+ then: { target: "1:a", native: { action: "key", key: "return" }, web: { action: "press", key: "Enter" } } });+ expect(() => typePlan({ window: w, text: "x", replace: true })).toThrow("replace needs the field as target.");+ expect(pressPlan({ window: w, key: "ctrl+shift+Tab" })).toMatchObject({ native: { action: "key", key: "tab", modifiers: ["ctrl", "shift"] }, web: { action: "press", key: "Control+Shift+Tab" } });+ expect(pressPlan({ window: w, key: "w", hold_ms: 500 }).native).toEqual({ action: "hold", keys: ["w"], duration_ms: 500 });+ expect(pressPlan({ window: w, key: "f5" }).web).toEqual({ action: "press", key: "F5" });+ expect(() => pressPlan({ window: w, key: "hyper+a" })).toThrow("Modifiers are ctrl, shift, alt and win.");+ expect(selectPlan({ window: w, target: "e3", values: ["Red"] })).toMatchObject({ native: { action: "select", option: "Red" }, web: { action: "select", values: ["Red"] } });+ expect(selectPlan({ window: w, target: "1:a", checked: false })).toMatchObject({ native: { action: "uncheck" }, web: { action: "uncheck" } });+ expect(selectPlan({ window: w, menu: ["File", "Save"] })).toMatchObject({ native: { action: "menu", path: ["File", "Save"] } });+ expect(() => selectPlan({ window: w, target: "1:a", checked: true, values: ["a"] })).toThrow("select takes one of values, checked or menu");+ expect(timelinePlan({ window: w, events: [{ at_ms: 0, type: "key_down", key: "a" }, { at_ms: 5, type: "key_up", key: "a" }] }).native).toMatchObject({ action: "timeline" });+});++test("typed input refuses mismatched targets before anything reaches the machine", async () => {+ const f = fixture("closed");+ const window = at(`computer:window/${await f.window()}`);+ await expect(f.resources.input(at("computer:machine/local"), cwd, clickPlan({ window: "x", target: "1:a" }))).rejects.toThrow("Give the window address");+ await expect(f.resources.input(window, cwd, clickPlan({ window: "x", target: "e4" }))).rejects.toThrow("e4 is a page ref from a DOM Read; this window is not a browser page");+ await expect(f.resources.input(window, cwd, clickPlan({ window: "x", x: 1, y: 2, space: "page" }))).rejects.toThrow("space applies to browser windows only");+ await expect(f.resources.input(window, cwd, clickPlan({ window: "x", target: "9:z" }))).rejects.toThrow("9:z is not a ref of this window's latest Read");+ await expect(f.resources.input(window, cwd, { ...hoverPlan({ window: "x", target: "getByText('a')" }), foreground: false })).rejects.toThrow("hover on a native window moves the real pointer");+});+/** A Chrome window (pid 4100) attached to a fake Playwright worker that records page actions. */function browserFixture() {const target = { pid: 4100, window_id: 77, title: "Page - Google Chrome", app_name: "chrome.exe" };@@ -119,6 +164,22 @@ test("computer_browser runs a page action on the window and returns the page's nexpect(f.page).toHaveBeenCalledTimes(1);});+test("typed input on a browser window goes to Playwright with page refs and locators", async () => {+ const f = browserFixture();+ const window = await f.window();+ const clicked = await f.resources.input(window, cwd, clickPlan({ window: "x", target: "e1", modifiers: ["shift"] }));+ expect(clicked.details).toMatchObject({ status: "delivered", machine: "local" });+ expect(f.page).toHaveBeenLastCalledWith("act", { action: "click", target: { ref: "e1" }, modifiers: ["Shift"] }, undefined);+ // The snapshot of the first action is the window's current observation, so the next one compares with it.+ await f.resources.input(window, cwd, typePlan({ window: "x", target: "getByLabel('Name')", text: "Ada", submit: true }));+ expect(f.page.mock.calls.slice(-2).map(call => call[1])).toEqual([+ { action: "type", text: "Ada", target: { locator: "internal:label=\"Name\"i" } },+ { action: "press", key: "Enter", target: { locator: "internal:label=\"Name\"i" } }]);+ await expect(f.resources.input(window, cwd, clickPlan({ window: "x", x: 5, y: 5 }))).rejects.toThrow("click at a point of a browser page is not available yet");+ await expect(f.resources.input(window, cwd, selectPlan({ window: "x", menu: ["File"] }))).rejects.toThrow("menu works on native windows");+ expect(f.page).toHaveBeenCalledTimes(3);+});+/** One window (pid 10, id 7) a session can list and observe; the host decides how a close request ends. */function fixture(outcome: "closed" | "open" | "refused") {let open = true, mode: ComputerMode = "yolo";diff --git a/test/live/typed-input.live.ts b/test/live/typed-input.live.tsnew file mode 100644index 0000000..b877f3c--- /dev/null+++ b/test/live/typed-input.live.ts@@ -0,0 +1,87 @@+import { afterAll, expect, test } from "vitest";+import { requireComputerAddress, type InputPlan } from "../../src/computer-resources.ts";+import { clickPlan, dragPlan, hoverPlan, pressPlan, scrollPlan, selectPlan, timelinePlan, typePlan, waitCommand } from "../../src/computer-tools.ts";+import { presentComputerResult } from "../../src/provider.ts";+import { deleteComputer } from "../../src/provider.ts";+import { connectLive, nativeForm, openWindow, serveFixture, WINDOWS_POWERSHELL_PATH } from "./remote.ts";++// LPT-927: the typed input tools on a native WinForms window and on a Playwright browser page of the configured machine.+const live = await connectLive();+const fixture = await serveFixture(live.runner);+afterAll(async () => { await live.close(); fixture.close(); });+const timings: string[] = [];+const act = async (window: string, plan: InputPlan) => {+ const started = Date.now();+ const result = await live.resources.input(requireComputerAddress(window), live.cwd, plan);+ const body = presentComputerResult(result);+ timings.push(`${plan.label}${plan.target ? ` ${plan.target}` : plan.point ? ` ${plan.point.x},${plan.point.y}` : ""}: ${result.details.status} in ${Date.now() - started} ms`);+ console.log(timings.at(-1));+ return { result, body };+};+const holds = async (window: string, conditions: Parameters<typeof waitCommand>[0]["conditions"]) => {+ const result = await live.insert(window, waitCommand({ window, timeout_ms: 3000, conditions }));+ return (result.details.verification as { verdict?: string } | undefined)?.verdict;+};+afterAll(() => console.log(timings.join("\n")));++test("one action vocabulary drives a native window by locator, ref and point", async () => {+ const title = `pi-input-${Date.now()}`;+ const window = await openWindow(live, live.machine, WINDOWS_POWERSHELL_PATH, nativeForm(title), title);+ try {+ const button = "getByRole('button', { name: /^Count/ })";+ expect((await act(window, clickPlan({ window, target: button }))).result.details.status).toBe("delivered");+ expect(await holds(window, [{ target: "getByRole('button', { name: 'Count 1' })", is: "exists" }])).toBe("satisfied");+ // A full Read gives the button's place in the image; a later UI-only Read keeps that image as the space of points.+ const full = await live.read(window, ["full"]);+ const element = (full.details.ui as { elements: { hint: string; role: string; name?: string; bounds: { window_image?: { x: number; y: number; width: number; height: number } } }[] })+ .elements.find(item => /Count 1/.test(item.name ?? ""))!;+ const box = element.bounds.window_image!;+ const centre = { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) };+ const aria = await live.read(window, ["aria"]);+ const ref = /button "Count 1" \[ref=([0-9a-z]+:[a-z]+)\]/.exec(presentComputerResult(aria))![1];+ expect((await act(window, clickPlan({ window, target: ref }))).result.details.status).toBe("delivered");+ expect(await holds(window, [{ target: "getByRole('button', { name: 'Count 2' })", is: "exists" }])).toBe("satisfied");+ await live.read(window, ["aria"]);+ // The UI-only Read has no pixels, so the point click captures the window first and keeps the full Read's scale.+ expect((await act(window, clickPlan({ window, ...centre }))).result.details.status).toBe("delivered");+ expect(await holds(window, [{ target: "getByRole('button', { name: 'Count 3' })", is: "exists" }])).toBe("satisfied");+ const field = "getByRole('textbox', { name: 'Field' })";+ expect((await act(window, typePlan({ window, target: field, text: "hello" }))).result.details.status).toBe("delivered");+ expect(await holds(window, [{ target: field, value: "hello" }])).toBe("satisfied");+ expect((await act(window, typePlan({ window, target: field, text: "world", replace: true, submit: true }))).result.details.status).toBe("delivered");+ expect(await holds(window, [{ target: field, value: "world" }])).toBe("satisfied");+ expect((await act(window, pressPlan({ window, key: "ctrl+a" }))).result.details.status).toBe("delivered");+ expect((await act(window, pressPlan({ window, key: "Backspace" }))).result.details.status).toBe("delivered");+ // An empty field's value falls back to its name in conditions, so its aria line shows the emptiness: no ": value".+ expect(presentComputerResult(await live.read(window, ["aria"]))).toMatch(/textbox "Field" \[ref=/);+ expect((await act(window, selectPlan({ window, target: "getByRole('list', { name: 'Pick' })", values: ["beta"] }))).result.details.status).toBe("delivered");+ expect(await holds(window, [{ target: "getByRole('option', { name: 'beta' })", is: "selected" }])).toBe("satisfied");+ expect((await act(window, scrollPlan({ window, target: "getByRole('list', { name: 'Pick' })", delta_y: 1 }))).result.details.status).not.toBe("unknown");+ expect((await act(window, scrollPlan({ window, x: centre.x, y: centre.y + 60, delta_y: 1 }))).result.details.status).toBe("delivered");+ expect((await act(window, hoverPlan({ window, target: button }))).result.details.status).toBe("delivered");+ expect((await act(window, dragPlan({ window, from: { x: centre.x, y: centre.y }, to: { x: centre.x + 40, y: centre.y } }))).result.details.status).toBe("delivered");+ const timeline = await act(window, timelinePlan({ window, events: [{ at_ms: 0, type: "move", x: centre.x, y: centre.y }, { at_ms: 50, type: "button_down" }, { at_ms: 100, type: "button_up" }] }));+ expect(timeline.result.details.status, timeline.body).toBe("delivered");+ expect(await holds(window, [{ target: "getByRole('button', { name: /^Count [45]$/ })", is: "exists" }])).toBe("satisfied");+ } finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }+});++test("the same vocabulary drives a browser page through Playwright by locator and page ref", async () => {+ const opened = await live.insert(live.machine, { action: "open_browser", url: `${fixture.base}/wait` });+ expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");+ const window = `computer:window/${opened.details.window_ref}`;+ try {+ expect((await act(window, typePlan({ window, target: "getByLabel('Name')", text: "Ada" }))).result.details.status).toBe("delivered");+ const dom = presentComputerResult(await live.read(window, ["dom"]));+ const save = /button "Save" \[ref=(e\d+)\]/.exec(dom)![1];+ const clicked = await act(window, clickPlan({ window, target: save }));+ expect(clicked.result.details.status, clicked.body).toBe("delivered");+ expect(await holds(window, [{ target: { text: "saved Ada" }, is: "visible" }])).toBe("satisfied");+ expect((await act(window, typePlan({ window, target: "getByLabel('Name')", text: "Grace", replace: true, submit: true }))).result.details.status).toBe("delivered");+ expect((await act(window, pressPlan({ window, key: "ctrl+a" }))).result.details.status).toBe("delivered");+ expect((await act(window, hoverPlan({ window, target: "getByRole('button', { name: 'Unlock later' })" }))).result.details.status).toBe("delivered");+ expect((await act(window, scrollPlan({ window, delta_y: 2 }))).result.details.status).toBe("delivered");+ const refused = await live.resources.input(requireComputerAddress(window), live.cwd, timelinePlan({ window, events: [{ at_ms: 0, type: "key_down", key: "a" }, { at_ms: 10, type: "key_up", key: "a" }] })).catch((error: Error) => error.message);+ expect(String(refused)).toContain("timeline on browser pages");+ } finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }+});