Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

feat: typed input tools on native windows and browser elements (LPT-927 part a)

alexshpunt-clanker[bot]f6c72d670f2e
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index 1944b5b..8078bfb 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -49,7 +49,38 @@ const BATCH_STEPS = 200;
/** Native actions a batch may run; each later step re-resolves its target in a fresh UI walk. */
const NATIVE_BATCH_ACTIONS = ["click", "double_click", "right_click", "type_text", "key", "hotkey", "scroll", "invoke", "select", "set_value"];
/** Native actions that take a Playwright locator or JSON target. */
-const NATIVE_LOCATOR_ACTIONS = ["click", "double_click", "right_click", "hover", "type_text", "scroll", "invoke", "select", "set_value", "check", "uncheck"];
+const NATIVE_LOCATOR_ACTIONS = ["click", "double_click", "right_click", "hover", "type_text", "key", "scroll", "invoke", "select", "set_value", "check", "uncheck"];
+/** A point in pixels of a window's latest image, or in CSS pixels of a browser page. */
+export interface InputPoint { x: number; y: number }
+/** One typed input on a window, built by a computer_* input tool; the window decides which backend runs it. */
+export interface InputPlan {
+ /** The tool's action for refusals, such as click. */
+ label: string;
+ /** A ref of the window's latest Read (a native ref such as 1:a, a page ref such as e12) or a Playwright locator. */
+ target?: string;
+ point?: InputPoint;
+ /** drag: where it ends. */
+ end?: { target?: string; point?: InputPoint };
+ /** Browser windows: page (CSS pixels of the page, the default) or window (pixels of the window's latest image). */
+ space?: "page" | "window";
+ /** The native command without its target, or why native windows cannot run it. */
+ native: Record<string, unknown> | string;
+ /** The Playwright action without its target, or why browser pages cannot run it. */
+ web: Record<string, unknown> | string;
+ /** true sends native input in the foreground at once; false never leaves the background. */
+ foreground?: boolean;
+ /** Runs after this one is delivered, such as Enter after typing. */
+ then?: InputPlan;
+}
+/** Native input that moves the real pointer, so it always goes in the foreground. */
+const FOREGROUND_ONLY = ["hover", "move", "timeline"];
+/** Native actions that only a locator target runs; a ref of the latest Read becomes the locator of its element. */
+const LOCATOR_ONLY = ["check", "uncheck"];
+/** What a target string is: a native ref of a window Read (1:a), a page ref of a DOM Read (e12, f1e3), or a Playwright locator. */
+export function refKind(target: unknown): "hint" | "page" | "locator" | undefined {
+ if (typeof target !== "string") return undefined;
+ return /^[0-9a-z]+:[a-z]+$/.test(target) ? "hint" : /^(?:f\d+)?e\d+$/.test(target) ? "page" : "locator";
+}
/** Playwright verbs with a different native name. */
const LOCATOR_VERBS: Record<string, string> = { fill: "set_value", dblclick: "double_click" };
/**
@@ -287,6 +318,8 @@ export class ComputerResources {
private readonly starting = new Map<string, Promise<OpenedMachine>>();
/** Sandbox launches and machine starts still in progress; a cancelled one finishes its own cleanup before it settles. */
private readonly opening = new Set<Promise<unknown>>();
+ /** Windows that refused background input once; typed input to them goes in the foreground. */
+ private readonly foreground = new Set<string>();
/** Short aliases of this run's long addresses (computer:zinpc/w3qk). */
private readonly shorts = new ShortAddresses();
/** Window names the agent gave, with the window's title when it was named. */
@@ -877,6 +910,127 @@ export class ComputerResources {
const result = await session.actPage(address.ref, action, signal);
return this.stamp(place, result.details.dom ? this.remember({ window_ref: address.ref }, result) : result);
}
+ /**
+ * One typed input on a window by its address. The window picks the backend: Playwright for page refs, locators
+ * and CSS points of a browser page, the native driver and UI Automation otherwise. Native points are pixels of the
+ * window's latest image; without a current one the window is captured first. The receipt names its machine.
+ */
+ async input(address: ComputerAddress, cwd: string, plan: InputPlan, signal?: AbortSignal): Promise<ComputerResult> {
+ if (address.kind !== "window") throw new ComputerRequestError("Give the window address from Search, Read or a name, such as computer:zinpc/w3qk or computer:game.");
+ const place = this.place(address);
+ const session = await this.sessionFor(cwd, { window_ref: address.ref }, signal);
+ let result = await this.inputStep(address.ref, session, plan, cwd, signal);
+ // A follow-up such as Enter after typing runs only after the first part was delivered.
+ if (plan.then && result.details.status === "delivered") {
+ const next = await this.inputStep(address.ref, session, plan.then, cwd, signal);
+ result = { ...next, content: [...result.content, ...next.content] };
+ }
+ return this.stamp(place, result);
+ }
+ private async inputStep(window: string, session: ComputerSession, plan: InputPlan, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {
+ const ref = refKind(plan.target);
+ const browser = session.isBrowserWindow(window);
+ if (browser && plan.space !== "window" && ref !== "hint") return this.pageInput(window, session, plan, cwd, signal);
+ if (plan.space !== undefined && !browser) throw new ComputerRequestError("space applies to browser windows only; points on other windows are pixels of the window's latest image.");
+ if (ref === "page") throw new ComputerRequestError(`${plan.target} is a page ref from a DOM Read; ${browser ? "omit space: \"window\" to act on the page" : "this window is not a browser page, so use a ref from its latest Read or a Playwright locator"}.`);
+ return this.nativeStep(window, session, plan, cwd, signal);
+ }
+
+ /** Typed input on the active page of a browser window through Playwright; a current DOM observation gives the change list. */
+ private async pageInput(window: string, session: ComputerSession, plan: InputPlan, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {
+ if (typeof plan.web === "string") throw new ComputerRequestError(plan.web);
+ if (plan.point || plan.end) throw new ComputerRequestError(`${plan.label} at a point of a browser page is not available yet; give a target, or space: "window" for window pixels.`);
+ const target = typeof plan.target === "string" ? refKind(plan.target) === "page" ? { ref: plan.target } : plan.target : undefined;
+ const command = { ...plan.web, ...(target !== undefined ? { target } : {}) };
+ let action: WebAction;
+ try { action = validateWebAction(command); }
+ catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }
+ const dom = session.inputView(window).dom;
+ if (dom && this.snapshots.has(dom)) return this.performAt({ kind: "observation", ref: dom, source: computerSource("observation", dom) }, cwd, JSON.stringify(command), "insert", signal);
+ this.retireWindow(window);
+ const result = await session.actPage(window, action, signal);
+ return result.details.dom ? this.remember({ window_ref: window }, result) : result;
+ }
+
+ /**
+ * Typed input on a native window: a locator resolves in fresh UI walks, a ref of the latest Read and points use the
+ * window's current observation, captured first when there is none with the pixels a point needs. A background
+ * refusal, which sends nothing, is retried at once in the foreground, and the window keeps foreground delivery.
+ */
+ private async nativeStep(window: string, session: ComputerSession, plan: InputPlan, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {
+ if (typeof plan.native === "string") throw new ComputerRequestError(plan.native);
+ const command: Record<string, unknown> = structuredClone(plan.native);
+ const action = String(command.action);
+ const pointerOnly = FOREGROUND_ONLY.includes(action);
+ if (pointerOnly && plan.foreground === false) throw new ComputerRequestError(`${plan.label} on a native window moves the real pointer, so it needs the foreground.`);
+ const foreground = plan.foreground ?? (pointerOnly || this.foreground.has(window));
+ if (foreground && !isSemanticAction(action)) command.delivery_mode = "foreground";
+ let target = plan.target;
+ let view = session.inputView(window);
+ // Actions that only exist on locators take a ref of the latest Read as the locator that names its element.
+ if (refKind(target) === "hint" && (LOCATOR_ONLY.includes(action) || command.option !== undefined)) target = suggestLocator(this.hinted(view, target as string));
+ if (command.option !== undefined) {
+ if (refKind(target) !== "locator") throw new ComputerRequestError(`${plan.label} with values needs the list as target.`);
+ target = `${target}.getByRole("option", { name: ${JSON.stringify(command.option)}, exact: true })`;
+ delete command.option;
+ }
+ let result: ComputerResult;
+ // What a foreground retry sends: the input observation is spent by then, so its refs become locators or points.
+ let retry = (): InputPlan => plan;
+ if (refKind(target) === "locator") {
+ if (plan.end) throw new ComputerRequestError(`${plan.label} takes refs of the window's latest Read or points, not locators.`);
+ result = await this.performAt({ kind: "window", ref: window, source: computerSource("window", window) }, cwd, JSON.stringify({ ...command, target }), "insert", signal);
+ } else {
+ const hint = refKind(target) === "hint" ? target as string : undefined;
+ // Only a locator waits for its element; refs and points act on the current observation at once.
+ delete command.timeout_ms;
+ const pixels = plan.point !== undefined || plan.end !== undefined || command.unit === "wheel_detents" || action === "timeline";
+ if (hint) this.hinted(view, hint);
+ else if (!view.native || pixels && view.native.width === undefined) {
+ const captured = await this.read({ kind: "window", ref: window, source: computerSource("window", window) }, cwd, ["image"], signal);
+ if (!captured || captured.details.status !== "observed") return captured ?? { content: [{ type: "text", text: "The window could not be captured; nothing was sent." }], details: { status: "refused", mode: this.getMode(), window_ref: window }, isError: true };
+ view = session.inputView(window);
+ }
+ const native = view.native!;
+ if (pixels && native.width === undefined) throw new ComputerRequestError(`${plan.label} at a point needs the window's pixels: Read the window with an image, or give a locator.`);
+ // Points are in the latest image the agent saw; the current observation can be a capture of another size.
+ const sx = native.width && view.seen ? native.width / view.seen.width : 1, sy = native.height && view.seen ? native.height / view.seen.height : 1;
+ const at = (point: InputPoint) => ({ x: Math.min(native.width! - 1, Math.round(point.x * sx)), y: Math.min(native.height! - 1, Math.round(point.y * sy)) });
+ const seen = (point: InputPoint) => ({ x: point.x / sx, y: point.y / sy });
+ const centre = (ref: string) => {
+ const box = this.hinted(view, ref).bounds.window_image;
+ if (!box || native.width === undefined) throw new ComputerRequestError(`${ref} has no place in the window's image; give a point or a locator.`);
+ return { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) };
+ };
+ if (action === "drag") {
+ const from = hint ? centre(hint) : plan.point ? at(plan.point) : undefined;
+ const end = plan.end?.target !== undefined ? refKind(plan.end.target) === "hint" ? centre(plan.end.target as string) : undefined : plan.end?.point ? at(plan.end.point) : undefined;
+ if (!from || !end) throw new ComputerRequestError("drag on a native window goes from and to refs of the window's latest Read or points.");
+ Object.assign(command, { from_x: from.x, from_y: from.y, to_x: end.x, to_y: end.y });
+ if (Array.isArray(command.via)) command.via = (command.via as InputPoint[]).map(at);
+ retry = () => ({ ...plan, target: undefined, point: seen(from), end: { point: seen(end) } });
+ } else if (action === "timeline") {
+ command.events = (command.events as Record<string, unknown>[]).map(event => typeof event.x === "number" && typeof event.y === "number" ? { ...event, ...at(event as unknown as InputPoint) } : event);
+ } else if (plan.point) Object.assign(command, at(plan.point));
+ else if (command.unit === "wheel_detents") Object.assign(command, { x: Math.floor(native.width! / 2), y: Math.floor(native.height! / 2) });
+ if (hint && action !== "drag") retry = () => NATIVE_LOCATOR_ACTIONS.includes(LOCATOR_VERBS[action] ?? action) ? { ...plan, target: suggestLocator(this.hinted(view, hint)) }
+ : { ...plan, target: undefined, point: seen(centre(hint)) };
+ const observation = { kind: "observation" as const, ref: native.id, source: computerSource("observation", native.id), ...(hint && action !== "drag" ? { hint } : {}) };
+ result = await this.performAt(observation, cwd, JSON.stringify(command), "insert", signal);
+ }
+ const capability = result.details.action_capability;
+ if (foreground || plan.foreground === false || result.details.status !== "refused" || capability?.background !== "unavailable") return result;
+ this.foreground.add(window);
+ const again = await this.nativeStep(window, session, { ...retry(), foreground: true }, cwd, signal);
+ return { ...again, content: [{ type: "text", text: `The window refused background input (${capability.code}), so this call sent it in the foreground; later input to this window goes in the foreground.` }, ...again.content] };
+ }
+
+ /** The element of a ref in the window's current observation. */
+ private hinted(view: ReturnType<ComputerSession["inputView"]>, hint: string): UiElement {
+ const element = view.native?.ui?.elements.find(item => item.hint === hint);
+ if (!element) throw new ComputerRequestError(`${hint} is not a ref of this window's latest Read, or input since then spent it: Read the window again, or give a Playwright locator.`);
+ return element;
+ }
/** Dispatch exactly one literal Write or JSON Insert against its address-bound target; the receipt names its machine. */
async perform(address: ComputerAddress, cwd: string, value: string, tool: "write" | "insert", signal?: AbortSignal): Promise<ComputerResult> {
const place = this.place(address);
diff --git a/src/computer-tools.ts b/src/computer-tools.ts
index cbae9fa..8c96c6b 100644
--- a/src/computer-tools.ts
+++ b/src/computer-tools.ts
@@ -1,7 +1,7 @@
import { Type, type JsonValue, type Static } from "@earendil-works/pi-ai";
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
import { Text } from "@earendil-works/pi-tui";
-import { ComputerRequestError, type ComputerResources } from "./computer-resources.ts";
+import { ComputerRequestError, type ComputerResources, type InputPlan } from "./computer-resources.ts";
import type { ComputerAddress } from "./computer-address.ts";
import { computerActionRenderer } from "./computer-action-renderer.ts";
import { parseComputerViews } from "./computer-views.ts";
@@ -242,6 +242,188 @@ export function recordCommand(params: Static<typeof recordParameters>): { comman
return { command: { action: "audio_record", ...fields } };
}
+const inputWindow = Type.String({ description: "Window address from Search, Read or a name, such as computer:zinpc/w3qk or computer:game." });
+const inputTarget = (what: string) => Type.Optional(Type.String({ description: `${what}: a ref from the window's latest Read (1:a in a window Read, e12 in a page snapshot) or a Playwright locator such as getByRole('button', { name: 'Save' }).` }));
+const coordinate = (axis: string) => Type.Optional(Type.Number({ minimum: 0, description: `${axis} of a point instead of target: pixels of the window's latest image, or CSS pixels of a browser page.` }));
+const pointSpace = Type.Optional(Type.Union([Type.Literal("page"), Type.Literal("window")], { description: "Browser windows: page (default) takes refs, locators and CSS points of the page; window takes refs and points of a window Read, such as the browser's own toolbar." }));
+const foregroundField = Type.Optional(Type.Boolean({ description: "Native windows: true sends input in the foreground at once (it activates the window and moves the real pointer); false never leaves the background. By default input starts in the background and moves to the foreground when the window refuses it." }));
+const modifiersField = Type.Optional(Type.Array(Type.Union([Type.Literal("ctrl"), Type.Literal("shift"), Type.Literal("alt"), Type.Literal("win")]), { maxItems: 4, description: "Keys held during the action." }));
+const locatorTimeout = Type.Optional(Type.Integer({ minimum: 0, maximum: 60000, description: "How long a locator target may take to match one ready element, default 5000." }));
+const PAGE_MODIFIERS: Record<string, string> = { ctrl: "Control", shift: "Shift", alt: "Alt", win: "Meta" };
+
+type Endpoint = { target?: string; x?: number; y?: number };
+/** The target or point of an action; required unless the action can go to the focused element. */
+function aim(params: Endpoint, label: string, required: boolean): Pick<InputPlan, "target" | "point"> {
+ if ((params.x === undefined) !== (params.y === undefined)) throw new ComputerRequestError(`${label} needs both x and y for a point.`);
+ const point = params.x !== undefined ? { x: params.x, y: params.y! } : undefined;
+ if (point && params.target !== undefined) throw new ComputerRequestError(`${label} takes a target or a point, not both.`);
+ if (required && !point && params.target === undefined) throw new ComputerRequestError(`${label} needs a target or a point (x and y).`);
+ return { ...(params.target !== undefined ? { target: params.target } : {}), ...(point ? { point } : {}) };
+}
+/** Fields every input plan shares. */
+const shared = (params: { space?: "page" | "window"; foreground?: boolean }) =>
+ ({ ...(params.space ? { space: params.space } : {}), ...(params.foreground !== undefined ? { foreground: params.foreground } : {}) });
+const timeout = (params: { timeout_ms?: number }) => params.timeout_ms !== undefined ? { timeout_ms: params.timeout_ms } : {};
+const pageModifiers = (modifiers?: string[]) => modifiers?.length ? { modifiers: modifiers.map(key => PAGE_MODIFIERS[key]) } : {};
+
+const clickParameters = Type.Object({
+ window: inputWindow, target: inputTarget("The element to click"), x: coordinate("Left"), y: coordinate("Top"), space: pointSpace,
+ button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")], { description: "Mouse button, default left." })),
+ count: Type.Optional(Type.Integer({ minimum: 1, maximum: 3, description: "Clicks: 2 is a double click, 3 a triple click (browser pages). Default 1." })),
+ modifiers: modifiersField, timeout_ms: locatorTimeout, foreground: foregroundField,
+}, { additionalProperties: false });
+
+/** What computer_click sends to a native window and to a browser page. */
+export function clickPlan(params: Static<typeof clickParameters>): InputPlan {
+ const count = params.count ?? 1, button = params.button ?? "left", modifiers = params.modifiers?.length ? { modifiers: params.modifiers } : {};
+ const native = count === 3 ? "A triple click works on browser pages only."
+ : button === "right" ? count === 2 ? "A right double click works on browser pages only." : { action: "right_click", ...modifiers, ...timeout(params) }
+ : count === 2 ? button === "middle" ? "A middle double click works on browser pages only." : { action: "double_click", ...modifiers, ...timeout(params) }
+ : { action: "click", ...(button === "middle" ? { button } : {}), ...modifiers, ...timeout(params) };
+ return { label: "click", ...aim(params, "click", true), ...shared(params), native,
+ web: { action: "click", ...(button !== "left" ? { button } : {}), ...(count > 1 ? { click_count: count } : {}), ...pageModifiers(params.modifiers), ...timeout(params) } };
+}
+
+const hoverParameters = Type.Object({
+ window: inputWindow, target: inputTarget("The element to point at"), x: coordinate("Left"), y: coordinate("Top"), space: pointSpace,
+ dwell_ms: Type.Optional(Type.Integer({ minimum: 1, maximum: 10000, description: "Native windows: how long the pointer rests there before the result, default 300." })),
+ timeout_ms: locatorTimeout,
+}, { additionalProperties: false });
+
+/** What computer_hover sends; on a native window it moves the real pointer, so it runs in the foreground. */
+export function hoverPlan(params: Static<typeof hoverParameters>): InputPlan {
+ return { label: "hover", ...aim(params, "hover", true), ...shared(params),
+ native: { action: "hover", dwell_ms: params.dwell_ms ?? 300, ...timeout(params) }, web: { action: "hover", ...timeout(params) } };
+}
+
+const endpoint = (what: string) => Type.Object({
+ target: Type.Optional(Type.String({ description: `${what}: a ref from the window's latest Read, or a Playwright locator on browser pages.` })),
+ x: Type.Optional(Type.Number({ minimum: 0 })), y: Type.Optional(Type.Number({ minimum: 0 })),
+}, { additionalProperties: false, description: `${what}: a target, or a point (x and y) in pixels of the window's latest image or CSS pixels of a browser page.` });
+const dragParameters = Type.Object({
+ window: inputWindow, from: endpoint("Where the drag starts"), to: endpoint("Where the drag ends"), space: pointSpace,
+ via: Type.Optional(Type.Array(Type.Object({ x: Type.Number({ minimum: 0 }), y: Type.Number({ minimum: 0 }) }, { additionalProperties: false }),
+ { minItems: 1, maxItems: 254, description: "Points the pointer passes through on the way, in order, for a path rather than a straight line; left button only." })),
+ duration_ms: Type.Optional(Type.Integer({ minimum: 0, maximum: 10000, description: "How long the move takes, default 500." })),
+ button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")], { description: "Button held during the drag, default left." })),
+ modifiers: modifiersField, foreground: foregroundField,
+}, { additionalProperties: false });
+
+/** What computer_drag sends to a native window. */
+export function dragPlan(params: Static<typeof dragParameters>): InputPlan {
+ const end = aim(params.to, "to", true);
+ return { label: "drag", ...aim(params.from, "from", true), end, ...shared(params),
+ native: { action: "drag", ...(params.duration_ms !== undefined ? { duration_ms: params.duration_ms } : {}), ...(params.button && params.button !== "left" ? { button: params.button } : {}),
+ ...(params.modifiers?.length ? { modifiers: params.modifiers } : {}), ...(params.via ? { via: params.via } : {}) },
+ web: "drag on browser pages is not available yet; use space: \"window\" for window pixels." };
+}
+
+const scrollParameters = Type.Object({
+ window: inputWindow, target: inputTarget("The element to scroll; omit for the middle of the window or page"), x: coordinate("Left"), y: coordinate("Top"), space: pointSpace,
+ delta_x: Type.Optional(Type.Integer({ minimum: -50, maximum: 50, description: "Wheel notches to the right (negative: left)." })),
+ delta_y: Type.Optional(Type.Integer({ minimum: -50, maximum: 50, description: "Wheel notches down (negative: up). One notch scrolls about 100 CSS pixels in a browser; apps decide their own distance." })),
+ timeout_ms: locatorTimeout, foreground: foregroundField,
+}, { additionalProperties: false });
+
+/** What computer_scroll sends: wheel notches at a point, or an accessibility scroll of a native element target. */
+export function scrollPlan(params: Static<typeof scrollParameters>): InputPlan {
+ const dx = params.delta_x ?? 0, dy = params.delta_y ?? 0;
+ if (!dx && !dy) throw new ComputerRequestError("scroll needs delta_x or delta_y.");
+ const aimed = aim(params, "scroll", false);
+ const native = aimed.target === undefined ? { action: "scroll", ...(dx ? { delta_x: dx } : {}), ...(dy ? { delta_y: dy } : {}), unit: "wheel_detents" }
+ : dx && dy ? "Scroll a native element along one axis at a time, or give a point."
+ : { action: "scroll", direction: dy > 0 ? "down" : dy < 0 ? "up" : dx > 0 ? "right" : "left", amount: Math.abs(dx || dy), ...timeout(params) };
+ return { label: "scroll", ...aimed, ...shared(params), native,
+ web: { action: "scroll", ...(dx ? { delta_x: dx * 100 } : {}), ...(dy ? { delta_y: dy * 100 } : {}), ...timeout(params) } };
+}
+
+const typeParameters = Type.Object({
+ window: inputWindow, text: Type.String({ maxLength: 20000, description: "Text to type, as is: newlines and quotes stay text. Native windows take up to 2000 characters." }),
+ target: inputTarget("The field to type into; omit for the focused element"), space: pointSpace,
+ replace: Type.Optional(Type.Boolean({ description: "Replace the field's whole value instead of typing at the caret; needs target." })),
+ submit: Type.Optional(Type.Boolean({ description: "Press Enter after the text." })),
+ delay_ms: Type.Optional(Type.Integer({ minimum: 0, maximum: 200, description: "Native windows: pause between characters." })),
+ timeout_ms: locatorTimeout, foreground: foregroundField,
+}, { additionalProperties: false });
+
+/** What computer_type sends, with Enter as a second step when submit is set. */
+export function typePlan(params: Static<typeof typeParameters>): InputPlan {
+ if (params.replace && params.target === undefined) throw new ComputerRequestError("replace needs the field as target.");
+ // Enter goes to the same field: a semantic replace does not move the keyboard focus there.
+ const enter: InputPlan = { label: "submit", ...(params.target !== undefined ? { target: params.target } : {}), ...shared(params), native: { action: "key", key: "return", ...timeout(params) }, web: { action: "press", key: "Enter", ...timeout(params) } };
+ return { label: "type", ...aim(params, "type", false), ...shared(params),
+ native: params.replace ? { action: "set_value", text: params.text, ...timeout(params) } : { action: "type_text", text: params.text, ...(params.delay_ms !== undefined ? { delay_ms: params.delay_ms } : {}), ...timeout(params) },
+ web: params.delay_ms !== undefined ? "delay_ms applies to native windows." : { action: params.replace ? "fill" : "type", text: params.text, ...timeout(params) },
+ ...(params.submit ? { then: enter } : {}) };
+}
+
+const pressParameters = Type.Object({
+ window: inputWindow, key: Type.String({ maxLength: 64, description: "A key with optional modifiers joined by +, such as Enter, ctrl+s, shift+Tab, F5, ArrowDown or a." }),
+ target: inputTarget("The element to focus first; omit for the focused element"), space: pointSpace,
+ hold_ms: Type.Optional(Type.Integer({ minimum: 1, maximum: 10000, description: "Native windows: hold the keys down this long, for apps and games that read held keys." })),
+ foreground: foregroundField,
+}, { additionalProperties: false });
+const PAGE_KEYS: Record<string, string> = { enter: "Enter", return: "Enter", esc: "Escape", escape: "Escape", tab: "Tab", space: "Space", backspace: "Backspace", delete: "Delete", del: "Delete",
+ up: "ArrowUp", down: "ArrowDown", left: "ArrowLeft", right: "ArrowRight", home: "Home", end: "End", pageup: "PageUp", pagedown: "PageDown", pgup: "PageUp", pgdn: "PageDown" };
+
+/** What computer_press sends: one key or chord, or keys held for hold_ms on a native window. */
+export function pressPlan(params: Static<typeof pressParameters>): InputPlan {
+ const parts = params.key.split("+").map(part => part.trim());
+ if (parts.some(part => !part)) throw new ComputerRequestError("key joins a key and its modifiers with +, such as ctrl+s.");
+ const key = parts.at(-1)!, held = parts.slice(0, -1).map(part => part.toLowerCase().replace(/^(control)$/, "ctrl").replace(/^(meta|windows|cmd)$/, "win"));
+ if (held.some(part => !PAGE_MODIFIERS[part])) throw new ComputerRequestError("Modifiers are ctrl, shift, alt and win.");
+ const native = params.hold_ms !== undefined ? { action: "hold", keys: [...held, key.toLowerCase()], duration_ms: params.hold_ms }
+ : { action: "key", key: key.toLowerCase(), ...(held.length ? { modifiers: held } : {}) };
+ const pageKey = PAGE_KEYS[key.toLowerCase()] ?? (/^f\d{1,2}$/i.test(key) ? key.toUpperCase() : key);
+ return { label: "press", ...aim(params, "press", false), ...shared(params), native,
+ web: params.hold_ms !== undefined ? "hold_ms on browser pages is not available yet." : { action: "press", key: [...held.map(part => PAGE_MODIFIERS[part]), pageKey].join("+") } };
+}
+
+const selectParameters = Type.Object({
+ window: inputWindow, target: inputTarget("The list, select, checkbox or option; not used with menu"), space: pointSpace,
+ values: Type.Optional(Type.Array(Type.String(), { minItems: 1, maxItems: 64, description: "Options to choose in the target list, by label or value. Native lists take one; omit values to select the target option itself." })),
+ checked: Type.Optional(Type.Boolean({ description: "Set the target checkbox, radio or toggle to checked (true) or unchecked (false); nothing is sent when it already is." })),
+ menu: Type.Optional(Type.Array(Type.String(), { minItems: 1, maxItems: 16, description: "Native windows: menu labels from the menu bar to the item, such as [\"File\", \"Save As...\"]." })),
+ timeout_ms: locatorTimeout,
+}, { additionalProperties: false });
+
+/** What computer_select sends: options of a list, a checked state, or a menu item. */
+export function selectPlan(params: Static<typeof selectParameters>): InputPlan {
+ const chosen = [params.values && "values", params.checked !== undefined && "checked", params.menu && "menu"].filter(Boolean);
+ if (chosen.length > 1) throw new ComputerRequestError(`select takes one of values, checked or menu, not ${chosen.join(" and ")}.`);
+ if (params.menu) {
+ if (params.target !== undefined) throw new ComputerRequestError("menu names its item by path; omit target.");
+ return { label: "select", ...shared(params), native: { action: "menu", path: params.menu }, web: "menu works on native windows; use a target on browser pages." };
+ }
+ if (params.target === undefined) throw new ComputerRequestError("select needs a target.");
+ const base = { label: "select", target: params.target, ...shared(params) };
+ if (params.checked !== undefined) return { ...base, native: { action: params.checked ? "check" : "uncheck", ...timeout(params) }, web: { action: params.checked ? "check" : "uncheck", ...timeout(params) } };
+ if (params.values) return { ...base, native: params.values.length > 1 ? "A native list takes one value at a time." : { action: "select", option: params.values[0], ...timeout(params) },
+ web: { action: "select", values: params.values, ...timeout(params) } };
+ return { ...base, native: { action: "select", ...timeout(params) }, web: "Give values: the options to choose in this select element." };
+}
+
+const timelineParameters = Type.Object({
+ window: inputWindow, space: pointSpace,
+ events: Type.Array(Type.Object({
+ at_ms: Type.Integer({ minimum: 0, maximum: 120000, description: "Time from the start; never decreases." }),
+ type: Type.Union([Type.Literal("key_down"), Type.Literal("key_up"), Type.Literal("button_down"), Type.Literal("button_up"), Type.Literal("move"), Type.Literal("move_relative"), Type.Literal("wheel"), Type.Literal("pen_down"), Type.Literal("pen_move"), Type.Literal("pen_up")]),
+ key: Type.Optional(Type.String({ description: "key_down, key_up." })),
+ button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle"), Type.Literal("x1"), Type.Literal("x2")], { description: "button_down, button_up; default left." })),
+ x: Type.Optional(Type.Number({ minimum: 0, description: "move and pen events: pixels of the window's latest image." })),
+ y: Type.Optional(Type.Number({ minimum: 0 })),
+ dx: Type.Optional(Type.Integer({ description: "move_relative: raw mouse counts; wheel: notches." })),
+ dy: Type.Optional(Type.Integer()),
+ pressure: Type.Optional(Type.Number({ minimum: 0, maximum: 1, description: "Pen events." })),
+ tilt_x: Type.Optional(Type.Integer({ minimum: -90, maximum: 90 })), tilt_y: Type.Optional(Type.Integer({ minimum: -90, maximum: 90 })),
+ eraser: Type.Optional(Type.Boolean()), barrel: Type.Optional(Type.Boolean()),
+ }, { additionalProperties: false }), { minItems: 1, maxItems: 5000, description: "Timed events. Every key and button pressed must be released; anything still held is released when the call ends." }),
+}, { additionalProperties: false });
+
+/** What computer_timeline sends to a native window, in the foreground. */
+export function timelinePlan(params: Static<typeof timelineParameters>): InputPlan {
+ return { label: "timeline", ...shared(params), native: { action: "timeline", events: params.events }, web: "timeline on browser pages is not available yet; use space: \"window\" for window pixels." };
+}
/**
* Register the typed computer_* tools that replace Insert commands one cluster at a time. They are deferred: a
* tool_search for computer loads them, so sessions that never use the computer pay nothing for them.
@@ -359,6 +541,70 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
: resources.perform(address, ctx.cwd, JSON.stringify(run.command), "insert", signal)));
},
});
+ /** Run one typed input on a window: the plan is built and checked before anything reaches the machine. */
+ const input = (window: string, action: string, build: () => InputPlan, cwd: string, signal?: AbortSignal) => {
+ let plan: InputPlan | undefined;
+ return computerAction(resources, window, action, () => { plan = build(); return resources.address(window); },
+ address => resources.input(address, cwd, plan!, signal)).then(actionResult);
+ };
+ const aimed = (args: Record<string, unknown>) => args.target !== undefined ? String(args.target) : args.x !== undefined ? `${args.x},${args.y}` : "";
+ const inputAnnotations = { readOnlyHint: false, destructiveHint: true, openWorldHint: true };
+ pi.registerTool({
+ ...common, name: "computer_click", label: "Computer click",
+ description: "Use computer_click to click an element or a point of a window: any button, double or triple clicks, with modifier keys. The result lists what changed.",
+ parameters: clickParameters, annotations: inputAnnotations,
+ ...rendered("computer_click", args => `${args.window} ${aimed(args)}${args.count ? ` x${args.count}` : ""}${args.button && args.button !== "left" ? ` ${args.button}` : ""}`),
+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "click", () => clickPlan(params), ctx.cwd, signal); },
+ });
+ pi.registerTool({
+ ...common, name: "computer_hover", label: "Computer hover",
+ description: "Use computer_hover to rest the pointer on an element or a point without clicking, for tooltips, hover menus and hover states.",
+ parameters: hoverParameters, annotations: inputAnnotations,
+ ...rendered("computer_hover", args => `${args.window} ${aimed(args)}`),
+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "hover", () => hoverPlan(params), ctx.cwd, signal); },
+ });
+ pi.registerTool({
+ ...common, name: "computer_drag", label: "Computer drag",
+ description: "Use computer_drag to press a button at one element or point, move to another, optionally through a path, and release there, for sliders, drag and drop and drawing.",
+ parameters: dragParameters, annotations: inputAnnotations,
+ ...rendered("computer_drag", args => `${args.window} ${aimed(args.from as Record<string, unknown>)} → ${aimed(args.to as Record<string, unknown>)}`),
+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "drag", () => dragPlan(params), ctx.cwd, signal); },
+ });
+ pi.registerTool({
+ ...common, name: "computer_scroll", label: "Computer scroll",
+ description: "Use computer_scroll to turn the mouse wheel over an element, a point or the middle of a window or page.",
+ parameters: scrollParameters, annotations: inputAnnotations,
+ ...rendered("computer_scroll", args => `${args.window} ${aimed(args)} ${args.delta_x ?? 0},${args.delta_y ?? 0}`),
+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "scroll", () => scrollPlan(params), ctx.cwd, signal); },
+ });
+ pi.registerTool({
+ ...common, name: "computer_type", label: "Computer type",
+ description: "Use computer_type to type text into a field or the focused element, or to replace a field's value, optionally pressing Enter after it.",
+ parameters: typeParameters, annotations: inputAnnotations,
+ ...rendered("computer_type", args => `${args.window} ${aimed(args)} ${String(args.text ?? "").length} chars${args.submit ? " + Enter" : ""}`),
+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "type", () => typePlan(params), ctx.cwd, signal); },
+ });
+ pi.registerTool({
+ ...common, name: "computer_press", label: "Computer press",
+ description: "Use computer_press to press a key or shortcut such as Enter or ctrl+s, or to hold keys down for a time.",
+ parameters: pressParameters, annotations: inputAnnotations,
+ ...rendered("computer_press", args => `${args.window} ${args.key}${args.hold_ms ? ` for ${args.hold_ms} ms` : ""}`),
+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "press", () => pressPlan(params), ctx.cwd, signal); },
+ });
+ pi.registerTool({
+ ...common, name: "computer_select", label: "Computer select",
+ description: "Use computer_select to choose options in a list or select element, set a checkbox, radio or toggle, or open an app's menu item.",
+ parameters: selectParameters, annotations: inputAnnotations,
+ ...rendered("computer_select", args => `${args.window} ${args.menu ? (args.menu as string[]).join(" > ") : `${args.target ?? ""} ${args.values ? (args.values as string[]).join(", ") : args.checked !== undefined ? (args.checked ? "checked" : "unchecked") : ""}`}`),
+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "select", () => selectPlan(params), ctx.cwd, signal); },
+ });
+ pi.registerTool({
+ ...common, name: "computer_timeline", label: "Computer timeline",
+ description: "Use computer_timeline to play timed key, button, pointer, wheel and pen events in one call, for games, held keys across moves and pressure-sensitive strokes.",
+ parameters: timelineParameters, annotations: inputAnnotations,
+ ...rendered("computer_timeline", args => `${args.window} ${(args.events as unknown[] | undefined)?.length ?? 0} events`),
+ async execute(_id, params, signal, _update, ctx) { return input(params.window, "timeline", () => timelinePlan(params), ctx.cwd, signal); },
+ });
pi.registerTool({
...common, name: "computer_display", label: "Computer display",
description: "Use computer_display to change a display's resolution in YOLO mode, for example to test an app at another size; closing computer use restores it.",
diff --git a/src/session.ts b/src/session.ts
index 14acf09..fb91dd9 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -146,7 +146,10 @@ interface Observation { texts?: Map<string, string>; id: string; tokens: Set<str
interface WindowState { id: string; nativeId: string; kind: "restore" | "management" }
/** The unannotated pixels of a window's latest capture with pixels, kept as the baseline of its next pixel comparison. */
export interface WindowFrame { observation_id: string; png: Buffer; size: { maxLongEdge: number; scale: number } }
-interface WindowEntry { ref: string; target: WindowTarget; identity: string; owned: boolean; observation?: Observation; windowState?: WindowState; dom?: { id: string; page: WebDom }; web?: WebBrowser; detail?: DetailContext; lastAction?: GuideMark; frame?: WindowFrame }
+interface WindowEntry { ref: string; target: WindowTarget; identity: string; owned: boolean; observation?: Observation;
+ /** Size of the window's latest image capture: points of typed input are in its pixels. Outlives the observation. */
+ seen?: { width: number; height: number };
+ windowState?: WindowState; dom?: { id: string; page: WebDom }; web?: WebBrowser; detail?: DetailContext; lastAction?: GuideMark; frame?: WindowFrame }
interface App { name: string; launch_path?: string; bundle_id?: string; kind?: string }
/** A discovery query: `regex:<pattern>` is a case-insensitive regular expression, anything else a case-insensitive substring. */
export function textMatcher(query?: string): (text: string) => boolean {
@@ -1029,6 +1032,7 @@ export class ComputerSession {
const full = [element.label, element.value].filter((item): item is string => typeof item === "string" && item.length > 1024);
if (full.length && ui.elements[index]) texts.set(ui.elements[index].hint, full.join("\n").slice(0, ELEMENT_TEXT_LIMIT));
});
+ if (hasImage && typeof state.screenshot_width === "number" && typeof state.screenshot_height === "number") entry.seen = { width: state.screenshot_width, height: state.screenshot_height };
entry.observation = { id, ui, settings, ...(texts.size ? { texts } : {}),
width: hasImage && typeof state.screenshot_width === "number" ? state.screenshot_width : undefined,
height: hasImage && typeof state.screenshot_height === "number" ? state.screenshot_height : undefined,
@@ -1790,6 +1794,17 @@ export class ComputerSession {
}
/** Whether a window still holds this observation for input. */
holdsObservation(windowRef: string, id: string): boolean { return this.entries.get(windowRef)?.observation?.id === id; }
+ /**
+ * What typed input can use in a window now: its current observation (a DOM snapshot of a browser page, or a native
+ * capture with its size and, until the first input spends them, its UI hints) and the size of its latest image,
+ * which is the space of the agent's points.
+ */
+ inputView(ref: string): { dom?: string; native?: { id: string; width?: number; height?: number; ui?: UiResource }; seen?: { width: number; height: number } } {
+ const entry = this.entries.get(ref), observed = entry?.observation;
+ return { ...(entry?.dom ? { dom: entry.dom.id } : {}),
+ ...(observed ? { native: { id: observed.id, width: observed.width, height: observed.height, ...(observed.ui && !observed.pixelsOnly ? { ui: observed.ui } : {}) } } : {}),
+ ...(entry?.seen ? { seen: { ...entry.seen } } : {}) };
+ }
/** Whether this session issued a still-registered window reference. */
hasWindow(ref: string): boolean { return this.entries.has(ref); }
/** Whether this session issued a still-registered display reference. */
diff --git a/src/web-browser.ts b/src/web-browser.ts
index c972950..0e856e1 100644
--- a/src/web-browser.ts
+++ b/src/web-browser.ts
@@ -30,6 +30,8 @@ export interface WebAction {
| "upload" | "wait_download" | "cancel_download" | "evaluate";
target?: WebTarget;
button?: "left" | "right" | "middle"; click_count?: number;
+ /** Keys held during a click or hover. */
+ modifiers?: ("Alt" | "Control" | "Meta" | "Shift")[];
text?: string; key?: string; values?: string[];
delta_x?: number; delta_y?: number;
/** Element state for wait; load state for wait_load. */
@@ -174,10 +176,10 @@ export function validateWebSnapshot(value: unknown = {}): WebSnapshotOptions {
}
const ACTIONS: Record<WebAction["action"], { fields: readonly string[]; target: "required" | "optional" | "none" }> = {
- click: { fields: ["button", "click_count"], target: "required" },
- hover: { fields: [], target: "required" },
+ click: { fields: ["button", "click_count", "modifiers"], target: "required" },
+ hover: { fields: ["modifiers"], target: "required" },
fill: { fields: ["text"], target: "required" },
- type: { fields: ["text"], target: "required" },
+ type: { fields: ["text"], target: "optional" },
press: { fields: ["key"], target: "optional" },
check: { fields: [], target: "required" },
uncheck: { fields: [], target: "required" },
@@ -219,6 +221,11 @@ export function validateWebAction(value: unknown): WebAction {
action.button = command.button as WebAction["button"];
}
if (command.click_count !== undefined) action.click_count = integer(command.click_count, 1, 3, "click_count");
+ if (command.modifiers !== undefined) {
+ if (!Array.isArray(command.modifiers) || command.modifiers.length > 4 || new Set(command.modifiers).size !== command.modifiers.length || command.modifiers.some(key => !["Alt", "Control", "Meta", "Shift"].includes(key as string)))
+ throw Error("modifiers must be unique Alt, Control, Meta or Shift.");
+ action.modifiers = command.modifiers as WebAction["modifiers"];
+ }
if (name === "fill" || name === "type") {
if (typeof command.text !== "string" || command.text.length > 20_000) throw Error(`${name} needs text of at most 20000 characters.`);
action.text = command.text;
diff --git a/src/web-worker.ts b/src/web-worker.ts
index 7c4e161..20a5b69 100644
--- a/src/web-worker.ts
+++ b/src/web-worker.ts
@@ -449,10 +449,10 @@ async function act(action: WebAction) {
const target = action.target && locate(page, action.target);
const run = async (): Promise<unknown> => {
switch (action.action) {
- case "click": return target!.click({ button: action.button, clickCount: action.click_count, timeout });
- case "hover": return target!.hover({ timeout });
+ case "click": return target!.click({ button: action.button, clickCount: action.click_count, modifiers: action.modifiers, timeout });
+ case "hover": return target!.hover({ modifiers: action.modifiers, timeout });
case "fill": return target!.fill(action.text!, { timeout });
- case "type": return target!.pressSequentially(action.text!, { timeout });
+ case "type": return target ? target.pressSequentially(action.text!, { timeout }) : page.keyboard.type(action.text!);
case "press": return target ? target.press(action.key!, { timeout }) : page.keyboard.press(action.key!);
case "check": return target!.check({ timeout });
case "uncheck": return target!.uncheck({ timeout });
diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.ts
index bc5578d..45f0fb6 100644
--- a/test/computer-tools.test.ts
+++ b/test/computer-tools.test.ts
@@ -1,6 +1,6 @@
import { expect, test, vi } from "vitest";
-import { requireComputerAddress, ComputerResources } from "../src/computer-resources.ts";
-import { clipboardCommand, filesCommand, openCommand, recordCommand, waitCommand, windowCommand } from "../src/computer-tools.ts";
+import { requireComputerAddress, ComputerResources, refKind } from "../src/computer-resources.ts";
+import { clickPlan, clipboardCommand, dragPlan, filesCommand, hoverPlan, openCommand, pressPlan, recordCommand, scrollPlan, selectPlan, timelinePlan, typePlan, waitCommand, windowCommand } from "../src/computer-tools.ts";
import type { WebBrowser, WebLauncher } from "../src/web-browser.ts";
import type { MachineHost } from "../src/machine-host.ts";
import { mkdtemp, readFile, rm } from "node:fs/promises";
@@ -80,6 +80,51 @@ test("computer_record maps video and audio to machine commands and sequence to t
expect(() => recordCommand({ at: "m", action: "video_stop", seconds: 3 })).toThrow("video_stop takes no seconds.");
});
+test("input tools build one plan for native windows and one for browser pages", () => {
+ const w = "computer:w";
+ expect(refKind("1:a")).toBe("hint");
+ expect(refKind("2b:ac")).toBe("hint");
+ expect(refKind("e12")).toBe("page");
+ expect(refKind("f1e3")).toBe("page");
+ expect(refKind("getByRole('button', { name: 'Save' })")).toBe("locator");
+ expect(clickPlan({ window: w, target: "1:a" })).toEqual({ label: "click", target: "1:a", native: { action: "click" }, web: { action: "click" } });
+ expect(clickPlan({ window: w, x: 10, y: 20, count: 2, modifiers: ["ctrl"] })).toEqual({ label: "click", point: { x: 10, y: 20 },
+ native: { action: "double_click", modifiers: ["ctrl"] }, web: { action: "click", click_count: 2, modifiers: ["Control"] } });
+ expect(clickPlan({ window: w, target: "e1", button: "right" }).native).toEqual({ action: "right_click" });
+ expect(clickPlan({ window: w, target: "e1", count: 3 }).native).toBe("A triple click works on browser pages only.");
+ expect(() => clickPlan({ window: w })).toThrow("click needs a target or a point (x and y).");
+ expect(() => clickPlan({ window: w, x: 1 })).toThrow("click needs both x and y for a point.");
+ expect(() => clickPlan({ window: w, x: 1, y: 2, target: "1:a" })).toThrow("click takes a target or a point, not both.");
+ expect(hoverPlan({ window: w, target: "1:a" }).native).toEqual({ action: "hover", dwell_ms: 300 });
+ expect(dragPlan({ window: w, from: { target: "1:a" }, to: { x: 5, y: 6 }, via: [{ x: 3, y: 4 }] })).toMatchObject({ target: "1:a", end: { point: { x: 5, y: 6 } }, native: { action: "drag", via: [{ x: 3, y: 4 }] } });
+ expect(scrollPlan({ window: w, delta_y: -2 })).toMatchObject({ native: { action: "scroll", delta_y: -2, unit: "wheel_detents" }, web: { action: "scroll", delta_y: -200 } });
+ expect(scrollPlan({ window: w, target: "1:a", delta_x: 3 }).native).toEqual({ action: "scroll", direction: "right", amount: 3 });
+ expect(() => scrollPlan({ window: w })).toThrow("scroll needs delta_x or delta_y.");
+ const typed = typePlan({ window: w, target: "1:a", text: "Ada", replace: true, submit: true });
+ expect(typed).toMatchObject({ native: { action: "set_value", text: "Ada" }, web: { action: "fill", text: "Ada" },
+ then: { target: "1:a", native: { action: "key", key: "return" }, web: { action: "press", key: "Enter" } } });
+ expect(() => typePlan({ window: w, text: "x", replace: true })).toThrow("replace needs the field as target.");
+ expect(pressPlan({ window: w, key: "ctrl+shift+Tab" })).toMatchObject({ native: { action: "key", key: "tab", modifiers: ["ctrl", "shift"] }, web: { action: "press", key: "Control+Shift+Tab" } });
+ expect(pressPlan({ window: w, key: "w", hold_ms: 500 }).native).toEqual({ action: "hold", keys: ["w"], duration_ms: 500 });
+ expect(pressPlan({ window: w, key: "f5" }).web).toEqual({ action: "press", key: "F5" });
+ expect(() => pressPlan({ window: w, key: "hyper+a" })).toThrow("Modifiers are ctrl, shift, alt and win.");
+ expect(selectPlan({ window: w, target: "e3", values: ["Red"] })).toMatchObject({ native: { action: "select", option: "Red" }, web: { action: "select", values: ["Red"] } });
+ expect(selectPlan({ window: w, target: "1:a", checked: false })).toMatchObject({ native: { action: "uncheck" }, web: { action: "uncheck" } });
+ expect(selectPlan({ window: w, menu: ["File", "Save"] })).toMatchObject({ native: { action: "menu", path: ["File", "Save"] } });
+ expect(() => selectPlan({ window: w, target: "1:a", checked: true, values: ["a"] })).toThrow("select takes one of values, checked or menu");
+ expect(timelinePlan({ window: w, events: [{ at_ms: 0, type: "key_down", key: "a" }, { at_ms: 5, type: "key_up", key: "a" }] }).native).toMatchObject({ action: "timeline" });
+});
+
+test("typed input refuses mismatched targets before anything reaches the machine", async () => {
+ const f = fixture("closed");
+ const window = at(`computer:window/${await f.window()}`);
+ await expect(f.resources.input(at("computer:machine/local"), cwd, clickPlan({ window: "x", target: "1:a" }))).rejects.toThrow("Give the window address");
+ await expect(f.resources.input(window, cwd, clickPlan({ window: "x", target: "e4" }))).rejects.toThrow("e4 is a page ref from a DOM Read; this window is not a browser page");
+ await expect(f.resources.input(window, cwd, clickPlan({ window: "x", x: 1, y: 2, space: "page" }))).rejects.toThrow("space applies to browser windows only");
+ await expect(f.resources.input(window, cwd, clickPlan({ window: "x", target: "9:z" }))).rejects.toThrow("9:z is not a ref of this window's latest Read");
+ await expect(f.resources.input(window, cwd, { ...hoverPlan({ window: "x", target: "getByText('a')" }), foreground: false })).rejects.toThrow("hover on a native window moves the real pointer");
+});
+
/** A Chrome window (pid 4100) attached to a fake Playwright worker that records page actions. */
function browserFixture() {
const target = { pid: 4100, window_id: 77, title: "Page - Google Chrome", app_name: "chrome.exe" };
@@ -119,6 +164,22 @@ test("computer_browser runs a page action on the window and returns the page's n
expect(f.page).toHaveBeenCalledTimes(1);
});
+test("typed input on a browser window goes to Playwright with page refs and locators", async () => {
+ const f = browserFixture();
+ const window = await f.window();
+ const clicked = await f.resources.input(window, cwd, clickPlan({ window: "x", target: "e1", modifiers: ["shift"] }));
+ expect(clicked.details).toMatchObject({ status: "delivered", machine: "local" });
+ expect(f.page).toHaveBeenLastCalledWith("act", { action: "click", target: { ref: "e1" }, modifiers: ["Shift"] }, undefined);
+ // The snapshot of the first action is the window's current observation, so the next one compares with it.
+ await f.resources.input(window, cwd, typePlan({ window: "x", target: "getByLabel('Name')", text: "Ada", submit: true }));
+ expect(f.page.mock.calls.slice(-2).map(call => call[1])).toEqual([
+ { action: "type", text: "Ada", target: { locator: "internal:label=\"Name\"i" } },
+ { action: "press", key: "Enter", target: { locator: "internal:label=\"Name\"i" } }]);
+ await expect(f.resources.input(window, cwd, clickPlan({ window: "x", x: 5, y: 5 }))).rejects.toThrow("click at a point of a browser page is not available yet");
+ await expect(f.resources.input(window, cwd, selectPlan({ window: "x", menu: ["File"] }))).rejects.toThrow("menu works on native windows");
+ expect(f.page).toHaveBeenCalledTimes(3);
+});
+
/** One window (pid 10, id 7) a session can list and observe; the host decides how a close request ends. */
function fixture(outcome: "closed" | "open" | "refused") {
let open = true, mode: ComputerMode = "yolo";
diff --git a/test/live/typed-input.live.ts b/test/live/typed-input.live.ts
new file mode 100644
index 0000000..b877f3c
--- /dev/null
+++ b/test/live/typed-input.live.ts
@@ -0,0 +1,87 @@
+import { afterAll, expect, test } from "vitest";
+import { requireComputerAddress, type InputPlan } from "../../src/computer-resources.ts";
+import { clickPlan, dragPlan, hoverPlan, pressPlan, scrollPlan, selectPlan, timelinePlan, typePlan, waitCommand } from "../../src/computer-tools.ts";
+import { presentComputerResult } from "../../src/provider.ts";
+import { deleteComputer } from "../../src/provider.ts";
+import { connectLive, nativeForm, openWindow, serveFixture, WINDOWS_POWERSHELL_PATH } from "./remote.ts";
+
+// LPT-927: the typed input tools on a native WinForms window and on a Playwright browser page of the configured machine.
+const live = await connectLive();
+const fixture = await serveFixture(live.runner);
+afterAll(async () => { await live.close(); fixture.close(); });
+const timings: string[] = [];
+const act = async (window: string, plan: InputPlan) => {
+ const started = Date.now();
+ const result = await live.resources.input(requireComputerAddress(window), live.cwd, plan);
+ const body = presentComputerResult(result);
+ timings.push(`${plan.label}${plan.target ? ` ${plan.target}` : plan.point ? ` ${plan.point.x},${plan.point.y}` : ""}: ${result.details.status} in ${Date.now() - started} ms`);
+ console.log(timings.at(-1));
+ return { result, body };
+};
+const holds = async (window: string, conditions: Parameters<typeof waitCommand>[0]["conditions"]) => {
+ const result = await live.insert(window, waitCommand({ window, timeout_ms: 3000, conditions }));
+ return (result.details.verification as { verdict?: string } | undefined)?.verdict;
+};
+afterAll(() => console.log(timings.join("\n")));
+
+test("one action vocabulary drives a native window by locator, ref and point", async () => {
+ const title = `pi-input-${Date.now()}`;
+ const window = await openWindow(live, live.machine, WINDOWS_POWERSHELL_PATH, nativeForm(title), title);
+ try {
+ const button = "getByRole('button', { name: /^Count/ })";
+ expect((await act(window, clickPlan({ window, target: button }))).result.details.status).toBe("delivered");
+ expect(await holds(window, [{ target: "getByRole('button', { name: 'Count 1' })", is: "exists" }])).toBe("satisfied");
+ // A full Read gives the button's place in the image; a later UI-only Read keeps that image as the space of points.
+ const full = await live.read(window, ["full"]);
+ const element = (full.details.ui as { elements: { hint: string; role: string; name?: string; bounds: { window_image?: { x: number; y: number; width: number; height: number } } }[] })
+ .elements.find(item => /Count 1/.test(item.name ?? ""))!;
+ const box = element.bounds.window_image!;
+ const centre = { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) };
+ const aria = await live.read(window, ["aria"]);
+ const ref = /button "Count 1" \[ref=([0-9a-z]+:[a-z]+)\]/.exec(presentComputerResult(aria))![1];
+ expect((await act(window, clickPlan({ window, target: ref }))).result.details.status).toBe("delivered");
+ expect(await holds(window, [{ target: "getByRole('button', { name: 'Count 2' })", is: "exists" }])).toBe("satisfied");
+ await live.read(window, ["aria"]);
+ // The UI-only Read has no pixels, so the point click captures the window first and keeps the full Read's scale.
+ expect((await act(window, clickPlan({ window, ...centre }))).result.details.status).toBe("delivered");
+ expect(await holds(window, [{ target: "getByRole('button', { name: 'Count 3' })", is: "exists" }])).toBe("satisfied");
+ const field = "getByRole('textbox', { name: 'Field' })";
+ expect((await act(window, typePlan({ window, target: field, text: "hello" }))).result.details.status).toBe("delivered");
+ expect(await holds(window, [{ target: field, value: "hello" }])).toBe("satisfied");
+ expect((await act(window, typePlan({ window, target: field, text: "world", replace: true, submit: true }))).result.details.status).toBe("delivered");
+ expect(await holds(window, [{ target: field, value: "world" }])).toBe("satisfied");
+ expect((await act(window, pressPlan({ window, key: "ctrl+a" }))).result.details.status).toBe("delivered");
+ expect((await act(window, pressPlan({ window, key: "Backspace" }))).result.details.status).toBe("delivered");
+ // An empty field's value falls back to its name in conditions, so its aria line shows the emptiness: no ": value".
+ expect(presentComputerResult(await live.read(window, ["aria"]))).toMatch(/textbox "Field" \[ref=/);
+ expect((await act(window, selectPlan({ window, target: "getByRole('list', { name: 'Pick' })", values: ["beta"] }))).result.details.status).toBe("delivered");
+ expect(await holds(window, [{ target: "getByRole('option', { name: 'beta' })", is: "selected" }])).toBe("satisfied");
+ expect((await act(window, scrollPlan({ window, target: "getByRole('list', { name: 'Pick' })", delta_y: 1 }))).result.details.status).not.toBe("unknown");
+ expect((await act(window, scrollPlan({ window, x: centre.x, y: centre.y + 60, delta_y: 1 }))).result.details.status).toBe("delivered");
+ expect((await act(window, hoverPlan({ window, target: button }))).result.details.status).toBe("delivered");
+ expect((await act(window, dragPlan({ window, from: { x: centre.x, y: centre.y }, to: { x: centre.x + 40, y: centre.y } }))).result.details.status).toBe("delivered");
+ const timeline = await act(window, timelinePlan({ window, events: [{ at_ms: 0, type: "move", x: centre.x, y: centre.y }, { at_ms: 50, type: "button_down" }, { at_ms: 100, type: "button_up" }] }));
+ expect(timeline.result.details.status, timeline.body).toBe("delivered");
+ expect(await holds(window, [{ target: "getByRole('button', { name: /^Count [45]$/ })", is: "exists" }])).toBe("satisfied");
+ } finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }
+});
+
+test("the same vocabulary drives a browser page through Playwright by locator and page ref", async () => {
+ const opened = await live.insert(live.machine, { action: "open_browser", url: `${fixture.base}/wait` });
+ expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");
+ const window = `computer:window/${opened.details.window_ref}`;
+ try {
+ expect((await act(window, typePlan({ window, target: "getByLabel('Name')", text: "Ada" }))).result.details.status).toBe("delivered");
+ const dom = presentComputerResult(await live.read(window, ["dom"]));
+ const save = /button "Save" \[ref=(e\d+)\]/.exec(dom)![1];
+ const clicked = await act(window, clickPlan({ window, target: save }));
+ expect(clicked.result.details.status, clicked.body).toBe("delivered");
+ expect(await holds(window, [{ target: { text: "saved Ada" }, is: "visible" }])).toBe("satisfied");
+ expect((await act(window, typePlan({ window, target: "getByLabel('Name')", text: "Grace", replace: true, submit: true }))).result.details.status).toBe("delivered");
+ expect((await act(window, pressPlan({ window, key: "ctrl+a" }))).result.details.status).toBe("delivered");
+ expect((await act(window, hoverPlan({ window, target: "getByRole('button', { name: 'Unlock later' })" }))).result.details.status).toBe("delivered");
+ expect((await act(window, scrollPlan({ window, delta_y: 2 }))).result.details.status).toBe("delivered");
+ const refused = await live.resources.input(requireComputerAddress(window), live.cwd, timelinePlan({ window, events: [{ at_ms: 0, type: "key_down", key: "a" }, { at_ms: 10, type: "key_up", key: "a" }] })).catch((error: Error) => error.message);
+ expect(String(refused)).toContain("timeline on browser pages");
+ } finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }
+});