Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Address native windows with Playwright locators (LPT-911)

alexshpunt-clanker[bot]f281250dd285
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index 7875858..b1c2046 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -12,9 +12,10 @@ import { parseWindowManagement, type WindowManagementCommand } from "./window-ma
import { validateWebAction, validateWebLaunch, type WebAction } from "./web-browser.ts";
import { combine, evaluateNative, type WaitSpec } from "./ui-conditions.ts";
import { diffDom, diffUi } from "./successor.ts";
-import type { UiResource } from "./ui-resources.ts";
+import type { UiElement, UiResource } from "./ui-resources.ts";
import { displayMode } from "./display-mode.ts";
import { isSemanticAction } from "./semantic-actions.ts";
+import { accessibleName, ariaRole, checkedState, isVisible, nativeSelector, queryNative, suggestLocator, type ParsedSelector } from "./native-locator.ts";
import { SETUP_PARTS, SetupError, runSetup, type SetupPart } from "./setup.ts";
/** The machine this extension runs on, by its address name. */
@@ -34,6 +35,10 @@ function commandObject(value: string): Record<string, unknown> {
const BATCH_STEPS = 200;
/** Native actions a batch may run; each later step re-resolves its target in a fresh UI walk. */
const NATIVE_BATCH_ACTIONS = ["click", "double_click", "right_click", "type_text", "key", "hotkey", "scroll", "invoke", "select", "set_value"];
+/** Native actions that take a Playwright locator or JSON target. */
+const NATIVE_LOCATOR_ACTIONS = ["click", "double_click", "right_click", "hover", "type_text", "scroll", "invoke", "select", "set_value", "check", "uncheck"];
+/** Playwright verbs with a different native name. */
+const LOCATOR_VERBS: Record<string, string> = { fill: "set_value", dblclick: "double_click" };
/** An action's optional postcondition and successor request. */
interface FollowUp { expect?: WaitSpec; successor: boolean; preexisting?: boolean }
type ObservationTarget = { window_ref: string; display?: never } | { display: string; window_ref?: never };
@@ -702,8 +707,12 @@ export class ComputerResources {
only(command, ["action"]);
return session.attachBrowser(address.ref, signal);
}
+ if (command.target !== undefined && command.action !== "wait_for" && command.action !== "verify") {
+ if (session.isBrowserWindow(address.ref)) throw new ComputerRequestError("For a browser window, Read it with dom and Insert the action into that DOM observation; Playwright resolves the target there.");
+ return this.locatorAction(address.ref, session, command, signal);
+ }
if (command.action !== "wait_for" && command.action !== "verify")
- throw new ComputerRequestError("A window address accepts only wait_for, verify, attach_browser or kill. Input needs the observation address returned by a fresh window Read.");
+ throw new ComputerRequestError("A window address accepts an action with a target (a Playwright locator such as getByRole(\"button\", { name: \"Save\" })), wait_for, verify, attach_browser or kill. Other input needs the observation address returned by a fresh window Read.");
let spec;
try { spec = session.waitSpec(address.ref, command); }
catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }
@@ -734,6 +743,12 @@ export class ComputerResources {
const result = await this.batch(address, snapshot, command, cwd, signal);
return follow ? this.followUp(window, snapshot, result, follow, cwd, signal) : result;
}
+ if (command.target !== undefined && !snapshot.result.details.dom) {
+ if (address.hint !== undefined) throw new ComputerRequestError("Choose the address hint or a JSON target, not both.");
+ this.snapshots.delete(address.ref);
+ const result = await this.locatorAction(window, await this.sessionFor(cwd, { window_ref: window }), command, signal);
+ return follow ? this.followUp(window, snapshot, result, follow, cwd, signal) : result;
+ }
}
const result = await this.act(address, snapshot, input, tool, cwd, signal);
return follow && window ? this.followUp(window, snapshot, result, follow, cwd, signal) : result;
@@ -834,10 +849,86 @@ export class ComputerResources {
return { ...result, details };
}
+ /**
+ * Resolve a Playwright locator the way Playwright's actions do: take fresh UI walks until exactly one element
+ * matches and is ready (visible; enabled for input; for pointer input also at the same place in two walks), or
+ * until the timeout. Several matches refuse at once with each match's locator, like Playwright's strict mode.
+ */
+ private async resolveNative(window: string, session: ComputerSession, selector: ParsedSelector, options: { timeout_ms: number; enabled: boolean; stable: boolean; describe: string }, signal?: AbortSignal):
+ Promise<{ element: UiElement; observation: ComputerResult } | { refused: string; observation?: ComputerResult }> {
+ const deadline = performance.now() + options.timeout_ms;
+ let previous: UiElement | undefined, reason = "";
+ for (let attempt = 0; ; attempt++) {
+ if (signal?.aborted) return { refused: "Cancelled while waiting for the target; nothing was sent." };
+ this.retireWindow(window);
+ const fresh = this.remember({ window_ref: window }, await session.observeQuiet(window, signal));
+ if (fresh.details.status !== "observed" || !fresh.details.ui) return { refused: `No current UI sample of the window: ${computerResultText(fresh).split("\n")[0]}`, observation: fresh };
+ const ui = fresh.details.ui;
+ let found: UiElement[];
+ try { found = queryNative(ui, selector); }
+ catch (error) { return { refused: error instanceof Error ? error.message : String(error), observation: fresh }; }
+ if (found.length > 1) {
+ const listed = found.slice(0, 8).map(item => ` ${item.hint} ${ariaRole(item)} ${JSON.stringify(accessibleName(item))} → ${suggestLocator(item)}`).join("\n");
+ return { refused: `Strict mode: ${options.describe} matches ${found.length} elements; nothing was sent. Narrow it, for example with exact, filter or nth:\n${listed}${found.length > 8 ? `\n …and ${found.length - 8} more` : ""}`, observation: fresh };
+ }
+ const element = found[0];
+ if (!element) reason = ui.complete === true && !ui.truncated ? "no element matches it" : "no element matches it in the walked part of the window (the UI walk was incomplete)";
+ else if (!isVisible(element)) reason = "the element is not visible (offscreen)";
+ else if (options.enabled && element.enabled === false) reason = "the element is disabled";
+ else if (options.stable && previous && JSON.stringify(previous.bounds.native_screen) !== JSON.stringify(element.bounds.native_screen)) reason = "the element is still moving";
+ else if (!options.stable || previous || !element.bounds.native_screen) return { element, observation: fresh };
+ else reason = "the element is still moving";
+ previous = element && accessibleName(element) === accessibleName(previous ?? element) ? element : undefined;
+ const left = deadline - performance.now();
+ if (left <= 0) return { refused: `Timed out after ${options.timeout_ms} ms waiting for ${options.describe}: ${reason}. Nothing was sent.`, observation: fresh };
+ await new Promise(resolve => setTimeout(resolve, Math.min(left, [0, 20, 50, 100, 100, 250][attempt] ?? 500)));
+ }
+ }
+
+ /**
+ * One native action addressed by a Playwright locator or JSON target, without a prior Read: resolve it in a fresh
+ * walk with auto-waiting, then act on that element. `check`/`uncheck` act only when the state differs and confirm it.
+ */
+ private async locatorAction(window: string, session: ComputerSession, command: Record<string, unknown>, signal?: AbortSignal): Promise<ComputerResult> {
+ const { target, timeout_ms: timeout, ...rest } = command;
+ let selector: ParsedSelector;
+ try { selector = nativeSelector(target); }
+ catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }
+ if (timeout !== undefined && (!Number.isSafeInteger(timeout) || (timeout as number) < 0 || (timeout as number) > 60_000)) throw new ComputerRequestError("timeout_ms must be an integer 0..60000.");
+ if (["hint", "element_token", "x", "y", "window_ref", "observation_id"].some(key => key in rest)) throw new ComputerRequestError("Choose target or a hint, token or pixels, not both.");
+ const verb = String(rest.action);
+ const action = LOCATOR_VERBS[verb] ?? verb;
+ if (!NATIVE_LOCATOR_ACTIONS.includes(action)) throw new ComputerRequestError(`A target takes one of ${[...NATIVE_LOCATOR_ACTIONS, ...Object.keys(LOCATOR_VERBS)].join(", ")}.`);
+ const describe = typeof target === "string" ? target : JSON.stringify(target);
+ const pointer = ["click", "double_click", "right_click", "hover", "check", "uncheck"].includes(action);
+ const resolved = await this.resolveNative(window, session, selector, { timeout_ms: (timeout as number | undefined) ?? 5_000, enabled: action !== "hover", stable: pointer, describe }, signal);
+ const mode = this.getMode();
+ if ("refused" in resolved) return { content: [{ type: "text", text: resolved.refused }], details: { status: "refused", mode, window_ref: window }, isError: true };
+ const { element, observation } = resolved;
+ const named = `${ariaRole(element)} ${JSON.stringify(accessibleName(element))} (${suggestLocator(element)})`;
+ const done = (status: "delivered" | "unknown", text: string): ComputerResult => ({ content: [{ type: "text", text }], details: { status, mode, window_ref: window }, ...(status === "delivered" ? {} : { isError: true }) });
+ if (action === "check" || action === "uncheck") {
+ const want = action === "check", state = checkedState(element);
+ if (state === null) return { content: [{ type: "text", text: `${named} reports no checked state; ${action} needs a checkbox, radio or toggle. Nothing was sent.` }], details: { status: "refused", mode, window_ref: window }, isError: true };
+ if (state === want) return done("delivered", `${named} is already ${want ? "checked" : "unchecked"}; nothing was sent.`);
+ }
+ const nativeCommand = { ...rest, action: action === "check" || action === "uncheck" ? "click" : action, element_token: element.element_token,
+ window_ref: window, observation_id: observation.details.observation_id };
+ this.snapshots.delete(observation.details.observation_id!);
+ const result = await session.act(nativeCommand as Action, signal);
+ const prefix = { type: "text" as const, text: `Target ${describe} resolved to ${named}.` };
+ if ((action !== "check" && action !== "uncheck") || result.details.status !== "delivered") return { ...result, content: [prefix, ...result.content] };
+ // Like Playwright's check(): confirm the new state, and say so when the app did not change it.
+ const after = await this.resolveNative(window, session, selector, { timeout_ms: 2_000, enabled: false, stable: false, describe }, signal);
+ const state = "element" in after ? checkedState(after.element) : null;
+ return state === (action === "check") ? { ...result, content: [prefix, ...result.content, { type: "text", text: `It is now ${action}ed.` }] }
+ : done("unknown", `${prefix.text} The click was delivered, but the element is ${state === null ? "no longer readable" : state === "mixed" ? "mixed" : state ? "checked" : "unchecked"} afterwards. Read the window before deciding; do not replay blindly.`);
+ }
+
/**
* Run up to 16 known steps in one call and stop at the first step that is not delivered or satisfied.
- * Browser steps are Playwright actions and conditions. Native steps re-resolve `target` {role,name,text}
- * against a fresh UI walk before each step; only the first step may use a hint or pixels of the current observation.
+ * Browser steps are Playwright actions and conditions. Native steps resolve `target` (a Playwright locator or
+ * JSON target) in fresh UI walks with auto-waiting; only the first step may use a hint or pixels of the current observation.
*/
private async batch(address: Extract<ComputerAddress, { kind: "observation" }>, snapshot: Snapshot, command: Record<string, unknown>, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {
only(command, ["action", "steps"]);
@@ -846,27 +937,27 @@ export class ComputerResources {
if (!Array.isArray(command.steps) || command.steps.length < 1 || command.steps.length > BATCH_STEPS) throw new ComputerRequestError(`batch needs steps: 1..${BATCH_STEPS} step objects.`);
const session = await this.sessionFor(cwd, { window_ref: window });
const browser = snapshot.result.details.dom !== undefined;
- type Step = { kind: "wait"; spec: WaitSpec; action: string } | { kind: "dom"; action: WebAction } | { kind: "native"; command: Record<string, unknown>; target?: Record<string, string> };
+ type Step = { kind: "wait"; spec: WaitSpec; action: string } | { kind: "dom"; action: WebAction } | { kind: "native"; command: Record<string, unknown>; target?: ParsedSelector; describe?: string; timeout?: number };
const steps: Step[] = command.steps.map((raw, index): Step => {
const step = object(raw);
const label = `steps[${index}]`;
try {
if (step.action === "wait_for" || step.action === "verify") return { kind: "wait", spec: session.waitSpec(window, step), action: String(step.action) };
if (browser) return { kind: "dom", action: validateWebAction(step) };
+ if (typeof step.action === "string" && LOCATOR_VERBS[step.action]) step.action = LOCATOR_VERBS[step.action];
if (!NATIVE_BATCH_ACTIONS.includes(String(step.action))) throw Error(`action must be one of ${NATIVE_BATCH_ACTIONS.join(", ")}, wait_for or verify.`);
if (["window_ref", "observation_id", "sequence"].some(key => key in step)) throw Error("identity fields and nested sequences are not allowed.");
const pinned = ["hint", "element_token", "x", "y"].some(key => key in step);
- if (pinned && index > 0) throw Error("only the first step may use a hint, token or pixels; later steps use target {role,name,text}.");
- let target: Record<string, string> | undefined;
+ if (pinned && index > 0) throw Error("only the first step may use a hint, token or pixels; later steps use a target (a Playwright locator or JSON target).");
+ let target: ParsedSelector | undefined;
if (step.target !== undefined) {
if (pinned) throw Error("choose target or a hint/pixel target, not both.");
- const value = object(step.target);
- if (!Object.keys(value).length || Object.entries(value).some(([key, item]) => !["role", "name", "text"].includes(key) || typeof item !== "string" || !item))
- throw Error("target needs non-empty role, name and/or text.");
- target = value as Record<string, string>;
+ target = nativeSelector(step.target);
}
- const { target: _target, ...nativeStep } = step;
- return { kind: "native", command: nativeStep, ...(target ? { target } : {}) };
+ const timeout = step.timeout_ms;
+ if (timeout !== undefined && (!Number.isSafeInteger(timeout) || (timeout as number) < 0 || (timeout as number) > 60_000)) throw Error("timeout_ms must be an integer 0..60000.");
+ const { target: raw, timeout_ms: _timeout, ...nativeStep } = step;
+ return { kind: "native", command: nativeStep, ...(target ? { target, describe: typeof raw === "string" ? raw : JSON.stringify(raw), timeout: (timeout as number | undefined) ?? 5_000 } : {}) };
} catch (error) { throw new ComputerRequestError(`${label}: ${error instanceof Error ? error.message : String(error)}`); }
});
this.snapshots.delete(address.ref);
@@ -893,19 +984,22 @@ export class ComputerResources {
} else {
let element: string | undefined;
if (index > 0 || step.target) {
- // Never reuse a stale observation: take a fresh UI walk and resolve the target in it.
- this.retireWindow(window);
- const fresh = this.remember({ window_ref: window }, await session.observeQuiet(window, signal));
- if (fresh.details.status !== "observed" || !fresh.details.ui) { last = fresh; rows.push({ index, action: String(step.command.action), outcome: "refused", reason: "No current UI sample." }); stop = `No current UI before step ${index}.`; break; }
- issued = fresh; observation = fresh.details.observation_id!;
+ // Never reuse a stale observation: a target resolves in fresh UI walks; other steps take one fresh walk.
if (step.target) {
- const found = fresh.details.ui.elements.filter(item => (!step.target!.role || item.role?.toLowerCase() === step.target!.role.toLowerCase())
- && (!step.target!.name || item.name === step.target!.name) && (!step.target!.text || `${item.name ?? ""}\n${item.value ?? ""}`.includes(step.target!.text)));
- if (found.length !== 1) {
- rows.push({ index, action: String(step.command.action), outcome: "refused", reason: found.length ? `${found.length} elements match the target.` : "No element matches the target in the current UI walk." });
+ const action = String(step.command.action), pointer = ["click", "double_click", "right_click"].includes(action);
+ const resolved = await this.resolveNative(window, session, step.target, { timeout_ms: step.timeout!, enabled: true, stable: pointer, describe: step.describe! }, signal);
+ if ("refused" in resolved) {
+ if (resolved.observation) last = resolved.observation;
+ rows.push({ index, action, outcome: "refused", reason: resolved.refused.slice(0, 600) });
stop = `Target not resolved at step ${index}; nothing was sent for it.`; break;
}
- element = found[0].element_token;
+ issued = resolved.observation; observation = resolved.observation.details.observation_id!;
+ element = resolved.element.element_token;
+ } else {
+ this.retireWindow(window);
+ const fresh = this.remember({ window_ref: window }, await session.observeQuiet(window, signal));
+ if (fresh.details.status !== "observed" || !fresh.details.ui) { last = fresh; rows.push({ index, action: String(step.command.action), outcome: "refused", reason: "No current UI sample." }); stop = `No current UI before step ${index}.`; break; }
+ issued = fresh; observation = fresh.details.observation_id!;
}
}
this.snapshots.delete(observation);
diff --git a/src/native-locator.ts b/src/native-locator.ts
new file mode 100644
index 0000000..d45f962
--- /dev/null
+++ b/src/native-locator.ts
@@ -0,0 +1,253 @@
+import { createRequire } from "node:module";
+import type { UiElement, UiResource } from "./ui-resources.ts";
+
+/** Playwright's own selector and locator parsers; loaded on first use because playwright-core takes about a second to load. */
+interface PlaywrightIso {
+ unsafeLocatorOrSelectorAsSelector(language: "javascript", locator: string, testIdAttribute: string): string;
+ parseSelector(selector: string): ParsedSelector;
+ parseAttributeSelector(selector: string, allowUnquotedStrings: boolean): { name: string; attributes: Attribute[] };
+ asLocator(language: "javascript", selector: string): string;
+}
+let loaded: PlaywrightIso | undefined;
+const iso = (): PlaywrightIso => loaded ??= (createRequire(import.meta.url)("playwright-core/lib/coreBundle") as { iso: PlaywrightIso }).iso;
+
+/**
+ * Playwright locators over a native UI Automation snapshot.
+ *
+ * Playwright parses the locator (`getByRole("button", { name: "Save" }).nth(1)`) into its internal selector
+ * (`internal:role=button[name="Save"i] >> nth=1`); this module evaluates that selector against the UIA tree
+ * with Playwright's semantics: chained parts search inside the previous matches, names and texts match a
+ * case-insensitive substring unless exact, the role engine skips hidden elements unless `includeHidden`.
+ */
+
+/** A parsed Playwright selector, as `iso.parseSelector` returns it. */
+export interface ParsedSelector { parts: { name: string; body: unknown; source: string }[] }
+/** The JSON target shared with browser pages; on native windows `ref` is an observation hint and `test_id` an AutomationId. */
+export interface NativeTarget {
+ ref?: string; role?: string; name?: string; label?: string; text?: string; placeholder?: string; test_id?: string; exact?: boolean;
+}
+
+/** UIA control type → ARIA roles, most specific first. The first one names the element in snapshots and suggestions. */
+const ROLES: Record<string, string[]> = {
+ appbar: ["toolbar"], button: ["button"], calendar: ["grid"], checkbox: ["checkbox"], combobox: ["combobox"], custom: ["generic"],
+ datagrid: ["grid", "table"], dataitem: ["row"], document: ["document"], edit: ["textbox"], group: ["group"], header: ["rowgroup"],
+ headeritem: ["columnheader"], hyperlink: ["link"], image: ["img"], list: ["list", "listbox"], listitem: ["listitem", "option"],
+ menu: ["menu"], menubar: ["menubar"], menuitem: ["menuitem"], pane: ["generic"], progressbar: ["progressbar"], radiobutton: ["radio"],
+ scrollbar: ["scrollbar"], semanticzoom: ["generic"], separator: ["separator"], slider: ["slider"], spinner: ["spinbutton"],
+ splitbutton: ["button"], statusbar: ["status"], tab: ["tablist"], tabitem: ["tab"], table: ["table", "grid"], text: ["text"],
+ thumb: ["generic"], titlebar: ["titlebar"], toolbar: ["toolbar"], tooltip: ["tooltip"], tree: ["tree"], treeitem: ["treeitem"],
+ window: ["window", "dialog"],
+};
+/** Roles Playwright's getByLabel looks at: labelled form controls. */
+const LABELLED = new Set(["textbox", "combobox", "checkbox", "radio", "slider", "spinbutton", "listbox", "switch"]);
+/** Roles whose `checked` state is a checkbox state; for buttons the same UIA toggle is `pressed`. */
+const CHECKABLE = new Set(["checkbox", "radio", "menuitem", "switch", "treeitem", "listitem"]);
+
+/** ARIA roles of a native element, primary first; the UIA control type name also matches. */
+export function ariaRoles(element: Pick<UiElement, "role">): string[] {
+ const control = (element.role ?? "").replace(/\s+/g, "").toLowerCase();
+ return ROLES[control] ?? (control ? [control] : ["generic"]);
+}
+export const ariaRole = (element: Pick<UiElement, "role">): string => ariaRoles(element)[0];
+
+/** The accessible name: UIA Name when the runtime reports it, otherwise the driver's label. */
+export const accessibleName = (element: UiElement): string => element.uia_name ?? element.name ?? "";
+
+/** Checkbox state: the UIA toggle when reported; older runtimes fold it into `selected` for checkable controls. */
+export function checkedState(element: UiElement): boolean | "mixed" | null {
+ if (element.checked === "on") return true;
+ if (element.checked === "off") return false;
+ if (element.checked === "mixed") return "mixed";
+ return element.checked === undefined && ["checkbox", "radio"].includes(ariaRole(element)) ? element.selected : null;
+}
+/** Hidden means UIA IsOffscreen; unknown visibility counts as visible, as an element without layout info would in a page. */
+export const isVisible = (element: UiElement): boolean => element.offscreen !== true;
+
+const normalize = (value: string) => value.replace(/\s+/g, " ").trim();
+
+/** A Playwright string or regex matcher: `"x"i` substring, `"x"s` exact, `/re/flags`, or an attribute value. */
+type Matcher = (value: string) => boolean;
+function stringMatcher(value: string | RegExp, exact: boolean): Matcher {
+ if (value instanceof RegExp) return text => value.test(normalize(text));
+ const wanted = normalize(value);
+ return exact ? text => normalize(text) === wanted : text => normalize(text).toLowerCase().includes(wanted.toLowerCase());
+}
+function textMatcher(body: unknown): Matcher {
+ const source = String(body);
+ const regex = /^\/(.*)\/([a-z]*)$/s.exec(source);
+ if (regex) return stringMatcher(new RegExp(regex[1], regex[2]), false);
+ const quoted = /^(".*")([is]?)$/s.exec(source);
+ if (!quoted) throw Error(`Unsupported text matcher ${source}.`);
+ return stringMatcher(JSON.parse(quoted[1]) as string, quoted[2] === "s");
+}
+type Attribute = { name: string; op: string; value: unknown; caseSensitive: boolean };
+function attributeMatcher(attribute: Attribute): Matcher {
+ if (attribute.op !== "=") throw Error(`Attribute operator ${attribute.op} is not supported for native windows.`);
+ return stringMatcher(attribute.value as string | RegExp, attribute.caseSensitive);
+}
+const parseAttributes = (body: unknown): { name: string; attributes: Attribute[] } => iso().parseAttributeSelector(String(body), true);
+
+/** Index, parent links and texts of one snapshot, so selectors evaluate without repeated scans. */
+class Tree {
+ readonly order = new Map<string, number>();
+ readonly byHint = new Map<string, UiElement>();
+ constructor(readonly elements: readonly UiElement[]) {
+ elements.forEach((element, index) => { this.order.set(element.hint, index); this.byHint.set(element.hint, element); });
+ }
+ /** Whether `element` lies strictly inside `root`. */
+ inside(element: UiElement, root: UiElement): boolean {
+ for (let parent = element.parent_hint; parent; parent = this.byHint.get(parent)?.parent_hint ?? null) if (parent === root.hint) return true;
+ return false;
+ }
+ descendants(roots: readonly UiElement[] | null): UiElement[] {
+ return roots === null ? [...this.elements] : this.elements.filter(element => roots.some(root => this.inside(element, root)));
+ }
+ /** Own text plus the text of every descendant, as Playwright's has-text sees an element's text content. */
+ subtreeText(element: UiElement): string {
+ return [element, ...this.descendants([element])].flatMap(item => [accessibleName(item), item.value ?? ""]).filter(Boolean).join(" ");
+ }
+ sort(elements: Iterable<UiElement>): UiElement[] {
+ return [...new Set(elements)].sort((a, b) => this.order.get(a.hint)! - this.order.get(b.hint)!);
+ }
+}
+
+/** One query engine: does `element` match this selector part? */
+function engine(part: ParsedSelector["parts"][number]): (element: UiElement) => boolean {
+ switch (part.name) {
+ case "internal:role": case "role": {
+ const { name: role, attributes } = parseAttributes(part.body);
+ const wanted = role.toLowerCase();
+ let includeHidden = false;
+ const checks: ((element: UiElement) => boolean)[] = [element => ariaRoles(element).includes(wanted) || (element.role ?? "").toLowerCase() === wanted];
+ for (const attribute of attributes) {
+ const value = attribute.value;
+ switch (attribute.name) {
+ case "name": { const match = attributeMatcher(attribute); checks.push(element => match(accessibleName(element))); break; }
+ case "include-hidden": includeHidden = value === true || value === "true"; break;
+ case "checked": checks.push(element => CHECKABLE.has(ariaRole(element)) && String(checkedState(element)) === String(value)); break;
+ case "pressed": checks.push(element => ariaRole(element) === "button" && element.checked !== undefined && String(checkedState({ ...element, role: "CheckBox" })) === String(value)); break;
+ case "disabled": checks.push(element => element.enabled !== null && String(element.enabled === false) === String(value)); break;
+ case "selected": checks.push(element => element.selected !== null && String(element.selected) === String(value)); break;
+ case "expanded": checks.push(element => element.expanded !== undefined && String(element.expanded) === String(value)); break;
+ default: throw Error(`getByRole option ${attribute.name} is not available for native windows.`);
+ }
+ }
+ return element => (includeHidden || isVisible(element)) && checks.every(check => check(element));
+ }
+ case "internal:text": { const match = textMatcher(part.body); return element => match(accessibleName(element)) || (ariaRole(element) === "text" && match(element.value ?? "")); }
+ case "internal:label": { const match = textMatcher(part.body); return element => LABELLED.has(ariaRole(element)) && match(accessibleName(element)); }
+ case "internal:testid": {
+ const [attribute] = parseAttributes(part.body).attributes;
+ const match = attributeMatcher(attribute);
+ return element => element.automation_id !== undefined && match(element.automation_id);
+ }
+ case "internal:attr": {
+ const [attribute] = parseAttributes(part.body).attributes;
+ const match = attributeMatcher(attribute);
+ if (attribute.name === "title" || attribute.name === "placeholder") return element => element.help_text !== undefined && match(element.help_text);
+ if (attribute.name === "alt") return element => ariaRole(element) === "img" && match(accessibleName(element));
+ throw Error(`Attribute ${attribute.name} is not available for native windows.`);
+ }
+ default:
+ throw Error(`Selector engine "${part.name}" is not available for native windows. Use getByRole, getByText, getByLabel, getByTestId (AutomationId), getByTitle, filter, nth, and or or.`);
+ }
+}
+const nested = (body: unknown): ParsedSelector => {
+ const parsed = (body as { parsed?: ParsedSelector })?.parsed;
+ if (!parsed) throw Error("A nested locator could not be read.");
+ return parsed;
+};
+
+/** Evaluate a parsed selector from `roots` (the whole window when null) in tree order. */
+function evaluate(tree: Tree, selector: ParsedSelector, roots: readonly UiElement[] | null): UiElement[] {
+ let current: UiElement[] | null = null;
+ let scope = roots;
+ for (const part of selector.parts) {
+ const all = () => current ?? tree.descendants(scope);
+ switch (part.name) {
+ case "nth": {
+ const index = Number(part.body), list = all();
+ const item = index < 0 ? list[list.length + index] : list[index];
+ current = item ? [item] : [];
+ break;
+ }
+ case "visible": current = all().filter(element => isVisible(element) === (String(part.body) === "true")); break;
+ case "internal:has-text": case "internal:has-not-text": {
+ const match = textMatcher(part.body), keep = part.name === "internal:has-text";
+ current = all().filter(element => match(tree.subtreeText(element)) === keep);
+ break;
+ }
+ case "internal:has": case "internal:has-not": {
+ const inner = nested(part.body), keep = part.name === "internal:has";
+ current = all().filter(element => (evaluate(tree, inner, [element]).length > 0) === keep);
+ break;
+ }
+ case "internal:and": { const other = new Set(evaluate(tree, nested(part.body), scope)); current = all().filter(element => other.has(element)); break; }
+ case "internal:or": current = tree.sort([...all(), ...evaluate(tree, nested(part.body), scope)]); break;
+ case "internal:describe": break;
+ default: {
+ const match = engine(part);
+ scope = current ?? roots;
+ current = tree.descendants(scope).filter(match);
+ }
+ }
+ }
+ return current ?? tree.descendants(roots);
+}
+
+/** Playwright's internal selector for a JSON target, in the same shape browser pages use. */
+export function targetSelector(target: NativeTarget): string {
+ const quote = (value: string, exact: boolean) => `${JSON.stringify(value)}${exact ? "s" : "i"}`;
+ const exact = target.exact === true;
+ const parts: string[] = [];
+ if (target.role !== undefined) parts.push(`internal:role=${target.role}${target.name !== undefined ? `[name=${quote(target.name, exact)}]` : ""}`);
+ if (target.label !== undefined) parts.push(`internal:label=${quote(target.label, exact)}`);
+ if (target.text !== undefined) parts.push(`internal:text=${quote(target.text, exact)}`);
+ if (target.placeholder !== undefined) parts.push(`internal:attr=[placeholder=${quote(target.placeholder, exact)}]`);
+ if (target.test_id !== undefined) parts.push(`internal:testid=[data-testid=${quote(target.test_id, true)}]`);
+ if (!parts.length) throw Error("target needs role, label, text, placeholder or test_id.");
+ return parts.slice(1).reduce((all, part) => `${all} >> internal:and=${JSON.stringify(part)}`, parts[0]);
+}
+
+/** Playwright's internal selector for a locator such as `getByRole("button", { name: "Save" }).nth(1)`. */
+export function locatorSelector(locator: string): string {
+ if (typeof locator !== "string" || !locator.trim() || locator.length > 2000) throw Error("A locator must be a Playwright locator string of at most 2000 characters.");
+ try { return iso().unsafeLocatorOrSelectorAsSelector("javascript", locator.trim().replace(/^(?:page|window|this)\./, ""), "data-testid"); }
+ catch (error) { throw Error(`The locator could not be parsed: ${error instanceof Error ? error.message : String(error)}`); }
+}
+
+/** Parse a Playwright locator for a native window. CSS and XPath parse here but refuse when evaluated. */
+export function parseLocator(locator: string): ParsedSelector {
+ return iso().parseSelector(locatorSelector(locator));
+}
+
+const TARGET_FIELDS = ["role", "name", "label", "text", "placeholder", "test_id", "exact"] as const;
+/**
+ * A native target: a Playwright locator string, or the JSON target browser pages take. Hints are snapshot-local,
+ * so `ref` refuses here: targets resolve in fresh UI walks.
+ */
+export function nativeSelector(value: unknown, what = "target"): ParsedSelector {
+ if (typeof value === "string") return parseLocator(value);
+ if (!value || typeof value !== "object" || Array.isArray(value)) throw Error(`${what} must be a Playwright locator string or an object with ${TARGET_FIELDS.join(", ")}.`);
+ const target = value as Record<string, unknown>;
+ if ("ref" in target) throw Error(`${what}.ref names an element of one observation; use a hint in the observation address, or a locator.`);
+ const extra = Object.keys(target).filter(key => !(TARGET_FIELDS as readonly string[]).includes(key));
+ if (extra.length) throw Error(`${what} does not accept ${extra.join(", ")}; use ${TARGET_FIELDS.join(", ")} or a locator string.`);
+ for (const key of TARGET_FIELDS) if (key !== "exact" && target[key] !== undefined && (typeof target[key] !== "string" || !target[key] || (target[key] as string).length > 512))
+ throw Error(`${what}.${key} must be a non-empty string of at most 512 characters.`);
+ if (target.exact !== undefined && typeof target.exact !== "boolean") throw Error(`${what}.exact must be boolean.`);
+ if (target.name !== undefined && target.role === undefined) throw Error(`${what}.name requires ${what}.role.`);
+ return iso().parseSelector(targetSelector(target as NativeTarget));
+}
+
+/** Elements of one snapshot that a parsed selector matches, in tree order. */
+export function queryNative(ui: Pick<UiResource, "elements">, selector: ParsedSelector): UiElement[] {
+ return evaluate(new Tree(ui.elements), selector, null);
+}
+
+/** A Playwright locator that names this element, for strict-mode errors and listings. */
+export function suggestLocator(element: UiElement): string {
+ const role = ariaRole(element), name = accessibleName(element);
+ if (element.automation_id && !/^\d+$/.test(element.automation_id)) return `getByTestId(${JSON.stringify(element.automation_id)})`;
+ return iso().asLocator("javascript", `internal:role=${role}${name ? `[name=${JSON.stringify(name)}s]` : ""}`);
+}
diff --git a/src/session.ts b/src/session.ts
index e958b7c..c10ea37 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -1145,6 +1145,11 @@ export class ComputerSession {
});
}
+ /** Whether this window is a browser this session opened or attached, whose pages Playwright drives. */
+ isBrowserWindow(ref: string): boolean {
+ return this.entries.get(ref)?.web !== undefined;
+ }
+
/** Validate a wait_for/verify command for this window: owned browsers take Playwright targets and page checks. */
waitSpec(ref: string, command: Record<string, unknown>): WaitSpec {
return validateWait(command, this.entries.get(ref)?.web !== undefined);
diff --git a/src/ui-conditions.ts b/src/ui-conditions.ts
index 4731ba0..b0880c1 100644
--- a/src/ui-conditions.ts
+++ b/src/ui-conditions.ts
@@ -1,17 +1,20 @@
-import type { UiElement, UiResource } from "./ui-resources.ts";
+import type { UiResource } from "./ui-resources.ts";
import { validateWebTarget, type WebTarget } from "./web-browser.ts";
+import { checkedState, isVisible, nativeSelector, queryNative, type ParsedSelector } from "./native-locator.ts";
/** Three-valued verdict. `unknown` means the evidence cannot decide, never a hidden `false`. */
export type Verdict = "satisfied" | "unsatisfied" | "unknown";
-/** Element state checked by a condition. Native UI reports only role, name, value, enabled and selected. */
+/** Element state checked by a condition. A state the native runtime does not report is unknown, never false. */
export type ElementState = "exists" | "absent" | "visible" | "hidden" | "enabled" | "disabled" | "selected" | "checked" | "unchecked" | "focused";
/**
- * One condition. Native windows match `target` by `role`, exact `name` or `text` contained in name or value;
- * browser pages use any Playwright target. `value` and `contains` compare the element's value (or text).
+ * One condition. `target` is a Playwright locator string or the JSON target; native windows evaluate it over
+ * the UIA tree with Playwright's semantics. `value` and `contains` compare the element's value (or text).
*/
export interface UiCondition {
- target?: { role?: string; name?: string; text?: string } | WebTarget;
+ target?: WebTarget;
+ /** The native target as Playwright's parsed selector. */
+ selector?: ParsedSelector;
is?: ElementState;
value?: string;
contains?: string;
@@ -27,12 +30,8 @@ export interface WaitResult {
}
const STATES: readonly ElementState[] = ["exists", "absent", "visible", "hidden", "enabled", "disabled", "selected", "checked", "unchecked", "focused"];
-const NATIVE_STATES: readonly ElementState[] = ["exists", "absent", "enabled", "disabled", "selected"];
+const NATIVE_STATES: readonly ElementState[] = ["exists", "absent", "visible", "hidden", "enabled", "disabled", "selected", "checked", "unchecked", "focused"];
-const text = (value: unknown, what: string): string => {
- if (typeof value !== "string" || !value || value.length > 512) throw Error(`${what} must be a non-empty string of at most 512 characters.`);
- return value;
-};
const integer = (value: unknown, min: number, max: number, fallback: number, what: string): number => {
if (value === undefined) return fallback;
if (!Number.isSafeInteger(value) || (value as number) < min || (value as number) > max) throw Error(`${what} must be an integer ${min}..${max}.`);
@@ -65,18 +64,14 @@ export function validateWait(command: Record<string, unknown>, browser: boolean)
}
if (value.target !== undefined) {
if (browser) condition.target = validateWebTarget(value.target);
- else {
- const target = value.target as Record<string, unknown>;
- if (!target || typeof target !== "object" || Array.isArray(target) || Object.keys(target).some(key => !["role", "name", "text"].includes(key)) || !Object.keys(target).length)
- throw Error(`conditions[${index}].target needs role, name and/or text for a native window.`);
- condition.target = Object.fromEntries(Object.entries(target).map(([key, item]) => [key, text(item, `conditions[${index}].target.${key}`)]));
- }
+ else condition.selector = nativeSelector(value.target, `conditions[${index}].target`);
}
const checks = [condition.is, condition.value, condition.contains].filter(item => item !== undefined).length;
if (checks > 1) throw Error(`conditions[${index}] takes one of is, value or contains; use separate conditions for several checks.`);
const elementCheck = checks === 1;
const pageCheck = condition.url_contains !== undefined || condition.title !== undefined;
- if (elementCheck === pageCheck || (elementCheck && !condition.target) || (pageCheck && condition.target))
+ const targeted = condition.target !== undefined || condition.selector !== undefined;
+ if (elementCheck === pageCheck || (elementCheck && !targeted) || (pageCheck && targeted))
throw Error(`conditions[${index}] needs a target with is/value/contains, or a page check (url_contains/title) without a target.`);
return condition;
});
@@ -90,13 +85,6 @@ export function combine(results: readonly ConditionResult[]): Verdict {
return results.some(r => r.verdict === "unsatisfied") ? "unsatisfied" : results.some(r => r.verdict === "unknown") ? "unknown" : "satisfied";
}
-function matches(element: UiElement, target: { role?: string; name?: string; text?: string }): boolean {
- if (target.role !== undefined && element.role?.toLowerCase() !== target.role.toLowerCase()) return false;
- if (target.name !== undefined && element.name !== target.name) return false;
- if (target.text !== undefined && !`${element.name ?? ""}\n${element.value ?? ""}`.includes(target.text)) return false;
- return true;
-}
-
/**
* Evaluate native conditions against one bounded UI walk. Absence is decided only when the walk is
* complete and untruncated; otherwise a missing element is `unknown`. Several matches are ambiguous.
@@ -104,8 +92,13 @@ function matches(element: UiElement, target: { role?: string; name?: string; tex
export function evaluateNative(ui: UiResource, conditions: readonly UiCondition[], texts?: ReadonlyMap<string, string>): ConditionResult[] {
const complete = ui.complete === true && !ui.truncated;
return conditions.map((condition, index) => {
- const target = condition.target as { role?: string; name?: string; text?: string };
- const found = ui.elements.filter(element => matches(element, target));
+ const found = queryNative(ui, condition.selector!);
+ // Like Playwright's toBeHidden: no matching element, or one that is not visible.
+ if (condition.is === "hidden") {
+ if (found.length > 1) return { index, verdict: "unknown", reason: `${found.length} elements match; narrow the target.` };
+ if (found.length === 1) return { index, verdict: isVisible(found[0]) ? "unsatisfied" : "satisfied", observed: isVisible(found[0]) ? "visible" : "offscreen" };
+ return complete ? { index, verdict: "satisfied", observed: "no match" } : { index, verdict: "unknown", reason: "No match in an incomplete UI walk; the element may exist beyond it." };
+ }
if (condition.is === "absent")
return found.length ? { index, verdict: "unsatisfied", observed: `${found.length} match(es)` }
: complete ? { index, verdict: "satisfied", observed: "no match" }
@@ -123,6 +116,13 @@ export function evaluateNative(ui: UiResource, conditions: readonly UiCondition[
if (condition.is === "enabled") return known(element.enabled, true);
if (condition.is === "disabled") return known(element.enabled, false);
if (condition.is === "selected") return known(element.selected, true);
+ if (condition.is === "visible") return { index, verdict: isVisible(element) ? "satisfied" : "unsatisfied", observed: isVisible(element) ? "visible" : "offscreen" };
+ if (condition.is === "checked" || condition.is === "unchecked") {
+ const state = checkedState(element);
+ return state === null ? { index, verdict: "unknown", reason: "The native UI reports no checked state for this element." }
+ : { index, verdict: (state === true) === (condition.is === "checked") && state !== "mixed" ? "satisfied" : "unsatisfied", observed: String(state) };
+ }
+ if (condition.is === "focused") return known(element.focused ?? null, true);
if (condition.value !== undefined || condition.contains !== undefined) {
if (observed === null) return { index, verdict: "unknown", reason: "The element reports no value or name." };
if (element.text_truncated && full === undefined) return { index, verdict: "unknown", observed, reason: "The element text was truncated at 1024 characters." };
diff --git a/src/ui-resources.ts b/src/ui-resources.ts
index 8f529d6..81c37d2 100644
--- a/src/ui-resources.ts
+++ b/src/ui-resources.ts
@@ -11,6 +11,11 @@ export interface UiElement {
role: string | null; name: string | null; value: string | null; text_truncated: boolean;
enabled: boolean | null; selected: boolean | null; actions: string[]; depth: number | null;
bounds: { native_screen?: ScreenRect; window_image?: ScreenRect };
+ /** Structural fields from runtimes that report the full UIA tree; absent when the runtime does not report them. */
+ uia_name?: string; automation_id?: string; class_name?: string; help_text?: string;
+ checked?: "on" | "off" | "mixed"; expanded?: boolean; offscreen?: boolean; focused?: boolean;
+ password?: boolean; read_only?: boolean; required?: boolean; actionable?: boolean;
+ range?: { value: number; min: number; max: number };
hint_display: "pending" | "shown" | "disabled" | "no_bounds" | "outside_image" | "collision" | "off";
}
/** Bounded native UIA projection. Completeness is not inferred from an empty search. */
@@ -60,6 +65,19 @@ export function validateUiCapture(options: UiCaptureOptions): void {
throw Error("UI-only observation has no image hints or guides.");
}
+/** Optional structural fields, copied only when the runtime reports them with the expected type. */
+function structural(e: Record<string, unknown>): Partial<UiElement> {
+ const out: Partial<UiElement> = {};
+ for (const [from, to] of [["name", "uia_name"], ["automation_id", "automation_id"], ["class_name", "class_name"], ["help_text", "help_text"]] as const)
+ if (typeof e[from] === "string" && e[from]) out[to] = (e[from] as string).slice(0, 1024);
+ if (e.checked === "on" || e.checked === "off" || e.checked === "mixed") out.checked = e.checked;
+ for (const key of ["expanded", "offscreen", "focused", "password", "read_only", "required", "actionable"] as const)
+ if (typeof e[key] === "boolean") out[key] = e[key] as boolean;
+ const range = e.range as { value?: unknown; min?: unknown; max?: unknown } | undefined;
+ if (range && [range.value, range.min, range.max].every(Number.isFinite)) out.range = { value: range.value as number, min: range.min as number, max: range.max as number };
+ return out;
+}
+
/** Adapt reported properties only. Native bounds are never treated as local image coordinates. */
export function createUiResource(state: Record<string, unknown>, window: string, observation: string, serial: number): UiResource {
const native = (Array.isArray(state.elements) ? state.elements : []) as Record<string, unknown>[];
@@ -74,6 +92,7 @@ export function createUiResource(state: Record<string, unknown>, window: string,
actions: (Array.isArray(e.actions) ? e.actions : []).filter((a): a is string => typeof a === "string").slice(0, 32),
bounds: { ...(rect(e.frame) ? { native_screen: rect(e.frame) } : {}), ...(rect(e.screenshot_frame) ? { window_image: rect(e.screenshot_frame) } : {}) },
hint_display: e.enabled === false ? "disabled" : rect(e.screenshot_frame) ? "pending" : "no_bounds",
+ ...structural(e),
}));
return { window_ref: window, observation_id: observation, snapshot_id: state.snapshot_id ?? null,
captured_at: new Date().toISOString(), complete: state.truncated === true ? false : bool(state.elements_complete),
diff --git a/src/web-browser.ts b/src/web-browser.ts
index 2ebc9fe..c972950 100644
--- a/src/web-browser.ts
+++ b/src/web-browser.ts
@@ -4,12 +4,15 @@ import { promisify } from "node:util";
import type { CommandRunner } from "./runner.ts";
import { ATTACH_APPROVAL_MS } from "./browser-command-line.ts";
import { findWindowsNode } from "./windows-node.ts";
+import { locatorSelector } from "./native-locator.ts";
const run = promisify(execFile);
/** An element chosen by Playwright: an aria ref from the latest DOM snapshot, or locator predicates that must all match. */
export interface WebTarget {
ref?: string; role?: string; name?: string; label?: string; text?: string; placeholder?: string; test_id?: string; css?: string;
+ /** Playwright's internal selector, parsed from a locator string such as getByRole("button", { name: "Save" }). */
+ locator?: string;
exact?: boolean;
/** CSS selectors of nested iframes, outermost first. Refs already carry their frame. */
frames?: string[];
@@ -134,6 +137,8 @@ function httpUrl(value: unknown): string {
const PREDICATES = ["role", "name", "label", "text", "placeholder", "test_id", "css"] as const;
/** Check target shape only; Playwright resolves, waits for and disambiguates the element. */
export function validateWebTarget(value: unknown): WebTarget {
+ // A Playwright locator string, such as getByRole("button", { name: "Save" }).nth(1), in Playwright's own syntax.
+ if (typeof value === "string") return { locator: locatorSelector(value) };
const target = record(value, "target");
only(target, ["ref", ...PREDICATES, "exact", "frames"], "target");
if (target.ref !== undefined) {
diff --git a/src/web-worker.ts b/src/web-worker.ts
index 76bb87f..8fe7e8b 100644
--- a/src/web-worker.ts
+++ b/src/web-worker.ts
@@ -137,6 +137,7 @@ function untilDialog<T>(tab: Tab, operation: Promise<T>): Promise<T | "dialog">
function locate(page: Page, target: WebTarget): Locator {
if (target.ref) return page.getByRef(target.ref);
+ if (target.locator) return page.locator(target.locator);
let root: Page | FrameLocator = page;
for (const frame of target.frames ?? []) root = root.frameLocator(frame);
const exact = target.exact;
diff --git a/test/native-locator.test.ts b/test/native-locator.test.ts
new file mode 100644
index 0000000..97022f1
--- /dev/null
+++ b/test/native-locator.test.ts
@@ -0,0 +1,97 @@
+import { beforeAll, expect, test } from "vitest";
+import { ariaRole, parseLocator, queryNative, suggestLocator, targetSelector } from "../src/native-locator.ts";
+import type { UiElement } from "../src/ui-resources.ts";
+
+/** The WinForms fixture of the live checks, as a runtime with the full UIA tree reports it. */
+function element(hint: string, role: string, name: string, extra: Partial<UiElement> = {}): UiElement {
+ return { hint, element_token: `s1:${hint}`, index: null, parent_hint: "w", role, name, value: null, text_truncated: false,
+ enabled: true, selected: null, actions: [], depth: 1, bounds: {}, hint_display: "off", uia_name: name, ...extra };
+}
+const tree = { elements: [
+ element("w", "Window", "Fixture", { parent_hint: null, depth: 0 }),
+ element("count", "Button", "Count 0", { automation_id: "countButton" }),
+ element("off", "Button", "Disabled", { enabled: false }),
+ element("name", "Edit", "Name", { value: "Ada Lovelace" }),
+ element("agree", "CheckBox", "Agree", { checked: "on" }),
+ element("opts", "Group", "Options"),
+ element("ok", "Button", "OK", { parent_hint: "opts", depth: 2 }),
+ element("pick", "List", "Pick"),
+ element("alpha", "ListItem", "alpha", { parent_hint: "pick", depth: 2, selected: false }),
+ element("beta", "ListItem", "beta", { parent_hint: "pick", depth: 2, selected: true }),
+ element("hidden", "ListItem", "gamma", { parent_hint: "pick", depth: 2, offscreen: true }),
+ element("status", "Text", "clicked 3"),
+ element("save", "Button", "Save", { help_text: "Save the file" }),
+ element("saveAs", "Button", "Save as…"),
+] };
+// playwright-core loads on first use; under a parallel test run that can take several seconds.
+beforeAll(() => { parseLocator("getByRole('button')"); }, 60_000);
+const hints = (locator: string) => queryNative(tree, parseLocator(locator)).map(found => found.hint);
+
+test("getByRole matches ARIA or UIA roles and names the Playwright way: substring by default, exact on request", () => {
+ expect(hints(`getByRole("button", { name: "Count 0" })`)).toEqual(["count"]);
+ expect(hints(`getByRole("Button", { name: "count" })`)).toEqual(["count"]);
+ expect(hints(`getByRole("button", { name: "Save" })`)).toEqual(["save", "saveAs"]);
+ expect(hints(`getByRole("button", { name: "Save", exact: true })`)).toEqual(["save"]);
+ expect(hints(`getByRole("button", { name: /^save$/i })`)).toEqual(["save"]);
+ expect(hints(`getByRole("textbox")`)).toEqual(["name"]);
+ expect(hints(`getByRole("option")`)).toEqual(["alpha", "beta"]);
+});
+
+test("role states filter like Playwright's options; hidden elements need includeHidden", () => {
+ expect(hints(`getByRole("checkbox", { checked: true })`)).toEqual(["agree"]);
+ expect(hints(`getByRole("checkbox", { checked: false })`)).toEqual([]);
+ expect(hints(`getByRole("button", { disabled: true })`)).toEqual(["off"]);
+ expect(hints(`getByRole("listitem", { selected: true })`)).toEqual(["beta"]);
+ expect(hints(`getByRole("listitem", { includeHidden: true })`)).toEqual(["alpha", "beta", "hidden"]);
+ expect(() => hints(`getByRole("heading", { level: 2 })`)).toThrow(/level is not available for native windows/);
+});
+
+test("text, label, test id and title locators read the matching UIA properties", () => {
+ expect(hints(`getByText("clicked")`)).toEqual(["status"]);
+ expect(hints(`getByText("Clicked 3", { exact: true })`)).toEqual([]);
+ expect(hints(`getByText("clicked 3", { exact: true })`)).toEqual(["status"]);
+ expect(hints(`getByLabel("Name")`)).toEqual(["name"]);
+ expect(hints(`getByLabel("Agree")`)).toEqual(["agree"]);
+ expect(hints(`getByTestId("countButton")`)).toEqual(["count"]);
+ expect(hints(`getByTitle("Save the file")`)).toEqual(["save"]);
+});
+
+test("chaining searches inside the previous matches; filter, nth, and, or compose", () => {
+ expect(hints(`getByRole("group", { name: "Options" }).getByRole("button")`)).toEqual(["ok"]);
+ expect(hints(`getByRole("list").getByRole("listitem").filter({ hasText: "beta" })`)).toEqual(["beta"]);
+ expect(hints(`getByRole("listitem").filter({ hasNotText: "beta" })`)).toEqual(["alpha"]);
+ expect(hints(`getByRole("group").filter({ has: getByRole("button", { name: "OK" }) })`)).toEqual(["opts"]);
+ expect(hints(`getByRole("list").filter({ hasText: "beta" })`)).toEqual(["pick"]);
+ expect(hints(`getByRole("button").first()`)).toEqual(["count"]);
+ expect(hints(`getByRole("button").last()`)).toEqual(["saveAs"]);
+ expect(hints(`getByRole("button").nth(2)`)).toEqual(["ok"]);
+ expect(hints(`getByRole("button").and(getByTitle("Save the file"))`)).toEqual(["save"]);
+ expect(hints(`getByRole("checkbox").or(getByRole("textbox"))`)).toEqual(["name", "agree"]);
+ expect(hints(`page.getByRole("button", { name: "OK" })`)).toEqual(["ok"]);
+});
+
+test("CSS, XPath and unparsable locators refuse instead of guessing", () => {
+ expect(() => hints(`locator("css=div")`)).toThrow(/not available for native windows/);
+ expect(() => hints(`locator("//div")`)).toThrow(/not available for native windows/);
+ expect(() => hints(`getByRole(`)).toThrow(/could not be parsed/);
+});
+
+test("the JSON target shared with browser pages becomes the same Playwright selector", () => {
+ const byTarget = (target: Parameters<typeof targetSelector>[0]) => queryNative(tree, parseLocator(targetSelector(target))).map(found => found.hint);
+ expect(byTarget({ role: "button", name: "Save" })).toEqual(["save", "saveAs"]);
+ expect(byTarget({ role: "button", name: "Save", exact: true })).toEqual(["save"]);
+ expect(byTarget({ role: "textbox", label: "Name" })).toEqual(["name"]);
+ expect(byTarget({ text: "clicked" })).toEqual(["status"]);
+ expect(byTarget({ test_id: "countButton" })).toEqual(["count"]);
+ expect(() => targetSelector({})).toThrow(/needs role/);
+});
+
+test("suggested locators name an element with Playwright syntax", () => {
+ const byHint = (hint: string) => tree.elements.find(found => found.hint === hint)!;
+ expect(suggestLocator(byHint("count"))).toBe(`getByTestId("countButton")`);
+ expect(suggestLocator(byHint("ok"))).toBe(`getByRole('button', { name: 'OK', exact: true })`);
+ expect(ariaRole(byHint("name"))).toBe("textbox");
+ // Older runtimes fold the checkbox toggle into `selected`.
+ const old = { elements: [element("c", "CheckBox", "Agree", { uia_name: undefined, selected: true })] };
+ expect(queryNative(old, parseLocator(`getByRole("checkbox", { name: "Agree", checked: true })`)).length).toBe(1);
+});
diff --git a/test/ui-conditions.test.ts b/test/ui-conditions.test.ts
index 11f540c..43aa57c 100644
--- a/test/ui-conditions.test.ts
+++ b/test/ui-conditions.test.ts
@@ -1,14 +1,16 @@
-import { expect, test } from "vitest";
+import { beforeAll, expect, test } from "vitest";
+import { parseLocator } from "../src/native-locator.ts";
import { combine, evaluateNative, validateWait } from "../src/ui-conditions.ts";
import type { UiElement, UiResource } from "../src/ui-resources.ts";
+beforeAll(() => { parseLocator("getByRole('button')"); }, 60_000);
const element = (fields: Partial<UiElement>): UiElement => ({ hint: "1:a", element_token: "t", index: 0, parent_hint: null, role: null, name: null, value: null,
text_truncated: false, enabled: true, selected: null, actions: [], depth: 0, bounds: {}, hint_display: "off", ...fields });
const ui = (elements: UiElement[], complete: boolean | null): UiResource => ({ window_ref: "w", observation_id: "o", snapshot_id: null, captured_at: "",
complete, truncated: false, truncation_reason: null, native_total: elements.length, collected: elements.length, matches: elements.length, projection_truncated: false, elements });
test("native absence is decided only by a complete walk; a missing element in a partial walk stays unknown", () => {
- const conditions = validateWait({ action: "verify", conditions: [{ target: { role: "Button", name: "Gone" }, is: "absent" }, { target: { name: "Status" }, value: "ok" }] }, false).conditions;
+ const conditions = validateWait({ action: "verify", conditions: [{ target: { role: "Button", name: "Gone" }, is: "absent" }, { target: { text: "Status" }, value: "ok" }] }, false).conditions;
expect(evaluateNative(ui([], true), conditions).map(r => r.verdict)).toEqual(["satisfied", "unsatisfied"]);
expect(evaluateNative(ui([], false), conditions).map(r => r.verdict)).toEqual(["unknown", "unknown"]);
expect(evaluateNative(ui([], null), conditions).map(r => r.verdict)).toEqual(["unknown", "unknown"]);
@@ -20,12 +22,12 @@ test("native checks compare reported properties and keep ambiguity and missing p
const check = (condition: object) => evaluateNative(tree, validateWait({ action: "verify", conditions: [condition] }, false).conditions)[0];
expect(check({ target: { role: "Button", name: "Save" }, is: "disabled" }).verdict).toBe("satisfied");
expect(check({ target: { role: "Button", name: "Save" }, is: "enabled" }).verdict).toBe("unsatisfied");
- expect(check({ target: { name: "Status" }, value: "saved Ada" }).verdict).toBe("satisfied");
- expect(check({ target: { name: "Status" }, contains: "Bob" }).verdict).toBe("unsatisfied");
+ expect(check({ target: { text: "Status" }, value: "saved Ada" }).verdict).toBe("satisfied");
+ expect(check({ target: "getByText('Status')", contains: "Bob" }).verdict).toBe("unsatisfied");
expect(check({ target: { role: "Button" }, is: "exists" }).verdict).toBe("unknown");
- expect(check({ target: { name: "Save" }, is: "selected" }).verdict).toBe("unknown");
+ expect(check({ target: { role: "button", name: "Save", exact: true }, is: "selected" }).verdict).toBe("unknown");
// Truncated text cannot prove an exact value either way.
- expect(check({ target: { name: "Long" }, value: "x" }).verdict).toBe("unknown");
+ expect(check({ target: { text: "Long" }, value: "x" }).verdict).toBe("unknown");
});
test("one unsatisfied condition decides; otherwise any unknown keeps the result unknown", () => {
@@ -38,7 +40,9 @@ test("wait commands are bounded and reject states the backend cannot report", ()
expect(validateWait({ action: "wait_for", conditions: [{ target: { role: "status" }, value: "ok" }] }, true))
.toMatchObject({ timeout_ms: 5000, interval_ms: 250, stable: 1 });
expect(validateWait({ action: "verify", conditions: [{ url_contains: "/done" }] }, true)).toMatchObject({ timeout_ms: 0, stable: 1 });
- expect(() => validateWait({ action: "wait_for", conditions: [{ target: { role: "Document" }, is: "visible" }] }, false)).toThrow(/native windows/);
+ expect(validateWait({ action: "wait_for", conditions: [{ target: { role: "Document" }, is: "visible" }] }, false).conditions).toHaveLength(1);
+ expect(() => validateWait({ action: "wait_for", conditions: [{ target: { css: "div" }, is: "visible" }] }, false)).toThrow(/does not accept css/);
+ expect(() => validateWait({ action: "wait_for", conditions: [{ target: { name: "Save" }, is: "visible" }] }, false)).toThrow(/name requires/);
expect(() => validateWait({ action: "wait_for", conditions: [{ url_contains: "/x" }] }, false)).toThrow(/does not accept: url_contains/);
expect(() => validateWait({ action: "wait_for", conditions: [{ target: { role: "status" } }] }, true)).toThrow(/needs a target with/);
expect(() => validateWait({ action: "wait_for", conditions: [{ target: { role: "status" }, value: "a", contains: "b" }] }, true)).toThrow(/one of is, value or contains/);