Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Address native windows with Playwright locators (LPT-911)
alexshpunt-clanker[bot]f281250dd285
diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex 7875858..b1c2046 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -12,9 +12,10 @@ import { parseWindowManagement, type WindowManagementCommand } from "./window-maimport { validateWebAction, validateWebLaunch, type WebAction } from "./web-browser.ts";import { combine, evaluateNative, type WaitSpec } from "./ui-conditions.ts";import { diffDom, diffUi } from "./successor.ts";-import type { UiResource } from "./ui-resources.ts";+import type { UiElement, UiResource } from "./ui-resources.ts";import { displayMode } from "./display-mode.ts";import { isSemanticAction } from "./semantic-actions.ts";+import { accessibleName, ariaRole, checkedState, isVisible, nativeSelector, queryNative, suggestLocator, type ParsedSelector } from "./native-locator.ts";import { SETUP_PARTS, SetupError, runSetup, type SetupPart } from "./setup.ts";/** The machine this extension runs on, by its address name. */@@ -34,6 +35,10 @@ function commandObject(value: string): Record<string, unknown> {const BATCH_STEPS = 200;/** Native actions a batch may run; each later step re-resolves its target in a fresh UI walk. */const NATIVE_BATCH_ACTIONS = ["click", "double_click", "right_click", "type_text", "key", "hotkey", "scroll", "invoke", "select", "set_value"];+/** Native actions that take a Playwright locator or JSON target. */+const NATIVE_LOCATOR_ACTIONS = ["click", "double_click", "right_click", "hover", "type_text", "scroll", "invoke", "select", "set_value", "check", "uncheck"];+/** Playwright verbs with a different native name. */+const LOCATOR_VERBS: Record<string, string> = { fill: "set_value", dblclick: "double_click" };/** An action's optional postcondition and successor request. */interface FollowUp { expect?: WaitSpec; successor: boolean; preexisting?: boolean }type ObservationTarget = { window_ref: string; display?: never } | { display: string; window_ref?: never };@@ -702,8 +707,12 @@ export class ComputerResources {only(command, ["action"]);return session.attachBrowser(address.ref, signal);}+ if (command.target !== undefined && command.action !== "wait_for" && command.action !== "verify") {+ if (session.isBrowserWindow(address.ref)) throw new ComputerRequestError("For a browser window, Read it with dom and Insert the action into that DOM observation; Playwright resolves the target there.");+ return this.locatorAction(address.ref, session, command, signal);+ }if (command.action !== "wait_for" && command.action !== "verify")- throw new ComputerRequestError("A window address accepts only wait_for, verify, attach_browser or kill. Input needs the observation address returned by a fresh window Read.");+ throw new ComputerRequestError("A window address accepts an action with a target (a Playwright locator such as getByRole(\"button\", { name: \"Save\" })), wait_for, verify, attach_browser or kill. Other input needs the observation address returned by a fresh window Read.");let spec;try { spec = session.waitSpec(address.ref, command); }catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }@@ -734,6 +743,12 @@ export class ComputerResources {const result = await this.batch(address, snapshot, command, cwd, signal);return follow ? this.followUp(window, snapshot, result, follow, cwd, signal) : result;}+ if (command.target !== undefined && !snapshot.result.details.dom) {+ if (address.hint !== undefined) throw new ComputerRequestError("Choose the address hint or a JSON target, not both.");+ this.snapshots.delete(address.ref);+ const result = await this.locatorAction(window, await this.sessionFor(cwd, { window_ref: window }), command, signal);+ return follow ? this.followUp(window, snapshot, result, follow, cwd, signal) : result;+ }}const result = await this.act(address, snapshot, input, tool, cwd, signal);return follow && window ? this.followUp(window, snapshot, result, follow, cwd, signal) : result;@@ -834,10 +849,86 @@ export class ComputerResources {return { ...result, details };}+ /**+ * Resolve a Playwright locator the way Playwright's actions do: take fresh UI walks until exactly one element+ * matches and is ready (visible; enabled for input; for pointer input also at the same place in two walks), or+ * until the timeout. Several matches refuse at once with each match's locator, like Playwright's strict mode.+ */+ private async resolveNative(window: string, session: ComputerSession, selector: ParsedSelector, options: { timeout_ms: number; enabled: boolean; stable: boolean; describe: string }, signal?: AbortSignal):+ Promise<{ element: UiElement; observation: ComputerResult } | { refused: string; observation?: ComputerResult }> {+ const deadline = performance.now() + options.timeout_ms;+ let previous: UiElement | undefined, reason = "";+ for (let attempt = 0; ; attempt++) {+ if (signal?.aborted) return { refused: "Cancelled while waiting for the target; nothing was sent." };+ this.retireWindow(window);+ const fresh = this.remember({ window_ref: window }, await session.observeQuiet(window, signal));+ if (fresh.details.status !== "observed" || !fresh.details.ui) return { refused: `No current UI sample of the window: ${computerResultText(fresh).split("\n")[0]}`, observation: fresh };+ const ui = fresh.details.ui;+ let found: UiElement[];+ try { found = queryNative(ui, selector); }+ catch (error) { return { refused: error instanceof Error ? error.message : String(error), observation: fresh }; }+ if (found.length > 1) {+ const listed = found.slice(0, 8).map(item => ` ${item.hint} ${ariaRole(item)} ${JSON.stringify(accessibleName(item))} → ${suggestLocator(item)}`).join("\n");+ return { refused: `Strict mode: ${options.describe} matches ${found.length} elements; nothing was sent. Narrow it, for example with exact, filter or nth:\n${listed}${found.length > 8 ? `\n …and ${found.length - 8} more` : ""}`, observation: fresh };+ }+ const element = found[0];+ if (!element) reason = ui.complete === true && !ui.truncated ? "no element matches it" : "no element matches it in the walked part of the window (the UI walk was incomplete)";+ else if (!isVisible(element)) reason = "the element is not visible (offscreen)";+ else if (options.enabled && element.enabled === false) reason = "the element is disabled";+ else if (options.stable && previous && JSON.stringify(previous.bounds.native_screen) !== JSON.stringify(element.bounds.native_screen)) reason = "the element is still moving";+ else if (!options.stable || previous || !element.bounds.native_screen) return { element, observation: fresh };+ else reason = "the element is still moving";+ previous = element && accessibleName(element) === accessibleName(previous ?? element) ? element : undefined;+ const left = deadline - performance.now();+ if (left <= 0) return { refused: `Timed out after ${options.timeout_ms} ms waiting for ${options.describe}: ${reason}. Nothing was sent.`, observation: fresh };+ await new Promise(resolve => setTimeout(resolve, Math.min(left, [0, 20, 50, 100, 100, 250][attempt] ?? 500)));+ }+ }++ /**+ * One native action addressed by a Playwright locator or JSON target, without a prior Read: resolve it in a fresh+ * walk with auto-waiting, then act on that element. `check`/`uncheck` act only when the state differs and confirm it.+ */+ private async locatorAction(window: string, session: ComputerSession, command: Record<string, unknown>, signal?: AbortSignal): Promise<ComputerResult> {+ const { target, timeout_ms: timeout, ...rest } = command;+ let selector: ParsedSelector;+ try { selector = nativeSelector(target); }+ catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }+ if (timeout !== undefined && (!Number.isSafeInteger(timeout) || (timeout as number) < 0 || (timeout as number) > 60_000)) throw new ComputerRequestError("timeout_ms must be an integer 0..60000.");+ if (["hint", "element_token", "x", "y", "window_ref", "observation_id"].some(key => key in rest)) throw new ComputerRequestError("Choose target or a hint, token or pixels, not both.");+ const verb = String(rest.action);+ const action = LOCATOR_VERBS[verb] ?? verb;+ if (!NATIVE_LOCATOR_ACTIONS.includes(action)) throw new ComputerRequestError(`A target takes one of ${[...NATIVE_LOCATOR_ACTIONS, ...Object.keys(LOCATOR_VERBS)].join(", ")}.`);+ const describe = typeof target === "string" ? target : JSON.stringify(target);+ const pointer = ["click", "double_click", "right_click", "hover", "check", "uncheck"].includes(action);+ const resolved = await this.resolveNative(window, session, selector, { timeout_ms: (timeout as number | undefined) ?? 5_000, enabled: action !== "hover", stable: pointer, describe }, signal);+ const mode = this.getMode();+ if ("refused" in resolved) return { content: [{ type: "text", text: resolved.refused }], details: { status: "refused", mode, window_ref: window }, isError: true };+ const { element, observation } = resolved;+ const named = `${ariaRole(element)} ${JSON.stringify(accessibleName(element))} (${suggestLocator(element)})`;+ const done = (status: "delivered" | "unknown", text: string): ComputerResult => ({ content: [{ type: "text", text }], details: { status, mode, window_ref: window }, ...(status === "delivered" ? {} : { isError: true }) });+ if (action === "check" || action === "uncheck") {+ const want = action === "check", state = checkedState(element);+ if (state === null) return { content: [{ type: "text", text: `${named} reports no checked state; ${action} needs a checkbox, radio or toggle. Nothing was sent.` }], details: { status: "refused", mode, window_ref: window }, isError: true };+ if (state === want) return done("delivered", `${named} is already ${want ? "checked" : "unchecked"}; nothing was sent.`);+ }+ const nativeCommand = { ...rest, action: action === "check" || action === "uncheck" ? "click" : action, element_token: element.element_token,+ window_ref: window, observation_id: observation.details.observation_id };+ this.snapshots.delete(observation.details.observation_id!);+ const result = await session.act(nativeCommand as Action, signal);+ const prefix = { type: "text" as const, text: `Target ${describe} resolved to ${named}.` };+ if ((action !== "check" && action !== "uncheck") || result.details.status !== "delivered") return { ...result, content: [prefix, ...result.content] };+ // Like Playwright's check(): confirm the new state, and say so when the app did not change it.+ const after = await this.resolveNative(window, session, selector, { timeout_ms: 2_000, enabled: false, stable: false, describe }, signal);+ const state = "element" in after ? checkedState(after.element) : null;+ return state === (action === "check") ? { ...result, content: [prefix, ...result.content, { type: "text", text: `It is now ${action}ed.` }] }+ : done("unknown", `${prefix.text} The click was delivered, but the element is ${state === null ? "no longer readable" : state === "mixed" ? "mixed" : state ? "checked" : "unchecked"} afterwards. Read the window before deciding; do not replay blindly.`);+ }+/*** Run up to 16 known steps in one call and stop at the first step that is not delivered or satisfied.- * Browser steps are Playwright actions and conditions. Native steps re-resolve `target` {role,name,text}- * against a fresh UI walk before each step; only the first step may use a hint or pixels of the current observation.+ * Browser steps are Playwright actions and conditions. Native steps resolve `target` (a Playwright locator or+ * JSON target) in fresh UI walks with auto-waiting; only the first step may use a hint or pixels of the current observation.*/private async batch(address: Extract<ComputerAddress, { kind: "observation" }>, snapshot: Snapshot, command: Record<string, unknown>, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {only(command, ["action", "steps"]);@@ -846,27 +937,27 @@ export class ComputerResources {if (!Array.isArray(command.steps) || command.steps.length < 1 || command.steps.length > BATCH_STEPS) throw new ComputerRequestError(`batch needs steps: 1..${BATCH_STEPS} step objects.`);const session = await this.sessionFor(cwd, { window_ref: window });const browser = snapshot.result.details.dom !== undefined;- type Step = { kind: "wait"; spec: WaitSpec; action: string } | { kind: "dom"; action: WebAction } | { kind: "native"; command: Record<string, unknown>; target?: Record<string, string> };+ type Step = { kind: "wait"; spec: WaitSpec; action: string } | { kind: "dom"; action: WebAction } | { kind: "native"; command: Record<string, unknown>; target?: ParsedSelector; describe?: string; timeout?: number };const steps: Step[] = command.steps.map((raw, index): Step => {const step = object(raw);const label = `steps[${index}]`;try {if (step.action === "wait_for" || step.action === "verify") return { kind: "wait", spec: session.waitSpec(window, step), action: String(step.action) };if (browser) return { kind: "dom", action: validateWebAction(step) };+ if (typeof step.action === "string" && LOCATOR_VERBS[step.action]) step.action = LOCATOR_VERBS[step.action];if (!NATIVE_BATCH_ACTIONS.includes(String(step.action))) throw Error(`action must be one of ${NATIVE_BATCH_ACTIONS.join(", ")}, wait_for or verify.`);if (["window_ref", "observation_id", "sequence"].some(key => key in step)) throw Error("identity fields and nested sequences are not allowed.");const pinned = ["hint", "element_token", "x", "y"].some(key => key in step);- if (pinned && index > 0) throw Error("only the first step may use a hint, token or pixels; later steps use target {role,name,text}.");- let target: Record<string, string> | undefined;+ if (pinned && index > 0) throw Error("only the first step may use a hint, token or pixels; later steps use a target (a Playwright locator or JSON target).");+ let target: ParsedSelector | undefined;if (step.target !== undefined) {if (pinned) throw Error("choose target or a hint/pixel target, not both.");- const value = object(step.target);- if (!Object.keys(value).length || Object.entries(value).some(([key, item]) => !["role", "name", "text"].includes(key) || typeof item !== "string" || !item))- throw Error("target needs non-empty role, name and/or text.");- target = value as Record<string, string>;+ target = nativeSelector(step.target);}- const { target: _target, ...nativeStep } = step;- return { kind: "native", command: nativeStep, ...(target ? { target } : {}) };+ const timeout = step.timeout_ms;+ if (timeout !== undefined && (!Number.isSafeInteger(timeout) || (timeout as number) < 0 || (timeout as number) > 60_000)) throw Error("timeout_ms must be an integer 0..60000.");+ const { target: raw, timeout_ms: _timeout, ...nativeStep } = step;+ return { kind: "native", command: nativeStep, ...(target ? { target, describe: typeof raw === "string" ? raw : JSON.stringify(raw), timeout: (timeout as number | undefined) ?? 5_000 } : {}) };} catch (error) { throw new ComputerRequestError(`${label}: ${error instanceof Error ? error.message : String(error)}`); }});this.snapshots.delete(address.ref);@@ -893,19 +984,22 @@ export class ComputerResources {} else {let element: string | undefined;if (index > 0 || step.target) {- // Never reuse a stale observation: take a fresh UI walk and resolve the target in it.- this.retireWindow(window);- const fresh = this.remember({ window_ref: window }, await session.observeQuiet(window, signal));- if (fresh.details.status !== "observed" || !fresh.details.ui) { last = fresh; rows.push({ index, action: String(step.command.action), outcome: "refused", reason: "No current UI sample." }); stop = `No current UI before step ${index}.`; break; }- issued = fresh; observation = fresh.details.observation_id!;+ // Never reuse a stale observation: a target resolves in fresh UI walks; other steps take one fresh walk.if (step.target) {- const found = fresh.details.ui.elements.filter(item => (!step.target!.role || item.role?.toLowerCase() === step.target!.role.toLowerCase())- && (!step.target!.name || item.name === step.target!.name) && (!step.target!.text || `${item.name ?? ""}\n${item.value ?? ""}`.includes(step.target!.text)));- if (found.length !== 1) {- rows.push({ index, action: String(step.command.action), outcome: "refused", reason: found.length ? `${found.length} elements match the target.` : "No element matches the target in the current UI walk." });+ const action = String(step.command.action), pointer = ["click", "double_click", "right_click"].includes(action);+ const resolved = await this.resolveNative(window, session, step.target, { timeout_ms: step.timeout!, enabled: true, stable: pointer, describe: step.describe! }, signal);+ if ("refused" in resolved) {+ if (resolved.observation) last = resolved.observation;+ rows.push({ index, action, outcome: "refused", reason: resolved.refused.slice(0, 600) });stop = `Target not resolved at step ${index}; nothing was sent for it.`; break;}- element = found[0].element_token;+ issued = resolved.observation; observation = resolved.observation.details.observation_id!;+ element = resolved.element.element_token;+ } else {+ this.retireWindow(window);+ const fresh = this.remember({ window_ref: window }, await session.observeQuiet(window, signal));+ if (fresh.details.status !== "observed" || !fresh.details.ui) { last = fresh; rows.push({ index, action: String(step.command.action), outcome: "refused", reason: "No current UI sample." }); stop = `No current UI before step ${index}.`; break; }+ issued = fresh; observation = fresh.details.observation_id!;}}this.snapshots.delete(observation);diff --git a/src/native-locator.ts b/src/native-locator.tsnew file mode 100644index 0000000..d45f962--- /dev/null+++ b/src/native-locator.ts@@ -0,0 +1,253 @@+import { createRequire } from "node:module";+import type { UiElement, UiResource } from "./ui-resources.ts";++/** Playwright's own selector and locator parsers; loaded on first use because playwright-core takes about a second to load. */+interface PlaywrightIso {+ unsafeLocatorOrSelectorAsSelector(language: "javascript", locator: string, testIdAttribute: string): string;+ parseSelector(selector: string): ParsedSelector;+ parseAttributeSelector(selector: string, allowUnquotedStrings: boolean): { name: string; attributes: Attribute[] };+ asLocator(language: "javascript", selector: string): string;+}+let loaded: PlaywrightIso | undefined;+const iso = (): PlaywrightIso => loaded ??= (createRequire(import.meta.url)("playwright-core/lib/coreBundle") as { iso: PlaywrightIso }).iso;++/**+ * Playwright locators over a native UI Automation snapshot.+ *+ * Playwright parses the locator (`getByRole("button", { name: "Save" }).nth(1)`) into its internal selector+ * (`internal:role=button[name="Save"i] >> nth=1`); this module evaluates that selector against the UIA tree+ * with Playwright's semantics: chained parts search inside the previous matches, names and texts match a+ * case-insensitive substring unless exact, the role engine skips hidden elements unless `includeHidden`.+ */++/** A parsed Playwright selector, as `iso.parseSelector` returns it. */+export interface ParsedSelector { parts: { name: string; body: unknown; source: string }[] }+/** The JSON target shared with browser pages; on native windows `ref` is an observation hint and `test_id` an AutomationId. */+export interface NativeTarget {+ ref?: string; role?: string; name?: string; label?: string; text?: string; placeholder?: string; test_id?: string; exact?: boolean;+}++/** UIA control type → ARIA roles, most specific first. The first one names the element in snapshots and suggestions. */+const ROLES: Record<string, string[]> = {+ appbar: ["toolbar"], button: ["button"], calendar: ["grid"], checkbox: ["checkbox"], combobox: ["combobox"], custom: ["generic"],+ datagrid: ["grid", "table"], dataitem: ["row"], document: ["document"], edit: ["textbox"], group: ["group"], header: ["rowgroup"],+ headeritem: ["columnheader"], hyperlink: ["link"], image: ["img"], list: ["list", "listbox"], listitem: ["listitem", "option"],+ menu: ["menu"], menubar: ["menubar"], menuitem: ["menuitem"], pane: ["generic"], progressbar: ["progressbar"], radiobutton: ["radio"],+ scrollbar: ["scrollbar"], semanticzoom: ["generic"], separator: ["separator"], slider: ["slider"], spinner: ["spinbutton"],+ splitbutton: ["button"], statusbar: ["status"], tab: ["tablist"], tabitem: ["tab"], table: ["table", "grid"], text: ["text"],+ thumb: ["generic"], titlebar: ["titlebar"], toolbar: ["toolbar"], tooltip: ["tooltip"], tree: ["tree"], treeitem: ["treeitem"],+ window: ["window", "dialog"],+};+/** Roles Playwright's getByLabel looks at: labelled form controls. */+const LABELLED = new Set(["textbox", "combobox", "checkbox", "radio", "slider", "spinbutton", "listbox", "switch"]);+/** Roles whose `checked` state is a checkbox state; for buttons the same UIA toggle is `pressed`. */+const CHECKABLE = new Set(["checkbox", "radio", "menuitem", "switch", "treeitem", "listitem"]);++/** ARIA roles of a native element, primary first; the UIA control type name also matches. */+export function ariaRoles(element: Pick<UiElement, "role">): string[] {+ const control = (element.role ?? "").replace(/\s+/g, "").toLowerCase();+ return ROLES[control] ?? (control ? [control] : ["generic"]);+}+export const ariaRole = (element: Pick<UiElement, "role">): string => ariaRoles(element)[0];++/** The accessible name: UIA Name when the runtime reports it, otherwise the driver's label. */+export const accessibleName = (element: UiElement): string => element.uia_name ?? element.name ?? "";++/** Checkbox state: the UIA toggle when reported; older runtimes fold it into `selected` for checkable controls. */+export function checkedState(element: UiElement): boolean | "mixed" | null {+ if (element.checked === "on") return true;+ if (element.checked === "off") return false;+ if (element.checked === "mixed") return "mixed";+ return element.checked === undefined && ["checkbox", "radio"].includes(ariaRole(element)) ? element.selected : null;+}+/** Hidden means UIA IsOffscreen; unknown visibility counts as visible, as an element without layout info would in a page. */+export const isVisible = (element: UiElement): boolean => element.offscreen !== true;++const normalize = (value: string) => value.replace(/\s+/g, " ").trim();++/** A Playwright string or regex matcher: `"x"i` substring, `"x"s` exact, `/re/flags`, or an attribute value. */+type Matcher = (value: string) => boolean;+function stringMatcher(value: string | RegExp, exact: boolean): Matcher {+ if (value instanceof RegExp) return text => value.test(normalize(text));+ const wanted = normalize(value);+ return exact ? text => normalize(text) === wanted : text => normalize(text).toLowerCase().includes(wanted.toLowerCase());+}+function textMatcher(body: unknown): Matcher {+ const source = String(body);+ const regex = /^\/(.*)\/([a-z]*)$/s.exec(source);+ if (regex) return stringMatcher(new RegExp(regex[1], regex[2]), false);+ const quoted = /^(".*")([is]?)$/s.exec(source);+ if (!quoted) throw Error(`Unsupported text matcher ${source}.`);+ return stringMatcher(JSON.parse(quoted[1]) as string, quoted[2] === "s");+}+type Attribute = { name: string; op: string; value: unknown; caseSensitive: boolean };+function attributeMatcher(attribute: Attribute): Matcher {+ if (attribute.op !== "=") throw Error(`Attribute operator ${attribute.op} is not supported for native windows.`);+ return stringMatcher(attribute.value as string | RegExp, attribute.caseSensitive);+}+const parseAttributes = (body: unknown): { name: string; attributes: Attribute[] } => iso().parseAttributeSelector(String(body), true);++/** Index, parent links and texts of one snapshot, so selectors evaluate without repeated scans. */+class Tree {+ readonly order = new Map<string, number>();+ readonly byHint = new Map<string, UiElement>();+ constructor(readonly elements: readonly UiElement[]) {+ elements.forEach((element, index) => { this.order.set(element.hint, index); this.byHint.set(element.hint, element); });+ }+ /** Whether `element` lies strictly inside `root`. */+ inside(element: UiElement, root: UiElement): boolean {+ for (let parent = element.parent_hint; parent; parent = this.byHint.get(parent)?.parent_hint ?? null) if (parent === root.hint) return true;+ return false;+ }+ descendants(roots: readonly UiElement[] | null): UiElement[] {+ return roots === null ? [...this.elements] : this.elements.filter(element => roots.some(root => this.inside(element, root)));+ }+ /** Own text plus the text of every descendant, as Playwright's has-text sees an element's text content. */+ subtreeText(element: UiElement): string {+ return [element, ...this.descendants([element])].flatMap(item => [accessibleName(item), item.value ?? ""]).filter(Boolean).join(" ");+ }+ sort(elements: Iterable<UiElement>): UiElement[] {+ return [...new Set(elements)].sort((a, b) => this.order.get(a.hint)! - this.order.get(b.hint)!);+ }+}++/** One query engine: does `element` match this selector part? */+function engine(part: ParsedSelector["parts"][number]): (element: UiElement) => boolean {+ switch (part.name) {+ case "internal:role": case "role": {+ const { name: role, attributes } = parseAttributes(part.body);+ const wanted = role.toLowerCase();+ let includeHidden = false;+ const checks: ((element: UiElement) => boolean)[] = [element => ariaRoles(element).includes(wanted) || (element.role ?? "").toLowerCase() === wanted];+ for (const attribute of attributes) {+ const value = attribute.value;+ switch (attribute.name) {+ case "name": { const match = attributeMatcher(attribute); checks.push(element => match(accessibleName(element))); break; }+ case "include-hidden": includeHidden = value === true || value === "true"; break;+ case "checked": checks.push(element => CHECKABLE.has(ariaRole(element)) && String(checkedState(element)) === String(value)); break;+ case "pressed": checks.push(element => ariaRole(element) === "button" && element.checked !== undefined && String(checkedState({ ...element, role: "CheckBox" })) === String(value)); break;+ case "disabled": checks.push(element => element.enabled !== null && String(element.enabled === false) === String(value)); break;+ case "selected": checks.push(element => element.selected !== null && String(element.selected) === String(value)); break;+ case "expanded": checks.push(element => element.expanded !== undefined && String(element.expanded) === String(value)); break;+ default: throw Error(`getByRole option ${attribute.name} is not available for native windows.`);+ }+ }+ return element => (includeHidden || isVisible(element)) && checks.every(check => check(element));+ }+ case "internal:text": { const match = textMatcher(part.body); return element => match(accessibleName(element)) || (ariaRole(element) === "text" && match(element.value ?? "")); }+ case "internal:label": { const match = textMatcher(part.body); return element => LABELLED.has(ariaRole(element)) && match(accessibleName(element)); }+ case "internal:testid": {+ const [attribute] = parseAttributes(part.body).attributes;+ const match = attributeMatcher(attribute);+ return element => element.automation_id !== undefined && match(element.automation_id);+ }+ case "internal:attr": {+ const [attribute] = parseAttributes(part.body).attributes;+ const match = attributeMatcher(attribute);+ if (attribute.name === "title" || attribute.name === "placeholder") return element => element.help_text !== undefined && match(element.help_text);+ if (attribute.name === "alt") return element => ariaRole(element) === "img" && match(accessibleName(element));+ throw Error(`Attribute ${attribute.name} is not available for native windows.`);+ }+ default:+ throw Error(`Selector engine "${part.name}" is not available for native windows. Use getByRole, getByText, getByLabel, getByTestId (AutomationId), getByTitle, filter, nth, and or or.`);+ }+}+const nested = (body: unknown): ParsedSelector => {+ const parsed = (body as { parsed?: ParsedSelector })?.parsed;+ if (!parsed) throw Error("A nested locator could not be read.");+ return parsed;+};++/** Evaluate a parsed selector from `roots` (the whole window when null) in tree order. */+function evaluate(tree: Tree, selector: ParsedSelector, roots: readonly UiElement[] | null): UiElement[] {+ let current: UiElement[] | null = null;+ let scope = roots;+ for (const part of selector.parts) {+ const all = () => current ?? tree.descendants(scope);+ switch (part.name) {+ case "nth": {+ const index = Number(part.body), list = all();+ const item = index < 0 ? list[list.length + index] : list[index];+ current = item ? [item] : [];+ break;+ }+ case "visible": current = all().filter(element => isVisible(element) === (String(part.body) === "true")); break;+ case "internal:has-text": case "internal:has-not-text": {+ const match = textMatcher(part.body), keep = part.name === "internal:has-text";+ current = all().filter(element => match(tree.subtreeText(element)) === keep);+ break;+ }+ case "internal:has": case "internal:has-not": {+ const inner = nested(part.body), keep = part.name === "internal:has";+ current = all().filter(element => (evaluate(tree, inner, [element]).length > 0) === keep);+ break;+ }+ case "internal:and": { const other = new Set(evaluate(tree, nested(part.body), scope)); current = all().filter(element => other.has(element)); break; }+ case "internal:or": current = tree.sort([...all(), ...evaluate(tree, nested(part.body), scope)]); break;+ case "internal:describe": break;+ default: {+ const match = engine(part);+ scope = current ?? roots;+ current = tree.descendants(scope).filter(match);+ }+ }+ }+ return current ?? tree.descendants(roots);+}++/** Playwright's internal selector for a JSON target, in the same shape browser pages use. */+export function targetSelector(target: NativeTarget): string {+ const quote = (value: string, exact: boolean) => `${JSON.stringify(value)}${exact ? "s" : "i"}`;+ const exact = target.exact === true;+ const parts: string[] = [];+ if (target.role !== undefined) parts.push(`internal:role=${target.role}${target.name !== undefined ? `[name=${quote(target.name, exact)}]` : ""}`);+ if (target.label !== undefined) parts.push(`internal:label=${quote(target.label, exact)}`);+ if (target.text !== undefined) parts.push(`internal:text=${quote(target.text, exact)}`);+ if (target.placeholder !== undefined) parts.push(`internal:attr=[placeholder=${quote(target.placeholder, exact)}]`);+ if (target.test_id !== undefined) parts.push(`internal:testid=[data-testid=${quote(target.test_id, true)}]`);+ if (!parts.length) throw Error("target needs role, label, text, placeholder or test_id.");+ return parts.slice(1).reduce((all, part) => `${all} >> internal:and=${JSON.stringify(part)}`, parts[0]);+}++/** Playwright's internal selector for a locator such as `getByRole("button", { name: "Save" }).nth(1)`. */+export function locatorSelector(locator: string): string {+ if (typeof locator !== "string" || !locator.trim() || locator.length > 2000) throw Error("A locator must be a Playwright locator string of at most 2000 characters.");+ try { return iso().unsafeLocatorOrSelectorAsSelector("javascript", locator.trim().replace(/^(?:page|window|this)\./, ""), "data-testid"); }+ catch (error) { throw Error(`The locator could not be parsed: ${error instanceof Error ? error.message : String(error)}`); }+}++/** Parse a Playwright locator for a native window. CSS and XPath parse here but refuse when evaluated. */+export function parseLocator(locator: string): ParsedSelector {+ return iso().parseSelector(locatorSelector(locator));+}++const TARGET_FIELDS = ["role", "name", "label", "text", "placeholder", "test_id", "exact"] as const;+/**+ * A native target: a Playwright locator string, or the JSON target browser pages take. Hints are snapshot-local,+ * so `ref` refuses here: targets resolve in fresh UI walks.+ */+export function nativeSelector(value: unknown, what = "target"): ParsedSelector {+ if (typeof value === "string") return parseLocator(value);+ if (!value || typeof value !== "object" || Array.isArray(value)) throw Error(`${what} must be a Playwright locator string or an object with ${TARGET_FIELDS.join(", ")}.`);+ const target = value as Record<string, unknown>;+ if ("ref" in target) throw Error(`${what}.ref names an element of one observation; use a hint in the observation address, or a locator.`);+ const extra = Object.keys(target).filter(key => !(TARGET_FIELDS as readonly string[]).includes(key));+ if (extra.length) throw Error(`${what} does not accept ${extra.join(", ")}; use ${TARGET_FIELDS.join(", ")} or a locator string.`);+ for (const key of TARGET_FIELDS) if (key !== "exact" && target[key] !== undefined && (typeof target[key] !== "string" || !target[key] || (target[key] as string).length > 512))+ throw Error(`${what}.${key} must be a non-empty string of at most 512 characters.`);+ if (target.exact !== undefined && typeof target.exact !== "boolean") throw Error(`${what}.exact must be boolean.`);+ if (target.name !== undefined && target.role === undefined) throw Error(`${what}.name requires ${what}.role.`);+ return iso().parseSelector(targetSelector(target as NativeTarget));+}++/** Elements of one snapshot that a parsed selector matches, in tree order. */+export function queryNative(ui: Pick<UiResource, "elements">, selector: ParsedSelector): UiElement[] {+ return evaluate(new Tree(ui.elements), selector, null);+}++/** A Playwright locator that names this element, for strict-mode errors and listings. */+export function suggestLocator(element: UiElement): string {+ const role = ariaRole(element), name = accessibleName(element);+ if (element.automation_id && !/^\d+$/.test(element.automation_id)) return `getByTestId(${JSON.stringify(element.automation_id)})`;+ return iso().asLocator("javascript", `internal:role=${role}${name ? `[name=${JSON.stringify(name)}s]` : ""}`);+}diff --git a/src/session.ts b/src/session.tsindex e958b7c..c10ea37 100644--- a/src/session.ts+++ b/src/session.ts@@ -1145,6 +1145,11 @@ export class ComputerSession {});}+ /** Whether this window is a browser this session opened or attached, whose pages Playwright drives. */+ isBrowserWindow(ref: string): boolean {+ return this.entries.get(ref)?.web !== undefined;+ }+/** Validate a wait_for/verify command for this window: owned browsers take Playwright targets and page checks. */waitSpec(ref: string, command: Record<string, unknown>): WaitSpec {return validateWait(command, this.entries.get(ref)?.web !== undefined);diff --git a/src/ui-conditions.ts b/src/ui-conditions.tsindex 4731ba0..b0880c1 100644--- a/src/ui-conditions.ts+++ b/src/ui-conditions.ts@@ -1,17 +1,20 @@-import type { UiElement, UiResource } from "./ui-resources.ts";+import type { UiResource } from "./ui-resources.ts";import { validateWebTarget, type WebTarget } from "./web-browser.ts";+import { checkedState, isVisible, nativeSelector, queryNative, type ParsedSelector } from "./native-locator.ts";/** Three-valued verdict. `unknown` means the evidence cannot decide, never a hidden `false`. */export type Verdict = "satisfied" | "unsatisfied" | "unknown";-/** Element state checked by a condition. Native UI reports only role, name, value, enabled and selected. */+/** Element state checked by a condition. A state the native runtime does not report is unknown, never false. */export type ElementState = "exists" | "absent" | "visible" | "hidden" | "enabled" | "disabled" | "selected" | "checked" | "unchecked" | "focused";/**- * One condition. Native windows match `target` by `role`, exact `name` or `text` contained in name or value;- * browser pages use any Playwright target. `value` and `contains` compare the element's value (or text).+ * One condition. `target` is a Playwright locator string or the JSON target; native windows evaluate it over+ * the UIA tree with Playwright's semantics. `value` and `contains` compare the element's value (or text).*/export interface UiCondition {- target?: { role?: string; name?: string; text?: string } | WebTarget;+ target?: WebTarget;+ /** The native target as Playwright's parsed selector. */+ selector?: ParsedSelector;is?: ElementState;value?: string;contains?: string;@@ -27,12 +30,8 @@ export interface WaitResult {}const STATES: readonly ElementState[] = ["exists", "absent", "visible", "hidden", "enabled", "disabled", "selected", "checked", "unchecked", "focused"];-const NATIVE_STATES: readonly ElementState[] = ["exists", "absent", "enabled", "disabled", "selected"];+const NATIVE_STATES: readonly ElementState[] = ["exists", "absent", "visible", "hidden", "enabled", "disabled", "selected", "checked", "unchecked", "focused"];-const text = (value: unknown, what: string): string => {- if (typeof value !== "string" || !value || value.length > 512) throw Error(`${what} must be a non-empty string of at most 512 characters.`);- return value;-};const integer = (value: unknown, min: number, max: number, fallback: number, what: string): number => {if (value === undefined) return fallback;if (!Number.isSafeInteger(value) || (value as number) < min || (value as number) > max) throw Error(`${what} must be an integer ${min}..${max}.`);@@ -65,18 +64,14 @@ export function validateWait(command: Record<string, unknown>, browser: boolean)}if (value.target !== undefined) {if (browser) condition.target = validateWebTarget(value.target);- else {- const target = value.target as Record<string, unknown>;- if (!target || typeof target !== "object" || Array.isArray(target) || Object.keys(target).some(key => !["role", "name", "text"].includes(key)) || !Object.keys(target).length)- throw Error(`conditions[${index}].target needs role, name and/or text for a native window.`);- condition.target = Object.fromEntries(Object.entries(target).map(([key, item]) => [key, text(item, `conditions[${index}].target.${key}`)]));- }+ else condition.selector = nativeSelector(value.target, `conditions[${index}].target`);}const checks = [condition.is, condition.value, condition.contains].filter(item => item !== undefined).length;if (checks > 1) throw Error(`conditions[${index}] takes one of is, value or contains; use separate conditions for several checks.`);const elementCheck = checks === 1;const pageCheck = condition.url_contains !== undefined || condition.title !== undefined;- if (elementCheck === pageCheck || (elementCheck && !condition.target) || (pageCheck && condition.target))+ const targeted = condition.target !== undefined || condition.selector !== undefined;+ if (elementCheck === pageCheck || (elementCheck && !targeted) || (pageCheck && targeted))throw Error(`conditions[${index}] needs a target with is/value/contains, or a page check (url_contains/title) without a target.`);return condition;});@@ -90,13 +85,6 @@ export function combine(results: readonly ConditionResult[]): Verdict {return results.some(r => r.verdict === "unsatisfied") ? "unsatisfied" : results.some(r => r.verdict === "unknown") ? "unknown" : "satisfied";}-function matches(element: UiElement, target: { role?: string; name?: string; text?: string }): boolean {- if (target.role !== undefined && element.role?.toLowerCase() !== target.role.toLowerCase()) return false;- if (target.name !== undefined && element.name !== target.name) return false;- if (target.text !== undefined && !`${element.name ?? ""}\n${element.value ?? ""}`.includes(target.text)) return false;- return true;-}-/*** Evaluate native conditions against one bounded UI walk. Absence is decided only when the walk is* complete and untruncated; otherwise a missing element is `unknown`. Several matches are ambiguous.@@ -104,8 +92,13 @@ function matches(element: UiElement, target: { role?: string; name?: string; texexport function evaluateNative(ui: UiResource, conditions: readonly UiCondition[], texts?: ReadonlyMap<string, string>): ConditionResult[] {const complete = ui.complete === true && !ui.truncated;return conditions.map((condition, index) => {- const target = condition.target as { role?: string; name?: string; text?: string };- const found = ui.elements.filter(element => matches(element, target));+ const found = queryNative(ui, condition.selector!);+ // Like Playwright's toBeHidden: no matching element, or one that is not visible.+ if (condition.is === "hidden") {+ if (found.length > 1) return { index, verdict: "unknown", reason: `${found.length} elements match; narrow the target.` };+ if (found.length === 1) return { index, verdict: isVisible(found[0]) ? "unsatisfied" : "satisfied", observed: isVisible(found[0]) ? "visible" : "offscreen" };+ return complete ? { index, verdict: "satisfied", observed: "no match" } : { index, verdict: "unknown", reason: "No match in an incomplete UI walk; the element may exist beyond it." };+ }if (condition.is === "absent")return found.length ? { index, verdict: "unsatisfied", observed: `${found.length} match(es)` }: complete ? { index, verdict: "satisfied", observed: "no match" }@@ -123,6 +116,13 @@ export function evaluateNative(ui: UiResource, conditions: readonly UiCondition[if (condition.is === "enabled") return known(element.enabled, true);if (condition.is === "disabled") return known(element.enabled, false);if (condition.is === "selected") return known(element.selected, true);+ if (condition.is === "visible") return { index, verdict: isVisible(element) ? "satisfied" : "unsatisfied", observed: isVisible(element) ? "visible" : "offscreen" };+ if (condition.is === "checked" || condition.is === "unchecked") {+ const state = checkedState(element);+ return state === null ? { index, verdict: "unknown", reason: "The native UI reports no checked state for this element." }+ : { index, verdict: (state === true) === (condition.is === "checked") && state !== "mixed" ? "satisfied" : "unsatisfied", observed: String(state) };+ }+ if (condition.is === "focused") return known(element.focused ?? null, true);if (condition.value !== undefined || condition.contains !== undefined) {if (observed === null) return { index, verdict: "unknown", reason: "The element reports no value or name." };if (element.text_truncated && full === undefined) return { index, verdict: "unknown", observed, reason: "The element text was truncated at 1024 characters." };diff --git a/src/ui-resources.ts b/src/ui-resources.tsindex 8f529d6..81c37d2 100644--- a/src/ui-resources.ts+++ b/src/ui-resources.ts@@ -11,6 +11,11 @@ export interface UiElement {role: string | null; name: string | null; value: string | null; text_truncated: boolean;enabled: boolean | null; selected: boolean | null; actions: string[]; depth: number | null;bounds: { native_screen?: ScreenRect; window_image?: ScreenRect };+ /** Structural fields from runtimes that report the full UIA tree; absent when the runtime does not report them. */+ uia_name?: string; automation_id?: string; class_name?: string; help_text?: string;+ checked?: "on" | "off" | "mixed"; expanded?: boolean; offscreen?: boolean; focused?: boolean;+ password?: boolean; read_only?: boolean; required?: boolean; actionable?: boolean;+ range?: { value: number; min: number; max: number };hint_display: "pending" | "shown" | "disabled" | "no_bounds" | "outside_image" | "collision" | "off";}/** Bounded native UIA projection. Completeness is not inferred from an empty search. */@@ -60,6 +65,19 @@ export function validateUiCapture(options: UiCaptureOptions): void {throw Error("UI-only observation has no image hints or guides.");}+/** Optional structural fields, copied only when the runtime reports them with the expected type. */+function structural(e: Record<string, unknown>): Partial<UiElement> {+ const out: Partial<UiElement> = {};+ for (const [from, to] of [["name", "uia_name"], ["automation_id", "automation_id"], ["class_name", "class_name"], ["help_text", "help_text"]] as const)+ if (typeof e[from] === "string" && e[from]) out[to] = (e[from] as string).slice(0, 1024);+ if (e.checked === "on" || e.checked === "off" || e.checked === "mixed") out.checked = e.checked;+ for (const key of ["expanded", "offscreen", "focused", "password", "read_only", "required", "actionable"] as const)+ if (typeof e[key] === "boolean") out[key] = e[key] as boolean;+ const range = e.range as { value?: unknown; min?: unknown; max?: unknown } | undefined;+ if (range && [range.value, range.min, range.max].every(Number.isFinite)) out.range = { value: range.value as number, min: range.min as number, max: range.max as number };+ return out;+}+/** Adapt reported properties only. Native bounds are never treated as local image coordinates. */export function createUiResource(state: Record<string, unknown>, window: string, observation: string, serial: number): UiResource {const native = (Array.isArray(state.elements) ? state.elements : []) as Record<string, unknown>[];@@ -74,6 +92,7 @@ export function createUiResource(state: Record<string, unknown>, window: string,actions: (Array.isArray(e.actions) ? e.actions : []).filter((a): a is string => typeof a === "string").slice(0, 32),bounds: { ...(rect(e.frame) ? { native_screen: rect(e.frame) } : {}), ...(rect(e.screenshot_frame) ? { window_image: rect(e.screenshot_frame) } : {}) },hint_display: e.enabled === false ? "disabled" : rect(e.screenshot_frame) ? "pending" : "no_bounds",+ ...structural(e),}));return { window_ref: window, observation_id: observation, snapshot_id: state.snapshot_id ?? null,captured_at: new Date().toISOString(), complete: state.truncated === true ? false : bool(state.elements_complete),diff --git a/src/web-browser.ts b/src/web-browser.tsindex 2ebc9fe..c972950 100644--- a/src/web-browser.ts+++ b/src/web-browser.ts@@ -4,12 +4,15 @@ import { promisify } from "node:util";import type { CommandRunner } from "./runner.ts";import { ATTACH_APPROVAL_MS } from "./browser-command-line.ts";import { findWindowsNode } from "./windows-node.ts";+import { locatorSelector } from "./native-locator.ts";const run = promisify(execFile);/** An element chosen by Playwright: an aria ref from the latest DOM snapshot, or locator predicates that must all match. */export interface WebTarget {ref?: string; role?: string; name?: string; label?: string; text?: string; placeholder?: string; test_id?: string; css?: string;+ /** Playwright's internal selector, parsed from a locator string such as getByRole("button", { name: "Save" }). */+ locator?: string;exact?: boolean;/** CSS selectors of nested iframes, outermost first. Refs already carry their frame. */frames?: string[];@@ -134,6 +137,8 @@ function httpUrl(value: unknown): string {const PREDICATES = ["role", "name", "label", "text", "placeholder", "test_id", "css"] as const;/** Check target shape only; Playwright resolves, waits for and disambiguates the element. */export function validateWebTarget(value: unknown): WebTarget {+ // A Playwright locator string, such as getByRole("button", { name: "Save" }).nth(1), in Playwright's own syntax.+ if (typeof value === "string") return { locator: locatorSelector(value) };const target = record(value, "target");only(target, ["ref", ...PREDICATES, "exact", "frames"], "target");if (target.ref !== undefined) {diff --git a/src/web-worker.ts b/src/web-worker.tsindex 76bb87f..8fe7e8b 100644--- a/src/web-worker.ts+++ b/src/web-worker.ts@@ -137,6 +137,7 @@ function untilDialog<T>(tab: Tab, operation: Promise<T>): Promise<T | "dialog">function locate(page: Page, target: WebTarget): Locator {if (target.ref) return page.getByRef(target.ref);+ if (target.locator) return page.locator(target.locator);let root: Page | FrameLocator = page;for (const frame of target.frames ?? []) root = root.frameLocator(frame);const exact = target.exact;diff --git a/test/native-locator.test.ts b/test/native-locator.test.tsnew file mode 100644index 0000000..97022f1--- /dev/null+++ b/test/native-locator.test.ts@@ -0,0 +1,97 @@+import { beforeAll, expect, test } from "vitest";+import { ariaRole, parseLocator, queryNative, suggestLocator, targetSelector } from "../src/native-locator.ts";+import type { UiElement } from "../src/ui-resources.ts";++/** The WinForms fixture of the live checks, as a runtime with the full UIA tree reports it. */+function element(hint: string, role: string, name: string, extra: Partial<UiElement> = {}): UiElement {+ return { hint, element_token: `s1:${hint}`, index: null, parent_hint: "w", role, name, value: null, text_truncated: false,+ enabled: true, selected: null, actions: [], depth: 1, bounds: {}, hint_display: "off", uia_name: name, ...extra };+}+const tree = { elements: [+ element("w", "Window", "Fixture", { parent_hint: null, depth: 0 }),+ element("count", "Button", "Count 0", { automation_id: "countButton" }),+ element("off", "Button", "Disabled", { enabled: false }),+ element("name", "Edit", "Name", { value: "Ada Lovelace" }),+ element("agree", "CheckBox", "Agree", { checked: "on" }),+ element("opts", "Group", "Options"),+ element("ok", "Button", "OK", { parent_hint: "opts", depth: 2 }),+ element("pick", "List", "Pick"),+ element("alpha", "ListItem", "alpha", { parent_hint: "pick", depth: 2, selected: false }),+ element("beta", "ListItem", "beta", { parent_hint: "pick", depth: 2, selected: true }),+ element("hidden", "ListItem", "gamma", { parent_hint: "pick", depth: 2, offscreen: true }),+ element("status", "Text", "clicked 3"),+ element("save", "Button", "Save", { help_text: "Save the file" }),+ element("saveAs", "Button", "Save as…"),+] };+// playwright-core loads on first use; under a parallel test run that can take several seconds.+beforeAll(() => { parseLocator("getByRole('button')"); }, 60_000);+const hints = (locator: string) => queryNative(tree, parseLocator(locator)).map(found => found.hint);++test("getByRole matches ARIA or UIA roles and names the Playwright way: substring by default, exact on request", () => {+ expect(hints(`getByRole("button", { name: "Count 0" })`)).toEqual(["count"]);+ expect(hints(`getByRole("Button", { name: "count" })`)).toEqual(["count"]);+ expect(hints(`getByRole("button", { name: "Save" })`)).toEqual(["save", "saveAs"]);+ expect(hints(`getByRole("button", { name: "Save", exact: true })`)).toEqual(["save"]);+ expect(hints(`getByRole("button", { name: /^save$/i })`)).toEqual(["save"]);+ expect(hints(`getByRole("textbox")`)).toEqual(["name"]);+ expect(hints(`getByRole("option")`)).toEqual(["alpha", "beta"]);+});++test("role states filter like Playwright's options; hidden elements need includeHidden", () => {+ expect(hints(`getByRole("checkbox", { checked: true })`)).toEqual(["agree"]);+ expect(hints(`getByRole("checkbox", { checked: false })`)).toEqual([]);+ expect(hints(`getByRole("button", { disabled: true })`)).toEqual(["off"]);+ expect(hints(`getByRole("listitem", { selected: true })`)).toEqual(["beta"]);+ expect(hints(`getByRole("listitem", { includeHidden: true })`)).toEqual(["alpha", "beta", "hidden"]);+ expect(() => hints(`getByRole("heading", { level: 2 })`)).toThrow(/level is not available for native windows/);+});++test("text, label, test id and title locators read the matching UIA properties", () => {+ expect(hints(`getByText("clicked")`)).toEqual(["status"]);+ expect(hints(`getByText("Clicked 3", { exact: true })`)).toEqual([]);+ expect(hints(`getByText("clicked 3", { exact: true })`)).toEqual(["status"]);+ expect(hints(`getByLabel("Name")`)).toEqual(["name"]);+ expect(hints(`getByLabel("Agree")`)).toEqual(["agree"]);+ expect(hints(`getByTestId("countButton")`)).toEqual(["count"]);+ expect(hints(`getByTitle("Save the file")`)).toEqual(["save"]);+});++test("chaining searches inside the previous matches; filter, nth, and, or compose", () => {+ expect(hints(`getByRole("group", { name: "Options" }).getByRole("button")`)).toEqual(["ok"]);+ expect(hints(`getByRole("list").getByRole("listitem").filter({ hasText: "beta" })`)).toEqual(["beta"]);+ expect(hints(`getByRole("listitem").filter({ hasNotText: "beta" })`)).toEqual(["alpha"]);+ expect(hints(`getByRole("group").filter({ has: getByRole("button", { name: "OK" }) })`)).toEqual(["opts"]);+ expect(hints(`getByRole("list").filter({ hasText: "beta" })`)).toEqual(["pick"]);+ expect(hints(`getByRole("button").first()`)).toEqual(["count"]);+ expect(hints(`getByRole("button").last()`)).toEqual(["saveAs"]);+ expect(hints(`getByRole("button").nth(2)`)).toEqual(["ok"]);+ expect(hints(`getByRole("button").and(getByTitle("Save the file"))`)).toEqual(["save"]);+ expect(hints(`getByRole("checkbox").or(getByRole("textbox"))`)).toEqual(["name", "agree"]);+ expect(hints(`page.getByRole("button", { name: "OK" })`)).toEqual(["ok"]);+});++test("CSS, XPath and unparsable locators refuse instead of guessing", () => {+ expect(() => hints(`locator("css=div")`)).toThrow(/not available for native windows/);+ expect(() => hints(`locator("//div")`)).toThrow(/not available for native windows/);+ expect(() => hints(`getByRole(`)).toThrow(/could not be parsed/);+});++test("the JSON target shared with browser pages becomes the same Playwright selector", () => {+ const byTarget = (target: Parameters<typeof targetSelector>[0]) => queryNative(tree, parseLocator(targetSelector(target))).map(found => found.hint);+ expect(byTarget({ role: "button", name: "Save" })).toEqual(["save", "saveAs"]);+ expect(byTarget({ role: "button", name: "Save", exact: true })).toEqual(["save"]);+ expect(byTarget({ role: "textbox", label: "Name" })).toEqual(["name"]);+ expect(byTarget({ text: "clicked" })).toEqual(["status"]);+ expect(byTarget({ test_id: "countButton" })).toEqual(["count"]);+ expect(() => targetSelector({})).toThrow(/needs role/);+});++test("suggested locators name an element with Playwright syntax", () => {+ const byHint = (hint: string) => tree.elements.find(found => found.hint === hint)!;+ expect(suggestLocator(byHint("count"))).toBe(`getByTestId("countButton")`);+ expect(suggestLocator(byHint("ok"))).toBe(`getByRole('button', { name: 'OK', exact: true })`);+ expect(ariaRole(byHint("name"))).toBe("textbox");+ // Older runtimes fold the checkbox toggle into `selected`.+ const old = { elements: [element("c", "CheckBox", "Agree", { uia_name: undefined, selected: true })] };+ expect(queryNative(old, parseLocator(`getByRole("checkbox", { name: "Agree", checked: true })`)).length).toBe(1);+});diff --git a/test/ui-conditions.test.ts b/test/ui-conditions.test.tsindex 11f540c..43aa57c 100644--- a/test/ui-conditions.test.ts+++ b/test/ui-conditions.test.ts@@ -1,14 +1,16 @@-import { expect, test } from "vitest";+import { beforeAll, expect, test } from "vitest";+import { parseLocator } from "../src/native-locator.ts";import { combine, evaluateNative, validateWait } from "../src/ui-conditions.ts";import type { UiElement, UiResource } from "../src/ui-resources.ts";+beforeAll(() => { parseLocator("getByRole('button')"); }, 60_000);const element = (fields: Partial<UiElement>): UiElement => ({ hint: "1:a", element_token: "t", index: 0, parent_hint: null, role: null, name: null, value: null,text_truncated: false, enabled: true, selected: null, actions: [], depth: 0, bounds: {}, hint_display: "off", ...fields });const ui = (elements: UiElement[], complete: boolean | null): UiResource => ({ window_ref: "w", observation_id: "o", snapshot_id: null, captured_at: "",complete, truncated: false, truncation_reason: null, native_total: elements.length, collected: elements.length, matches: elements.length, projection_truncated: false, elements });test("native absence is decided only by a complete walk; a missing element in a partial walk stays unknown", () => {- const conditions = validateWait({ action: "verify", conditions: [{ target: { role: "Button", name: "Gone" }, is: "absent" }, { target: { name: "Status" }, value: "ok" }] }, false).conditions;+ const conditions = validateWait({ action: "verify", conditions: [{ target: { role: "Button", name: "Gone" }, is: "absent" }, { target: { text: "Status" }, value: "ok" }] }, false).conditions;expect(evaluateNative(ui([], true), conditions).map(r => r.verdict)).toEqual(["satisfied", "unsatisfied"]);expect(evaluateNative(ui([], false), conditions).map(r => r.verdict)).toEqual(["unknown", "unknown"]);expect(evaluateNative(ui([], null), conditions).map(r => r.verdict)).toEqual(["unknown", "unknown"]);@@ -20,12 +22,12 @@ test("native checks compare reported properties and keep ambiguity and missing pconst check = (condition: object) => evaluateNative(tree, validateWait({ action: "verify", conditions: [condition] }, false).conditions)[0];expect(check({ target: { role: "Button", name: "Save" }, is: "disabled" }).verdict).toBe("satisfied");expect(check({ target: { role: "Button", name: "Save" }, is: "enabled" }).verdict).toBe("unsatisfied");- expect(check({ target: { name: "Status" }, value: "saved Ada" }).verdict).toBe("satisfied");- expect(check({ target: { name: "Status" }, contains: "Bob" }).verdict).toBe("unsatisfied");+ expect(check({ target: { text: "Status" }, value: "saved Ada" }).verdict).toBe("satisfied");+ expect(check({ target: "getByText('Status')", contains: "Bob" }).verdict).toBe("unsatisfied");expect(check({ target: { role: "Button" }, is: "exists" }).verdict).toBe("unknown");- expect(check({ target: { name: "Save" }, is: "selected" }).verdict).toBe("unknown");+ expect(check({ target: { role: "button", name: "Save", exact: true }, is: "selected" }).verdict).toBe("unknown");// Truncated text cannot prove an exact value either way.- expect(check({ target: { name: "Long" }, value: "x" }).verdict).toBe("unknown");+ expect(check({ target: { text: "Long" }, value: "x" }).verdict).toBe("unknown");});test("one unsatisfied condition decides; otherwise any unknown keeps the result unknown", () => {@@ -38,7 +40,9 @@ test("wait commands are bounded and reject states the backend cannot report", ()expect(validateWait({ action: "wait_for", conditions: [{ target: { role: "status" }, value: "ok" }] }, true)).toMatchObject({ timeout_ms: 5000, interval_ms: 250, stable: 1 });expect(validateWait({ action: "verify", conditions: [{ url_contains: "/done" }] }, true)).toMatchObject({ timeout_ms: 0, stable: 1 });- expect(() => validateWait({ action: "wait_for", conditions: [{ target: { role: "Document" }, is: "visible" }] }, false)).toThrow(/native windows/);+ expect(validateWait({ action: "wait_for", conditions: [{ target: { role: "Document" }, is: "visible" }] }, false).conditions).toHaveLength(1);+ expect(() => validateWait({ action: "wait_for", conditions: [{ target: { css: "div" }, is: "visible" }] }, false)).toThrow(/does not accept css/);+ expect(() => validateWait({ action: "wait_for", conditions: [{ target: { name: "Save" }, is: "visible" }] }, false)).toThrow(/name requires/);expect(() => validateWait({ action: "wait_for", conditions: [{ url_contains: "/x" }] }, false)).toThrow(/does not accept: url_contains/);expect(() => validateWait({ action: "wait_for", conditions: [{ target: { role: "status" } }] }, true)).toThrow(/needs a target with/);expect(() => validateWait({ action: "wait_for", conditions: [{ target: { role: "status" }, value: "a", contains: "b" }] }, true)).toThrow(/one of is, value or contains/);