Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

20.3 KB
Raw Blame
import { createRequire } from "node:module";
import type { UiElement, UiResource } from "./ui-resources.ts";
import type { AriaTreeNode } from "./aria-match.ts";

/** Playwright's own selector and locator parsers; loaded on first use because playwright-core takes about a second to load. */
interface PlaywrightIso {
  unsafeLocatorOrSelectorAsSelector(language: "javascript", locator: string, testIdAttribute: string): string;
  parseSelector(selector: string): ParsedSelector;
  parseAttributeSelector(selector: string, allowUnquotedStrings: boolean): { name: string; attributes: Attribute[] };
  asLocator(language: "javascript", selector: string): string;
  renderAriaSnapshotAsYaml(nodes: AriaNode[]): string;
}
/** The node shape Playwright's aria snapshot renderer takes. */
interface AriaNode {
  role: string; name?: string; text?: string; ref?: string; children?: AriaNode[];
  checked?: boolean | "mixed"; pressed?: boolean | "mixed"; disabled?: boolean; expanded?: boolean; selected?: boolean;
}
let loaded: PlaywrightIso | undefined;
const iso = (): PlaywrightIso => loaded ??= (createRequire(import.meta.url)("playwright-core/lib/coreBundle") as { iso: PlaywrightIso }).iso;

/**
 * Playwright locators over a native UI Automation snapshot.
 *
 * Playwright parses the locator (`getByRole("button", { name: "Save" }).nth(1)`) into its internal selector
 * (`internal:role=button[name="Save"i] >> nth=1`); this module evaluates that selector against the UIA tree
 * with Playwright's semantics: chained parts search inside the previous matches, names and texts match a
 * case-insensitive substring unless exact. UIA reports scrolled-out elements as offscreen; like Playwright, which
 * still finds and scrolls to them, locators match them, and only visible/hidden checks treat them as not visible.
 */

/** A parsed Playwright selector, as `iso.parseSelector` returns it. */
export interface ParsedSelector { parts: { name: string; body: unknown; source: string }[] }
/** The JSON target shared with browser pages; on native windows `ref` is an observation hint and `test_id` an AutomationId. */
export interface NativeTarget {
  ref?: string; role?: string; name?: string; label?: string; text?: string; placeholder?: string; test_id?: string; exact?: boolean;
}

/** UIA control type → ARIA roles, most specific first. The first one names the element in snapshots and suggestions. */
const ROLES: Record<string, string[]> = {
  appbar: ["toolbar"], button: ["button"], calendar: ["grid"], checkbox: ["checkbox"], combobox: ["combobox"], custom: ["generic"],
  datagrid: ["grid", "table"], dataitem: ["row"], document: ["document"], edit: ["textbox"], group: ["group"], header: ["rowgroup"],
  headeritem: ["columnheader"], hyperlink: ["link"], image: ["img"], list: ["list", "listbox"], listitem: ["listitem", "option"],
  menu: ["menu"], menubar: ["menubar"], menuitem: ["menuitem"], pane: ["generic"], progressbar: ["progressbar"], radiobutton: ["radio"],
  scrollbar: ["scrollbar"], semanticzoom: ["generic"], separator: ["separator"], slider: ["slider"], spinner: ["spinbutton"],
  splitbutton: ["button"], statusbar: ["status"], tab: ["tablist"], tabitem: ["tab"], table: ["table", "grid"], text: ["text"],
  thumb: ["generic"], titlebar: ["titlebar"], toolbar: ["toolbar"], tooltip: ["tooltip"], tree: ["tree"], treeitem: ["treeitem"],
  window: ["window", "dialog"],
};
/** Roles Playwright's getByLabel looks at: labelled form controls. */
const LABELLED = new Set(["textbox", "combobox", "checkbox", "radio", "slider", "spinbutton", "listbox", "switch"]);
/**
 * Roles whose UIA name is their own visible text, as an element's text content is on a page. Containers such as
 * windows, lists and menu bars are left out: their name is a label, and some apps borrow it from a nearby control.
 */
const CONTENT = new Set(["text", "button", "link", "menuitem", "listitem", "option", "tab", "treeitem", "checkbox", "radio",
  "cell", "gridcell", "columnheader", "rowheader", "heading", "tooltip", "group"]);
/** Roles whose `checked` state is a checkbox state; for buttons the same UIA toggle is `pressed`. */
const CHECKABLE = new Set(["checkbox", "radio", "menuitem", "switch", "treeitem", "listitem"]);

/** ARIA roles of a native element, primary first; the UIA control type name also matches. */
export function ariaRoles(element: Pick<UiElement, "role">): string[] {
  const control = (element.role ?? "").replace(/\s+/g, "").toLowerCase();
  return ROLES[control] ?? (control ? [control] : ["generic"]);
}
export const ariaRole = (element: Pick<UiElement, "role">): string => ariaRoles(element)[0];

/** The accessible name: UIA Name when the runtime reports it, otherwise the driver's label. */
export const accessibleName = (element: UiElement): string => element.uia_name ?? element.name ?? "";

/** Checkbox state: the UIA toggle when reported; older runtimes fold it into `selected` for checkable controls. */
export function checkedState(element: UiElement): boolean | "mixed" | null {
  if (element.checked === "on") return true;
  if (element.checked === "off") return false;
  if (element.checked === "mixed") return "mixed";
  return element.checked === undefined && ["checkbox", "radio"].includes(ariaRole(element)) ? element.selected : null;
}
/** Not visible means UIA IsOffscreen (scrolled out, collapsed or minimized); unknown visibility counts as visible. */
export const isVisible = (element: UiElement): boolean => element.offscreen !== true;

const normalize = (value: string) => value.replace(/\s+/g, " ").trim();

/** A Playwright string or regex matcher: `"x"i` substring, `"x"s` exact, `/re/flags`, or an attribute value. */
type Matcher = (value: string) => boolean;
function stringMatcher(value: string | RegExp, exact: boolean): Matcher {
  if (value instanceof RegExp) return text => value.test(normalize(text));
  const wanted = normalize(value);
  return exact ? text => normalize(text) === wanted : text => normalize(text).toLowerCase().includes(wanted.toLowerCase());
}
function textMatcher(body: unknown): Matcher {
  const source = String(body);
  const regex = /^\/(.*)\/([a-z]*)$/s.exec(source);
  if (regex) return stringMatcher(new RegExp(regex[1], regex[2]), false);
  const quoted = /^(".*")([is]?)$/s.exec(source);
  if (!quoted) throw Error(`Unsupported text matcher ${source}.`);
  return stringMatcher(JSON.parse(quoted[1]) as string, quoted[2] === "s");
}
type Attribute = { name: string; op: string; value: unknown; caseSensitive: boolean };
function attributeMatcher(attribute: Attribute): Matcher {
  if (attribute.op !== "=") throw Error(`Attribute operator ${attribute.op} is not supported for native windows.`);
  return stringMatcher(attribute.value as string | RegExp, attribute.caseSensitive);
}
const parseAttributes = (body: unknown): { name: string; attributes: Attribute[] } => iso().parseAttributeSelector(String(body), true);

/** Index, parent links and texts of one snapshot, so selectors evaluate without repeated scans. */
class Tree {
  readonly order = new Map<string, number>();
  readonly byHint = new Map<string, UiElement>();
  private readonly children = new Map<string, UiElement[]>();
  private readonly texts = new Map<string, string>();
  constructor(readonly elements: readonly UiElement[]) {
    elements.forEach((element, index) => { this.order.set(element.hint, index); this.byHint.set(element.hint, element); });
    for (const element of elements) if (element.parent_hint && this.byHint.has(element.parent_hint))
      (this.children.get(element.parent_hint) ?? this.children.set(element.parent_hint, []).get(element.parent_hint)!).push(element);
  }
  /** Direct children in tree order. */
  childrenOf(element: UiElement): UiElement[] { return this.children.get(element.hint) ?? []; }
  /** Text content, as Playwright reads it: the visible text of this element and its descendants, without container labels. */
  textContent(element: UiElement): string {
    let text = this.texts.get(element.hint);
    if (text === undefined) {
      const role = ariaRole(element);
      const own = CONTENT.has(role) ? [accessibleName(element), role === "text" ? element.value ?? "" : ""] : [];
      text = normalize([...own, ...this.childrenOf(element).map(child => this.textContent(child))].filter(Boolean).join(" "));
      this.texts.set(element.hint, text);
    }
    return text;
  }
  /** Whether `element` lies strictly inside `root`. */
  inside(element: UiElement, root: UiElement): boolean {
    for (let parent = element.parent_hint; parent; parent = this.byHint.get(parent)?.parent_hint ?? null) if (parent === root.hint) return true;
    return false;
  }
  descendants(roots: readonly UiElement[] | null): UiElement[] {
    return roots === null ? [...this.elements] : this.elements.filter(element => roots.some(root => this.inside(element, root)));
  }
  sort(elements: Iterable<UiElement>): UiElement[] {
    return [...new Set(elements)].sort((a, b) => this.order.get(a.hint)! - this.order.get(b.hint)!);
  }
}

/** One query engine: does `element` match this selector part? */
function engine(part: ParsedSelector["parts"][number]): (element: UiElement) => boolean {
  switch (part.name) {
    case "internal:role": case "role": {
      const { name: role, attributes } = parseAttributes(part.body);
      const wanted = role.toLowerCase();
      const checks: ((element: UiElement) => boolean)[] = [element => ariaRoles(element).includes(wanted) || (element.role ?? "").toLowerCase() === wanted];
      for (const attribute of attributes) {
        const value = attribute.value;
        switch (attribute.name) {
          case "name": { const match = attributeMatcher(attribute); checks.push(element => match(accessibleName(element))); break; }
          case "include-hidden": break;
          case "checked": checks.push(element => CHECKABLE.has(ariaRole(element)) && String(checkedState(element)) === String(value)); break;
          case "pressed": checks.push(element => ariaRole(element) === "button" && element.checked !== undefined && String(checkedState({ ...element, role: "CheckBox" })) === String(value)); break;
          case "disabled": checks.push(element => element.enabled !== null && String(element.enabled === false) === String(value)); break;
          case "selected": checks.push(element => element.selected !== null && String(element.selected) === String(value)); break;
          case "expanded": checks.push(element => element.expanded !== undefined && String(element.expanded) === String(value)); break;
          default: throw Error(`getByRole option ${attribute.name} is not available for native windows.`);
        }
      }
      return element => checks.every(check => check(element));
    }
    case "internal:label": { const match = textMatcher(part.body); return element => LABELLED.has(ariaRole(element)) && match(accessibleName(element)); }
    case "internal:testid": {
      const [attribute] = parseAttributes(part.body).attributes;
      const match = attributeMatcher(attribute);
      return element => element.automation_id !== undefined && match(element.automation_id);
    }
    case "internal:attr": {
      const [attribute] = parseAttributes(part.body).attributes;
      const match = attributeMatcher(attribute);
      if (attribute.name === "title" || attribute.name === "placeholder") return element => element.help_text !== undefined && match(element.help_text);
      if (attribute.name === "alt") return element => ariaRole(element) === "img" && match(accessibleName(element));
      throw Error(`Attribute ${attribute.name} is not available for native windows.`);
    }
    default:
      throw Error(`Selector engine "${part.name}" is not available for native windows. Use getByRole, getByText, getByLabel, getByTestId (AutomationId), getByTitle, filter, nth, and or or.`);
  }
}
const nested = (body: unknown): ParsedSelector => {
  const parsed = (body as { parsed?: ParsedSelector })?.parsed;
  if (!parsed) throw Error("A nested locator could not be read.");
  return parsed;
};

/** Evaluate a parsed selector from `roots` (the whole window when null) in tree order. */
function evaluate(tree: Tree, selector: ParsedSelector, roots: readonly UiElement[] | null): UiElement[] {
  let current: UiElement[] | null = null;
  let scope = roots;
  for (const part of selector.parts) {
    const all = () => current ?? tree.descendants(scope);
    switch (part.name) {
      case "nth": {
        const index = Number(part.body), list = all();
        const item = index < 0 ? list[list.length + index] : list[index];
        current = item ? [item] : [];
        break;
      }
      case "visible": current = all().filter(element => isVisible(element) === (String(part.body) === "true")); break;
      case "internal:has-text": case "internal:has-not-text": {
        const match = textMatcher(part.body), keep = part.name === "internal:has-text";
        current = all().filter(element => match(tree.textContent(element)) === keep);
        break;
      }
      case "internal:has": case "internal:has-not": {
        const inner = nested(part.body), keep = part.name === "internal:has";
        current = all().filter(element => (evaluate(tree, inner, [element]).length > 0) === keep);
        break;
      }
      case "internal:and": { const other = new Set(evaluate(tree, nested(part.body), scope)); current = all().filter(element => other.has(element)); break; }
      case "internal:or": current = tree.sort([...all(), ...evaluate(tree, nested(part.body), scope)]); break;
      case "internal:describe": break;
      // Like Playwright's text engine: the smallest elements whose text content matches, not their ancestors.
      case "internal:text": {
        const match = textMatcher(part.body);
        scope = current ?? roots;
        current = tree.descendants(scope).filter(element => match(tree.textContent(element)) && !tree.childrenOf(element).some(child => match(tree.textContent(child))));
        break;
      }
      default: {
        const match = engine(part);
        scope = current ?? roots;
        current = tree.descendants(scope).filter(match);
      }
    }
  }
  return current ?? tree.descendants(roots);
}

/** Playwright's internal selector for a JSON target, in the same shape browser pages use. */
export function targetSelector(target: NativeTarget): string {
  const quote = (value: string, exact: boolean) => `${JSON.stringify(value)}${exact ? "s" : "i"}`;
  const exact = target.exact === true;
  const parts: string[] = [];
  if (target.role !== undefined) parts.push(`internal:role=${target.role}${target.name !== undefined ? `[name=${quote(target.name, exact)}]` : ""}`);
  if (target.label !== undefined) parts.push(`internal:label=${quote(target.label, exact)}`);
  if (target.text !== undefined) parts.push(`internal:text=${quote(target.text, exact)}`);
  if (target.placeholder !== undefined) parts.push(`internal:attr=[placeholder=${quote(target.placeholder, exact)}]`);
  if (target.test_id !== undefined) parts.push(`internal:testid=[data-testid=${quote(target.test_id, true)}]`);
  if (!parts.length) throw Error("target needs role, label, text, placeholder or test_id.");
  return parts.slice(1).reduce((all, part) => `${all} >> internal:and=${JSON.stringify(part)}`, parts[0]);
}

/** Playwright's internal selector for a locator such as `getByRole("button", { name: "Save" }).nth(1)`. */
export function locatorSelector(locator: string): string {
  if (typeof locator !== "string" || !locator.trim() || locator.length > 2000) throw Error("A locator must be a Playwright locator string of at most 2000 characters.");
  try { return iso().unsafeLocatorOrSelectorAsSelector("javascript", locator.trim().replace(/^(?:page|window|this)\./, ""), "data-testid"); }
  catch (error) { throw Error(`The locator could not be parsed: ${error instanceof Error ? error.message : String(error)}`); }
}

/** Parse a Playwright locator for a native window. CSS and XPath parse here but refuse when evaluated. */
export function parseLocator(locator: string): ParsedSelector {
  return iso().parseSelector(locatorSelector(locator));
}

const TARGET_FIELDS = ["role", "name", "label", "text", "placeholder", "test_id", "exact"] as const;
/**
 * A native target: a Playwright locator string, or the JSON target browser pages take. Hints are snapshot-local,
 * so `ref` refuses here: targets resolve in fresh UI walks.
 */
export function nativeSelector(value: unknown, what = "target"): ParsedSelector {
  if (typeof value === "string") return parseLocator(value);
  if (!value || typeof value !== "object" || Array.isArray(value)) throw Error(`${what} must be a Playwright locator string or an object with ${TARGET_FIELDS.join(", ")}.`);
  const target = value as Record<string, unknown>;
  if ("ref" in target) throw Error(`${what}.ref names an element of one observation; use a hint in the observation address, or a locator.`);
  const extra = Object.keys(target).filter(key => !(TARGET_FIELDS as readonly string[]).includes(key));
  if (extra.length) throw Error(`${what} does not accept ${extra.join(", ")}; use ${TARGET_FIELDS.join(", ")} or a locator string.`);
  for (const key of TARGET_FIELDS) if (key !== "exact" && target[key] !== undefined && (typeof target[key] !== "string" || !target[key] || (target[key] as string).length > 512))
    throw Error(`${what}.${key} must be a non-empty string of at most 512 characters.`);
  if (target.exact !== undefined && typeof target.exact !== "boolean") throw Error(`${what}.exact must be boolean.`);
  if (target.name !== undefined && target.role === undefined) throw Error(`${what}.name requires ${what}.role.`);
  return iso().parseSelector(targetSelector(target as NativeTarget));
}

/** Elements of one snapshot that a parsed selector matches, in tree order. */
export function queryNative(ui: Pick<UiResource, "elements">, selector: ParsedSelector): UiElement[] {
  return evaluate(new Tree(ui.elements), selector, null);
}

/**
 * The UIA tree as a Playwright aria snapshot, rendered by Playwright itself: `- button "Save" [ref=g:a]`,
 * `- textbox "Name": Ada`, `- checkbox "Agree" [checked]`, nested children, `- text: idle`. Refs are observation hints.
 */
export function nativeAriaSnapshot(ui: Pick<UiResource, "elements">, root?: UiElement): string {
  const { roots, nodes } = ariaNodes(ui);
  return iso().renderAriaSnapshotAsYaml(root ? [nodes.get(root.hint)!] : roots);
}

/** The tree toMatchAriaSnapshot checks: a fragment of the whole window, or of one element and its subtree. */
export function nativeAriaTree(ui: Pick<UiResource, "elements">, root?: UiElement): AriaTreeNode {
  const { roots, nodes } = ariaNodes(ui);
  const convert = (node: AriaNode): AriaTreeNode | string => node.role === "text" ? normalize(node.text ?? "") : {
    ...node, children: [...(node.text !== undefined ? [normalize(node.text)] : []), ...(node.children ?? []).map(convert)],
  };
  return { role: "fragment", children: (root ? [nodes.get(root.hint)!] : roots).map(convert) };
}

function ariaNodes(ui: Pick<UiResource, "elements">): { roots: AriaNode[]; nodes: Map<string, AriaNode> } {
  const nodes = new Map<string, AriaNode>(), roots: AriaNode[] = [];
  for (const element of ui.elements) {
    const role = ariaRole(element), name = accessibleName(element);
    if (role === "text") { nodes.set(element.hint, { role, text: name || element.value || "" }); continue; }
    const node: AriaNode = { role, ref: element.hint, ...(name ? { name } : {}) };
    if (role === "textbox" || role === "combobox" || role === "spinbutton" || role === "slider") { if (element.value) node.text = element.value; }
    const state = checkedState(element);
    if (state !== null) { if (role === "button") node.pressed = state; else node.checked = state; }
    if (element.enabled === false) node.disabled = true;
    if (element.expanded === true) node.expanded = true;
    if (element.selected === true && state === null) node.selected = true;
    nodes.set(element.hint, node);
  }
  for (const element of ui.elements) {
    const node = nodes.get(element.hint)!, parent = element.parent_hint ? nodes.get(element.parent_hint) : undefined;
    if (parent && parent.role !== "text") (parent.children ??= []).push(node); else roots.push(node);
  }
  return { roots, nodes };
}

/** A Playwright locator that names this element, for strict-mode errors and listings. */
export function suggestLocator(element: UiElement): string {
  const role = ariaRole(element), name = accessibleName(element);
  if (element.automation_id && !/^\d+$/.test(element.automation_id)) return `getByTestId(${JSON.stringify(element.automation_id)})`;
  return iso().asLocator("javascript", `internal:role=${role}${name ? `[name=${JSON.stringify(name)}s]` : ""}`);
}