Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
/**
* Playwright's toMatchAriaSnapshot matching, for native UIA trees and browser pages alike.
*
* The template is parsed by Playwright's own parser. The matching functions below are ported from
* Playwright's packages/injected/src/ariaSnapshot.ts (matchesNode, listEqual, containsList, matchesNodeDeep,
* matchesTextValue, matchesStringOrRegex), because Playwright ships them only inside its injected page script.
* Copyright (c) Microsoft Corporation, licensed under the Apache License, Version 2.0
* (https://www.apache.org/licenses/LICENSE-2.0). Changed: typed, and absent boolean states read as false.
*/
import { createRequire } from "node:module";
/** One node of a received aria tree; strings are text. */
export interface AriaTreeNode {
role: string; name?: string; children?: (AriaTreeNode | string)[];
checked?: boolean | "mixed"; pressed?: boolean | "mixed"; disabled?: boolean; expanded?: boolean; selected?: boolean; level?: number;
props?: { url?: string };
}
interface TextTemplate { raw: string; normalized: string }
interface TemplateNode {
kind: "role" | "text"; role?: string; name?: string | { pattern: string }; text?: TextTemplate;
children?: TemplateNode[]; containerMode?: "contain" | "equal" | "deep-equal"; props?: { url?: TextTemplate };
checked?: boolean | "mixed"; pressed?: boolean | "mixed"; disabled?: boolean; expanded?: boolean; selected?: boolean; level?: number; invalid?: unknown;
}
const load = () => {
const require = createRequire(import.meta.url);
const { iso } = require("playwright-core/lib/coreBundle") as { iso: { parseAriaSnapshotUnsafe(yaml: unknown, text: string): TemplateNode } };
const { yaml } = require("playwright-core/lib/utilsBundle") as { yaml: unknown };
return (text: string) => iso.parseAriaSnapshotUnsafe(yaml, text);
};
let parser: ((text: string) => TemplateNode) | undefined;
/** Parse an aria snapshot template with Playwright's parser; throws Playwright's message on bad YAML. */
export function parseAriaTemplate(text: string): TemplateNode {
return (parser ??= load())(text);
}
/** Whether `root` (usually a fragment holding the checked element) contains a subtree matching the template. */
export function matchesAriaTemplate(root: AriaTreeNode, template: string | TemplateNode): boolean {
return matchesNodeDeep(root, typeof template === "string" ? parseAriaTemplate(template) : template);
}
/** A received YAML snapshot (from a page) as a tree the matcher takes; the result is a fragment of its roots. */
export function ariaTreeFromYaml(text: string): AriaTreeNode {
const convert = (node: TemplateNode): AriaTreeNode | string => node.kind === "text" ? node.text!.normalized : {
role: node.role!, ...(typeof node.name === "string" && node.name ? { name: node.name } : {}),
...(node.checked !== undefined ? { checked: node.checked } : {}), ...(node.pressed !== undefined ? { pressed: node.pressed } : {}),
...(node.disabled ? { disabled: true } : {}), ...(node.expanded ? { expanded: true } : {}), ...(node.selected ? { selected: true } : {}),
...(node.level !== undefined ? { level: node.level } : {}), ...(node.props?.url ? { props: { url: node.props.url.normalized } } : {}),
children: (node.children ?? []).map(convert),
};
const parsed = parseAriaTemplate(text);
return parsed.role === "fragment" ? { role: "fragment", children: (parsed.children ?? []).map(convert) } : { role: "fragment", children: [convert(parsed)] };
}
function matchesStringOrRegex(text: string | undefined, template: TemplateNode["name"]): boolean {
if (!template) return true;
if (!text) return false;
if (typeof template === "string") return text === template;
return !!text.match(new RegExp(template.pattern));
}
function matchesTextValue(text: string | undefined, template: TextTemplate | undefined): boolean {
if (!template?.normalized) return true;
if (!text) return false;
if (text === template.normalized || text === template.raw) return true;
const { raw } = template;
if (!(raw.startsWith("/") && raw.endsWith("/") && raw.length > 1)) return false;
try { return !!text.match(new RegExp(raw.slice(1, -1))); } catch { return false; }
}
function matchesNode(node: AriaTreeNode | string, template: TemplateNode, isDeepEqual: boolean): boolean {
if (typeof node === "string" && template.kind === "text") return matchesTextValue(node, template.text);
if (node === null || typeof node !== "object" || template.kind !== "role") return false;
if (template.role !== "fragment" && template.role !== node.role) return false;
if (template.checked !== undefined && template.checked !== node.checked) return false;
if (template.disabled !== undefined && template.disabled !== (node.disabled ?? false)) return false;
if (template.expanded !== undefined && template.expanded !== (node.expanded ?? false)) return false;
if (template.invalid !== undefined) return false;
if (template.level !== undefined && template.level !== node.level) return false;
if (template.pressed !== undefined && template.pressed !== node.pressed) return false;
if (template.selected !== undefined && template.selected !== (node.selected ?? false)) return false;
if (!matchesStringOrRegex(node.name, template.name)) return false;
if (!matchesTextValue(node.props?.url, template.props?.url)) return false;
if (template.containerMode === "contain") return containsList(node.children ?? [], template.children ?? []);
if (template.containerMode === "equal") return listEqual(node.children ?? [], template.children ?? [], false);
if (template.containerMode === "deep-equal" || isDeepEqual) return listEqual(node.children ?? [], template.children ?? [], true);
return containsList(node.children ?? [], template.children ?? []);
}
function listEqual(children: (AriaTreeNode | string)[], template: TemplateNode[], isDeepEqual: boolean): boolean {
if (template.length !== children.length) return false;
return template.every((item, index) => matchesNode(children[index], item, isDeepEqual));
}
function containsList(children: (AriaTreeNode | string)[], template: TemplateNode[]): boolean {
if (template.length > children.length) return false;
const remaining = children.slice();
for (const item of template) {
let child = remaining.shift();
while (child !== undefined && !matchesNode(child, item, false)) child = remaining.shift();
if (child === undefined) return false;
}
return true;
}
function matchesNodeDeep(root: AriaTreeNode, template: TemplateNode): boolean {
const visit = (node: AriaTreeNode | string): boolean => matchesNode(node, template, false)
|| (typeof node !== "string" && (node.children ?? []).some(visit));
return visit(root);
}