Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

11.2 KB
Raw Blame
/**
 * Turning Playwright's aria snapshot into the flat element list the tool contract publishes.
 *
 * Playwright's `mode: "ai"` output is YAML: each entry is a mapping whose key describes the element
 * and whose value is that control's contents. A YAML parser preserves quoted values, colons, escaped
 * quotes, multi-line values and deeper indentation.
 *
 * Parsed under the failsafe schema, which resolves every scalar as a string. What a control contains
 * is text, and nothing here wants it typed.
 *
 * What remains to parse by hand is one descriptor per element,
 * `textbox "Customer name:" [ref=e5] [checked]`, and that is scanned rather than matched, because the
 * parts can contain one another: an accessible name can hold a bracket and a flag can hold a quote.
 *
 * This module has no Playwright import, so parser tests do not need a browser.
 */

import { parse as parseYaml } from "yaml";

/** How many elements a snapshot will describe. Bounded for the same reason the text extract is. */
const SNAPSHOT_ELEMENT_LIMIT = 200;

/**
 * The roles worth handing to a Bot that wants to act.
 *
 * Role allow-list, not a CSS selector list. `mode: "ai"` returns the whole accessible tree, which on
 * a real page is mostly headings, paragraphs and generic containers. What a Bot needs in order to fill
 * in a form is the controls, and naming them by ARIA role is both shorter and more correct than
 * guessing at tag names: a `div[role=button]` is a button here, and Playwright has already worked out
 * which is which.
 */
const INTERACTIVE_ROLES = new Set([
  "button",
  "checkbox",
  "combobox",
  "link",
  "listbox",
  "menuitem",
  "menuitemcheckbox",
  "menuitemradio",
  "option",
  "radio",
  "searchbox",
  "slider",
  "spinbutton",
  "switch",
  "tab",
  "textbox",
]);

/** The roles whose unchecked state is meaningful, so absence of `[checked]` means false, not unknown. */
const CHECKABLE_ROLES = new Set([
  "checkbox",
  "menuitemcheckbox",
  "menuitemradio",
  "radio",
  "switch",
]);

/**
 * One thing on the page a Bot can act on.
 *
 * Mirrors `SnapshotElement` in the server's published contract (`server/src/computer/schema.ts`), the
 * same way the workspace entry type does. Duplicated rather than shared because this process is a
 * separate deployable with no code in common with the server; the two must be changed together, and a
 * field added here and not there is invisible until a Bot asks for it.
 */
export type SnapshotElement = {
  ref: string;
  role: string;
  name: string;
  /** Present for form controls, so the Bot can tell an empty field from a filled one. */
  value?: string;
  disabled?: boolean;
  checked?: boolean;
  /**
   * A native dropdown's choices, by what each says. Its options carry no refs, so without this a Bot
   * saw `combobox "Industry"` and nothing it could choose; it chooses by sending one of these as text.
   */
  options?: string[];
};

/** The most choices one dropdown reports, so a list of every country cannot crowd out the form. */
const OPTION_LIMIT = 50;

/** The descriptor half of an entry: everything before the colon. */
type Descriptor = {
  role: string;
  name: string;
  flags: Map<string, string>;
};

/**
 * Read `textbox "Customer name:" [ref=e5] [checked]`.
 *
 * Scanned left to right rather than matched with a pattern, because the parts can contain each other:
 * an accessible name can contain a bracket, and a flag value can contain a quote. A single expression
 * that handles every page is not maintainable.
 */
export function parseDescriptor(text: string): Descriptor | null {
  const trimmed = text.trim();
  if (!trimmed) return null;

  // The role runs up to the first space, quote or bracket.
  let index = 0;
  while (
    index < trimmed.length &&
    trimmed[index] !== " " &&
    trimmed[index] !== '"' &&
    trimmed[index] !== "["
  ) {
    index += 1;
  }
  const role = trimmed.slice(0, index);
  if (!role) return null;

  let name = "";
  const flags = new Map<string, string>();

  while (index < trimmed.length) {
    const char = trimmed[index];

    if (char === '"') {
      // The accessible name, consuming escapes so a name containing a quote survives intact.
      index += 1;
      let collected = "";
      while (index < trimmed.length && trimmed[index] !== '"') {
        if (trimmed[index] === "\\" && index + 1 < trimmed.length) {
          collected += trimmed[index + 1];
          index += 2;
          continue;
        }
        collected += trimmed[index];
        index += 1;
      }
      index += 1;
      name = collected;
      continue;
    }

    if (char === "[") {
      index += 1;
      let collected = "";
      while (index < trimmed.length && trimmed[index] !== "]") {
        collected += trimmed[index];
        index += 1;
      }
      index += 1;
      const equals = collected.indexOf("=");
      if (equals === -1) {
        flags.set(collected.trim(), "");
      } else {
        flags.set(
          collected.slice(0, equals).trim(),
          collected.slice(equals + 1).trim(),
        );
      }
      continue;
    }

    index += 1;
  }

  return { role, name, flags };
}

/**
 * The first `limit` UTF-16 code units of `text`, one fewer when the cut would split a character.
 *
 * `slice` counts code units and an emoji is two, so a limit landing between the halves leaves a lone
 * high surrogate last: JSON carries it as a bare `\ud83d` and UTF-8 as U+FFFD, and the Bot reads a
 * broken character that is not on the page. The server's `cutAtCodeUnits` is the same rule; this
 * process shares no code with the server, so it is repeated here rather than imported.
 *
 * Exported for `index.ts`, which cuts the readable page text the same way, and so that the rule has
 * its own tests. It lives here rather than beside that caller because this module imports no
 * Playwright: `index.ts` does, at load, so a helper declared there could not be tested at all.
 */
export function cutAtCodeUnits(text: string, limit: number): string {
  const sliced = text.slice(0, limit);
  const last = sliced.charCodeAt(sliced.length - 1);
  return last >= 0xd800 && last <= 0xdbff ? sliced.slice(0, -1) : sliced;
}

/** Build an element from a descriptor and whatever YAML gave as its value, or null if not actionable. */
function toElement(
  descriptor: Descriptor,
  value: unknown,
): SnapshotElement | null {
  if (!INTERACTIVE_ROLES.has(descriptor.role)) return null;

  const ref = descriptor.flags.get("ref");
  // No ref means nothing can be done to it, so it is noise in a list whose entire purpose is acting.
  if (!ref) return null;

  const element: SnapshotElement = {
    ref,
    role: descriptor.role,
    name: cutAtCodeUnits(descriptor.name, 200),
  };

  // Values arrive as text, with quoting and escapes already resolved.
  if (typeof value === "string") {
    const text = value.trim();
    if (text) element.value = cutAtCodeUnits(text, 200);
  }

  if (descriptor.flags.has("disabled")) element.disabled = true;

  // A native `<select>`: its value in the YAML is the list of its options, not text. The option marked
  // `[selected]` is what the field shows, which is the value a person reads off it.
  if (descriptor.role === "combobox" || descriptor.role === "listbox") {
    const list = optionsOf(value);
    if (list) {
      element.options = list.options;
      if (list.selected !== undefined && element.value === undefined)
        element.value = list.selected;
    }
  }

  // Playwright emits `[checked]` only when something is checked, so absence is ambiguous on its own: a
  // Bot cannot tell an unticked box from a control that does not tick. Reported as false for the roles
  // that can be checked, and left off entirely for the ones that cannot.
  //
  // `[checked=mixed]` is the third state and the one spelling Playwright gives the flag a value for:
  // `aria-checked="mixed"`, which is what the box above a partly-ticked list carries. It is not
  // ticked, and a Bot told it was left the rows underneath unselected and reported the job done. The
  // contract this fills in is `checked?: boolean`, so "neither" has to be said as false — which is
  // also the answer that gets the right action, since clicking a mixed box ticks it.
  if (descriptor.flags.has("checked")) {
    const state = descriptor.flags.get("checked");
    element.checked = state !== "mixed" && state !== "false";
  } else if (CHECKABLE_ROLES.has(descriptor.role)) {
    element.checked = false;
  }

  return element;
}

/**
 * The options listed under a dropdown, and which one is selected.
 *
 * Only options with no ref of their own: one with a ref is a custom listbox's option, already an
 * element the Bot clicks, and listing it twice would offer two ways to do one thing.
 */
function optionsOf(
  value: unknown,
): { options: string[]; selected?: string } | null {
  if (!Array.isArray(value)) return null;
  const options: string[] = [];
  let selected: string | undefined;
  for (const child of value) {
    const key =
      typeof child === "string"
        ? child
        : child && typeof child === "object"
          ? Object.keys(child)[0]
          : undefined;
    const descriptor = key ? parseDescriptor(key) : null;
    if (descriptor?.role !== "option" || descriptor.flags.has("ref")) continue;
    const name = cutAtCodeUnits(descriptor.name, 200);
    if (descriptor.flags.has("selected") && selected === undefined)
      selected = name;
    if (options.length < OPTION_LIMIT) options.push(name);
  }
  return options.length ? { options, selected } : null;
}

/** Turn an aria snapshot into the flat element list the tool contract publishes. */
export function parseAriaSnapshot(yaml: string): {
  elements: SnapshotElement[];
  truncated: boolean;
} {
  const elements: SnapshotElement[] = [];
  let truncated = false;

  const push = (element: SnapshotElement): void => {
    if (elements.length >= SNAPSHOT_ELEMENT_LIMIT) {
      truncated = true;
      return;
    }
    elements.push(element);
  };

  let tree: unknown;
  try {
    tree = parseYaml(yaml, { schema: "failsafe" });
  } catch {
    // A snapshot that will not parse yields no elements rather than throwing. The caller's next move is
    // to take another one, and an exception here would reach the Bot as a broken computer.
    return { elements: [], truncated: false };
  }

  /** Depth first: the tree nests by containment, and a flat list is what a Bot acts on. */
  const walk = (node: unknown): void => {
    if (truncated) return;

    if (Array.isArray(node)) {
      for (const child of node) walk(child);
      return;
    }

    if (typeof node === "string") {
      // An entry with no value: `- button "Submit order" [ref=e44]`.
      const descriptor = parseDescriptor(node);
      const element = descriptor ? toElement(descriptor, undefined) : null;
      if (element) push(element);
      return;
    }

    if (node && typeof node === "object") {
      for (const [key, value] of Object.entries(node)) {
        const descriptor = parseDescriptor(key);
        const element = descriptor ? toElement(descriptor, value) : null;
        if (element) push(element);
        // Descend regardless of whether this entry was actionable: a `group "Pizza Size"` is not, and
        // its radios are.
        if (value && typeof value === "object") walk(value);
      }
    }
  };

  walk(tree);
  return { elements, truncated };
}