Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

14.4 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import {
  cutAtCodeUnits,
  parseAriaSnapshot,
  parseDescriptor,
} from "../src/aria-snapshot";

/**
 * The parser, tested against captured Playwright output.
 *
 * The fixture below is copied from `ariaSnapshot({ mode: "ai" })` against httpbin's form in the
 * container, after filling one field and ticking two boxes. Captured output matters because the real
 * shape includes nested wrappers and quoted text that plausible hand-written YAML can miss.
 *
 * Note what the real output shows that a plausible guess did not: flags come in any order
 * (`[checked] [active] [ref=e19]`, with ref last), Playwright quotes any value containing a colon, and
 * the tree nests several levels through `generic` and `paragraph` wrappers that carry refs of their own.
 */
const CAPTURED = `- generic [ref=e2]:
  - paragraph [ref=e3]:
    - generic [ref=e4]:
      - text: "Customer name:"
      - textbox "Customer name:" [ref=e5]: Katherine Johnson
  - paragraph [ref=e6]:
    - generic [ref=e7]:
      - text: "Telephone:"
      - textbox "Telephone:" [ref=e8]
  - group "Pizza Size" [ref=e12]:
    - paragraph [ref=e14]:
      - generic [ref=e15]:
        - radio "Small" [ref=e16]
        - text: Small
    - paragraph [ref=e17]:
      - generic [ref=e18]:
        - radio "Medium" [checked] [active] [ref=e19]
        - text: Medium
  - group "Pizza Toppings" [ref=e23]:
    - paragraph [ref=e25]:
      - generic [ref=e26]:
        - checkbox "Bacon" [ref=e27]
        - text: Bacon
    - paragraph [ref=e28]:
      - generic [ref=e29]:
        - checkbox "Extra Cheese" [checked] [ref=e30]
        - text: Extra Cheese
  - button "Submit order" [ref=e44]`;

describe("parseAriaSnapshot, against captured output", () => {
  test("keeps the controls and drops the scaffolding", () => {
    const { elements } = parseAriaSnapshot(CAPTURED);
    // `generic`, `paragraph`, `group` and `text` all carry refs but are not things a Bot can act on,
    // and a list full of them is what makes a model pick the wrong element.
    expect(elements.map((e) => e.role)).toEqual([
      "textbox",
      "textbox",
      "radio",
      "radio",
      "checkbox",
      "checkbox",
      "button",
    ]);
  });

  test("finds controls nested several levels deep", () => {
    const names = parseAriaSnapshot(CAPTURED).elements.map((e) => e.name);
    // The radios live under group > paragraph > generic. A parser reading only the top level would
    // return two textboxes and call that the page.
    expect(names).toContain("Small");
    expect(names).toContain("Extra Cheese");
  });

  test("reads ref, role and accessible name", () => {
    const { elements } = parseAriaSnapshot(CAPTURED);
    expect(elements[0]).toMatchObject({
      ref: "e5",
      role: "textbox",
      name: "Customer name:",
    });
    expect(elements.at(-1)).toMatchObject({
      ref: "e44",
      role: "button",
      name: "Submit order",
    });
  });

  test("reads a control's value, and omits it when empty", () => {
    const { elements } = parseAriaSnapshot(CAPTURED);
    expect(elements[0]?.value).toBe("Katherine Johnson");
    expect(elements[1]?.value).toBeUndefined();
  });

  test("reports checked AND unchecked for things that can be checked", () => {
    const byName = new Map(
      parseAriaSnapshot(CAPTURED).elements.map((e) => [e.name, e]),
    );
    expect(byName.get("Medium")?.checked).toBe(true);
    expect(byName.get("Extra Cheese")?.checked).toBe(true);
    // Playwright emits nothing for an unchecked control; false is inferred so a Bot can see the state
    // rather than assuming it.
    expect(byName.get("Small")?.checked).toBe(false);
    expect(byName.get("Bacon")?.checked).toBe(false);
    // A button cannot be checked, so the field is absent rather than false.
    expect(byName.get("Submit order")).not.toHaveProperty("checked");
  });

  /**
   * A half-ticked "select all", captured the same way the fixture above was.
   *
   * `ariaSnapshot({ mode: "ai" })` against a fieldset of three boxes in Chromium 151, with the first
   * one's `indeterminate` set — which is what a box above a partly-ticked list carries. Playwright
   * writes that as `[checked=mixed]`: a value, where an ordinary tick is the bare `[checked]`. It is
   * the only flag in this output that carries one, which is the whole of the bug.
   *
   * Captured rather than written, for the reason the note at the top of this file gives. Guessed at,
   * the entry loses the `generic` wrapper each box sits inside and the `text` node beside it, and the
   * guess would pass while the shipped parser walked something else.
   */
  const MIXED = `- group "Toppings" [ref=e2]:
  - generic [ref=e4]:
    - checkbox "Select all" [checked=mixed] [ref=e5]
    - text: Select all
  - generic [ref=e6]:
    - checkbox "Bacon" [checked] [ref=e7]
    - text: Bacon
  - generic [ref=e8]:
    - checkbox "Extra Cheese" [ref=e9]
    - text: Extra Cheese`;

  test("a half-ticked box is not reported as ticked", () => {
    const byName = new Map(
      parseAriaSnapshot(MIXED).elements.map((e) => [e.name, e]),
    );
    // Not checked, so a Bot asked to tick it clicks it. Told it was already checked, it left the
    // rows underneath unselected and said they were done.
    expect(byName.get("Select all")?.checked).toBe(false);
    // And the two beside it are unchanged, so this is not a swap.
    expect(byName.get("Bacon")?.checked).toBe(true);
    expect(byName.get("Extra Cheese")?.checked).toBe(false);
  });

  test("a half-ticked box is still a control a Bot can act on", () => {
    const [first] = parseAriaSnapshot(MIXED).elements;
    // The ref is what a click needs, and it sits after the flag Playwright gave a value to.
    expect(first).toMatchObject({ ref: "e5", role: "checkbox" });
  });

  test("empty and unparseable input produce no elements rather than throwing", () => {
    expect(parseAriaSnapshot("").elements).toEqual([]);
    expect(parseAriaSnapshot("\t- [[[ not yaml").elements).toEqual([]);
    expect(parseAriaSnapshot("").truncated).toBe(false);
  });

  test("the element list is bounded, and says when it was cut", () => {
    const many = Array.from(
      { length: 250 },
      (_, index) => `- button "B${index}" [ref=e${index}]`,
    ).join("\n");
    const { elements, truncated } = parseAriaSnapshot(many);
    expect(elements).toHaveLength(200);
    expect(truncated).toBe(true);
  });
});

/**
 * A name or value longer than the 200 kept is cut, and the cut counts UTF-16 code units.
 *
 * An emoji is two of them. With its first half as the 200th unit, a plain `slice` leaves a lone high
 * surrogate as the last character of what the Bot is handed: JSON carries it as a bare `\ud83d` and
 * UTF-8 as U+FFFD, so the Bot reads back a broken character that is not on the page, in text it may
 * well have typed itself.
 */
describe("a name or value too long to keep whole", () => {
  const CUT_THROUGH = `${"a".repeat(199)}😀 and the rest`;
  const ENDS_INSIDE = `${"a".repeat(198)}😀 and the rest`;

  test("a name cut inside a character loses the whole character, not half of it", () => {
    const { elements } = parseAriaSnapshot(`- link "${CUT_THROUGH}" [ref=e1]`);
    expect(elements[0]?.name).toBe("a".repeat(199));
  });

  test("a value cut inside a character loses the whole character, not half of it", () => {
    const { elements } = parseAriaSnapshot(
      `- textbox "Message" [ref=e1]: ${CUT_THROUGH}`,
    );
    expect(elements[0]?.value).toBe("a".repeat(199));
  });

  test("a character that ends at the cut is kept", () => {
    const { elements } = parseAriaSnapshot(
      `- textbox "${ENDS_INSIDE}" [ref=e1]: ${ENDS_INSIDE}`,
    );
    expect(elements[0]?.name).toBe(`${"a".repeat(198)}😀`);
    expect(elements[0]?.value).toBe(`${"a".repeat(198)}😀`);
  });
});

/**
 * The cut itself, at any limit.
 *
 * Tested directly as well as through the parser because `index.ts` cuts the readable page text with
 * it at 6000 rather than 200, and that caller imports Playwright at load, so there is no test that
 * can reach it. The rule is the thing worth pinning, so it is pinned where it is declared.
 */
describe("cutting at code units", () => {
  const EMOJI = "\u{1F600}";

  test("a limit landing between the halves of a character drops the character", () => {
    const text = `${"a".repeat(5999)}${EMOJI}tail`;
    const cut = cutAtCodeUnits(text, 6000);
    expect(cut).toBe("a".repeat(5999));
    // Not a lone high surrogate, which is what a bare `slice` leaves and what reads as U+FFFD.
    expect(cut.charCodeAt(cut.length - 1)).toBeLessThan(0xd800);
  });

  test("a character that ends exactly at the limit is kept whole", () => {
    const text = `${"a".repeat(5998)}${EMOJI}tail`;
    expect(cutAtCodeUnits(text, 6000)).toBe(`${"a".repeat(5998)}${EMOJI}`);
  });

  test("text that fits is returned unchanged, emoji and all", () => {
    const text = `hello ${EMOJI} world`;
    expect(cutAtCodeUnits(text, 6000)).toBe(text);
  });

  test("empty text is empty rather than a thrown index", () => {
    // `charCodeAt(-1)` is NaN and every comparison against it is false, so this returns "".
    expect(cutAtCodeUnits("", 6000)).toBe("");
  });

  test("a low surrogate last is a whole character and is kept", () => {
    // The guard must look only for an UNPAIRED high surrogate. A complete pair ends on its low half,
    // and dropping that would cost a character the limit had room for.
    expect(cutAtCodeUnits(EMOJI, 2)).toBe(EMOJI);
  });
});

describe("values a real parser handles and a pattern got wrong", () => {
  test("a quoted numeric value is not left with its quotes", () => {
    // Numeric-looking text remains a string, so one-time codes are not coerced.
    const { elements } = parseAriaSnapshot(
      '- textbox "Code" [ref=e1]: "123456"',
    );
    expect(elements[0]?.value).toBe("123456");
  });

  test("a value containing a colon survives", () => {
    const { elements } = parseAriaSnapshot(
      '- textbox "Homepage" [ref=e1]: "https://example.com:8443/path"',
    );
    expect(elements[0]?.value).toBe("https://example.com:8443/path");
  });

  test("an escaped quote inside a value survives", () => {
    const { elements } = parseAriaSnapshot(
      '- textbox "Note" [ref=e1]: "she said \\"yes\\""',
    );
    expect(elements[0]?.value).toBe('she said "yes"');
  });

  /**
   * Playwright quotes a value only when it has to, so the ones a Bot most often needs to read back
   * arrive bare: a telephone number, a postcode, an order reference.
   */
  test.each([
    ["555-0142", "a telephone number keeps both groups"],
    ["90210-1234", "a postcode keeps its extension"],
    ["0142-555", "a reference keeps its leading zero"],
    ["007", "a padded number keeps its padding"],
    ["2026-08-16", "a date stays the text on the page"],
    ["20:30", "a time is not read as a number"],
    ["1.2.3", "a version is not read as a number"],
    ["true", "a field holding a word is that word"],
  ])("an unquoted %s survives: %s", (written) => {
    const { elements } = parseAriaSnapshot(
      `- textbox "Field" [ref=e1]: ${written}`,
    );
    expect(elements[0]?.value).toBe(written);
  });
});

describe("parseDescriptor", () => {
  test("flags are read in any order, including ref last", () => {
    // Exactly what the captured output does: `[checked] [active] [ref=e19]`.
    const descriptor = parseDescriptor(
      'radio "Medium" [checked] [active] [ref=e19]',
    );
    expect(descriptor?.role).toBe("radio");
    expect(descriptor?.name).toBe("Medium");
    expect(descriptor?.flags.get("ref")).toBe("e19");
    expect(descriptor?.flags.has("checked")).toBe(true);
  });

  test("an escaped quote inside a name is unescaped", () => {
    const descriptor = parseDescriptor(
      'button "Delete \\"draft\\" now" [ref=e2]',
    );
    expect(descriptor?.name).toBe('Delete "draft" now');
  });

  test("a bracket inside the name is not mistaken for a flag", () => {
    // The reason this is scanned rather than matched: the parts can contain each other.
    const descriptor = parseDescriptor('button "Save [draft]" [ref=e3]');
    expect(descriptor?.name).toBe("Save [draft]");
    expect(descriptor?.flags.get("ref")).toBe("e3");
  });

  test("an unnamed control still parses", () => {
    const descriptor = parseDescriptor("button [ref=e1]");
    expect(descriptor).toMatchObject({ role: "button", name: "" });
    expect(descriptor?.flags.get("ref")).toBe("e1");
  });

  test("junk yields nothing rather than a half-built descriptor", () => {
    expect(parseDescriptor("")).toBeNull();
    expect(parseDescriptor("   ")).toBeNull();
  });
});

/**
 * Captured from `ariaSnapshot({ mode: "ai" })` in this package's Playwright: two native `<select>`s
 * and a custom listbox. A native select's options carry no refs, so they used to be dropped and a Bot
 * saw `combobox "Industry"` with nothing it could choose and no way to tell what it was set to.
 */
const DROPDOWNS = `- generic [active] [ref=e1]:
  - generic [ref=e2]:
    - text: Industry
    - combobox "Industry" [ref=e3]:
      - option "Select an industry" [selected]
      - option "Logistics"
      - option "Retail"
  - generic [ref=e4]:
    - text: Stage
    - combobox "Stage" [ref=e5]:
      - option "Prospect" [selected]
      - option "Qualified"
  - listbox "Owner" [ref=e6]:
    - option "Alex" [ref=e7]`;

describe("parseAriaSnapshot, dropdowns", () => {
  test("a native select reports its options and the one it shows", () => {
    const { elements } = parseAriaSnapshot(DROPDOWNS);
    expect(elements.find((element) => element.ref === "e3")).toEqual({
      ref: "e3",
      role: "combobox",
      name: "Industry",
      options: ["Select an industry", "Logistics", "Retail"],
      value: "Select an industry",
    });
    expect(elements.find((element) => element.ref === "e5")?.value).toBe(
      "Prospect",
    );
  });

  test("a custom listbox's options stay elements of their own, listed once", () => {
    const { elements } = parseAriaSnapshot(DROPDOWNS);
    expect(elements.find((element) => element.ref === "e6")).toEqual({
      ref: "e6",
      role: "listbox",
      name: "Owner",
    });
    expect(elements.find((element) => element.ref === "e7")).toEqual({
      ref: "e7",
      role: "option",
      name: "Alex",
    });
  });

  test("a very long list is cut to fifty choices", () => {
    const many = Array.from(
      { length: 120 },
      (_, index) => `      - option "Country ${index}"`,
    ).join("\n");
    const { elements } = parseAriaSnapshot(
      `- combobox "Country" [ref=e9]:\n${many}`,
    );
    expect(elements[0]?.options).toHaveLength(50);
  });
});