Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { describe, expect, test } from "bun:test";
import {
cutAtCodeUnits,
parseAriaSnapshot,
parseDescriptor,
} from "../src/aria-snapshot";
/**
* The parser, tested against captured Playwright output.
*
* The fixture below is copied from `ariaSnapshot({ mode: "ai" })` against httpbin's form in the
* container, after filling one field and ticking two boxes. Captured output matters because the real
* shape includes nested wrappers and quoted text that plausible hand-written YAML can miss.
*
* Note what the real output shows that a plausible guess did not: flags come in any order
* (`[checked] [active] [ref=e19]`, with ref last), Playwright quotes any value containing a colon, and
* the tree nests several levels through `generic` and `paragraph` wrappers that carry refs of their own.
*/
const CAPTURED = `- generic [ref=e2]:
- paragraph [ref=e3]:
- generic [ref=e4]:
- text: "Customer name:"
- textbox "Customer name:" [ref=e5]: Katherine Johnson
- paragraph [ref=e6]:
- generic [ref=e7]:
- text: "Telephone:"
- textbox "Telephone:" [ref=e8]
- group "Pizza Size" [ref=e12]:
- paragraph [ref=e14]:
- generic [ref=e15]:
- radio "Small" [ref=e16]
- text: Small
- paragraph [ref=e17]:
- generic [ref=e18]:
- radio "Medium" [checked] [active] [ref=e19]
- text: Medium
- group "Pizza Toppings" [ref=e23]:
- paragraph [ref=e25]:
- generic [ref=e26]:
- checkbox "Bacon" [ref=e27]
- text: Bacon
- paragraph [ref=e28]:
- generic [ref=e29]:
- checkbox "Extra Cheese" [checked] [ref=e30]
- text: Extra Cheese
- button "Submit order" [ref=e44]`;
describe("parseAriaSnapshot, against captured output", () => {
test("keeps the controls and drops the scaffolding", () => {
const { elements } = parseAriaSnapshot(CAPTURED);
// `generic`, `paragraph`, `group` and `text` all carry refs but are not things a Bot can act on,
// and a list full of them is what makes a model pick the wrong element.
expect(elements.map((e) => e.role)).toEqual([
"textbox",
"textbox",
"radio",
"radio",
"checkbox",
"checkbox",
"button",
]);
});
test("finds controls nested several levels deep", () => {
const names = parseAriaSnapshot(CAPTURED).elements.map((e) => e.name);
// The radios live under group > paragraph > generic. A parser reading only the top level would
// return two textboxes and call that the page.
expect(names).toContain("Small");
expect(names).toContain("Extra Cheese");
});
test("reads ref, role and accessible name", () => {
const { elements } = parseAriaSnapshot(CAPTURED);
expect(elements[0]).toMatchObject({
ref: "e5",
role: "textbox",
name: "Customer name:",
});
expect(elements.at(-1)).toMatchObject({
ref: "e44",
role: "button",
name: "Submit order",
});
});
test("reads a control's value, and omits it when empty", () => {
const { elements } = parseAriaSnapshot(CAPTURED);
expect(elements[0]?.value).toBe("Katherine Johnson");
expect(elements[1]?.value).toBeUndefined();
});
test("reports checked AND unchecked for things that can be checked", () => {
const byName = new Map(
parseAriaSnapshot(CAPTURED).elements.map((e) => [e.name, e]),
);
expect(byName.get("Medium")?.checked).toBe(true);
expect(byName.get("Extra Cheese")?.checked).toBe(true);
// Playwright emits nothing for an unchecked control; false is inferred so a Bot can see the state
// rather than assuming it.
expect(byName.get("Small")?.checked).toBe(false);
expect(byName.get("Bacon")?.checked).toBe(false);
// A button cannot be checked, so the field is absent rather than false.
expect(byName.get("Submit order")).not.toHaveProperty("checked");
});
/**
* A half-ticked "select all", captured the same way the fixture above was.
*
* `ariaSnapshot({ mode: "ai" })` against a fieldset of three boxes in Chromium 151, with the first
* one's `indeterminate` set — which is what a box above a partly-ticked list carries. Playwright
* writes that as `[checked=mixed]`: a value, where an ordinary tick is the bare `[checked]`. It is
* the only flag in this output that carries one, which is the whole of the bug.
*
* Captured rather than written, for the reason the note at the top of this file gives. Guessed at,
* the entry loses the `generic` wrapper each box sits inside and the `text` node beside it, and the
* guess would pass while the shipped parser walked something else.
*/
const MIXED = `- group "Toppings" [ref=e2]:
- generic [ref=e4]:
- checkbox "Select all" [checked=mixed] [ref=e5]
- text: Select all
- generic [ref=e6]:
- checkbox "Bacon" [checked] [ref=e7]
- text: Bacon
- generic [ref=e8]:
- checkbox "Extra Cheese" [ref=e9]
- text: Extra Cheese`;
test("a half-ticked box is not reported as ticked", () => {
const byName = new Map(
parseAriaSnapshot(MIXED).elements.map((e) => [e.name, e]),
);
// Not checked, so a Bot asked to tick it clicks it. Told it was already checked, it left the
// rows underneath unselected and said they were done.
expect(byName.get("Select all")?.checked).toBe(false);
// And the two beside it are unchanged, so this is not a swap.
expect(byName.get("Bacon")?.checked).toBe(true);
expect(byName.get("Extra Cheese")?.checked).toBe(false);
});
test("a half-ticked box is still a control a Bot can act on", () => {
const [first] = parseAriaSnapshot(MIXED).elements;
// The ref is what a click needs, and it sits after the flag Playwright gave a value to.
expect(first).toMatchObject({ ref: "e5", role: "checkbox" });
});
test("empty and unparseable input produce no elements rather than throwing", () => {
expect(parseAriaSnapshot("").elements).toEqual([]);
expect(parseAriaSnapshot("\t- [[[ not yaml").elements).toEqual([]);
expect(parseAriaSnapshot("").truncated).toBe(false);
});
test("the element list is bounded, and says when it was cut", () => {
const many = Array.from(
{ length: 250 },
(_, index) => `- button "B${index}" [ref=e${index}]`,
).join("\n");
const { elements, truncated } = parseAriaSnapshot(many);
expect(elements).toHaveLength(200);
expect(truncated).toBe(true);
});
});
/**
* A name or value longer than the 200 kept is cut, and the cut counts UTF-16 code units.
*
* An emoji is two of them. With its first half as the 200th unit, a plain `slice` leaves a lone high
* surrogate as the last character of what the Bot is handed: JSON carries it as a bare `\ud83d` and
* UTF-8 as U+FFFD, so the Bot reads back a broken character that is not on the page, in text it may
* well have typed itself.
*/
describe("a name or value too long to keep whole", () => {
const CUT_THROUGH = `${"a".repeat(199)}😀 and the rest`;
const ENDS_INSIDE = `${"a".repeat(198)}😀 and the rest`;
test("a name cut inside a character loses the whole character, not half of it", () => {
const { elements } = parseAriaSnapshot(`- link "${CUT_THROUGH}" [ref=e1]`);
expect(elements[0]?.name).toBe("a".repeat(199));
});
test("a value cut inside a character loses the whole character, not half of it", () => {
const { elements } = parseAriaSnapshot(
`- textbox "Message" [ref=e1]: ${CUT_THROUGH}`,
);
expect(elements[0]?.value).toBe("a".repeat(199));
});
test("a character that ends at the cut is kept", () => {
const { elements } = parseAriaSnapshot(
`- textbox "${ENDS_INSIDE}" [ref=e1]: ${ENDS_INSIDE}`,
);
expect(elements[0]?.name).toBe(`${"a".repeat(198)}😀`);
expect(elements[0]?.value).toBe(`${"a".repeat(198)}😀`);
});
});
/**
* The cut itself, at any limit.
*
* Tested directly as well as through the parser because `index.ts` cuts the readable page text with
* it at 6000 rather than 200, and that caller imports Playwright at load, so there is no test that
* can reach it. The rule is the thing worth pinning, so it is pinned where it is declared.
*/
describe("cutting at code units", () => {
const EMOJI = "\u{1F600}";
test("a limit landing between the halves of a character drops the character", () => {
const text = `${"a".repeat(5999)}${EMOJI}tail`;
const cut = cutAtCodeUnits(text, 6000);
expect(cut).toBe("a".repeat(5999));
// Not a lone high surrogate, which is what a bare `slice` leaves and what reads as U+FFFD.
expect(cut.charCodeAt(cut.length - 1)).toBeLessThan(0xd800);
});
test("a character that ends exactly at the limit is kept whole", () => {
const text = `${"a".repeat(5998)}${EMOJI}tail`;
expect(cutAtCodeUnits(text, 6000)).toBe(`${"a".repeat(5998)}${EMOJI}`);
});
test("text that fits is returned unchanged, emoji and all", () => {
const text = `hello ${EMOJI} world`;
expect(cutAtCodeUnits(text, 6000)).toBe(text);
});
test("empty text is empty rather than a thrown index", () => {
// `charCodeAt(-1)` is NaN and every comparison against it is false, so this returns "".
expect(cutAtCodeUnits("", 6000)).toBe("");
});
test("a low surrogate last is a whole character and is kept", () => {
// The guard must look only for an UNPAIRED high surrogate. A complete pair ends on its low half,
// and dropping that would cost a character the limit had room for.
expect(cutAtCodeUnits(EMOJI, 2)).toBe(EMOJI);
});
});
describe("values a real parser handles and a pattern got wrong", () => {
test("a quoted numeric value is not left with its quotes", () => {
// Numeric-looking text remains a string, so one-time codes are not coerced.
const { elements } = parseAriaSnapshot(
'- textbox "Code" [ref=e1]: "123456"',
);
expect(elements[0]?.value).toBe("123456");
});
test("a value containing a colon survives", () => {
const { elements } = parseAriaSnapshot(
'- textbox "Homepage" [ref=e1]: "https://example.com:8443/path"',
);
expect(elements[0]?.value).toBe("https://example.com:8443/path");
});
test("an escaped quote inside a value survives", () => {
const { elements } = parseAriaSnapshot(
'- textbox "Note" [ref=e1]: "she said \\"yes\\""',
);
expect(elements[0]?.value).toBe('she said "yes"');
});
/**
* Playwright quotes a value only when it has to, so the ones a Bot most often needs to read back
* arrive bare: a telephone number, a postcode, an order reference.
*/
test.each([
["555-0142", "a telephone number keeps both groups"],
["90210-1234", "a postcode keeps its extension"],
["0142-555", "a reference keeps its leading zero"],
["007", "a padded number keeps its padding"],
["2026-08-16", "a date stays the text on the page"],
["20:30", "a time is not read as a number"],
["1.2.3", "a version is not read as a number"],
["true", "a field holding a word is that word"],
])("an unquoted %s survives: %s", (written) => {
const { elements } = parseAriaSnapshot(
`- textbox "Field" [ref=e1]: ${written}`,
);
expect(elements[0]?.value).toBe(written);
});
});
describe("parseDescriptor", () => {
test("flags are read in any order, including ref last", () => {
// Exactly what the captured output does: `[checked] [active] [ref=e19]`.
const descriptor = parseDescriptor(
'radio "Medium" [checked] [active] [ref=e19]',
);
expect(descriptor?.role).toBe("radio");
expect(descriptor?.name).toBe("Medium");
expect(descriptor?.flags.get("ref")).toBe("e19");
expect(descriptor?.flags.has("checked")).toBe(true);
});
test("an escaped quote inside a name is unescaped", () => {
const descriptor = parseDescriptor(
'button "Delete \\"draft\\" now" [ref=e2]',
);
expect(descriptor?.name).toBe('Delete "draft" now');
});
test("a bracket inside the name is not mistaken for a flag", () => {
// The reason this is scanned rather than matched: the parts can contain each other.
const descriptor = parseDescriptor('button "Save [draft]" [ref=e3]');
expect(descriptor?.name).toBe("Save [draft]");
expect(descriptor?.flags.get("ref")).toBe("e3");
});
test("an unnamed control still parses", () => {
const descriptor = parseDescriptor("button [ref=e1]");
expect(descriptor).toMatchObject({ role: "button", name: "" });
expect(descriptor?.flags.get("ref")).toBe("e1");
});
test("junk yields nothing rather than a half-built descriptor", () => {
expect(parseDescriptor("")).toBeNull();
expect(parseDescriptor(" ")).toBeNull();
});
});
/**
* Captured from `ariaSnapshot({ mode: "ai" })` in this package's Playwright: two native `<select>`s
* and a custom listbox. A native select's options carry no refs, so they used to be dropped and a Bot
* saw `combobox "Industry"` with nothing it could choose and no way to tell what it was set to.
*/
const DROPDOWNS = `- generic [active] [ref=e1]:
- generic [ref=e2]:
- text: Industry
- combobox "Industry" [ref=e3]:
- option "Select an industry" [selected]
- option "Logistics"
- option "Retail"
- generic [ref=e4]:
- text: Stage
- combobox "Stage" [ref=e5]:
- option "Prospect" [selected]
- option "Qualified"
- listbox "Owner" [ref=e6]:
- option "Alex" [ref=e7]`;
describe("parseAriaSnapshot, dropdowns", () => {
test("a native select reports its options and the one it shows", () => {
const { elements } = parseAriaSnapshot(DROPDOWNS);
expect(elements.find((element) => element.ref === "e3")).toEqual({
ref: "e3",
role: "combobox",
name: "Industry",
options: ["Select an industry", "Logistics", "Retail"],
value: "Select an industry",
});
expect(elements.find((element) => element.ref === "e5")?.value).toBe(
"Prospect",
);
});
test("a custom listbox's options stay elements of their own, listed once", () => {
const { elements } = parseAriaSnapshot(DROPDOWNS);
expect(elements.find((element) => element.ref === "e6")).toEqual({
ref: "e6",
role: "listbox",
name: "Owner",
});
expect(elements.find((element) => element.ref === "e7")).toEqual({
ref: "e7",
role: "option",
name: "Alex",
});
});
test("a very long list is cut to fifty choices", () => {
const many = Array.from(
{ length: 120 },
(_, index) => ` - option "Country ${index}"`,
).join("\n");
const { elements } = parseAriaSnapshot(
`- combobox "Country" [ref=e9]:\n${many}`,
);
expect(elements[0]?.options).toHaveLength(50);
});
});