Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(agent): improve page-agent DOM extraction fidelity (#19711)

p0nyfd591b5adf34
diff --git a/frontend/src/plugins/agent/dom/domTree.test.ts b/frontend/src/plugins/agent/dom/domTree.test.ts
index 9f9a7f4e1b..b00eea10db 100644
--- a/frontend/src/plugins/agent/dom/domTree.test.ts
+++ b/frontend/src/plugins/agent/dom/domTree.test.ts
@@ -39,6 +39,623 @@ describe("extractDomTree", () => {
expect(getElementByRef("e9")).toBeUndefined();
});
+ test("preserves visible non-interactive text nodes with normalized whitespace", () => {
+ document.body.innerHTML = `
+ <main>
+ <section>
+ <h1> Instances </h1>
+ <div>
+ <span>Primary</span>
+ <span>US East</span>
+ </div>
+ <button aria-label="Create instance">Create</button>
+ <p style="display: none">Hidden text</p>
+ <div aria-hidden="true" style="display: none">
+ <span>Also hidden</span>
+ </div>
+ </section>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+
+ expect(count).toBe(1);
+ expect(tree).toContain("Instances");
+ expect(tree).toContain("Primary");
+ expect(tree).toContain("US East");
+ expect(tree).toContain("[e1]<button>Create instance</button>");
+ expect(tree).not.toContain("Hidden text");
+ expect(tree).not.toContain("Also hidden");
+ });
+
+ test("treats clickable containers with pointer cursor as interactive and preserves descendant text", () => {
+ document.body.innerHTML = `
+ <main>
+ <div style="cursor: pointer">
+ <span>Prod Primary</span>
+ <span>us-east-1</span>
+ </div>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+
+ expect(count).toBe(1);
+ expect(tree).toContain("[e1]<div>Prod Primary us-east-1</div>");
+ expect(tree).toContain(" Prod Primary");
+ expect(tree).toContain(" us-east-1");
+ expect(getElementByRef("e1")?.tag).toBe("div");
+ });
+
+ test("does not promote inherited pointer-cursor wrappers beneath clickable containers", () => {
+ document.body.innerHTML = `
+ <main>
+ <div style="cursor: pointer">
+ <span>Prod Primary</span>
+ <div>
+ <span>Healthy</span>
+ </div>
+ </div>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+ const suggestions = extractDomRefSuggestions();
+
+ expect(count).toBe(1);
+ expect(tree).toContain("[e1]<div>Prod Primary Healthy</div>");
+ expect(tree).not.toContain("[e2]");
+ expect(suggestions).toEqual([
+ {
+ ref: "e1",
+ tag: "div",
+ role: undefined,
+ label: "Prod Primary Healthy",
+ value: undefined,
+ },
+ ]);
+ });
+
+ test("skips disabled and hidden pointer-cursor containers", () => {
+ document.body.innerHTML = `
+ <main>
+ <div style="cursor: pointer" disabled>Disabled row</div>
+ <div style="cursor: pointer" aria-disabled="true">Aria disabled row</div>
+ <div style="cursor: pointer" inert>Inert row</div>
+ <div style="cursor: pointer" aria-hidden="true">Aria hidden row</div>
+ <div style="cursor: pointer">Active row</div>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+ const suggestions = extractDomRefSuggestions();
+
+ expect(count).toBe(1);
+ expect(tree).toContain("[e1]<div>Active row</div>");
+ expect(tree).toContain("Disabled row");
+ expect(tree).toContain("Aria disabled row");
+ expect(tree).not.toContain("Inert row");
+ expect(tree).not.toContain("Aria hidden row");
+ expect(suggestions).toEqual([
+ {
+ ref: "e1",
+ tag: "div",
+ role: undefined,
+ label: "Active row",
+ value: undefined,
+ },
+ ]);
+ });
+
+ test("treats contenteditable regions as interactive without duplicating descendants", () => {
+ document.body.innerHTML = `
+ <main>
+ <div contenteditable="true">Editable SQL</div>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+
+ expect(count).toBe(1);
+ expect(tree).toContain("[e1]<div>Editable SQL</div>");
+ expect(tree.match(/Editable SQL/g)).toHaveLength(1);
+ });
+
+ test("dedupes descendant text when it matches an interactive label", () => {
+ document.body.innerHTML = `
+ <main>
+ <div style="cursor: pointer">
+ <span>Prod Primary us-east-1</span>
+ </div>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+
+ expect(count).toBe(1);
+ expect(tree).toContain("[e1]<div>Prod Primary us-east-1</div>");
+ expect(tree.match(/Prod Primary us-east-1/g)).toHaveLength(1);
+ });
+
+ test("suppresses blank button refs while keeping meaningful native controls", () => {
+ document.body.innerHTML = `
+ <main>
+ <button><span aria-hidden="true"></span></button>
+ <button title="Refresh"></button>
+ <input type="checkbox" />
+ <input type="text" />
+ <select>
+ <option selected>Prod</option>
+ </select>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+ const suggestions = extractDomRefSuggestions();
+
+ expect(count).toBe(4);
+ expect(tree).toContain("[e1]<button>Refresh</button>");
+ expect(tree).toContain('[e2]<input value="unchecked">checkbox</input>');
+ expect(tree).toContain("[e3]<input>textbox</input>");
+ expect(tree).toContain('[e4]<select value="Prod">Prod</select>');
+ expect(tree).not.toContain("<button></button>");
+ expect(suggestions).toEqual([
+ {
+ ref: "e1",
+ tag: "button",
+ role: undefined,
+ label: "Refresh",
+ value: undefined,
+ },
+ {
+ ref: "e2",
+ tag: "input",
+ role: undefined,
+ label: "checkbox",
+ value: "unchecked",
+ },
+ {
+ ref: "e3",
+ tag: "input",
+ role: undefined,
+ label: "textbox",
+ value: undefined,
+ },
+ {
+ ref: "e4",
+ tag: "select",
+ role: undefined,
+ label: "Prod",
+ value: "Prod",
+ },
+ ]);
+ });
+
+ test("serializes tables with semantic header and row separators", () => {
+ document.body.innerHTML = `
+ <main>
+ <table>
+ <thead>
+ <tr>
+ <th>Instance</th>
+ <th>Environment</th>
+ <th>Status</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><div><span>Prod Primary</span></div></td>
+ <td><span>us-east-1</span></td>
+ <td><span>Healthy</span></td>
+ </tr>
+ <tr>
+ <td><a href="/instances/staging">Staging</a></td>
+ <td>us-west-2</td>
+ <td><button aria-label="Open action menu">Actions</button></td>
+ </tr>
+ </tbody>
+ </table>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+
+ expect(count).toBe(2);
+ expect(tree).toContain("<table>");
+ expect(tree).toContain("<thead>Instance | Environment | Status</thead>");
+ expect(tree).toContain("<tr>Prod Primary | us-east-1 | Healthy</tr>");
+ expect(tree).toContain("<tr>Staging | us-west-2 | Actions</tr>");
+ expect(tree).toContain("[e1]<a>Staging</a>");
+ expect(tree).toContain("[e2]<button>Open action menu</button>");
+ expect(tree).not.toContain("<div>Prod Primary");
+ expect(tree).not.toContain("<span>Healthy");
+ });
+
+ test("drops utility action cells from clickable row summaries while keeping controls actionable", () => {
+ document.body.innerHTML = `
+ <main>
+ <table>
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Enabled</th>
+ <th>Actions</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr style="cursor: pointer">
+ <td><div><span>Prod Primary</span></div></td>
+ <td>
+ <div class="n-switch n-switch--active">
+ <div class="n-switch__rail"></div>
+ </div>
+ </td>
+ <td>
+ <div>
+ <button></button>
+ </div>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+ const suggestions = extractDomRefSuggestions();
+
+ expect(count).toBe(3);
+ expect(tree).toContain("<thead>Name | Enabled | Actions</thead>");
+ expect(tree).toContain("[e1]<tr>Prod Primary | switch</tr>");
+ expect(tree).toContain('[e2]<div value="checked">switch</div>');
+ expect(tree).toContain("[e3]<button>Actions</button>");
+ expect(tree).not.toContain("More actions");
+ expect(tree).not.toContain("n-switch__rail");
+ expect(tree).not.toContain("Prod PrimaryProd Primary");
+ expect(suggestions).toEqual([
+ {
+ ref: "e1",
+ tag: "tr",
+ role: undefined,
+ label: "Prod Primary | switch",
+ value: undefined,
+ },
+ {
+ ref: "e2",
+ tag: "div",
+ role: undefined,
+ label: "switch",
+ value: "checked",
+ },
+ {
+ ref: "e3",
+ tag: "button",
+ role: undefined,
+ label: "Actions",
+ value: undefined,
+ },
+ ]);
+ });
+
+ test("drops utility selection cells from table row summaries while keeping checkboxes actionable", () => {
+ document.body.innerHTML = `
+ <main>
+ <table>
+ <thead>
+ <tr>
+ <th></th>
+ <th>Name</th>
+ <th>Status</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr style="cursor: pointer">
+ <td><input type="checkbox" checked /></td>
+ <td>Prod Primary</td>
+ <td>Healthy</td>
+ </tr>
+ </tbody>
+ </table>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+ const suggestions = extractDomRefSuggestions();
+
+ expect(count).toBe(2);
+ expect(tree).toContain("[e1]<tr>Prod Primary | Healthy</tr>");
+ expect(tree).toContain('[e2]<input value="checked">checkbox</input>');
+ expect(tree).not.toContain("checkbox | Prod Primary");
+ expect(suggestions).toEqual([
+ {
+ ref: "e1",
+ tag: "tr",
+ role: undefined,
+ label: "Prod Primary | Healthy",
+ value: undefined,
+ },
+ {
+ ref: "e2",
+ tag: "input",
+ role: undefined,
+ label: "checkbox",
+ value: "checked",
+ },
+ ]);
+ });
+
+ test("suppresses Naive input prefix text while preserving the textbox ref", () => {
+ document.body.innerHTML = `
+ <main>
+ <div class="n-input">
+ <div class="n-input-prefix">
+ <span>Filter</span>
+ </div>
+ <div class="n-input-wrapper">
+ <input placeholder="Search instances" />
+ </div>
+ </div>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+ const suggestions = extractDomRefSuggestions();
+
+ expect(count).toBe(1);
+ expect(tree).toContain("[e1]<input>Search instances</input>");
+ expect(tree).not.toContain("Filter");
+ expect(suggestions).toEqual([
+ {
+ ref: "e1",
+ tag: "input",
+ role: undefined,
+ label: "Search instances",
+ value: undefined,
+ },
+ ]);
+ });
+
+ test("serializes Naive select controls with sibling labels and selected values", () => {
+ document.body.innerHTML = `
+ <main>
+ <div class="flex items-center gap-x-2">
+ <div class="textinfolabel">Rows per page</div>
+ <div class="n-base-selection" role="combobox">
+ <div class="n-base-selection-label">50</div>
+ </div>
+ </div>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+ const suggestions = extractDomRefSuggestions();
+
+ expect(count).toBe(1);
+ expect(tree).toContain('[e1]<select value="50">Rows per page</select>');
+ expect(tree).not.toContain("<div>50</div>");
+ expect(suggestions).toEqual([
+ {
+ ref: "e1",
+ tag: "select",
+ role: "combobox",
+ label: "Rows per page",
+ value: "50",
+ },
+ ]);
+ });
+
+ test("prefers clickable list items over nested wrapper clones with the same label", () => {
+ document.body.innerHTML = `
+ <main>
+ <ul>
+ <li style="cursor: pointer">
+ <div role="button">
+ <span>Prod Primary</span>
+ </div>
+ </li>
+ </ul>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+ const suggestions = extractDomRefSuggestions();
+
+ expect(count).toBe(1);
+ expect(tree).toContain("[e1]<li>Prod Primary</li>");
+ expect(tree).not.toContain("[e2]");
+ expect(tree.match(/Prod Primary/g)).toHaveLength(1);
+ expect(suggestions).toEqual([
+ {
+ ref: "e1",
+ tag: "li",
+ role: undefined,
+ label: "Prod Primary",
+ value: undefined,
+ },
+ ]);
+ });
+
+ test("drops nested wrapper refs when a clickable row already captures the same content", () => {
+ document.body.innerHTML = `
+ <main>
+ <table>
+ <tbody>
+ <tr style="cursor: pointer">
+ <td>
+ <div role="button">
+ <span>Prod Primary</span>
+ </div>
+ </td>
+ <td><span>Healthy</span></td>
+ </tr>
+ </tbody>
+ </table>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+ const suggestions = extractDomRefSuggestions();
+
+ expect(count).toBe(1);
+ expect(tree).toContain("[e1]<tr>Prod Primary | Healthy</tr>");
+ expect(tree).not.toContain("[e2]<div>Prod Primary</div>");
+ expect(tree.match(/Prod Primary/g)).toHaveLength(1);
+ expect(suggestions).toEqual([
+ {
+ ref: "e1",
+ tag: "tr",
+ role: undefined,
+ label: "Prod Primary | Healthy",
+ value: undefined,
+ },
+ ]);
+ });
+
+ test("truncates long labels and values deterministically", () => {
+ const longText = "x".repeat(180);
+ document.body.innerHTML = `
+ <main>
+ <button>${longText}</button>
+ <input value="${longText}" />
+ </main>
+ `;
+
+ const { tree } = extractDomTree();
+
+ expect(tree).toContain(`${"x".repeat(120)}...`);
+ expect(tree).not.toContain("x".repeat(150));
+ });
+
+ test("drops plain text before interactive refs when the tree budget is hit", () => {
+ const textBlocks = Array.from(
+ { length: 140 },
+ (_, index) => `
+ <section>
+ <p>Context line ${index}</p>
+ <div>
+ <span>${"detail".repeat(12)} ${index}</span>
+ <span>${"detail".repeat(12)} duplicate ${index}</span>
+ </div>
+ </section>
+ `
+ ).join("");
+ document.body.innerHTML = `
+ <main>
+ ${textBlocks}
+ <button aria-label="Launch workflow">Launch</button>
+ </main>
+ `;
+
+ const { tree, count } = extractDomTree();
+
+ expect(count).toBe(1);
+ expect(tree).toContain("[e1]<button>Launch workflow</button>");
+ expect(tree).not.toContain("Context line 139");
+ expect(tree.split("\n").length).toBeLessThanOrEqual(120);
+ });
+
+ test("prioritizes main content over shell regions when the budget is hit", () => {
+ const headerButtons = Array.from(
+ { length: 60 },
+ (_, index) => `<button>Header action ${index}</button>`
+ ).join("");
+ const sidebarLinks = Array.from(
+ { length: 60 },
+ (_, index) => `<a href="/sidebar/${index}">Sidebar item ${index}</a>`
+ ).join("");
+ const tableRows = Array.from(
+ { length: 25 },
+ (_, index) => `
+ <tr>
+ <td>Instance ${index}</td>
+ <td>Prod</td>
+ <td>Healthy</td>
+ </tr>
+ `
+ ).join("");
+
+ document.body.innerHTML = `
+ <div data-label="bb-main-body-wrapper">
+ <nav data-label="bb-dashboard-header">${headerButtons}</nav>
+ <div>
+ <aside data-label="bb-dashboard-static-sidebar">${sidebarLinks}</aside>
+ <main id="bb-layout-main">
+ <button>Create instance</button>
+ <table>
+ <thead>
+ <tr>
+ <th>Instance</th>
+ <th>Environment</th>
+ <th>Status</th>
+ </tr>
+ </thead>
+ <tbody>
+ ${tableRows}
+ </tbody>
+ </table>
+ </main>
+ </div>
+ </div>
+ `;
+
+ const { tree } = extractDomTree();
+
+ expect(tree).toMatch(/\[e\d+\]<button>Create instance<\/button>/);
+ expect(tree).toContain("<thead>Instance | Environment | Status</thead>");
+ expect(tree).toContain("<tr>Instance 24 | Prod | Healthy</tr>");
+ expect(tree).toContain("Sidebar item 0");
+ expect(tree).not.toContain("Header action 59");
+ expect(tree).not.toContain("Sidebar item 59");
+ });
+
+ test("demotes overlay drawers behind main content when the budget is hit", () => {
+ const drawerButtons = Array.from(
+ { length: 80 },
+ (_, index) => `<button>Drawer action ${index}</button>`
+ ).join("");
+ const rows = Array.from(
+ { length: 20 },
+ (_, index) => `
+ <tr>
+ <td>Database ${index}</td>
+ <td>Production</td>
+ <td>Ready</td>
+ </tr>
+ `
+ ).join("");
+
+ document.body.innerHTML = `
+ <div data-label="bb-main-body-wrapper">
+ <main id="bb-layout-main">
+ <button aria-label="Create database">Create</button>
+ <table>
+ <thead>
+ <tr>
+ <th>Database</th>
+ <th>Environment</th>
+ <th>Status</th>
+ </tr>
+ </thead>
+ <tbody>
+ ${rows}
+ </tbody>
+ </table>
+ </main>
+ <div class="n-drawer" role="dialog" aria-modal="true">
+ ${drawerButtons}
+ </div>
+ </div>
+ `;
+
+ const { tree } = extractDomTree();
+
+ expect(tree).toMatch(/\[e\d+\]<button>Create database<\/button>/);
+ expect(tree).toContain("<thead>Database | Environment | Status</thead>");
+ expect(tree).toContain("<tr>Database 19 | Production | Ready</tr>");
+ expect(tree).toContain("Drawer action 0");
+ expect(tree).not.toContain("Drawer action 79");
+ });
+
test("returns structured DOM ref suggestions for visible interactive elements", () => {
document.body.innerHTML = `
<main>
diff --git a/frontend/src/plugins/agent/dom/domTree.ts b/frontend/src/plugins/agent/dom/domTree.ts
index a94ffa56f0..0ded18ca10 100644
--- a/frontend/src/plugins/agent/dom/domTree.ts
+++ b/frontend/src/plugins/agent/dom/domTree.ts
@@ -35,9 +35,126 @@ const INTERACTIVE_ROLES = new Set([
"slider",
]);
-const NAIVE_INTERACTIVE_CLASSES = ["n-button", "n-switch", "n-checkbox"];
+const NAIVE_INTERACTIVE_CLASSES = [
+ "n-button",
+ "n-switch",
+ "n-checkbox",
+ "n-base-selection",
+];
+
+const TERMINAL_INTERACTIVE_TAGS = new Set([
+ "A",
+ "BUTTON",
+ "INPUT",
+ "SELECT",
+ "TEXTAREA",
+ "SUMMARY",
+]);
+
+const STRUCTURAL_WRAPPER_TAGS = new Set(["DIV", "SPAN", "TD", "TH"]);
+const STATEFUL_INTERACTIVE_ROLES = new Set([
+ "checkbox",
+ "radio",
+ "switch",
+ "textbox",
+ "combobox",
+ "listbox",
+ "option",
+ "slider",
+]);
+const TEXTBOX_INPUT_TYPES = new Set([
+ "",
+ "email",
+ "number",
+ "password",
+ "search",
+ "tel",
+ "text",
+ "url",
+]);
+const CHECKABLE_INPUT_TYPES = new Set(["checkbox", "radio"]);
+const BUTTON_LIKE_INPUT_TYPES = new Set(["button", "image", "reset", "submit"]);
+const INTERACTIVE_STATE_ATTRIBUTES = [
+ "aria-checked",
+ "aria-expanded",
+ "aria-pressed",
+ "aria-selected",
+ "aria-current",
+];
const SKIP_TAGS = new Set(["SCRIPT", "STYLE", "NOSCRIPT", "SVG", "PATH"]);
+const TABLE_SECTION_TAGS = new Set(["THEAD", "TBODY", "TFOOT"]);
+const TABLE_CELL_TAGS = new Set(["TH", "TD"]);
+const MAX_TEXT_NODE_LENGTH = 160;
+const MAX_LABEL_LENGTH = 120;
+const MAX_EDITOR_PREVIEW_LENGTH = 160;
+const MAX_DOM_TREE_LINES = 120;
+const MAX_DOM_TREE_CHARS = 6000;
+const ACTION_LIKE_HEADERS = new Set([
+ "action",
+ "actions",
+ "operation",
+ "operations",
+ "menu",
+]);
+const NAIVE_CONTROL_TEXT_SELECTORS = [
+ ".n-input-prefix",
+ ".n-input-suffix",
+ ".n-input__placeholder",
+ ".n-base-selection-label",
+ ".n-base-selection-placeholder",
+ ".n-base-selection-tags",
+ ".n-base-selection-input",
+ ".n-base-suffix",
+ ".n-base-loading",
+].join(",");
+
+const MAIN_CONTENT_SELECTOR = [
+ "main",
+ '[role="main"]',
+ "#bb-layout-main",
+ '[data-label="bb-main-body-wrapper"]',
+].join(",");
+const SHELL_REGION_SELECTOR = [
+ "aside",
+ "nav",
+ "header",
+ "footer",
+ '[role="navigation"]',
+ '[role="banner"]',
+ '[role="contentinfo"]',
+ '[role="complementary"]',
+ '[data-label="bb-dashboard-static-sidebar"]',
+ '[data-label="bb-dashboard-header"]',
+].join(",");
+const OVERLAY_REGION_SELECTOR = [
+ '[role="dialog"]',
+ '[aria-modal="true"]',
+ ".n-drawer",
+ ".n-modal",
+ ".n-popover",
+ ".n-dropdown-menu",
+ ".n-base-select-menu",
+ ".n-dialog",
+].join(",");
+const SECONDARY_REGION_CAPS: Partial<
+ Record<LineRegion, { lines: number; chars: number }>
+> = {
+ overlay: { lines: 24, chars: 1600 },
+ shell: { lines: 18, chars: 1200 },
+};
+const LINE_KIND_PRIORITY: Record<DomLine["kind"], number> = {
+ interactive: 0,
+ context: 1,
+ text: 2,
+};
+const LINE_REGION_PRIORITY: Record<LineRegion, number> = {
+ main: 0,
+ default: 1,
+ overlay: 2,
+ shell: 3,
+};
+const elementRegionCache = new WeakMap<Element, LineRegion>();
const LANDMARK_TAGS = new Set([
"NAV",
@@ -65,17 +182,62 @@ function isLandmark(el: Element): boolean {
return false;
}
+function getElementRegion(el?: Element): LineRegion {
+ if (!el) return "default";
+
+ const cachedRegion = elementRegionCache.get(el);
+ if (cachedRegion) return cachedRegion;
+
+ let region: LineRegion = "default";
+ if (el.closest(OVERLAY_REGION_SELECTOR)) {
+ region = "overlay";
+ } else if (el.closest(SHELL_REGION_SELECTOR)) {
+ region = "shell";
+ } else if (el.closest(MAIN_CONTENT_SELECTOR)) {
+ region = "main";
+ }
+
+ elementRegionCache.set(el, region);
+ return region;
+}
+
+function getLineRegion(line: DomLine): LineRegion {
+ return getElementRegion(line.sourceElement);
+}
+
const elementRegistry = new Map<string, IndexedElement>();
let nextElementRef = 1;
+type LineRegion = "main" | "default" | "overlay" | "shell";
+
+interface DomLine {
+ text: string;
+ kind: "interactive" | "context" | "text";
+ entry?: IndexedElement;
+ dedupeKey?: string;
+ sourceElement?: Element;
+ signature?: string;
+ semanticTag?: string;
+}
+
+type InteractiveReason = "native" | "role" | "class" | "editable" | "pointer";
+
+interface WalkContext {
+ lines: DomLine[];
+ textSeenByParent: Map<string, Set<string>>;
+ parentPath: string;
+ interactiveLabels: string[];
+ hasInteractiveAncestor: boolean;
+}
+
function createElementRef(): string {
return `e${nextElementRef++}`;
}
-function registerElement(entry: Omit<IndexedElement, "ref">): IndexedElement {
- const registered = { ref: createElementRef(), ...entry };
- elementRegistry.set(registered.ref, registered);
- return registered;
+function createIndexedElement(
+ entry: Omit<IndexedElement, "ref">
+): IndexedElement {
+ return { ref: createElementRef(), ...entry };
}
export function getElementByRef(ref: string): IndexedElement | undefined {
@@ -91,6 +253,46 @@ function isVisible(el: Element): boolean {
);
}
+function hasHiddenAncestor(el: Element): boolean {
+ return el.closest('[hidden], [aria-hidden="true"], [inert]') !== null;
+}
+
+function isPerceivable(el: Element): boolean {
+ return isVisible(el) && !hasHiddenAncestor(el);
+}
+
+function hasDisabledState(el: Element): boolean {
+ if (el.matches(":disabled") || el.hasAttribute("disabled")) return true;
+ return el.closest('[aria-disabled="true"], [disabled]') !== null;
+}
+
+function isContentEditableElement(el: Element): boolean {
+ const contentEditable = el.getAttribute("contenteditable");
+ return (
+ el instanceof HTMLElement &&
+ (el.isContentEditable ||
+ contentEditable === "" ||
+ contentEditable === "true" ||
+ contentEditable === "plaintext-only")
+ );
+}
+
+function isHiddenInput(el: Element): boolean {
+ return el instanceof HTMLInputElement && el.type === "hidden";
+}
+
+function hasPointerCursor(el: Element): boolean {
+ return window.getComputedStyle(el).cursor === "pointer";
+}
+
+function isNaiveSelectElement(el: Element): boolean {
+ return el.classList.contains("n-base-selection");
+}
+
+function isNaiveControlTextElement(el: Element): boolean {
+ return Boolean(el.closest(NAIVE_CONTROL_TEXT_SELECTORS));
+}
+
function isMonacoEditor(el: Element): boolean {
return el.classList.contains("monaco-editor");
}
@@ -107,54 +309,165 @@ function getMonacoContent(el: Element): string | undefined {
return content || undefined;
}
-function isInteractive(el: Element): boolean {
- if (INTERACTIVE_TAGS.has(el.tagName)) return true;
+function getInteractiveReason(el: Element): InteractiveReason | undefined {
+ if (!isPerceivable(el) || hasDisabledState(el) || isHiddenInput(el))
+ return undefined;
+ if (INTERACTIVE_TAGS.has(el.tagName)) return "native";
const role = el.getAttribute("role");
- if (role && INTERACTIVE_ROLES.has(role)) return true;
+ if (role && INTERACTIVE_ROLES.has(role)) return "role";
for (const cls of NAIVE_INTERACTIVE_CLASSES) {
- if (el.classList.contains(cls)) return true;
+ if (el.classList.contains(cls)) return "class";
}
- return false;
+ if (isContentEditableElement(el)) return "editable";
+ if (hasPointerCursor(el)) return "pointer";
+
+ return undefined;
+}
+
+function isInteractive(el: Element): boolean {
+ return Boolean(getInteractiveReason(el));
+}
+
+function shouldRecurseIntoInteractive(el: Element): boolean {
+ if (TERMINAL_INTERACTIVE_TAGS.has(el.tagName)) return false;
+ if (isContentEditableElement(el)) return false;
+ return Array.from(el.children).length > 0;
+}
+
+function truncateText(text: string, maxLength: number): string {
+ if (text.length <= maxLength) return text;
+ return `${text.slice(0, maxLength)}...`;
+}
+
+function normalizeTextContent(text: string): string | undefined {
+ const normalized = text.trim().replace(/\s+/g, " ");
+ if (!normalized) return undefined;
+ return truncateText(normalized, MAX_TEXT_NODE_LENGTH);
+}
+
+function getSiblingControlLabel(el: Element): string | undefined {
+ const parent = el.parentElement;
+ if (!parent) return undefined;
+
+ for (
+ let sibling = el.previousElementSibling;
+ sibling;
+ sibling = sibling.previousElementSibling
+ ) {
+ if (!isPerceivable(sibling) || isInteractive(sibling)) continue;
+ const text = normalizeTextContent(sibling.textContent ?? "");
+ if (text) return truncateText(text, MAX_LABEL_LENGTH);
+ }
+
+ if (parent.children.length <= 3) {
+ for (
+ let sibling = parent.previousElementSibling;
+ sibling;
+ sibling = sibling.previousElementSibling
+ ) {
+ if (!isPerceivable(sibling) || isInteractive(sibling)) continue;
+ const text = normalizeTextContent(sibling.textContent ?? "");
+ if (text) return truncateText(text, MAX_LABEL_LENGTH);
+ }
+ }
+
+ return undefined;
+}
+
+function getTableColumnHeaderLabel(el: Element): string | undefined {
+ const cell = el.closest("td, th");
+ if (!cell || !cell.parentElement) return undefined;
+
+ const cells = Array.from(cell.parentElement.children).filter(
+ (child): child is Element =>
+ child instanceof Element && TABLE_CELL_TAGS.has(child.tagName)
+ );
+ const columnIndex = cells.indexOf(cell);
+ if (columnIndex < 0) return undefined;
+
+ const table = cell.closest("table");
+ if (!table) return undefined;
+
+ const headerRow = table.querySelector("thead tr");
+ if (!headerRow) return undefined;
+
+ const headerCells = Array.from(headerRow.children).filter(
+ (child): child is Element =>
+ child instanceof Element && TABLE_CELL_TAGS.has(child.tagName)
+ );
+ const headerText = normalizeTextContent(
+ headerCells[columnIndex]?.textContent ?? ""
+ );
+ return headerText ? truncateText(headerText, MAX_LABEL_LENGTH) : undefined;
}
function extractLabel(el: Element): string {
// aria-label
- const ariaLabel = el.getAttribute("aria-label");
- if (ariaLabel) return ariaLabel.trim();
+ const ariaLabel = normalizeTextContent(el.getAttribute("aria-label") ?? "");
+ if (ariaLabel) return truncateText(ariaLabel, MAX_LABEL_LENGTH);
// aria-labelledby
const labelledBy = el.getAttribute("aria-labelledby");
if (labelledBy) {
const parts = labelledBy
.split(/\s+/)
- .map((id) => document.getElementById(id)?.textContent?.trim())
- .filter(Boolean);
- if (parts.length > 0) return parts.join(" ");
+ .map((id) =>
+ normalizeTextContent(document.getElementById(id)?.textContent ?? "")
+ )
+ .filter((part): part is string => Boolean(part));
+ if (parts.length > 0) {
+ return truncateText(parts.join(" "), MAX_LABEL_LENGTH);
+ }
}
// placeholder
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
- if (el.placeholder) return el.placeholder.trim();
+ const placeholder = normalizeTextContent(el.placeholder);
+ if (placeholder) return truncateText(placeholder, MAX_LABEL_LENGTH);
}
// Naive UI form-item label
const formItem = el.closest(".n-form-item");
if (formItem) {
- const labelEl = formItem.querySelector(".n-form-item-label__text");
- if (labelEl?.textContent) return labelEl.textContent.trim();
+ const label = normalizeTextContent(
+ formItem.querySelector(".n-form-item-label__text")?.textContent ?? ""
+ );
+ if (label) return truncateText(label, MAX_LABEL_LENGTH);
+ }
+
+ if (el instanceof HTMLInputElement && BUTTON_LIKE_INPUT_TYPES.has(el.type)) {
+ const buttonValue = normalizeTextContent(el.value);
+ if (buttonValue) return truncateText(buttonValue, MAX_LABEL_LENGTH);
}
// title
- const title = el.getAttribute("title");
- if (title) return title.trim();
+ const title = normalizeTextContent(el.getAttribute("title") ?? "");
+ if (title) return truncateText(title, MAX_LABEL_LENGTH);
+
+ if (
+ el instanceof HTMLInputElement ||
+ el instanceof HTMLTextAreaElement ||
+ el instanceof HTMLSelectElement ||
+ isNaiveSelectElement(el)
+ ) {
+ const siblingLabel = getSiblingControlLabel(el);
+ if (siblingLabel) return siblingLabel;
+ }
+
+ const columnHeaderLabel = getTableColumnHeaderLabel(el);
+ if (columnHeaderLabel) {
+ const normalizedHeader = normalizeLineSignature(columnHeaderLabel);
+ if (normalizedHeader && ACTION_LIKE_HEADERS.has(normalizedHeader)) {
+ return truncateText(columnHeaderLabel, MAX_LABEL_LENGTH);
+ }
+ }
// text content (truncated)
- const text = (el.textContent ?? "").trim().replace(/\s+/g, " ");
- if (text.length > 80) return text.slice(0, 80) + "...";
- return text;
+ const text = normalizeTextContent(el.textContent ?? "");
+ return text ? truncateText(text, MAX_LABEL_LENGTH) : "";
}
function extractValue(el: Element): string | undefined {
@@ -162,20 +475,26 @@ function extractValue(el: Element): string | undefined {
if (el.type === "checkbox" || el.type === "radio") {
return el.checked ? "checked" : "unchecked";
}
- return el.value || undefined;
+ const value = normalizeTextContent(el.value);
+ return value ? truncateText(value, MAX_LABEL_LENGTH) : undefined;
}
if (el instanceof HTMLTextAreaElement) {
- return el.value || undefined;
+ const value = normalizeTextContent(el.value);
+ return value ? truncateText(value, MAX_LABEL_LENGTH) : undefined;
}
if (el instanceof HTMLSelectElement) {
- return el.options[el.selectedIndex]?.text || el.value || undefined;
+ const value = normalizeTextContent(
+ el.options[el.selectedIndex]?.text || el.value || ""
+ );
+ return value ? truncateText(value, MAX_LABEL_LENGTH) : undefined;
}
// Naive UI select
if (el.classList.contains("n-base-selection")) {
- const label = el.querySelector(".n-base-selection-label");
- const text = label?.textContent?.trim();
- if (text) return text;
+ const value = normalizeTextContent(
+ el.querySelector(".n-base-selection-label")?.textContent ?? ""
+ );
+ return value ? truncateText(value, MAX_LABEL_LENGTH) : undefined;
}
// Naive UI checkbox/switch
@@ -191,6 +510,92 @@ function extractValue(el: Element): string | undefined {
return undefined;
}
+function hasInteractiveState(el: Element): boolean {
+ return INTERACTIVE_STATE_ATTRIBUTES.some((name) => {
+ const value = el.getAttribute(name);
+ return value !== null && value !== "false";
+ });
+}
+
+function getFallbackLabel(el: Element, reason: InteractiveReason): string {
+ if (el instanceof HTMLInputElement) {
+ if (CHECKABLE_INPUT_TYPES.has(el.type)) return el.type;
+ if (TEXTBOX_INPUT_TYPES.has(el.type)) return "textbox";
+ if (BUTTON_LIKE_INPUT_TYPES.has(el.type)) {
+ return normalizeTextContent(el.value) ?? "";
+ }
+ }
+
+ if (el instanceof HTMLTextAreaElement) return "textbox";
+ if (el instanceof HTMLSelectElement || isNaiveSelectElement(el)) {
+ return "select";
+ }
+
+ const role = el.getAttribute("role") ?? "";
+ if (STATEFUL_INTERACTIVE_ROLES.has(role)) return role;
+ if (
+ reason === "class" &&
+ (el.classList.contains("n-checkbox") || el.classList.contains("n-switch"))
+ ) {
+ return el.classList.contains("n-switch") ? "switch" : "checkbox";
+ }
+
+ return "";
+}
+
+function shouldIndexInteractiveNode(
+ el: Element,
+ reason: InteractiveReason,
+ label: string,
+ value: string | undefined,
+ context: WalkContext
+): boolean {
+ if (reason === "pointer" && context.hasInteractiveAncestor) return false;
+
+ const hasDescriptor = Boolean(label || value || hasInteractiveState(el));
+
+ if (reason === "pointer") {
+ if (!hasDescriptor) return false;
+ if (STRUCTURAL_WRAPPER_TAGS.has(el.tagName)) {
+ const childCount = el.children.length;
+ const meaningfulDescendantCount = Array.from(el.children).filter(
+ (child) =>
+ !STRUCTURAL_WRAPPER_TAGS.has(child.tagName) || isInteractive(child)
+ ).length;
+ return (
+ childCount === 0 ||
+ meaningfulDescendantCount > 0 ||
+ el.tagName === "DIV"
+ );
+ }
+ return true;
+ }
+
+ if (el instanceof HTMLButtonElement || el instanceof HTMLAnchorElement) {
+ return hasDescriptor;
+ }
+
+ if (el instanceof HTMLInputElement) {
+ if (
+ CHECKABLE_INPUT_TYPES.has(el.type) ||
+ TEXTBOX_INPUT_TYPES.has(el.type)
+ ) {
+ return true;
+ }
+ if (BUTTON_LIKE_INPUT_TYPES.has(el.type)) {
+ return hasDescriptor;
+ }
+ return hasDescriptor;
+ }
+
+ if (el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement) {
+ return true;
+ }
+
+ if (reason === "editable") return true;
+
+ return hasDescriptor;
+}
function toDomRefSuggestion({
ref,
@@ -208,49 +613,714 @@ function toDomRefSuggestion({
};
}
-function walkDom(node: Element, depth: number, lines: string[]): void {
+function normalizeLineSignature(text: string): string | undefined {
+ return normalizeTextContent(text)?.toLowerCase();
+}
+
+function splitStructuredSignature(signature: string): string[] {
+ return signature
+ .split("|")
+ .map((segment) => normalizeLineSignature(segment) ?? "")
+ .filter(Boolean);
+}
+
+function classifyTextLine(
+ parent: Element,
+ depth: number,
+ text: string
+): DomLine["kind"] {
+ if (depth <= 1 || isLandmark(parent) || text.length <= 40) {
+ return "context";
+ }
+ return "text";
+}
+
+function pushTextLine(node: Text, depth: number, context: WalkContext): void {
+ if (!node.parentElement || !isPerceivable(node.parentElement)) return;
+ if (isNaiveControlTextElement(node.parentElement)) return;
+ const text = normalizeTextContent(node.textContent ?? "");
+ if (!text) return;
+ if (context.interactiveLabels.includes(text)) return;
+
+ const seenForParent =
+ context.textSeenByParent.get(context.parentPath) ?? new Set<string>();
+ if (seenForParent.has(text)) return;
+ seenForParent.add(text);
+ context.textSeenByParent.set(context.parentPath, seenForParent);
+
+ const indent = " ".repeat(depth);
+ context.lines.push({
+ text: `${indent}${text}`,
+ kind: classifyTextLine(node.parentElement, depth, text),
+ dedupeKey: `${depth}:${text}`,
+ sourceElement: node.parentElement,
+ signature: normalizeLineSignature(text),
+ semanticTag: node.parentElement.tagName.toLowerCase(),
+ });
+}
+
+function pushInteractiveLine(
+ depth: number,
+ entry: IndexedElement,
+ value: string | undefined,
+ context: WalkContext
+): void {
+ const indent = " ".repeat(depth);
+ const valueAttr = value ? ` value="${value}"` : "";
+ context.lines.push({
+ text: `${indent}[${entry.ref}]<${entry.tag}${valueAttr}>${entry.label}</${entry.tag}>`,
+ kind: "interactive",
+ entry,
+ sourceElement: entry.element,
+ signature: normalizeLineSignature(entry.label),
+ semanticTag: entry.tag,
+ });
+}
+
+function pushContextLine(
+ depth: number,
+ text: string,
+ context: WalkContext,
+ dedupeKey?: string,
+ sourceElement?: Element,
+ signature?: string,
+ semanticTag?: string
+): void {
+ const indent = " ".repeat(depth);
+ context.lines.push({
+ text: `${indent}${text}`,
+ kind: "context",
+ dedupeKey,
+ sourceElement,
+ signature,
+ semanticTag,
+ });
+}
+
+function collectSemanticTextSegments(node: Node, segments: string[]): void {
+ if (node instanceof Text) {
+ const text = normalizeTextContent(node.textContent ?? "");
+ if (text) segments.push(text);
+ return;
+ }
+
+ if (!(node instanceof Element)) return;
+ if (SKIP_TAGS.has(node.tagName)) return;
+ if (node.hasAttribute("data-agent-window")) return;
+ if (!isPerceivable(node)) return;
+
+ if (node instanceof HTMLInputElement) {
+ const text =
+ extractLabel(node) ||
+ extractValue(node) ||
+ getFallbackLabel(node, "native");
+ if (text) segments.push(text);
+ return;
+ }
+
+ if (
+ node instanceof HTMLTextAreaElement ||
+ node instanceof HTMLSelectElement
+ ) {
+ const text =
+ extractLabel(node) ||
+ extractValue(node) ||
+ getFallbackLabel(node, "native");
+ if (text) segments.push(text);
+ return;
+ }
+
+ if (isMonacoEditor(node)) {
+ const content = getMonacoContent(node);
+ if (content)
+ segments.push(truncateText(content, MAX_EDITOR_PREVIEW_LENGTH));
+ return;
+ }
+
+ const beforeChildSegments = segments.length;
+ Array.from(node.childNodes).forEach((child) => {
+ collectSemanticTextSegments(child, segments);
+ });
+
+ if (segments.length > beforeChildSegments) return;
+
+ const interactiveReason = getInteractiveReason(node);
+ if (!interactiveReason) return;
+
+ const text =
+ extractLabel(node) ||
+ getFallbackLabel(node, interactiveReason) ||
+ extractValue(node);
+ if (text) segments.push(text);
+}
+
+function collectNonInteractiveTextSegments(
+ node: Node,
+ segments: string[],
+ isRoot = false
+): void {
+ if (node instanceof Text) {
+ const text = normalizeTextContent(node.textContent ?? "");
+ if (text) segments.push(text);
+ return;
+ }
+
+ if (!(node instanceof Element)) return;
+ if (SKIP_TAGS.has(node.tagName)) return;
+ if (node.hasAttribute("data-agent-window")) return;
+ if (!isPerceivable(node)) return;
+ if (!isRoot && getInteractiveReason(node)) return;
+ if (isNaiveControlTextElement(node)) return;
+
+ Array.from(node.childNodes).forEach((child) => {
+ collectNonInteractiveTextSegments(child, segments);
+ });
+}
+
+function extractNonInteractiveText(node: Node): string | undefined {
+ const segments: string[] = [];
+ collectNonInteractiveTextSegments(node, segments, true);
+ const uniqueSegments = [...new Set(segments)];
+ return normalizeTextContent(uniqueSegments.join(" "));
+}
+
+function extractSemanticText(node: Node): string | undefined {
+ const segments: string[] = [];
+ collectSemanticTextSegments(node, segments);
+ const uniqueSegments: string[] = [];
+ const seen = new Set<string>();
+ for (const segment of segments) {
+ if (seen.has(segment)) continue;
+ seen.add(segment);
+ uniqueSegments.push(segment);
+ }
+
+ if (node instanceof Element && uniqueSegments.length === 0) {
+ const interactiveReason = getInteractiveReason(node);
+ if (interactiveReason) {
+ const label =
+ extractLabel(node) || getFallbackLabel(node, interactiveReason);
+ const value = extractValue(node);
+ const semanticText = normalizeTextContent(label || value || "");
+ if (semanticText) return semanticText;
+ }
+ }
+
+ return normalizeTextContent(uniqueSegments.join(" "));
+}
+
+function walkTableActionableDescendants(
+ node: Node,
+ depth: number,
+ context: WalkContext,
+ path: string
+): void {
+ if (!(node instanceof Element)) return;
+ if (SKIP_TAGS.has(node.tagName)) return;
+ if (node.hasAttribute("data-agent-window")) return;
+ if (!isPerceivable(node)) return;
+ if (
+ TABLE_SECTION_TAGS.has(node.tagName) ||
+ node.tagName === "TR" ||
+ TABLE_CELL_TAGS.has(node.tagName)
+ ) {
+ Array.from(node.childNodes).forEach((child, index) => {
+ walkTableActionableDescendants(
+ child,
+ depth,
+ context,
+ `${path}/${node.tagName.toLowerCase()}[${index}]`
+ );
+ });
+ return;
+ }
+
+ if (isMonacoEditor(node)) {
+ walkDomNode(node, depth, context, path);
+ return;
+ }
+
+ const interactiveReason = getInteractiveReason(node);
+ if (interactiveReason) {
+ const value = extractValue(node);
+ const label =
+ extractLabel(node) || getFallbackLabel(node, interactiveReason);
+ if (
+ shouldIndexInteractiveNode(node, interactiveReason, label, value, context)
+ ) {
+ walkDomNode(node, depth, context, path);
+ return;
+ }
+ }
+
+ Array.from(node.childNodes).forEach((child, index) => {
+ walkTableActionableDescendants(
+ child,
+ depth,
+ context,
+ `${path}/${node.tagName.toLowerCase()}[${index}]`
+ );
+ });
+}
+
+function isUtilityTableCell(
+ cell: Element,
+ columnHeader: string | undefined,
+ cellText: string
+): boolean {
+ if (!cellText) return false;
+
+ const normalizedHeader = normalizeLineSignature(columnHeader ?? "") ?? "";
+ if (normalizedHeader && !ACTION_LIKE_HEADERS.has(normalizedHeader)) {
+ return false;
+ }
+
+ if (
+ !cell.querySelector(
+ "a, button, input, select, textarea, .n-button, .n-switch, .n-checkbox, .n-base-selection"
+ )
+ ) {
+ return false;
+ }
+
+ return !extractNonInteractiveText(cell);
+}
+
+function walkTableNode(
+ table: Element,
+ depth: number,
+ context: WalkContext,
+ path: string
+): void {
+ pushContextLine(
+ depth,
+ "<table>",
+ context,
+ `${path}:table`,
+ table,
+ undefined,
+ "table"
+ );
+
+ const sectionChildren = Array.from(table.children).filter(
+ (child): child is Element =>
+ child instanceof Element && isPerceivable(child)
+ );
+
+ const rows = sectionChildren.flatMap((child) => {
+ if (TABLE_SECTION_TAGS.has(child.tagName)) {
+ return Array.from(child.children).filter(
+ (row): row is Element =>
+ row instanceof Element && row.tagName === "TR" && isPerceivable(row)
+ );
+ }
+ return child.tagName === "TR" ? [child] : [];
+ });
+
+ const headerLabels = rows.reduce<string[] | undefined>((labels, row) => {
+ if (labels) return labels;
+ const cells = Array.from(row.children).filter(
+ (cell): cell is Element =>
+ TABLE_CELL_TAGS.has(cell.tagName) && isPerceivable(cell)
+ );
+ const isHeaderRow =
+ row.parentElement?.tagName === "THEAD" ||
+ cells.every((cell) => cell.tagName === "TH");
+ if (!isHeaderRow) return labels;
+ return cells.map((cell) => extractSemanticText(cell) ?? "");
+ }, undefined);
+
+ rows.forEach((row, rowIndex) => {
+ const cells = Array.from(row.children).filter(
+ (cell): cell is Element =>
+ TABLE_CELL_TAGS.has(cell.tagName) && isPerceivable(cell)
+ );
+ if (cells.length === 0) return;
+
+ const cellTexts = cells.map((cell, cellIndex) => {
+ const cellText = extractSemanticText(cell) ?? "";
+ if (isUtilityTableCell(cell, headerLabels?.[cellIndex], cellText)) {
+ return "";
+ }
+ return cellText;
+ });
+ const rowLabel =
+ normalizeTextContent(cellTexts.filter(Boolean).join(" | ")) ??
+ extractLabel(row);
+ const isHeaderRow =
+ row.parentElement?.tagName === "THEAD" ||
+ cells.every((cell) => cell.tagName === "TH");
+
+ if (isHeaderRow) {
+ if (rowLabel) {
+ pushContextLine(
+ depth + 1,
+ `<thead>${rowLabel}</thead>`,
+ context,
+ `${path}:thead:${rowLabel}`,
+ row,
+ normalizeLineSignature(rowLabel),
+ "thead"
+ );
+ }
+ return;
+ }
+
+ const interactiveReason = getInteractiveReason(row);
+ let childContext = context;
+ if (interactiveReason) {
+ const value = extractValue(row);
+ const label =
+ rowLabel ||
+ extractLabel(row) ||
+ getFallbackLabel(row, interactiveReason);
+ if (
+ shouldIndexInteractiveNode(
+ row,
+ interactiveReason,
+ label,
+ value,
+ context
+ )
+ ) {
+ const entry = createIndexedElement({
+ tag: row.tagName.toLowerCase(),
+ role: row.getAttribute("role") ?? undefined,
+ label,
+ value,
+ element: row,
+ });
+ pushInteractiveLine(depth + 1, entry, value, context);
+ childContext = {
+ ...context,
+ interactiveLabels: label
+ ? [...context.interactiveLabels, label]
+ : context.interactiveLabels,
+ hasInteractiveAncestor: true,
+ };
+ }
+ }
+
+ if (childContext === context && rowLabel) {
+ pushContextLine(
+ depth + 1,
+ `<tr>${rowLabel}</tr>`,
+ context,
+ `${path}:tr:${rowIndex}:${rowLabel}`,
+ row,
+ normalizeLineSignature(rowLabel),
+ "tr"
+ );
+ }
+
+ cells.forEach((cell, cellIndex) => {
+ Array.from(cell.childNodes).forEach((child, childIndex) => {
+ walkTableActionableDescendants(
+ child,
+ depth + 2,
+ childContext,
+ `${path}/tr[${rowIndex}]/${cell.tagName.toLowerCase()}[${cellIndex}]/child[${childIndex}]`
+ );
+ });
+ });
+ });
+}
+
+function walkDomNode(
+ node: Node,
+ depth: number,
+ context: WalkContext,
+ path: string
+): void {
+ if (node instanceof Text) {
+ pushTextLine(node, depth, context);
+ return;
+ }
+
+ if (!(node instanceof Element)) return;
if (SKIP_TAGS.has(node.tagName)) return;
if (node.hasAttribute("data-agent-window")) return;
- if (!isVisible(node)) return;
+ if (!isPerceivable(node)) return;
+
+ if (node.tagName === "TABLE") {
+ walkTableNode(node, depth, context, path);
+ return;
+ }
// Monaco editor — register as a single interactive element
if (isMonacoEditor(node)) {
const content = getMonacoContent(node) ?? "";
- const preview =
- content.length > 200 ? content.slice(0, 200) + "..." : content;
+ const preview = truncateText(content, MAX_EDITOR_PREVIEW_LENGTH);
const label = content ? `SQL: ${preview}` : "empty editor";
- const entry = registerElement({
+ const entry = createIndexedElement({
tag: "editor",
label,
value: content,
element: node,
});
- const indent = " ".repeat(depth);
- lines.push(`${indent}[${entry.ref}]<editor>${label}</editor>`);
+ pushInteractiveLine(depth, entry, undefined, context);
return;
}
- if (isInteractive(node)) {
- const tag = node.tagName.toLowerCase();
- const label = extractLabel(node);
+ const interactiveReason = getInteractiveReason(node);
+ const isInteractiveNode = Boolean(interactiveReason);
+ let nextInteractiveLabels = context.interactiveLabels;
+ let hasInteractiveAncestor = context.hasInteractiveAncestor;
+ if (interactiveReason) {
+ const tag = isNaiveSelectElement(node)
+ ? "select"
+ : node.tagName.toLowerCase();
const value = extractValue(node);
- const role = node.getAttribute("role") ?? undefined;
- const entry = registerElement({ tag, role, label, value, element: node });
+ const rawLabel = extractLabel(node);
+ const label = rawLabel || getFallbackLabel(node, interactiveReason);
- const indent = " ".repeat(depth);
- const valueAttr = value ? ` value="${value}"` : "";
- lines.push(`${indent}[${entry.ref}]<${tag}${valueAttr}>${label}</${tag}>`);
+ if (
+ shouldIndexInteractiveNode(node, interactiveReason, label, value, context)
+ ) {
+ const role = node.getAttribute("role") ?? undefined;
+ const entry = createIndexedElement({
+ tag,
+ role,
+ label,
+ value,
+ element: node,
+ });
- // Don't recurse into interactive elements
- return;
+ pushInteractiveLine(depth, entry, value, context);
+ nextInteractiveLabels = label
+ ? [...context.interactiveLabels, label]
+ : context.interactiveLabels;
+ hasInteractiveAncestor = true;
+ }
+
+ if (!shouldRecurseIntoInteractive(node)) {
+ return;
+ }
+ }
+
+ const childDepth = isInteractiveNode
+ ? depth + 1
+ : isLandmark(node)
+ ? depth + 1
+ : depth;
+ const childContext: WalkContext = {
+ ...context,
+ parentPath: path,
+ interactiveLabels: nextInteractiveLabels,
+ hasInteractiveAncestor,
+ };
+ const childNodes = Array.from(node.childNodes);
+ childNodes.forEach((child, index) => {
+ walkDomNode(
+ child,
+ childDepth,
+ childContext,
+ `${path}/${node.tagName.toLowerCase()}[${index}]`
+ );
+ });
+}
+
+function dedupeAdjacentLines(lines: DomLine[]): DomLine[] {
+ const deduped: DomLine[] = [];
+ for (const line of lines) {
+ const previous = deduped[deduped.length - 1];
+ if (
+ previous &&
+ !line.entry &&
+ !previous.entry &&
+ previous.dedupeKey === line.dedupeKey
+ ) {
+ continue;
+ }
+ deduped.push(line);
+ }
+ return deduped;
+}
+
+const WRAPPER_TAGS = new Set(["div", "span"]);
+const SEMANTIC_LINE_TAG_SCORES = new Map([
+ ["table", 50],
+ ["thead", 80],
+ ["tbody", 60],
+ ["tfoot", 60],
+ ["tr", 90],
+ ["th", 85],
+ ["td", 80],
+ ["ul", 55],
+ ["ol", 55],
+ ["li", 88],
+ ["button", 85],
+ ["a", 85],
+ ["input", 85],
+ ["select", 85],
+ ["textarea", 85],
+ ["editor", 85],
+ ["summary", 82],
+]);
+
+function getLineValueScore(line: DomLine): number {
+ const kindScore =
+ line.kind === "interactive" ? 200 : line.kind === "context" ? 120 : 40;
+ const tagScore = line.semanticTag
+ ? (SEMANTIC_LINE_TAG_SCORES.get(line.semanticTag) ??
+ (WRAPPER_TAGS.has(line.semanticTag) ? 0 : 20))
+ : 0;
+ return kindScore + tagScore;
+}
+
+function canDropEquivalentLine(line: DomLine): boolean {
+ if (line.kind !== "interactive") return true;
+ return WRAPPER_TAGS.has(line.semanticTag ?? "") && !line.entry?.value;
+}
+
+function isAncestralDuplicate(keeper: DomLine, candidate: DomLine): boolean {
+ if (!keeper.signature || !candidate.signature) return false;
+ if (!keeper.sourceElement || !candidate.sourceElement) return false;
+ if (
+ keeper.sourceElement !== candidate.sourceElement &&
+ !keeper.sourceElement.contains(candidate.sourceElement) &&
+ !candidate.sourceElement.contains(keeper.sourceElement)
+ ) {
+ return false;
+ }
+
+ if (keeper.signature === candidate.signature) {
+ return true;
+ }
+
+ if (
+ candidate.kind === "interactive" &&
+ !WRAPPER_TAGS.has(candidate.semanticTag ?? "")
+ ) {
+ return false;
}
- // Only increment depth at landmark/semantic containers
- const childDepth = isLandmark(node) ? depth + 1 : depth;
- for (const child of Array.from(node.children)) {
- walkDom(child, childDepth, lines);
+ const keeperSegments = splitStructuredSignature(keeper.signature);
+ if (keeperSegments.length <= 1) return false;
+
+ const candidateSegments = splitStructuredSignature(candidate.signature);
+ return (
+ candidateSegments.length === 1 &&
+ keeperSegments.includes(candidateSegments[0])
+ );
+}
+
+function dedupeEquivalentAncestorLines(lines: DomLine[]): DomLine[] {
+ return lines.filter((line, index) => {
+ if (!canDropEquivalentLine(line)) return true;
+
+ return !lines.some((other, otherIndex) => {
+ if (index === otherIndex) return false;
+ if (!isAncestralDuplicate(other, line)) return false;
+
+ const otherScore = getLineValueScore(other);
+ const lineScore = getLineValueScore(line);
+ if (otherScore !== lineScore) {
+ return otherScore > lineScore;
+ }
+
+ const otherIsCloserSemanticAncestor = Boolean(
+ other.sourceElement &&
+ line.sourceElement &&
+ other.sourceElement.contains(line.sourceElement) &&
+ other.semanticTag &&
+ !WRAPPER_TAGS.has(other.semanticTag)
+ );
+ if (otherIsCloserSemanticAncestor) {
+ return true;
+ }
+
+ return otherIndex < index;
+ });
+ });
+}
+
+function getBudgetSelectionOrder(lines: DomLine[]): number[] {
+ return lines
+ .map((_, index) => index)
+ .sort((leftIndex, rightIndex) => {
+ const left = lines[leftIndex];
+ const right = lines[rightIndex];
+
+ const kindPriorityDiff =
+ LINE_KIND_PRIORITY[left.kind] - LINE_KIND_PRIORITY[right.kind];
+ if (kindPriorityDiff !== 0) return kindPriorityDiff;
+
+ const regionPriorityDiff =
+ LINE_REGION_PRIORITY[getLineRegion(left)] -
+ LINE_REGION_PRIORITY[getLineRegion(right)];
+ if (regionPriorityDiff !== 0) return regionPriorityDiff;
+
+ const valueScoreDiff = getLineValueScore(right) - getLineValueScore(left);
+ if (valueScoreDiff !== 0) return valueScoreDiff;
+
+ const lengthDiff = left.text.length - right.text.length;
+ if (lengthDiff !== 0) return lengthDiff;
+
+ return leftIndex - rightIndex;
+ });
+}
+
+function selectBudgetedLines(lines: DomLine[]): DomLine[] {
+ const selected = new Set<number>();
+ const selectionOrder = getBudgetSelectionOrder(lines);
+ let usedLines = 0;
+ let usedChars = 0;
+ const regionUsage: Record<LineRegion, { lines: number; chars: number }> = {
+ main: { lines: 0, chars: 0 },
+ default: { lines: 0, chars: 0 },
+ overlay: { lines: 0, chars: 0 },
+ shell: { lines: 0, chars: 0 },
+ };
+
+ const trySelect = (index: number, enforceRegionCaps: boolean): void => {
+ if (selected.has(index)) return;
+ const line = lines[index];
+ const cost = line.text.length + (usedLines > 0 ? 1 : 0);
+ if (
+ usedLines >= MAX_DOM_TREE_LINES ||
+ usedChars + cost > MAX_DOM_TREE_CHARS
+ ) {
+ return;
+ }
+
+ const region = getLineRegion(line);
+ const regionCap = enforceRegionCaps
+ ? SECONDARY_REGION_CAPS[region]
+ : undefined;
+ if (regionCap) {
+ const usage = regionUsage[region];
+ if (
+ usage.lines >= regionCap.lines ||
+ usage.chars + cost > regionCap.chars
+ ) {
+ return;
+ }
+ }
+
+ selected.add(index);
+ usedLines += 1;
+ usedChars += cost;
+ regionUsage[region].lines += 1;
+ regionUsage[region].chars += cost;
+ };
+
+ selectionOrder.forEach((index) => {
+ trySelect(index, true);
+ });
+
+ const hasPrimaryRegionSelection = Array.from(selected).some((index) => {
+ const region = getLineRegion(lines[index]);
+ return region === "main" || region === "default";
+ });
+ if (!hasPrimaryRegionSelection) {
+ selectionOrder.forEach((index) => {
+ trySelect(index, false);
+ });
}
+
+ return lines.filter((_, index) => selected.has(index));
}
function extractDomState(root?: Element): {
@@ -260,15 +1330,33 @@ function extractDomState(root?: Element): {
} {
elementRegistry.clear();
nextElementRef = 1;
- const lines: string[] = [];
const rootEl = root ?? document.body;
+ const lines: DomLine[] = [];
+ const context: WalkContext = {
+ lines,
+ textSeenByParent: new Map(),
+ parentPath: "root",
+ interactiveLabels: [],
+ hasInteractiveAncestor: false,
+ };
- for (const child of Array.from(rootEl.children)) {
- walkDom(child, 0, lines);
- }
+ Array.from(rootEl.childNodes).forEach((child, index) => {
+ walkDomNode(child, 0, context, `root[${index}]`);
+ });
+
+ const keptLines = selectBudgetedLines(
+ dedupeEquivalentAncestorLines(dedupeAdjacentLines(lines))
+ );
+ const keptEntries = keptLines
+ .flatMap((line) => (line.entry ? [line.entry] : []))
+ .map((entry) => [entry.ref, entry] as const);
+ elementRegistry.clear();
+ keptEntries.forEach(([ref, entry]) => {
+ elementRegistry.set(ref, entry);
+ });
return {
- tree: lines.join("\n"),
+ tree: keptLines.map((line) => line.text).join("\n"),
count: elementRegistry.size,
refs: Array.from(elementRegistry.values(), toDomRefSuggestion),
};
diff --git a/frontend/src/plugins/agent/dom/index.test.ts b/frontend/src/plugins/agent/dom/index.test.ts
index 590e51b41c..9cad54ad73 100644
--- a/frontend/src/plugins/agent/dom/index.test.ts
+++ b/frontend/src/plugins/agent/dom/index.test.ts
@@ -9,6 +9,7 @@ import {
afterEach(() => {
document.body.innerHTML = "";
vi.restoreAllMocks();
+ vi.unstubAllGlobals();
});
describe("lazyExtractDomRefSuggestions", () => {
@@ -41,6 +42,60 @@ describe("lazyExecuteDomAction", () => {
expect(result).toEqual({ success: true, message: "hello" });
});
+ test("clicks pointer-cursor containers via refs without changing ref lookup semantics", async () => {
+ vi.stubGlobal(
+ "MouseEvent",
+ class extends Event {
+ constructor(type: string, init?: EventInit) {
+ super(type, init);
+ }
+ }
+ );
+
+ const onClick = vi.fn();
+ const row = document.createElement("div");
+ row.style.cursor = "pointer";
+ row.textContent = "Prod Primary";
+ row.addEventListener("click", onClick);
+ document.body.append(row);
+
+ await expect(lazyExtractDomRefSuggestions()).resolves.toEqual([
+ {
+ ref: "e1",
+ tag: "div",
+ role: undefined,
+ label: "Prod Primary",
+ value: undefined,
+ },
+ ]);
+
+ await expect(
+ lazyExecuteDomAction({ type: "click", ref: "e1" })
+ ).resolves.toEqual({ success: true, message: "Clicked div" });
+ expect(onClick).toHaveBeenCalledTimes(1);
+ });
+
+ test("returns only the top-level ref for inherited pointer-cursor wrappers", async () => {
+ document.body.innerHTML = `
+ <div style="cursor: pointer">
+ <span>Prod Primary</span>
+ <div>
+ <span>Healthy</span>
+ </div>
+ </div>
+ `;
+
+ await expect(lazyExtractDomRefSuggestions()).resolves.toEqual([
+ {
+ ref: "e1",
+ tag: "div",
+ role: undefined,
+ label: "Prod Primary Healthy",
+ value: undefined,
+ },
+ ]);
+ });
+
test("returns refresh guidance for malformed, missing, and stale refs", async () => {
document.body.innerHTML = `<button>Save</button>`;
await lazyExtractDomTree();