Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(agent): improve page-agent DOM extraction fidelity (#19711)
p0nyfd591b5adf34
diff --git a/frontend/src/plugins/agent/dom/domTree.test.ts b/frontend/src/plugins/agent/dom/domTree.test.tsindex 9f9a7f4e1b..b00eea10db 100644--- a/frontend/src/plugins/agent/dom/domTree.test.ts+++ b/frontend/src/plugins/agent/dom/domTree.test.ts@@ -39,6 +39,623 @@ describe("extractDomTree", () => {expect(getElementByRef("e9")).toBeUndefined();});+ test("preserves visible non-interactive text nodes with normalized whitespace", () => {+ document.body.innerHTML = `+ <main>+ <section>+ <h1> Instances </h1>+ <div>+ <span>Primary</span>+ <span>US East</span>+ </div>+ <button aria-label="Create instance">Create</button>+ <p style="display: none">Hidden text</p>+ <div aria-hidden="true" style="display: none">+ <span>Also hidden</span>+ </div>+ </section>+ </main>+ `;++ const { tree, count } = extractDomTree();++ expect(count).toBe(1);+ expect(tree).toContain("Instances");+ expect(tree).toContain("Primary");+ expect(tree).toContain("US East");+ expect(tree).toContain("[e1]<button>Create instance</button>");+ expect(tree).not.toContain("Hidden text");+ expect(tree).not.toContain("Also hidden");+ });++ test("treats clickable containers with pointer cursor as interactive and preserves descendant text", () => {+ document.body.innerHTML = `+ <main>+ <div style="cursor: pointer">+ <span>Prod Primary</span>+ <span>us-east-1</span>+ </div>+ </main>+ `;++ const { tree, count } = extractDomTree();++ expect(count).toBe(1);+ expect(tree).toContain("[e1]<div>Prod Primary us-east-1</div>");+ expect(tree).toContain(" Prod Primary");+ expect(tree).toContain(" us-east-1");+ expect(getElementByRef("e1")?.tag).toBe("div");+ });++ test("does not promote inherited pointer-cursor wrappers beneath clickable containers", () => {+ document.body.innerHTML = `+ <main>+ <div style="cursor: pointer">+ <span>Prod Primary</span>+ <div>+ <span>Healthy</span>+ </div>+ </div>+ </main>+ `;++ const { tree, count } = extractDomTree();+ const suggestions = extractDomRefSuggestions();++ expect(count).toBe(1);+ expect(tree).toContain("[e1]<div>Prod Primary Healthy</div>");+ expect(tree).not.toContain("[e2]");+ expect(suggestions).toEqual([+ {+ ref: "e1",+ tag: "div",+ role: undefined,+ label: "Prod Primary Healthy",+ value: undefined,+ },+ ]);+ });++ test("skips disabled and hidden pointer-cursor containers", () => {+ document.body.innerHTML = `+ <main>+ <div style="cursor: pointer" disabled>Disabled row</div>+ <div style="cursor: pointer" aria-disabled="true">Aria disabled row</div>+ <div style="cursor: pointer" inert>Inert row</div>+ <div style="cursor: pointer" aria-hidden="true">Aria hidden row</div>+ <div style="cursor: pointer">Active row</div>+ </main>+ `;++ const { tree, count } = extractDomTree();+ const suggestions = extractDomRefSuggestions();++ expect(count).toBe(1);+ expect(tree).toContain("[e1]<div>Active row</div>");+ expect(tree).toContain("Disabled row");+ expect(tree).toContain("Aria disabled row");+ expect(tree).not.toContain("Inert row");+ expect(tree).not.toContain("Aria hidden row");+ expect(suggestions).toEqual([+ {+ ref: "e1",+ tag: "div",+ role: undefined,+ label: "Active row",+ value: undefined,+ },+ ]);+ });++ test("treats contenteditable regions as interactive without duplicating descendants", () => {+ document.body.innerHTML = `+ <main>+ <div contenteditable="true">Editable SQL</div>+ </main>+ `;++ const { tree, count } = extractDomTree();++ expect(count).toBe(1);+ expect(tree).toContain("[e1]<div>Editable SQL</div>");+ expect(tree.match(/Editable SQL/g)).toHaveLength(1);+ });++ test("dedupes descendant text when it matches an interactive label", () => {+ document.body.innerHTML = `+ <main>+ <div style="cursor: pointer">+ <span>Prod Primary us-east-1</span>+ </div>+ </main>+ `;++ const { tree, count } = extractDomTree();++ expect(count).toBe(1);+ expect(tree).toContain("[e1]<div>Prod Primary us-east-1</div>");+ expect(tree.match(/Prod Primary us-east-1/g)).toHaveLength(1);+ });++ test("suppresses blank button refs while keeping meaningful native controls", () => {+ document.body.innerHTML = `+ <main>+ <button><span aria-hidden="true"></span></button>+ <button title="Refresh"></button>+ <input type="checkbox" />+ <input type="text" />+ <select>+ <option selected>Prod</option>+ </select>+ </main>+ `;++ const { tree, count } = extractDomTree();+ const suggestions = extractDomRefSuggestions();++ expect(count).toBe(4);+ expect(tree).toContain("[e1]<button>Refresh</button>");+ expect(tree).toContain('[e2]<input value="unchecked">checkbox</input>');+ expect(tree).toContain("[e3]<input>textbox</input>");+ expect(tree).toContain('[e4]<select value="Prod">Prod</select>');+ expect(tree).not.toContain("<button></button>");+ expect(suggestions).toEqual([+ {+ ref: "e1",+ tag: "button",+ role: undefined,+ label: "Refresh",+ value: undefined,+ },+ {+ ref: "e2",+ tag: "input",+ role: undefined,+ label: "checkbox",+ value: "unchecked",+ },+ {+ ref: "e3",+ tag: "input",+ role: undefined,+ label: "textbox",+ value: undefined,+ },+ {+ ref: "e4",+ tag: "select",+ role: undefined,+ label: "Prod",+ value: "Prod",+ },+ ]);+ });++ test("serializes tables with semantic header and row separators", () => {+ document.body.innerHTML = `+ <main>+ <table>+ <thead>+ <tr>+ <th>Instance</th>+ <th>Environment</th>+ <th>Status</th>+ </tr>+ </thead>+ <tbody>+ <tr>+ <td><div><span>Prod Primary</span></div></td>+ <td><span>us-east-1</span></td>+ <td><span>Healthy</span></td>+ </tr>+ <tr>+ <td><a href="/instances/staging">Staging</a></td>+ <td>us-west-2</td>+ <td><button aria-label="Open action menu">Actions</button></td>+ </tr>+ </tbody>+ </table>+ </main>+ `;++ const { tree, count } = extractDomTree();++ expect(count).toBe(2);+ expect(tree).toContain("<table>");+ expect(tree).toContain("<thead>Instance | Environment | Status</thead>");+ expect(tree).toContain("<tr>Prod Primary | us-east-1 | Healthy</tr>");+ expect(tree).toContain("<tr>Staging | us-west-2 | Actions</tr>");+ expect(tree).toContain("[e1]<a>Staging</a>");+ expect(tree).toContain("[e2]<button>Open action menu</button>");+ expect(tree).not.toContain("<div>Prod Primary");+ expect(tree).not.toContain("<span>Healthy");+ });++ test("drops utility action cells from clickable row summaries while keeping controls actionable", () => {+ document.body.innerHTML = `+ <main>+ <table>+ <thead>+ <tr>+ <th>Name</th>+ <th>Enabled</th>+ <th>Actions</th>+ </tr>+ </thead>+ <tbody>+ <tr style="cursor: pointer">+ <td><div><span>Prod Primary</span></div></td>+ <td>+ <div class="n-switch n-switch--active">+ <div class="n-switch__rail"></div>+ </div>+ </td>+ <td>+ <div>+ <button></button>+ </div>+ </td>+ </tr>+ </tbody>+ </table>+ </main>+ `;++ const { tree, count } = extractDomTree();+ const suggestions = extractDomRefSuggestions();++ expect(count).toBe(3);+ expect(tree).toContain("<thead>Name | Enabled | Actions</thead>");+ expect(tree).toContain("[e1]<tr>Prod Primary | switch</tr>");+ expect(tree).toContain('[e2]<div value="checked">switch</div>');+ expect(tree).toContain("[e3]<button>Actions</button>");+ expect(tree).not.toContain("More actions");+ expect(tree).not.toContain("n-switch__rail");+ expect(tree).not.toContain("Prod PrimaryProd Primary");+ expect(suggestions).toEqual([+ {+ ref: "e1",+ tag: "tr",+ role: undefined,+ label: "Prod Primary | switch",+ value: undefined,+ },+ {+ ref: "e2",+ tag: "div",+ role: undefined,+ label: "switch",+ value: "checked",+ },+ {+ ref: "e3",+ tag: "button",+ role: undefined,+ label: "Actions",+ value: undefined,+ },+ ]);+ });++ test("drops utility selection cells from table row summaries while keeping checkboxes actionable", () => {+ document.body.innerHTML = `+ <main>+ <table>+ <thead>+ <tr>+ <th></th>+ <th>Name</th>+ <th>Status</th>+ </tr>+ </thead>+ <tbody>+ <tr style="cursor: pointer">+ <td><input type="checkbox" checked /></td>+ <td>Prod Primary</td>+ <td>Healthy</td>+ </tr>+ </tbody>+ </table>+ </main>+ `;++ const { tree, count } = extractDomTree();+ const suggestions = extractDomRefSuggestions();++ expect(count).toBe(2);+ expect(tree).toContain("[e1]<tr>Prod Primary | Healthy</tr>");+ expect(tree).toContain('[e2]<input value="checked">checkbox</input>');+ expect(tree).not.toContain("checkbox | Prod Primary");+ expect(suggestions).toEqual([+ {+ ref: "e1",+ tag: "tr",+ role: undefined,+ label: "Prod Primary | Healthy",+ value: undefined,+ },+ {+ ref: "e2",+ tag: "input",+ role: undefined,+ label: "checkbox",+ value: "checked",+ },+ ]);+ });++ test("suppresses Naive input prefix text while preserving the textbox ref", () => {+ document.body.innerHTML = `+ <main>+ <div class="n-input">+ <div class="n-input-prefix">+ <span>Filter</span>+ </div>+ <div class="n-input-wrapper">+ <input placeholder="Search instances" />+ </div>+ </div>+ </main>+ `;++ const { tree, count } = extractDomTree();+ const suggestions = extractDomRefSuggestions();++ expect(count).toBe(1);+ expect(tree).toContain("[e1]<input>Search instances</input>");+ expect(tree).not.toContain("Filter");+ expect(suggestions).toEqual([+ {+ ref: "e1",+ tag: "input",+ role: undefined,+ label: "Search instances",+ value: undefined,+ },+ ]);+ });++ test("serializes Naive select controls with sibling labels and selected values", () => {+ document.body.innerHTML = `+ <main>+ <div class="flex items-center gap-x-2">+ <div class="textinfolabel">Rows per page</div>+ <div class="n-base-selection" role="combobox">+ <div class="n-base-selection-label">50</div>+ </div>+ </div>+ </main>+ `;++ const { tree, count } = extractDomTree();+ const suggestions = extractDomRefSuggestions();++ expect(count).toBe(1);+ expect(tree).toContain('[e1]<select value="50">Rows per page</select>');+ expect(tree).not.toContain("<div>50</div>");+ expect(suggestions).toEqual([+ {+ ref: "e1",+ tag: "select",+ role: "combobox",+ label: "Rows per page",+ value: "50",+ },+ ]);+ });++ test("prefers clickable list items over nested wrapper clones with the same label", () => {+ document.body.innerHTML = `+ <main>+ <ul>+ <li style="cursor: pointer">+ <div role="button">+ <span>Prod Primary</span>+ </div>+ </li>+ </ul>+ </main>+ `;++ const { tree, count } = extractDomTree();+ const suggestions = extractDomRefSuggestions();++ expect(count).toBe(1);+ expect(tree).toContain("[e1]<li>Prod Primary</li>");+ expect(tree).not.toContain("[e2]");+ expect(tree.match(/Prod Primary/g)).toHaveLength(1);+ expect(suggestions).toEqual([+ {+ ref: "e1",+ tag: "li",+ role: undefined,+ label: "Prod Primary",+ value: undefined,+ },+ ]);+ });++ test("drops nested wrapper refs when a clickable row already captures the same content", () => {+ document.body.innerHTML = `+ <main>+ <table>+ <tbody>+ <tr style="cursor: pointer">+ <td>+ <div role="button">+ <span>Prod Primary</span>+ </div>+ </td>+ <td><span>Healthy</span></td>+ </tr>+ </tbody>+ </table>+ </main>+ `;++ const { tree, count } = extractDomTree();+ const suggestions = extractDomRefSuggestions();++ expect(count).toBe(1);+ expect(tree).toContain("[e1]<tr>Prod Primary | Healthy</tr>");+ expect(tree).not.toContain("[e2]<div>Prod Primary</div>");+ expect(tree.match(/Prod Primary/g)).toHaveLength(1);+ expect(suggestions).toEqual([+ {+ ref: "e1",+ tag: "tr",+ role: undefined,+ label: "Prod Primary | Healthy",+ value: undefined,+ },+ ]);+ });++ test("truncates long labels and values deterministically", () => {+ const longText = "x".repeat(180);+ document.body.innerHTML = `+ <main>+ <button>${longText}</button>+ <input value="${longText}" />+ </main>+ `;++ const { tree } = extractDomTree();++ expect(tree).toContain(`${"x".repeat(120)}...`);+ expect(tree).not.toContain("x".repeat(150));+ });++ test("drops plain text before interactive refs when the tree budget is hit", () => {+ const textBlocks = Array.from(+ { length: 140 },+ (_, index) => `+ <section>+ <p>Context line ${index}</p>+ <div>+ <span>${"detail".repeat(12)} ${index}</span>+ <span>${"detail".repeat(12)} duplicate ${index}</span>+ </div>+ </section>+ `+ ).join("");+ document.body.innerHTML = `+ <main>+ ${textBlocks}+ <button aria-label="Launch workflow">Launch</button>+ </main>+ `;++ const { tree, count } = extractDomTree();++ expect(count).toBe(1);+ expect(tree).toContain("[e1]<button>Launch workflow</button>");+ expect(tree).not.toContain("Context line 139");+ expect(tree.split("\n").length).toBeLessThanOrEqual(120);+ });++ test("prioritizes main content over shell regions when the budget is hit", () => {+ const headerButtons = Array.from(+ { length: 60 },+ (_, index) => `<button>Header action ${index}</button>`+ ).join("");+ const sidebarLinks = Array.from(+ { length: 60 },+ (_, index) => `<a href="/sidebar/${index}">Sidebar item ${index}</a>`+ ).join("");+ const tableRows = Array.from(+ { length: 25 },+ (_, index) => `+ <tr>+ <td>Instance ${index}</td>+ <td>Prod</td>+ <td>Healthy</td>+ </tr>+ `+ ).join("");++ document.body.innerHTML = `+ <div data-label="bb-main-body-wrapper">+ <nav data-label="bb-dashboard-header">${headerButtons}</nav>+ <div>+ <aside data-label="bb-dashboard-static-sidebar">${sidebarLinks}</aside>+ <main id="bb-layout-main">+ <button>Create instance</button>+ <table>+ <thead>+ <tr>+ <th>Instance</th>+ <th>Environment</th>+ <th>Status</th>+ </tr>+ </thead>+ <tbody>+ ${tableRows}+ </tbody>+ </table>+ </main>+ </div>+ </div>+ `;++ const { tree } = extractDomTree();++ expect(tree).toMatch(/\[e\d+\]<button>Create instance<\/button>/);+ expect(tree).toContain("<thead>Instance | Environment | Status</thead>");+ expect(tree).toContain("<tr>Instance 24 | Prod | Healthy</tr>");+ expect(tree).toContain("Sidebar item 0");+ expect(tree).not.toContain("Header action 59");+ expect(tree).not.toContain("Sidebar item 59");+ });++ test("demotes overlay drawers behind main content when the budget is hit", () => {+ const drawerButtons = Array.from(+ { length: 80 },+ (_, index) => `<button>Drawer action ${index}</button>`+ ).join("");+ const rows = Array.from(+ { length: 20 },+ (_, index) => `+ <tr>+ <td>Database ${index}</td>+ <td>Production</td>+ <td>Ready</td>+ </tr>+ `+ ).join("");++ document.body.innerHTML = `+ <div data-label="bb-main-body-wrapper">+ <main id="bb-layout-main">+ <button aria-label="Create database">Create</button>+ <table>+ <thead>+ <tr>+ <th>Database</th>+ <th>Environment</th>+ <th>Status</th>+ </tr>+ </thead>+ <tbody>+ ${rows}+ </tbody>+ </table>+ </main>+ <div class="n-drawer" role="dialog" aria-modal="true">+ ${drawerButtons}+ </div>+ </div>+ `;++ const { tree } = extractDomTree();++ expect(tree).toMatch(/\[e\d+\]<button>Create database<\/button>/);+ expect(tree).toContain("<thead>Database | Environment | Status</thead>");+ expect(tree).toContain("<tr>Database 19 | Production | Ready</tr>");+ expect(tree).toContain("Drawer action 0");+ expect(tree).not.toContain("Drawer action 79");+ });+test("returns structured DOM ref suggestions for visible interactive elements", () => {document.body.innerHTML = `<main>diff --git a/frontend/src/plugins/agent/dom/domTree.ts b/frontend/src/plugins/agent/dom/domTree.tsindex a94ffa56f0..0ded18ca10 100644--- a/frontend/src/plugins/agent/dom/domTree.ts+++ b/frontend/src/plugins/agent/dom/domTree.ts@@ -35,9 +35,126 @@ const INTERACTIVE_ROLES = new Set(["slider",]);-const NAIVE_INTERACTIVE_CLASSES = ["n-button", "n-switch", "n-checkbox"];+const NAIVE_INTERACTIVE_CLASSES = [+ "n-button",+ "n-switch",+ "n-checkbox",+ "n-base-selection",+];++const TERMINAL_INTERACTIVE_TAGS = new Set([+ "A",+ "BUTTON",+ "INPUT",+ "SELECT",+ "TEXTAREA",+ "SUMMARY",+]);++const STRUCTURAL_WRAPPER_TAGS = new Set(["DIV", "SPAN", "TD", "TH"]);+const STATEFUL_INTERACTIVE_ROLES = new Set([+ "checkbox",+ "radio",+ "switch",+ "textbox",+ "combobox",+ "listbox",+ "option",+ "slider",+]);+const TEXTBOX_INPUT_TYPES = new Set([+ "",+ "email",+ "number",+ "password",+ "search",+ "tel",+ "text",+ "url",+]);+const CHECKABLE_INPUT_TYPES = new Set(["checkbox", "radio"]);+const BUTTON_LIKE_INPUT_TYPES = new Set(["button", "image", "reset", "submit"]);+const INTERACTIVE_STATE_ATTRIBUTES = [+ "aria-checked",+ "aria-expanded",+ "aria-pressed",+ "aria-selected",+ "aria-current",+];const SKIP_TAGS = new Set(["SCRIPT", "STYLE", "NOSCRIPT", "SVG", "PATH"]);+const TABLE_SECTION_TAGS = new Set(["THEAD", "TBODY", "TFOOT"]);+const TABLE_CELL_TAGS = new Set(["TH", "TD"]);+const MAX_TEXT_NODE_LENGTH = 160;+const MAX_LABEL_LENGTH = 120;+const MAX_EDITOR_PREVIEW_LENGTH = 160;+const MAX_DOM_TREE_LINES = 120;+const MAX_DOM_TREE_CHARS = 6000;+const ACTION_LIKE_HEADERS = new Set([+ "action",+ "actions",+ "operation",+ "operations",+ "menu",+]);+const NAIVE_CONTROL_TEXT_SELECTORS = [+ ".n-input-prefix",+ ".n-input-suffix",+ ".n-input__placeholder",+ ".n-base-selection-label",+ ".n-base-selection-placeholder",+ ".n-base-selection-tags",+ ".n-base-selection-input",+ ".n-base-suffix",+ ".n-base-loading",+].join(",");++const MAIN_CONTENT_SELECTOR = [+ "main",+ '[role="main"]',+ "#bb-layout-main",+ '[data-label="bb-main-body-wrapper"]',+].join(",");+const SHELL_REGION_SELECTOR = [+ "aside",+ "nav",+ "header",+ "footer",+ '[role="navigation"]',+ '[role="banner"]',+ '[role="contentinfo"]',+ '[role="complementary"]',+ '[data-label="bb-dashboard-static-sidebar"]',+ '[data-label="bb-dashboard-header"]',+].join(",");+const OVERLAY_REGION_SELECTOR = [+ '[role="dialog"]',+ '[aria-modal="true"]',+ ".n-drawer",+ ".n-modal",+ ".n-popover",+ ".n-dropdown-menu",+ ".n-base-select-menu",+ ".n-dialog",+].join(",");+const SECONDARY_REGION_CAPS: Partial<+ Record<LineRegion, { lines: number; chars: number }>+> = {+ overlay: { lines: 24, chars: 1600 },+ shell: { lines: 18, chars: 1200 },+};+const LINE_KIND_PRIORITY: Record<DomLine["kind"], number> = {+ interactive: 0,+ context: 1,+ text: 2,+};+const LINE_REGION_PRIORITY: Record<LineRegion, number> = {+ main: 0,+ default: 1,+ overlay: 2,+ shell: 3,+};+const elementRegionCache = new WeakMap<Element, LineRegion>();const LANDMARK_TAGS = new Set(["NAV",@@ -65,17 +182,62 @@ function isLandmark(el: Element): boolean {return false;}+function getElementRegion(el?: Element): LineRegion {+ if (!el) return "default";++ const cachedRegion = elementRegionCache.get(el);+ if (cachedRegion) return cachedRegion;++ let region: LineRegion = "default";+ if (el.closest(OVERLAY_REGION_SELECTOR)) {+ region = "overlay";+ } else if (el.closest(SHELL_REGION_SELECTOR)) {+ region = "shell";+ } else if (el.closest(MAIN_CONTENT_SELECTOR)) {+ region = "main";+ }++ elementRegionCache.set(el, region);+ return region;+}++function getLineRegion(line: DomLine): LineRegion {+ return getElementRegion(line.sourceElement);+}+const elementRegistry = new Map<string, IndexedElement>();let nextElementRef = 1;+type LineRegion = "main" | "default" | "overlay" | "shell";++interface DomLine {+ text: string;+ kind: "interactive" | "context" | "text";+ entry?: IndexedElement;+ dedupeKey?: string;+ sourceElement?: Element;+ signature?: string;+ semanticTag?: string;+}++type InteractiveReason = "native" | "role" | "class" | "editable" | "pointer";++interface WalkContext {+ lines: DomLine[];+ textSeenByParent: Map<string, Set<string>>;+ parentPath: string;+ interactiveLabels: string[];+ hasInteractiveAncestor: boolean;+}+function createElementRef(): string {return `e${nextElementRef++}`;}-function registerElement(entry: Omit<IndexedElement, "ref">): IndexedElement {- const registered = { ref: createElementRef(), ...entry };- elementRegistry.set(registered.ref, registered);- return registered;+function createIndexedElement(+ entry: Omit<IndexedElement, "ref">+): IndexedElement {+ return { ref: createElementRef(), ...entry };}export function getElementByRef(ref: string): IndexedElement | undefined {@@ -91,6 +253,46 @@ function isVisible(el: Element): boolean {);}+function hasHiddenAncestor(el: Element): boolean {+ return el.closest('[hidden], [aria-hidden="true"], [inert]') !== null;+}++function isPerceivable(el: Element): boolean {+ return isVisible(el) && !hasHiddenAncestor(el);+}++function hasDisabledState(el: Element): boolean {+ if (el.matches(":disabled") || el.hasAttribute("disabled")) return true;+ return el.closest('[aria-disabled="true"], [disabled]') !== null;+}++function isContentEditableElement(el: Element): boolean {+ const contentEditable = el.getAttribute("contenteditable");+ return (+ el instanceof HTMLElement &&+ (el.isContentEditable ||+ contentEditable === "" ||+ contentEditable === "true" ||+ contentEditable === "plaintext-only")+ );+}++function isHiddenInput(el: Element): boolean {+ return el instanceof HTMLInputElement && el.type === "hidden";+}++function hasPointerCursor(el: Element): boolean {+ return window.getComputedStyle(el).cursor === "pointer";+}++function isNaiveSelectElement(el: Element): boolean {+ return el.classList.contains("n-base-selection");+}++function isNaiveControlTextElement(el: Element): boolean {+ return Boolean(el.closest(NAIVE_CONTROL_TEXT_SELECTORS));+}+function isMonacoEditor(el: Element): boolean {return el.classList.contains("monaco-editor");}@@ -107,54 +309,165 @@ function getMonacoContent(el: Element): string | undefined {return content || undefined;}-function isInteractive(el: Element): boolean {- if (INTERACTIVE_TAGS.has(el.tagName)) return true;+function getInteractiveReason(el: Element): InteractiveReason | undefined {+ if (!isPerceivable(el) || hasDisabledState(el) || isHiddenInput(el))+ return undefined;+ if (INTERACTIVE_TAGS.has(el.tagName)) return "native";const role = el.getAttribute("role");- if (role && INTERACTIVE_ROLES.has(role)) return true;+ if (role && INTERACTIVE_ROLES.has(role)) return "role";for (const cls of NAIVE_INTERACTIVE_CLASSES) {- if (el.classList.contains(cls)) return true;+ if (el.classList.contains(cls)) return "class";}- return false;+ if (isContentEditableElement(el)) return "editable";+ if (hasPointerCursor(el)) return "pointer";++ return undefined;+}++function isInteractive(el: Element): boolean {+ return Boolean(getInteractiveReason(el));+}++function shouldRecurseIntoInteractive(el: Element): boolean {+ if (TERMINAL_INTERACTIVE_TAGS.has(el.tagName)) return false;+ if (isContentEditableElement(el)) return false;+ return Array.from(el.children).length > 0;+}++function truncateText(text: string, maxLength: number): string {+ if (text.length <= maxLength) return text;+ return `${text.slice(0, maxLength)}...`;+}++function normalizeTextContent(text: string): string | undefined {+ const normalized = text.trim().replace(/\s+/g, " ");+ if (!normalized) return undefined;+ return truncateText(normalized, MAX_TEXT_NODE_LENGTH);+}++function getSiblingControlLabel(el: Element): string | undefined {+ const parent = el.parentElement;+ if (!parent) return undefined;++ for (+ let sibling = el.previousElementSibling;+ sibling;+ sibling = sibling.previousElementSibling+ ) {+ if (!isPerceivable(sibling) || isInteractive(sibling)) continue;+ const text = normalizeTextContent(sibling.textContent ?? "");+ if (text) return truncateText(text, MAX_LABEL_LENGTH);+ }++ if (parent.children.length <= 3) {+ for (+ let sibling = parent.previousElementSibling;+ sibling;+ sibling = sibling.previousElementSibling+ ) {+ if (!isPerceivable(sibling) || isInteractive(sibling)) continue;+ const text = normalizeTextContent(sibling.textContent ?? "");+ if (text) return truncateText(text, MAX_LABEL_LENGTH);+ }+ }++ return undefined;+}++function getTableColumnHeaderLabel(el: Element): string | undefined {+ const cell = el.closest("td, th");+ if (!cell || !cell.parentElement) return undefined;++ const cells = Array.from(cell.parentElement.children).filter(+ (child): child is Element =>+ child instanceof Element && TABLE_CELL_TAGS.has(child.tagName)+ );+ const columnIndex = cells.indexOf(cell);+ if (columnIndex < 0) return undefined;++ const table = cell.closest("table");+ if (!table) return undefined;++ const headerRow = table.querySelector("thead tr");+ if (!headerRow) return undefined;++ const headerCells = Array.from(headerRow.children).filter(+ (child): child is Element =>+ child instanceof Element && TABLE_CELL_TAGS.has(child.tagName)+ );+ const headerText = normalizeTextContent(+ headerCells[columnIndex]?.textContent ?? ""+ );+ return headerText ? truncateText(headerText, MAX_LABEL_LENGTH) : undefined;}function extractLabel(el: Element): string {// aria-label- const ariaLabel = el.getAttribute("aria-label");- if (ariaLabel) return ariaLabel.trim();+ const ariaLabel = normalizeTextContent(el.getAttribute("aria-label") ?? "");+ if (ariaLabel) return truncateText(ariaLabel, MAX_LABEL_LENGTH);// aria-labelledbyconst labelledBy = el.getAttribute("aria-labelledby");if (labelledBy) {const parts = labelledBy.split(/\s+/)- .map((id) => document.getElementById(id)?.textContent?.trim())- .filter(Boolean);- if (parts.length > 0) return parts.join(" ");+ .map((id) =>+ normalizeTextContent(document.getElementById(id)?.textContent ?? "")+ )+ .filter((part): part is string => Boolean(part));+ if (parts.length > 0) {+ return truncateText(parts.join(" "), MAX_LABEL_LENGTH);+ }}// placeholderif (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {- if (el.placeholder) return el.placeholder.trim();+ const placeholder = normalizeTextContent(el.placeholder);+ if (placeholder) return truncateText(placeholder, MAX_LABEL_LENGTH);}// Naive UI form-item labelconst formItem = el.closest(".n-form-item");if (formItem) {- const labelEl = formItem.querySelector(".n-form-item-label__text");- if (labelEl?.textContent) return labelEl.textContent.trim();+ const label = normalizeTextContent(+ formItem.querySelector(".n-form-item-label__text")?.textContent ?? ""+ );+ if (label) return truncateText(label, MAX_LABEL_LENGTH);+ }++ if (el instanceof HTMLInputElement && BUTTON_LIKE_INPUT_TYPES.has(el.type)) {+ const buttonValue = normalizeTextContent(el.value);+ if (buttonValue) return truncateText(buttonValue, MAX_LABEL_LENGTH);}// title- const title = el.getAttribute("title");- if (title) return title.trim();+ const title = normalizeTextContent(el.getAttribute("title") ?? "");+ if (title) return truncateText(title, MAX_LABEL_LENGTH);++ if (+ el instanceof HTMLInputElement ||+ el instanceof HTMLTextAreaElement ||+ el instanceof HTMLSelectElement ||+ isNaiveSelectElement(el)+ ) {+ const siblingLabel = getSiblingControlLabel(el);+ if (siblingLabel) return siblingLabel;+ }++ const columnHeaderLabel = getTableColumnHeaderLabel(el);+ if (columnHeaderLabel) {+ const normalizedHeader = normalizeLineSignature(columnHeaderLabel);+ if (normalizedHeader && ACTION_LIKE_HEADERS.has(normalizedHeader)) {+ return truncateText(columnHeaderLabel, MAX_LABEL_LENGTH);+ }+ }// text content (truncated)- const text = (el.textContent ?? "").trim().replace(/\s+/g, " ");- if (text.length > 80) return text.slice(0, 80) + "...";- return text;+ const text = normalizeTextContent(el.textContent ?? "");+ return text ? truncateText(text, MAX_LABEL_LENGTH) : "";}function extractValue(el: Element): string | undefined {@@ -162,20 +475,26 @@ function extractValue(el: Element): string | undefined {if (el.type === "checkbox" || el.type === "radio") {return el.checked ? "checked" : "unchecked";}- return el.value || undefined;+ const value = normalizeTextContent(el.value);+ return value ? truncateText(value, MAX_LABEL_LENGTH) : undefined;}if (el instanceof HTMLTextAreaElement) {- return el.value || undefined;+ const value = normalizeTextContent(el.value);+ return value ? truncateText(value, MAX_LABEL_LENGTH) : undefined;}if (el instanceof HTMLSelectElement) {- return el.options[el.selectedIndex]?.text || el.value || undefined;+ const value = normalizeTextContent(+ el.options[el.selectedIndex]?.text || el.value || ""+ );+ return value ? truncateText(value, MAX_LABEL_LENGTH) : undefined;}// Naive UI selectif (el.classList.contains("n-base-selection")) {- const label = el.querySelector(".n-base-selection-label");- const text = label?.textContent?.trim();- if (text) return text;+ const value = normalizeTextContent(+ el.querySelector(".n-base-selection-label")?.textContent ?? ""+ );+ return value ? truncateText(value, MAX_LABEL_LENGTH) : undefined;}// Naive UI checkbox/switch@@ -191,6 +510,92 @@ function extractValue(el: Element): string | undefined {return undefined;}+function hasInteractiveState(el: Element): boolean {+ return INTERACTIVE_STATE_ATTRIBUTES.some((name) => {+ const value = el.getAttribute(name);+ return value !== null && value !== "false";+ });+}++function getFallbackLabel(el: Element, reason: InteractiveReason): string {+ if (el instanceof HTMLInputElement) {+ if (CHECKABLE_INPUT_TYPES.has(el.type)) return el.type;+ if (TEXTBOX_INPUT_TYPES.has(el.type)) return "textbox";+ if (BUTTON_LIKE_INPUT_TYPES.has(el.type)) {+ return normalizeTextContent(el.value) ?? "";+ }+ }++ if (el instanceof HTMLTextAreaElement) return "textbox";+ if (el instanceof HTMLSelectElement || isNaiveSelectElement(el)) {+ return "select";+ }++ const role = el.getAttribute("role") ?? "";+ if (STATEFUL_INTERACTIVE_ROLES.has(role)) return role;+ if (+ reason === "class" &&+ (el.classList.contains("n-checkbox") || el.classList.contains("n-switch"))+ ) {+ return el.classList.contains("n-switch") ? "switch" : "checkbox";+ }++ return "";+}++function shouldIndexInteractiveNode(+ el: Element,+ reason: InteractiveReason,+ label: string,+ value: string | undefined,+ context: WalkContext+): boolean {+ if (reason === "pointer" && context.hasInteractiveAncestor) return false;++ const hasDescriptor = Boolean(label || value || hasInteractiveState(el));++ if (reason === "pointer") {+ if (!hasDescriptor) return false;+ if (STRUCTURAL_WRAPPER_TAGS.has(el.tagName)) {+ const childCount = el.children.length;+ const meaningfulDescendantCount = Array.from(el.children).filter(+ (child) =>+ !STRUCTURAL_WRAPPER_TAGS.has(child.tagName) || isInteractive(child)+ ).length;+ return (+ childCount === 0 ||+ meaningfulDescendantCount > 0 ||+ el.tagName === "DIV"+ );+ }+ return true;+ }++ if (el instanceof HTMLButtonElement || el instanceof HTMLAnchorElement) {+ return hasDescriptor;+ }++ if (el instanceof HTMLInputElement) {+ if (+ CHECKABLE_INPUT_TYPES.has(el.type) ||+ TEXTBOX_INPUT_TYPES.has(el.type)+ ) {+ return true;+ }+ if (BUTTON_LIKE_INPUT_TYPES.has(el.type)) {+ return hasDescriptor;+ }+ return hasDescriptor;+ }++ if (el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement) {+ return true;+ }++ if (reason === "editable") return true;++ return hasDescriptor;+}function toDomRefSuggestion({ref,@@ -208,49 +613,714 @@ function toDomRefSuggestion({};}-function walkDom(node: Element, depth: number, lines: string[]): void {+function normalizeLineSignature(text: string): string | undefined {+ return normalizeTextContent(text)?.toLowerCase();+}++function splitStructuredSignature(signature: string): string[] {+ return signature+ .split("|")+ .map((segment) => normalizeLineSignature(segment) ?? "")+ .filter(Boolean);+}++function classifyTextLine(+ parent: Element,+ depth: number,+ text: string+): DomLine["kind"] {+ if (depth <= 1 || isLandmark(parent) || text.length <= 40) {+ return "context";+ }+ return "text";+}++function pushTextLine(node: Text, depth: number, context: WalkContext): void {+ if (!node.parentElement || !isPerceivable(node.parentElement)) return;+ if (isNaiveControlTextElement(node.parentElement)) return;+ const text = normalizeTextContent(node.textContent ?? "");+ if (!text) return;+ if (context.interactiveLabels.includes(text)) return;++ const seenForParent =+ context.textSeenByParent.get(context.parentPath) ?? new Set<string>();+ if (seenForParent.has(text)) return;+ seenForParent.add(text);+ context.textSeenByParent.set(context.parentPath, seenForParent);++ const indent = " ".repeat(depth);+ context.lines.push({+ text: `${indent}${text}`,+ kind: classifyTextLine(node.parentElement, depth, text),+ dedupeKey: `${depth}:${text}`,+ sourceElement: node.parentElement,+ signature: normalizeLineSignature(text),+ semanticTag: node.parentElement.tagName.toLowerCase(),+ });+}++function pushInteractiveLine(+ depth: number,+ entry: IndexedElement,+ value: string | undefined,+ context: WalkContext+): void {+ const indent = " ".repeat(depth);+ const valueAttr = value ? ` value="${value}"` : "";+ context.lines.push({+ text: `${indent}[${entry.ref}]<${entry.tag}${valueAttr}>${entry.label}</${entry.tag}>`,+ kind: "interactive",+ entry,+ sourceElement: entry.element,+ signature: normalizeLineSignature(entry.label),+ semanticTag: entry.tag,+ });+}++function pushContextLine(+ depth: number,+ text: string,+ context: WalkContext,+ dedupeKey?: string,+ sourceElement?: Element,+ signature?: string,+ semanticTag?: string+): void {+ const indent = " ".repeat(depth);+ context.lines.push({+ text: `${indent}${text}`,+ kind: "context",+ dedupeKey,+ sourceElement,+ signature,+ semanticTag,+ });+}++function collectSemanticTextSegments(node: Node, segments: string[]): void {+ if (node instanceof Text) {+ const text = normalizeTextContent(node.textContent ?? "");+ if (text) segments.push(text);+ return;+ }++ if (!(node instanceof Element)) return;+ if (SKIP_TAGS.has(node.tagName)) return;+ if (node.hasAttribute("data-agent-window")) return;+ if (!isPerceivable(node)) return;++ if (node instanceof HTMLInputElement) {+ const text =+ extractLabel(node) ||+ extractValue(node) ||+ getFallbackLabel(node, "native");+ if (text) segments.push(text);+ return;+ }++ if (+ node instanceof HTMLTextAreaElement ||+ node instanceof HTMLSelectElement+ ) {+ const text =+ extractLabel(node) ||+ extractValue(node) ||+ getFallbackLabel(node, "native");+ if (text) segments.push(text);+ return;+ }++ if (isMonacoEditor(node)) {+ const content = getMonacoContent(node);+ if (content)+ segments.push(truncateText(content, MAX_EDITOR_PREVIEW_LENGTH));+ return;+ }++ const beforeChildSegments = segments.length;+ Array.from(node.childNodes).forEach((child) => {+ collectSemanticTextSegments(child, segments);+ });++ if (segments.length > beforeChildSegments) return;++ const interactiveReason = getInteractiveReason(node);+ if (!interactiveReason) return;++ const text =+ extractLabel(node) ||+ getFallbackLabel(node, interactiveReason) ||+ extractValue(node);+ if (text) segments.push(text);+}++function collectNonInteractiveTextSegments(+ node: Node,+ segments: string[],+ isRoot = false+): void {+ if (node instanceof Text) {+ const text = normalizeTextContent(node.textContent ?? "");+ if (text) segments.push(text);+ return;+ }++ if (!(node instanceof Element)) return;+ if (SKIP_TAGS.has(node.tagName)) return;+ if (node.hasAttribute("data-agent-window")) return;+ if (!isPerceivable(node)) return;+ if (!isRoot && getInteractiveReason(node)) return;+ if (isNaiveControlTextElement(node)) return;++ Array.from(node.childNodes).forEach((child) => {+ collectNonInteractiveTextSegments(child, segments);+ });+}++function extractNonInteractiveText(node: Node): string | undefined {+ const segments: string[] = [];+ collectNonInteractiveTextSegments(node, segments, true);+ const uniqueSegments = [...new Set(segments)];+ return normalizeTextContent(uniqueSegments.join(" "));+}++function extractSemanticText(node: Node): string | undefined {+ const segments: string[] = [];+ collectSemanticTextSegments(node, segments);+ const uniqueSegments: string[] = [];+ const seen = new Set<string>();+ for (const segment of segments) {+ if (seen.has(segment)) continue;+ seen.add(segment);+ uniqueSegments.push(segment);+ }++ if (node instanceof Element && uniqueSegments.length === 0) {+ const interactiveReason = getInteractiveReason(node);+ if (interactiveReason) {+ const label =+ extractLabel(node) || getFallbackLabel(node, interactiveReason);+ const value = extractValue(node);+ const semanticText = normalizeTextContent(label || value || "");+ if (semanticText) return semanticText;+ }+ }++ return normalizeTextContent(uniqueSegments.join(" "));+}++function walkTableActionableDescendants(+ node: Node,+ depth: number,+ context: WalkContext,+ path: string+): void {+ if (!(node instanceof Element)) return;+ if (SKIP_TAGS.has(node.tagName)) return;+ if (node.hasAttribute("data-agent-window")) return;+ if (!isPerceivable(node)) return;+ if (+ TABLE_SECTION_TAGS.has(node.tagName) ||+ node.tagName === "TR" ||+ TABLE_CELL_TAGS.has(node.tagName)+ ) {+ Array.from(node.childNodes).forEach((child, index) => {+ walkTableActionableDescendants(+ child,+ depth,+ context,+ `${path}/${node.tagName.toLowerCase()}[${index}]`+ );+ });+ return;+ }++ if (isMonacoEditor(node)) {+ walkDomNode(node, depth, context, path);+ return;+ }++ const interactiveReason = getInteractiveReason(node);+ if (interactiveReason) {+ const value = extractValue(node);+ const label =+ extractLabel(node) || getFallbackLabel(node, interactiveReason);+ if (+ shouldIndexInteractiveNode(node, interactiveReason, label, value, context)+ ) {+ walkDomNode(node, depth, context, path);+ return;+ }+ }++ Array.from(node.childNodes).forEach((child, index) => {+ walkTableActionableDescendants(+ child,+ depth,+ context,+ `${path}/${node.tagName.toLowerCase()}[${index}]`+ );+ });+}++function isUtilityTableCell(+ cell: Element,+ columnHeader: string | undefined,+ cellText: string+): boolean {+ if (!cellText) return false;++ const normalizedHeader = normalizeLineSignature(columnHeader ?? "") ?? "";+ if (normalizedHeader && !ACTION_LIKE_HEADERS.has(normalizedHeader)) {+ return false;+ }++ if (+ !cell.querySelector(+ "a, button, input, select, textarea, .n-button, .n-switch, .n-checkbox, .n-base-selection"+ )+ ) {+ return false;+ }++ return !extractNonInteractiveText(cell);+}++function walkTableNode(+ table: Element,+ depth: number,+ context: WalkContext,+ path: string+): void {+ pushContextLine(+ depth,+ "<table>",+ context,+ `${path}:table`,+ table,+ undefined,+ "table"+ );++ const sectionChildren = Array.from(table.children).filter(+ (child): child is Element =>+ child instanceof Element && isPerceivable(child)+ );++ const rows = sectionChildren.flatMap((child) => {+ if (TABLE_SECTION_TAGS.has(child.tagName)) {+ return Array.from(child.children).filter(+ (row): row is Element =>+ row instanceof Element && row.tagName === "TR" && isPerceivable(row)+ );+ }+ return child.tagName === "TR" ? [child] : [];+ });++ const headerLabels = rows.reduce<string[] | undefined>((labels, row) => {+ if (labels) return labels;+ const cells = Array.from(row.children).filter(+ (cell): cell is Element =>+ TABLE_CELL_TAGS.has(cell.tagName) && isPerceivable(cell)+ );+ const isHeaderRow =+ row.parentElement?.tagName === "THEAD" ||+ cells.every((cell) => cell.tagName === "TH");+ if (!isHeaderRow) return labels;+ return cells.map((cell) => extractSemanticText(cell) ?? "");+ }, undefined);++ rows.forEach((row, rowIndex) => {+ const cells = Array.from(row.children).filter(+ (cell): cell is Element =>+ TABLE_CELL_TAGS.has(cell.tagName) && isPerceivable(cell)+ );+ if (cells.length === 0) return;++ const cellTexts = cells.map((cell, cellIndex) => {+ const cellText = extractSemanticText(cell) ?? "";+ if (isUtilityTableCell(cell, headerLabels?.[cellIndex], cellText)) {+ return "";+ }+ return cellText;+ });+ const rowLabel =+ normalizeTextContent(cellTexts.filter(Boolean).join(" | ")) ??+ extractLabel(row);+ const isHeaderRow =+ row.parentElement?.tagName === "THEAD" ||+ cells.every((cell) => cell.tagName === "TH");++ if (isHeaderRow) {+ if (rowLabel) {+ pushContextLine(+ depth + 1,+ `<thead>${rowLabel}</thead>`,+ context,+ `${path}:thead:${rowLabel}`,+ row,+ normalizeLineSignature(rowLabel),+ "thead"+ );+ }+ return;+ }++ const interactiveReason = getInteractiveReason(row);+ let childContext = context;+ if (interactiveReason) {+ const value = extractValue(row);+ const label =+ rowLabel ||+ extractLabel(row) ||+ getFallbackLabel(row, interactiveReason);+ if (+ shouldIndexInteractiveNode(+ row,+ interactiveReason,+ label,+ value,+ context+ )+ ) {+ const entry = createIndexedElement({+ tag: row.tagName.toLowerCase(),+ role: row.getAttribute("role") ?? undefined,+ label,+ value,+ element: row,+ });+ pushInteractiveLine(depth + 1, entry, value, context);+ childContext = {+ ...context,+ interactiveLabels: label+ ? [...context.interactiveLabels, label]+ : context.interactiveLabels,+ hasInteractiveAncestor: true,+ };+ }+ }++ if (childContext === context && rowLabel) {+ pushContextLine(+ depth + 1,+ `<tr>${rowLabel}</tr>`,+ context,+ `${path}:tr:${rowIndex}:${rowLabel}`,+ row,+ normalizeLineSignature(rowLabel),+ "tr"+ );+ }++ cells.forEach((cell, cellIndex) => {+ Array.from(cell.childNodes).forEach((child, childIndex) => {+ walkTableActionableDescendants(+ child,+ depth + 2,+ childContext,+ `${path}/tr[${rowIndex}]/${cell.tagName.toLowerCase()}[${cellIndex}]/child[${childIndex}]`+ );+ });+ });+ });+}++function walkDomNode(+ node: Node,+ depth: number,+ context: WalkContext,+ path: string+): void {+ if (node instanceof Text) {+ pushTextLine(node, depth, context);+ return;+ }++ if (!(node instanceof Element)) return;if (SKIP_TAGS.has(node.tagName)) return;if (node.hasAttribute("data-agent-window")) return;- if (!isVisible(node)) return;+ if (!isPerceivable(node)) return;++ if (node.tagName === "TABLE") {+ walkTableNode(node, depth, context, path);+ return;+ }// Monaco editor — register as a single interactive elementif (isMonacoEditor(node)) {const content = getMonacoContent(node) ?? "";- const preview =- content.length > 200 ? content.slice(0, 200) + "..." : content;+ const preview = truncateText(content, MAX_EDITOR_PREVIEW_LENGTH);const label = content ? `SQL: ${preview}` : "empty editor";- const entry = registerElement({+ const entry = createIndexedElement({tag: "editor",label,value: content,element: node,});- const indent = " ".repeat(depth);- lines.push(`${indent}[${entry.ref}]<editor>${label}</editor>`);+ pushInteractiveLine(depth, entry, undefined, context);return;}- if (isInteractive(node)) {- const tag = node.tagName.toLowerCase();- const label = extractLabel(node);+ const interactiveReason = getInteractiveReason(node);+ const isInteractiveNode = Boolean(interactiveReason);+ let nextInteractiveLabels = context.interactiveLabels;+ let hasInteractiveAncestor = context.hasInteractiveAncestor;+ if (interactiveReason) {+ const tag = isNaiveSelectElement(node)+ ? "select"+ : node.tagName.toLowerCase();const value = extractValue(node);- const role = node.getAttribute("role") ?? undefined;- const entry = registerElement({ tag, role, label, value, element: node });+ const rawLabel = extractLabel(node);+ const label = rawLabel || getFallbackLabel(node, interactiveReason);- const indent = " ".repeat(depth);- const valueAttr = value ? ` value="${value}"` : "";- lines.push(`${indent}[${entry.ref}]<${tag}${valueAttr}>${label}</${tag}>`);+ if (+ shouldIndexInteractiveNode(node, interactiveReason, label, value, context)+ ) {+ const role = node.getAttribute("role") ?? undefined;+ const entry = createIndexedElement({+ tag,+ role,+ label,+ value,+ element: node,+ });- // Don't recurse into interactive elements- return;+ pushInteractiveLine(depth, entry, value, context);+ nextInteractiveLabels = label+ ? [...context.interactiveLabels, label]+ : context.interactiveLabels;+ hasInteractiveAncestor = true;+ }++ if (!shouldRecurseIntoInteractive(node)) {+ return;+ }+ }++ const childDepth = isInteractiveNode+ ? depth + 1+ : isLandmark(node)+ ? depth + 1+ : depth;+ const childContext: WalkContext = {+ ...context,+ parentPath: path,+ interactiveLabels: nextInteractiveLabels,+ hasInteractiveAncestor,+ };+ const childNodes = Array.from(node.childNodes);+ childNodes.forEach((child, index) => {+ walkDomNode(+ child,+ childDepth,+ childContext,+ `${path}/${node.tagName.toLowerCase()}[${index}]`+ );+ });+}++function dedupeAdjacentLines(lines: DomLine[]): DomLine[] {+ const deduped: DomLine[] = [];+ for (const line of lines) {+ const previous = deduped[deduped.length - 1];+ if (+ previous &&+ !line.entry &&+ !previous.entry &&+ previous.dedupeKey === line.dedupeKey+ ) {+ continue;+ }+ deduped.push(line);+ }+ return deduped;+}++const WRAPPER_TAGS = new Set(["div", "span"]);+const SEMANTIC_LINE_TAG_SCORES = new Map([+ ["table", 50],+ ["thead", 80],+ ["tbody", 60],+ ["tfoot", 60],+ ["tr", 90],+ ["th", 85],+ ["td", 80],+ ["ul", 55],+ ["ol", 55],+ ["li", 88],+ ["button", 85],+ ["a", 85],+ ["input", 85],+ ["select", 85],+ ["textarea", 85],+ ["editor", 85],+ ["summary", 82],+]);++function getLineValueScore(line: DomLine): number {+ const kindScore =+ line.kind === "interactive" ? 200 : line.kind === "context" ? 120 : 40;+ const tagScore = line.semanticTag+ ? (SEMANTIC_LINE_TAG_SCORES.get(line.semanticTag) ??+ (WRAPPER_TAGS.has(line.semanticTag) ? 0 : 20))+ : 0;+ return kindScore + tagScore;+}++function canDropEquivalentLine(line: DomLine): boolean {+ if (line.kind !== "interactive") return true;+ return WRAPPER_TAGS.has(line.semanticTag ?? "") && !line.entry?.value;+}++function isAncestralDuplicate(keeper: DomLine, candidate: DomLine): boolean {+ if (!keeper.signature || !candidate.signature) return false;+ if (!keeper.sourceElement || !candidate.sourceElement) return false;+ if (+ keeper.sourceElement !== candidate.sourceElement &&+ !keeper.sourceElement.contains(candidate.sourceElement) &&+ !candidate.sourceElement.contains(keeper.sourceElement)+ ) {+ return false;+ }++ if (keeper.signature === candidate.signature) {+ return true;+ }++ if (+ candidate.kind === "interactive" &&+ !WRAPPER_TAGS.has(candidate.semanticTag ?? "")+ ) {+ return false;}- // Only increment depth at landmark/semantic containers- const childDepth = isLandmark(node) ? depth + 1 : depth;- for (const child of Array.from(node.children)) {- walkDom(child, childDepth, lines);+ const keeperSegments = splitStructuredSignature(keeper.signature);+ if (keeperSegments.length <= 1) return false;++ const candidateSegments = splitStructuredSignature(candidate.signature);+ return (+ candidateSegments.length === 1 &&+ keeperSegments.includes(candidateSegments[0])+ );+}++function dedupeEquivalentAncestorLines(lines: DomLine[]): DomLine[] {+ return lines.filter((line, index) => {+ if (!canDropEquivalentLine(line)) return true;++ return !lines.some((other, otherIndex) => {+ if (index === otherIndex) return false;+ if (!isAncestralDuplicate(other, line)) return false;++ const otherScore = getLineValueScore(other);+ const lineScore = getLineValueScore(line);+ if (otherScore !== lineScore) {+ return otherScore > lineScore;+ }++ const otherIsCloserSemanticAncestor = Boolean(+ other.sourceElement &&+ line.sourceElement &&+ other.sourceElement.contains(line.sourceElement) &&+ other.semanticTag &&+ !WRAPPER_TAGS.has(other.semanticTag)+ );+ if (otherIsCloserSemanticAncestor) {+ return true;+ }++ return otherIndex < index;+ });+ });+}++function getBudgetSelectionOrder(lines: DomLine[]): number[] {+ return lines+ .map((_, index) => index)+ .sort((leftIndex, rightIndex) => {+ const left = lines[leftIndex];+ const right = lines[rightIndex];++ const kindPriorityDiff =+ LINE_KIND_PRIORITY[left.kind] - LINE_KIND_PRIORITY[right.kind];+ if (kindPriorityDiff !== 0) return kindPriorityDiff;++ const regionPriorityDiff =+ LINE_REGION_PRIORITY[getLineRegion(left)] -+ LINE_REGION_PRIORITY[getLineRegion(right)];+ if (regionPriorityDiff !== 0) return regionPriorityDiff;++ const valueScoreDiff = getLineValueScore(right) - getLineValueScore(left);+ if (valueScoreDiff !== 0) return valueScoreDiff;++ const lengthDiff = left.text.length - right.text.length;+ if (lengthDiff !== 0) return lengthDiff;++ return leftIndex - rightIndex;+ });+}++function selectBudgetedLines(lines: DomLine[]): DomLine[] {+ const selected = new Set<number>();+ const selectionOrder = getBudgetSelectionOrder(lines);+ let usedLines = 0;+ let usedChars = 0;+ const regionUsage: Record<LineRegion, { lines: number; chars: number }> = {+ main: { lines: 0, chars: 0 },+ default: { lines: 0, chars: 0 },+ overlay: { lines: 0, chars: 0 },+ shell: { lines: 0, chars: 0 },+ };++ const trySelect = (index: number, enforceRegionCaps: boolean): void => {+ if (selected.has(index)) return;+ const line = lines[index];+ const cost = line.text.length + (usedLines > 0 ? 1 : 0);+ if (+ usedLines >= MAX_DOM_TREE_LINES ||+ usedChars + cost > MAX_DOM_TREE_CHARS+ ) {+ return;+ }++ const region = getLineRegion(line);+ const regionCap = enforceRegionCaps+ ? SECONDARY_REGION_CAPS[region]+ : undefined;+ if (regionCap) {+ const usage = regionUsage[region];+ if (+ usage.lines >= regionCap.lines ||+ usage.chars + cost > regionCap.chars+ ) {+ return;+ }+ }++ selected.add(index);+ usedLines += 1;+ usedChars += cost;+ regionUsage[region].lines += 1;+ regionUsage[region].chars += cost;+ };++ selectionOrder.forEach((index) => {+ trySelect(index, true);+ });++ const hasPrimaryRegionSelection = Array.from(selected).some((index) => {+ const region = getLineRegion(lines[index]);+ return region === "main" || region === "default";+ });+ if (!hasPrimaryRegionSelection) {+ selectionOrder.forEach((index) => {+ trySelect(index, false);+ });}++ return lines.filter((_, index) => selected.has(index));}function extractDomState(root?: Element): {@@ -260,15 +1330,33 @@ function extractDomState(root?: Element): {} {elementRegistry.clear();nextElementRef = 1;- const lines: string[] = [];const rootEl = root ?? document.body;+ const lines: DomLine[] = [];+ const context: WalkContext = {+ lines,+ textSeenByParent: new Map(),+ parentPath: "root",+ interactiveLabels: [],+ hasInteractiveAncestor: false,+ };- for (const child of Array.from(rootEl.children)) {- walkDom(child, 0, lines);- }+ Array.from(rootEl.childNodes).forEach((child, index) => {+ walkDomNode(child, 0, context, `root[${index}]`);+ });++ const keptLines = selectBudgetedLines(+ dedupeEquivalentAncestorLines(dedupeAdjacentLines(lines))+ );+ const keptEntries = keptLines+ .flatMap((line) => (line.entry ? [line.entry] : []))+ .map((entry) => [entry.ref, entry] as const);+ elementRegistry.clear();+ keptEntries.forEach(([ref, entry]) => {+ elementRegistry.set(ref, entry);+ });return {- tree: lines.join("\n"),+ tree: keptLines.map((line) => line.text).join("\n"),count: elementRegistry.size,refs: Array.from(elementRegistry.values(), toDomRefSuggestion),};diff --git a/frontend/src/plugins/agent/dom/index.test.ts b/frontend/src/plugins/agent/dom/index.test.tsindex 590e51b41c..9cad54ad73 100644--- a/frontend/src/plugins/agent/dom/index.test.ts+++ b/frontend/src/plugins/agent/dom/index.test.ts@@ -9,6 +9,7 @@ import {afterEach(() => {document.body.innerHTML = "";vi.restoreAllMocks();+ vi.unstubAllGlobals();});describe("lazyExtractDomRefSuggestions", () => {@@ -41,6 +42,60 @@ describe("lazyExecuteDomAction", () => {expect(result).toEqual({ success: true, message: "hello" });});+ test("clicks pointer-cursor containers via refs without changing ref lookup semantics", async () => {+ vi.stubGlobal(+ "MouseEvent",+ class extends Event {+ constructor(type: string, init?: EventInit) {+ super(type, init);+ }+ }+ );++ const onClick = vi.fn();+ const row = document.createElement("div");+ row.style.cursor = "pointer";+ row.textContent = "Prod Primary";+ row.addEventListener("click", onClick);+ document.body.append(row);++ await expect(lazyExtractDomRefSuggestions()).resolves.toEqual([+ {+ ref: "e1",+ tag: "div",+ role: undefined,+ label: "Prod Primary",+ value: undefined,+ },+ ]);++ await expect(+ lazyExecuteDomAction({ type: "click", ref: "e1" })+ ).resolves.toEqual({ success: true, message: "Clicked div" });+ expect(onClick).toHaveBeenCalledTimes(1);+ });++ test("returns only the top-level ref for inherited pointer-cursor wrappers", async () => {+ document.body.innerHTML = `+ <div style="cursor: pointer">+ <span>Prod Primary</span>+ <div>+ <span>Healthy</span>+ </div>+ </div>+ `;++ await expect(lazyExtractDomRefSuggestions()).resolves.toEqual([+ {+ ref: "e1",+ tag: "div",+ role: undefined,+ label: "Prod Primary Healthy",+ value: undefined,+ },+ ]);+ });+test("returns refresh guidance for malformed, missing, and stale refs", async () => {document.body.innerHTML = `<button>Save</button>`;await lazyExtractDomTree();