Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Stop Task V3 menu picks landing on a shifted or guessed row number (SKY-17529) (#8767)
pedrohsdbe0aade09cbb3
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex d5e68ee3c..7f4119e04 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -70,6 +70,8 @@ ToolErrorClass = Literal["ref_not_in_latest","stale_mark","mark_not_in_latest",+ # A guessed menu number on an open menu whose rows were withheld, so never numbered.+ "rows_unlisted",# The target resolved, but the page will not let the act happen."disabled","not_editable",@@ -2263,7 +2265,12 @@ def _menu_note_record_fields(tool_name: str, result: ToolResult | None) -> dict[if isinstance(rows, int) and not isinstance(rows, bool):fields["menu_rows"] = rowsreason = data.get("withhold_reason")- if note == "withheld" and reason in ("declared_row_over_caps", "bare_text_beside", "single_row_pieces"):+ if note == "withheld" and reason in (+ "declared_row_over_caps",+ "bare_text_beside",+ "single_row_pieces",+ "long_row_unread",+ ):fields["withhold_reason"] = reasonreturn fieldsdiff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 43d69f08e..1f3218558 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -4084,6 +4084,12 @@ _LOOK_ENUM_JS = (_ACT_ATTR_RE = re.compile(r'\s*data-tv3-act="[^"]*"')_CLOSE_ATTR_RE = re.compile(r' data-tv3-close="[^"]*"')+# A withheld menu's rows carry a salted tag so no position can be guessed; printing it would hand the model one.+_SALTED_MENU_ATTR_RE = re.compile(r'\s*data-tv3-menu="w[^"]*"')+# Visible only: a closed menu's lingering rows are stale, not an open menu whose rows went unnumbered.+_SALTED_MENU_SELECTOR = '[data-tv3-menu^="w"]:visible'+# Every CSS spelling of a numbered menu tag: either quote, no quote, spaces inside, any case (HTML attribute names).+_NUMBERED_MENU_SELECTOR_RE = re.compile(r'^\[\s*data-tv3-menu\s*=\s*(["\']?)\s*\d+\s*\1\s*\]$', re.IGNORECASE)_ACT_SELECTOR_PREFIX = '[data-tv3-act="'# Write the act-by-mark attribute on an element handle the caller already resolved (Playwright's# engine, which pierces open shadow). Returns whether the node is still connected; a detached handle@@ -6909,19 +6915,54 @@ _FIND_MENU_JS = (if (!txt || txt.length > ROW_TEXT_MAX) return null;return { r, txt };};+ // Role-less lists are grouped only on the trigger's own ARIA word: its aria-controls target, or —+ // when it declares a listbox/combobox — the rows' nearest shared ancestor. A plain button gives+ // no such word, and guessing would merge unrelated option sets on the page.+ const trigDeclares = !!trigger && (/(^|\s)combobox(\s|$)/i.test(trigger.getAttribute('role') || '')+ || (trigger.getAttribute('aria-haspopup') || '').toLowerCase() === 'listbox');+ let controlled = null;+ try {+ // A plain toggle's aria-controls names a revealed panel, not a list: only a declared picker's+ // aria-controls is read as its option container.+ const cid = trigDeclares ? trigger.getAttribute('aria-controls') : null;+ const root = trigger.getRootNode();+ controlled = cid ? ((root && root.getElementById) ? root.getElementById(cid) : document.getElementById(cid)) : null;+ if (controlled && controlled.getBoundingClientRect().height > 500) controlled = null;+ } catch (e) { controlled = null; }+ const controlledList = !!controlled && /^(|listbox|menu|tree|grid)$/.test(controlled.getAttribute('role') || '');const rows = [];+ // Clickable rows left out of the menu (too long to be undeclared, or over the row caps), so a list they sit+ // in is not whole. Only an element's OWN pointer counts: a container's cursor is inherited by its paragraphs.+ const unreadLong = [];+ const ownClickable = (el) => {+ if (el.tagName === 'BUTTON' || el.tagName === 'A') return true;+ try {+ const up = el.parentElement;+ return getComputedStyle(el).cursor === 'pointer' && !(up && getComputedStyle(up).cursor === 'pointer');+ } catch (e) { return false; }+ };for (const el of pScopeAll()) {if (reuse !== 'any' && (preHas(el) || focusHas(el))) continue;const tag = el.tagName;const box = rowBox(el);- if (!box) continue;+ if (!box) {+ if (arg.saltUnlisted && vis(el.getBoundingClientRect()) && (el.innerText || '').trim() && ownClickable(el)) unreadLong.push(el);+ continue;+ }const { r, txt } = box;- // A long row is an option only when the page declares one on it, in it or around it: an inherited- // pointer cursor otherwise makes every paragraph of a clickable popover read as a menu row.+ // A long row is an option only where the page declares one on it, in it, around it, or by its list (a+ // listbox/menu, or the list a combobox controls): a pointer cursor alone makes every paragraph a row.if (txt.length > UNDECLARED_ROW_TEXT_MAX) {let declaresOption = false;- try { declaresOption = el.matches(DECLARED_ROW_SEL) || !!el.querySelector(DECLARED_ROW_SEL) || !!composedClosest(el, DECLARED_ROW_SEL); } catch (e) { declaresOption = false; }- if (!declaresOption) continue;+ try {+ declaresOption = el.matches(DECLARED_ROW_SEL) || !!el.querySelector(DECLARED_ROW_SEL) || !!composedClosest(el, DECLARED_ROW_SEL)+ || !!composedClosest(el, '[role="listbox"],[role="menu"]')+ || (controlledList && controlled !== el && pContains(controlled, el));+ } catch (e) { declaresOption = false; }+ if (!declaresOption) {+ if (arg.saltUnlisted && ownClickable(el)) unreadLong.push(el);+ continue;+ }}// Options are individually actionable rows. Requiring it per-row keeps a dialog's title/body// text from being listed as "options" (and a horizontal Confirm/Cancel button pair then fails@@ -6942,20 +6983,6 @@ _FIND_MENU_JS = (// Group by parent AND grandparent so both flat menus (card > button*N) and nested ones// (ul > li > button) find their shared container.const groups = new Map();- // Role-less lists are grouped only on the trigger's own ARIA word: its aria-controls target, or —- // when it declares a listbox/combobox — the rows' nearest shared ancestor. A plain button gives- // no such word, and guessing would merge unrelated option sets on the page.- const trigDeclares = !!trigger && (/(^|\s)combobox(\s|$)/i.test(trigger.getAttribute('role') || '')- || (trigger.getAttribute('aria-haspopup') || '').toLowerCase() === 'listbox');- let controlled = null;- try {- // A plain toggle's aria-controls names a revealed panel, not a list: only a declared picker's- // aria-controls is read as its option container.- const cid = trigDeclares ? trigger.getAttribute('aria-controls') : null;- const root = trigger.getRootNode();- controlled = cid ? ((root && root.getElementById) ? root.getElementById(cid) : document.getElementById(cid)) : null;- if (controlled && controlled.getBoundingClientRect().height > 500) controlled = null;- } catch (e) { controlled = null; }// parentElement is null at a shadow boundary (a ShadowRoot is not an Element), so a menu whose// rows are written straight into the root -- root.innerHTML = '<div role="option">...' -- would// group under nothing and never be found. The host stands in for the boundary.@@ -7147,15 +7174,21 @@ _FIND_MENU_JS = (listed.length = 0;for (const c of best.g) listed.push({ leaves: [c], reason: 'single_row_pieces' });}+ const longUnread = unreadLong.some((el) => pContains(best.p, el) && !best.g.some((c) => pContains(el, c.el) || pContains(c.el, el)));+ const withheld = listed.find((e) => e.reason) || (longUnread ? { reason: 'long_row_unread' } : null);pQSA('[data-tv3-menu]').forEach((e) => e.removeAttribute('data-tv3-menu'));+ // A withheld note lists no rows, so a numbered tag could only be clicked by guessing a position. `saltUnlisted`+ // callers salt those tags (still read by the click-commit checks; get_html strips them, _SALTED_MENU_ATTR_RE).+ // One salt per document, so a rescan of an unchanged menu writes the same tags (the page fingerprint hashes them).+ if (withheld && arg.saltUnlisted && !window.__tv3_menu_salt) window.__tv3_menu_salt = 'w' + Math.random().toString(36).slice(2, 8) + '-';+ const salt = withheld && arg.saltUnlisted ? window.__tv3_menu_salt : '';let n = 0;const tagOf = new Map();for (const c of best.g) {n++;- c.el.setAttribute('data-tv3-menu', String(n));+ c.el.setAttribute('data-tv3-menu', salt + String(n));tagOf.set(c, n);}- const withheld = listed.find((e) => e.reason);const options = [];for (const e of listed) {if (options.length >= 15) break;@@ -7327,9 +7360,11 @@ _MENU_OPTION_TEXTS_JS = (} catch (e) { /* attributes unreadable: no veto values */ }// The nearest tagged ancestor, so a caller can tell a node nested inside another candidate from a sibling.const outer = el.parentElement ? composedClosest(el.parentElement, '[data-tv3-' + attr + ']') : null;+ // A withheld menu's tags carry a random prefix (see _FIND_MENU_JS): the position is the trailing number.+ const num = (e) => parseInt(String(e.getAttribute('data-tv3-' + attr)).split('-').pop(), 10);return {- n: parseInt(el.getAttribute('data-tv3-' + attr), 10),- inside: outer ? parseInt(outer.getAttribute('data-tv3-' + attr), 10) : null,+ n: num(el),+ inside: outer ? num(outer) : null,text: (el.innerText || el.textContent || '').trim(),nav: nav,setsize: Number.isFinite(setsize) && setsize > 0 ? setsize : 0,@@ -10200,7 +10235,9 @@ def _menu_mark_parts(options: list[dict[str, Any]], cap: int, text_cap: int = _Mreturn parts-_MENU_WITHHOLD_REASONS = frozenset({"declared_row_over_caps", "bare_text_beside", "single_row_pieces"})+_MENU_WITHHOLD_REASONS = frozenset(+ {"declared_row_over_caps", "bare_text_beside", "single_row_pieces", "long_row_unread"}+)def _menu_note_fields(found: dict[str, Any]) -> dict[str, Any]:@@ -12985,6 +13022,7 @@ def build_browser_tools(# token belongs to the element, not the number -- but it is ours, not the page's, and it# costs truncation budget the model needs for real markup.html = _ACT_ATTR_RE.sub("", html)+ html = _SALTED_MENU_ATTR_RE.sub("", html)html = _mask_refs(html)scoped = bool(selector)windowed = _window(html, offset, lambda end, total: _markup_cut(end, total, scoped=scoped))@@ -13378,6 +13416,10 @@ def build_browser_tools(menu_note_census.update(_menu_note_fields(found))return _menu_open_note(found, selector, clicked_row=clicked_row)+ async def _click_menu_scan(page: Any, arg: dict[str, Any]) -> Any:+ # Every menu the click tool reports goes through here, so a withheld one never keeps guessable numbers.+ return await page.evaluate(_FIND_MENU_JS, {**arg, "saltUnlisted": True})+async def _click_reaction(page: Any, selector: str, pre: dict[str, Any], url_before: str, *, doc_planted: bool) -> tuple[str | None, str | None]:@@ -13464,7 +13506,7 @@ def build_browser_tools(# "committed" from "expanded" and the child rows can -- a cascading click that opened# them committed nothing yet.try:- found = await page.evaluate(_FIND_MENU_JS, await _probe_arg(page, selector))+ found = await _click_menu_scan(page, await _probe_arg(page, selector))except Exception:return Noneif isinstance(found, dict) and found.get("count"):@@ -13481,7 +13523,7 @@ def build_browser_tools(deadline = time.monotonic() + 2.4while True:try:- found = await page.evaluate(_FIND_MENU_JS, {"sel": selector, "el": None, "cascade": True})+ found = await _click_menu_scan(page, {"sel": selector, "el": None, "cascade": True})except Exception:return Noneif isinstance(found, dict) and found.get("count"):@@ -13591,7 +13633,7 @@ def build_browser_tools(after_raw = await page.evaluate(_MENU_AFTER_JS, await _probe_arg(page, selector))except Exception:return None, None- found = await page.evaluate(_FIND_MENU_JS, await _probe_arg(page, selector))+ found = await _click_menu_scan(page, await _probe_arg(page, selector))if isinstance(found, dict) and found.get("count"):return _menu_note(found, selector), Noneif isinstance(after_raw, dict) and not after_raw.get("stillOpen"):@@ -13600,11 +13642,18 @@ def build_browser_tools('an option\'s [data-tv3-menu="N"] selector while the menu is open.'), Nonereturn None, None- found = await page.evaluate(_FIND_MENU_JS, await _probe_arg(page, selector))+ found = await _click_menu_scan(page, await _probe_arg(page, selector))if isinstance(found, dict) and found.get("count"):return _menu_note(found, selector), Nonereturn None, None+ async def _rows_unlisted(page: Any, selector: str) -> bool:+ try:+ return await _holders(page, selector) == 0 and await _holders(page, _SALTED_MENU_SELECTOR) > 0+ except Exception:+ # Unreadable: the click proceeds as it did before this check, through the marker path's own errors.+ return False+async def click(args: dict[str, Any]) -> ToolResult:page, error = await _resolve_page()if error is not None:@@ -13612,6 +13661,13 @@ def build_browser_tools(selector = args.get("selector")if not selector:return ToolResult.error("click needs a selector, or mark=N from the last look().")+ if _NUMBERED_MENU_SELECTOR_RE.match(selector.strip()) and await _rows_unlisted(page, selector):+ # Not a page change: the menu is still open, its rows were simply never numbered.+ raise ToolRefusal(+ f"{selector} names no row: the open menu's options could not be listed, so its rows have no "+ "numbers. Re-observe and click the option you want by its ref.",+ error_class="rows_unlisted",+ )if _TV3_MARKER_SELECTOR_RE.match(selector.strip()):matches = await _marker_matches(page, selector)if matches == 0:@@ -19280,8 +19336,16 @@ def build_browser_tools(between two documents on the model's behalf is the wrong-element commit in a new costume."""found: list[Any] = []+ guessed_number = bool(_NUMBERED_MENU_SELECTOR_RE.match(selector.strip()))++ async def _answers(realm: Any) -> bool:+ if await _holders(realm, selector) > 0:+ return True+ # A realm whose menu was withheld salts its tags, but a guessed number still names one of its rows.+ return guessed_number and await _holders(realm, _SALTED_MENU_SELECTOR) > 0+try:- if await _holders(page, selector) > 0:+ if await _answers(page):found.append(page)except Exception:return page, None@@ -19294,7 +19358,7 @@ def build_browser_tools(unreadable = 0for frame in frames:try:- if await _holders(frame, selector) > 0:+ if await _answers(frame):found.append(frame)except Exception:# The page's own query already answered, so the selector is well formed and it is thisdiff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex 96e565897..7c8c3e5ea 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -30,6 +30,7 @@ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorRetryableTaskfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.taskv3 import loop as loop_module+from skyvern.forge.taskv3 import tools as taskv3_tools_modulefrom skyvern.forge.taskv3.auth_tools import _COMPLETION_BLOCKED, VerificationFailure, VerificationStatefrom skyvern.forge.taskv3.engine import MAX_TOKENS_CEILING, MAX_TOKENS_PER_ACTION_STEP, taskv3_runaway_backstopsfrom skyvern.forge.taskv3.goal_check import GoalVerdict@@ -13195,3 +13196,9 @@ async def test_goal_check_enforce_asks_once_per_finish_and_logs_the_recheck_fielassert first["second_verdict"] is Noneassert first["would_fail"] is Noneassert first["second_latency_s"] is None++[email protected]("reason", sorted(taskv3_tools_module._MENU_WITHHOLD_REASONS))+def test_every_menu_withhold_reason_reaches_the_call_record(reason: str) -> None:+ result = ToolResult.ok("menu", data={"menu_note": "withheld", "menu_rows": 3, "withhold_reason": reason})+ assert loop_module._menu_note_record_fields("click", result).get("withhold_reason") == reasondiff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex 52986a936..94124220a 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -10,6 +10,7 @@ import astimport asyncioimport contextlibimport html+import html as html_libimport inspectimport itertoolsimport json@@ -9502,6 +9503,12 @@ _CUSTOM_OPTION_ROW_JS = """(txt, onPick) => {pytest.param(_PLAIN_OPTION_ROW_JS, _LONG_MENU_OPTIONS, id="role-option-rows"),pytest.param(_CUSTOM_OPTION_ROW_JS, _LONG_MENU_OPTIONS, id="custom-option-element"),pytest.param(_PLAIN_OPTION_ROW_JS, _SHARED_PREFIX_MENU_OPTIONS, id="answers-alike-until-past-60-chars"),+ # No role in the row at all: only the listbox around it declares what it is.+ pytest.param(+ _CUSTOM_OPTION_ROW_JS.replace("row.appendChild(aria);", ""),+ _LONG_MENU_OPTIONS,+ id="roleless-row-in-a-listbox",+ ),],)async def test_dom_a_menu_lists_an_option_whose_caption_is_long_and_it_commits(@@ -9523,6 +9530,116 @@ async def test_dom_a_menu_lists_an_option_whose_caption_is_long_and_it_commits(assert await page.evaluate("() => window.__picked") == options[0]+# A combobox whose aria-controls list holds option components with no role anywhere in the row: each caption+# sits a few wrappers deep and the row inherits a pointer cursor. Only the list declares what its rows are.+_CONTROLLED_LIST_COMBOBOX_HTML = """+<!doctype html><html><body style="margin:0;font:13px sans-serif">+ <input id="combo" role="combobox" aria-haspopup="listbox" aria-controls="menu-q1" aria-expanded="false"+ style="position:absolute;top:40px;left:40px;width:420px;height:28px">+ <script>+ window.__picked = null;+ const OPTIONS = __OPTIONS__;+ const SHADOW = __SHADOW__;+ const combo = document.getElementById('combo');+ combo.addEventListener('click', () => {+ if (document.getElementById('menu-q1')) return;+ const list = document.createElement('div');+ list.id = 'menu-q1';+ list.setAttribute('style', 'position:absolute;top:74px;left:40px;width:760px;background:#fff;border:1px solid #ccc');+ for (const txt of OPTIONS) {+ const row = document.createElement('x-select-option');+ row.setAttribute('style', 'display:block;padding:4px 6px;cursor:pointer');+ row.innerHTML = '<div><div><x-typo><x-truncate></x-truncate></x-typo></div></div>';+ row.querySelector('x-truncate').textContent = txt;+ if (SHADOW) row.attachShadow({ mode: 'open' }).innerHTML = '<slot></slot><div role="option"></div>';+ row.addEventListener('click', () => { window.__picked = txt; combo.value = txt; list.remove(); });+ list.appendChild(row);+ }+ if (SHADOW) list.setAttribute('role', 'listbox');+ document.body.appendChild(list);+ combo.setAttribute('aria-expanded', 'true');+ });+ </script>+</body></html>+"""+++def _numbered_menu_tags(page: Any) -> Any:+ return page.evaluate(+ "() => Array.from(document.querySelectorAll('[data-tv3-menu]'))"+ ".filter((e) => /^\\d+$/.test(e.getAttribute('data-tv3-menu'))).length"+ )+++@_skip_no_browser+ ("options", "shadow"),+ [+ pytest.param(_LONG_MENU_OPTIONS, False, id="long-first-caption"),+ # The production shape: a listbox whose option components draw an empty role=option in their own shadow+ # root, beside the slotted caption rather than around it.+ pytest.param(_LONG_MENU_OPTIONS, True, id="long-first-caption-beside-a-shadow-option"),+ pytest.param(("Yes", "No", "Prefer not to say"), False, id="short-captions"),+ ],+)+async def test_dom_a_long_caption_in_a_comboboxs_controlled_list_keeps_every_menu_number_in_page_order(+ options: tuple[str, ...], shadow: bool+) -> None:+ # The long caption was dropped as an undeclared paragraph, so the rows after it took its numbers and+ # [data-tv3-menu="1"] committed the second option.+ html = _CONTROLLED_LIST_COMBOBOX_HTML.replace("__OPTIONS__", json.dumps(list(options))).replace(+ "__SHADOW__", json.dumps(shadow)+ )+ async with _content_page(html) as page:+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")+ opened = await click.handler({"selector": "#combo"})+ assert opened.status == "ok", opened.content+ assert [text for _, text in _menu_note_entries(opened.content)] == list(options), opened.content+ picked = await click.handler({"selector": '[data-tv3-menu="1"]'})+ assert picked.status == "ok", picked.content+ assert await page.evaluate("() => window.__picked") == options[0]+++@_skip_no_browser+async def test_dom_a_long_paragraph_in_a_panel_a_combobox_controls_is_not_a_menu_row() -> None:+ # Only a list the combobox controls declares its rows; a help panel's long clickable paragraph is not an option,+ # and since it may be one the finder cannot read, the menu is withheld rather than listed without it.+ html = (+ _CONTROLLED_LIST_COMBOBOX_HTML.replace("__OPTIONS__", json.dumps(["Alpha", "Beta"]))+ .replace("__SHADOW__", "false")+ .replace(+ "document.body.appendChild(list);",+ "list.setAttribute('role', 'region'); const p = document.createElement('div');"+ " p.setAttribute('style', 'cursor:pointer;padding:4px 6px');"+ " p.textContent = 'Need help choosing? Read the guidance on how each of these answers is used later on.';"+ " list.appendChild(p); document.body.appendChild(list);",+ )+ )+ async with _content_page(html) as page:+ opened = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#combo"})+ assert _menu_note_entries(opened.content) == [], opened.content+ assert (opened.data or {}).get("withhold_reason") == "long_row_unread", opened.data+++@_skip_no_browser+async def test_dom_a_native_select_lists_and_commits_a_long_option_by_its_label() -> None:+ html = (+ '<!doctype html><html><body><select id="s"><option value="">Choose</option>'+ + "".join(f'<option value="v{i}">{text}</option>' for i, text in enumerate(_LONG_MENU_OPTIONS))+ + "</select></body></html>"+ )+ async with _content_page(html) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ observed = await _tool(tools, "observe").handler({})+ assert _LONG_MENU_OPTIONS[0] in observed.content, observed.content+ r = await _tool(tools, "select_option").handler({"selector": "#s", "label": _LONG_MENU_OPTIONS[0]})+ assert r.status == "ok", r.content+ assert await page.evaluate("() => document.getElementById('s').value") == "v0"++# Rows whose text starts with '~' are rendered hidden, as a filtering widget hides what it filtered out._DECLARED_COUNT_ROW_JS = """(txt, onPick) => {const row = document.createElement('div');@@ -9691,7 +9808,12 @@ async def test_dom_a_menu_note_lists_no_rows_it_cannot_read_whole() -> None:# fragments as options. The note sends the model to observe instead.html = _two_line_menu_html(_TWO_LINE_OPTIONS, declared=False)async with _content_page(html) as page:- opened = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#trigger"})+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")+ opened = await click.handler({"selector": "#trigger"})+ # Nothing was listed, so no row may be reachable by guessing its position.+ assert await _numbered_menu_tags(page) == 0+ guessed = await click.handler({"selector": '[data-tv3-menu="1"]'})+ assert guessed.status == "error" and await page.evaluate("() => window.__picked") is None, guessed.content# The page-text delta after the note quotes the page itself; only the note must not list fragments.note = opened.content[: (opened.data or {}).get("delta_at", len(opened.content))]assert "could not be read as whole rows" in note, opened.content@@ -9700,6 +9822,152 @@ async def test_dom_a_menu_note_lists_no_rows_it_cannot_read_whole() -> None:assert (opened.data or {}).get("withhold_reason") == "bare_text_beside", opened.data+# Every valid CSS spelling (an unquoted number is not a valid identifier, so the page refuses it as invalid).+_GUESSED_MENU_NUMBER_SPELLINGS = (+ '[data-tv3-menu="2"]',+ "[data-tv3-menu='2']",+ '[ data-tv3-menu = "2" ]',+ '[DATA-TV3-MENU="2"]',+)+_MENU_TAGS_JS = (+ "() => Array.from(document.querySelectorAll('[data-tv3-menu]')).map((e) => e.getAttribute('data-tv3-menu'))"+)+++@_skip_no_browser+async def test_dom_a_withheld_menus_rows_answer_only_to_their_own_tag_and_no_output_prints_it() -> None:+ # A withheld menu's rows keep a tag so the click commit check still names the pick, but the tag is salted:+ # a guessed number matches nothing, and no output the model reads may print the salted tag back.+ html = _two_line_menu_html(_TWO_LINE_OPTIONS, declared=False)+ async with _content_page(html) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ click = _tool(tools, "click")+ opened = await click.handler({"selector": "#trigger"})+ assert "could not be read as whole rows" in opened.content, opened.content+ tag = await page.evaluate("() => document.querySelector('[data-tv3-menu]').getAttribute('data-tv3-menu')")+ salt = tag.rsplit("-", 1)[0]+ assert not tag.isdigit() and salt, tag+ outputs = [opened.content]+ for name, args in (+ ("observe", {}),+ ("look", {}),+ ("get_html", {"format": "html"}),+ ("get_html", {"selector": "#list", "format": "html"}),+ ("get_html", {"format": "text"}),+ ):+ outputs.append(str((await _tool(tools, name).handler(args)).content))+ for guess in _GUESSED_MENU_NUMBER_SPELLINGS:+ guessed = await click.handler({"selector": guess})+ # The menu is still open, so the refusal must not read as a page change (that resets the stall nudges).+ assert guessed.error_class == "rows_unlisted", guessed.content+ assert not (guessed.data or {}).get("page_state_changed"), guessed.data+ outputs.append(guessed.content)+ for out in outputs:+ assert salt not in out, out+ # A rescan of the unchanged menu writes the same tags, so the page fingerprint does not churn.+ before = await page.evaluate(_MENU_TAGS_JS)+ await page.evaluate(+ taskv3_tools._FIND_MENU_JS, {"sel": "#trigger", "el": None, "reuse": "any", "saltUnlisted": True}+ )+ assert await page.evaluate(_MENU_TAGS_JS) == before+ # A hidden menu's leftover salted rows are stale, not an open menu with unnumbered rows.+ await page.evaluate("() => { document.getElementById('list').style.display = 'none'; }")+ hidden_guess = await click.handler({"selector": '[data-tv3-menu="2"]'})+ assert hidden_guess.error_class != "rows_unlisted", hidden_guess.content+ await page.evaluate("() => { document.getElementById('list').style.display = ''; }")+ picked = await click.handler({"selector": f'[data-tv3-menu="{tag}"]'})+ assert "Selected option" in picked.content, picked.content+ assert await page.evaluate("() => window.__picked") is not None+++def _pointer_rows(*texts: str) -> str:+ return "".join(+ f'<div style="cursor:pointer;padding:3px 6px" onclick="window.__picked = this.textContent">{t}</div>'+ for t in texts+ )+++@_skip_no_browser+ ("body", "listed"),+ [+ # Nothing declares the long first row, so it cannot be listed; a note listing the other two as the whole+ # menu would number them 1 and 2 and send a pick of the first row to the second.+ pytest.param(_pointer_rows(_LONG_MENU_OPTIONS[0], "Visa required", "Visa later"), [], id="long-clickable-row"),+ pytest.param(+ '<p style="margin:2px 6px">'+ + _LONG_MENU_OPTIONS[0]+ + "</p>"+ + _pointer_rows("Visa required", "Visa later"),+ ["Visa required", "Visa later"],+ id="long-paragraph-that-is-not-a-row",+ ),+ # Over the row caps, so dropped before the long-row check: still a row the list leaves out.+ pytest.param(+ _pointer_rows("Unrestricted " * 18, "Visa required", "Visa later"), [], id="row-over-the-row-caps"+ ),+ ],+)+async def test_dom_a_popover_with_a_long_undeclared_row_is_withheld_not_listed_short(+ body: str, listed: list[str]+) -> None:+ async with _content_page(_popover_html(body)) as page:+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")+ opened = await click.handler({"selector": "#trigger"})+ assert [text for _, text in _menu_note_entries(opened.content)] == listed, opened.content+ if listed:+ return+ assert (opened.data or {}).get("withhold_reason") == "long_row_unread", opened.data+ assert await _numbered_menu_tags(page) == 0+ guessed = await click.handler({"selector": '[data-tv3-menu="1"]'})+ assert guessed.error_class == "rows_unlisted", guessed.content+ assert await page.evaluate("() => window.__picked") is None+++@_skip_no_browser+async def test_dom_a_pointer_container_with_a_long_help_paragraph_still_lists_its_rows() -> None:+ # The container's pointer cursor is inherited by its help paragraph; that alone must not withhold the menu.+ body = (+ '<p style="margin:2px 6px">'+ + _LONG_MENU_OPTIONS[0]+ + "</p>"+ + "".join(+ f'<div style="padding:3px 6px" onclick="window.__picked = this.textContent">{t}</div>'+ for t in ("Alpha", "Beta", "Gamma")+ )+ )+ async with _content_page(_popover_html(body, "cursor:pointer;")) as page:+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")+ opened = await click.handler({"selector": "#trigger"})+ assert [text for _, text in _menu_note_entries(opened.content)] == ["Alpha", "Beta", "Gamma"], opened.content+ await click.handler({"selector": '[data-tv3-menu="2"]'})+ assert await page.evaluate("() => window.__picked") == "Beta"+++@_skip_no_browser+async def test_dom_a_guessed_menu_number_stays_ambiguous_beside_a_child_frames_leftover_tag() -> None:+ # The page's withheld rows are salted, so a guessed "2" no longer matches there; a leftover "2" in a child+ # frame must not become the only match and take the click.+ frame_row = '<div data-tv3-menu="2" onclick="parent.__frame_clicked = true">Leftover</div>'+ html = _two_line_menu_html(_TWO_LINE_OPTIONS, declared=False).replace(+ "<script>",+ f'<iframe srcdoc="{html_lib.escape(frame_row)}" style="position:absolute;top:400px"></iframe><script>',+ 1,+ )+ async with _content_page(html) as page:+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")+ opened = await click.handler({"selector": "#trigger"})+ assert "could not be read as whole rows" in opened.content, opened.content+ for guess in _GUESSED_MENU_NUMBER_SPELLINGS:+ guessed = await click.handler({"selector": guess})+ assert guessed.status == "error", (guess, guessed.content)+ assert await page.evaluate("() => window.__frame_clicked === undefined && window.__picked === null"), guess++@_skip_no_browser@pytest.mark.asyncioasync def test_dom_a_lone_option_of_two_pieces_is_still_tagged_leaf_by_leaf() -> None:@@ -9715,6 +9983,7 @@ async def test_dom_a_lone_option_of_two_pieces_is_still_tagged_leaf_by_leaf() ->assert (opened.data or {}).get("withhold_reason") == "single_row_pieces", opened.dataassert opened.data["menu_rows"] == 1, opened.dataassert await page.evaluate("() => document.querySelectorAll('[data-tv3-menu]').length") == 2+ assert await _numbered_menu_tags(page) == 0@_skip_no_browser@@ -9810,11 +10079,20 @@ async def test_dom_a_roleless_popover_lists_its_clickable_rows_apart("<div><button>Copy</button> Ctrl+C</div><div><button>Paste</button> Ctrl+V</div>",id="button-beside-a-shortcut-hint",),+ pytest.param(+ "".join(+ f'<div role="option" style="height:100px"><div style="cursor:pointer">{name}</div>'+ f'<div style="cursor:pointer">{name} account</div></div>'+ for name in ("Northwind", "Contoso")+ ),+ id="declared-row-over-the-caps",+ ),],)async def test_dom_a_menu_note_withholds_leaves_beside_bare_text_since_it_may_be_their_identifier(body: str) -> None:async with _content_page(_popover_html(body)) as page:opened = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#trigger"})+ assert await _numbered_menu_tags(page) == 0assert "could not be read as whole rows" in opened.content, opened.contentassert "opened a menu of" not in opened.content, opened.content