Skip to content

Automate browser based workflows with AI

Stop Task V3 menu picks landing on a shifted or guessed row number (SKY-17529) (#8767)

pedrohsdbe0aade09cbb3
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index d5e68ee3c..7f4119e04 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -70,6 +70,8 @@ ToolErrorClass = Literal[
"ref_not_in_latest",
"stale_mark",
"mark_not_in_latest",
+ # A guessed menu number on an open menu whose rows were withheld, so never numbered.
+ "rows_unlisted",
# The target resolved, but the page will not let the act happen.
"disabled",
"not_editable",
@@ -2263,7 +2265,12 @@ def _menu_note_record_fields(tool_name: str, result: ToolResult | None) -> dict[
if isinstance(rows, int) and not isinstance(rows, bool):
fields["menu_rows"] = rows
reason = data.get("withhold_reason")
- if note == "withheld" and reason in ("declared_row_over_caps", "bare_text_beside", "single_row_pieces"):
+ if note == "withheld" and reason in (
+ "declared_row_over_caps",
+ "bare_text_beside",
+ "single_row_pieces",
+ "long_row_unread",
+ ):
fields["withhold_reason"] = reason
return fields
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 43d69f08e..1f3218558 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -4084,6 +4084,12 @@ _LOOK_ENUM_JS = (
_ACT_ATTR_RE = re.compile(r'\s*data-tv3-act="[^"]*"')
_CLOSE_ATTR_RE = re.compile(r' data-tv3-close="[^"]*"')
+# A withheld menu's rows carry a salted tag so no position can be guessed; printing it would hand the model one.
+_SALTED_MENU_ATTR_RE = re.compile(r'\s*data-tv3-menu="w[^"]*"')
+# Visible only: a closed menu's lingering rows are stale, not an open menu whose rows went unnumbered.
+_SALTED_MENU_SELECTOR = '[data-tv3-menu^="w"]:visible'
+# Every CSS spelling of a numbered menu tag: either quote, no quote, spaces inside, any case (HTML attribute names).
+_NUMBERED_MENU_SELECTOR_RE = re.compile(r'^\[\s*data-tv3-menu\s*=\s*(["\']?)\s*\d+\s*\1\s*\]$', re.IGNORECASE)
_ACT_SELECTOR_PREFIX = '[data-tv3-act="'
# Write the act-by-mark attribute on an element handle the caller already resolved (Playwright's
# engine, which pierces open shadow). Returns whether the node is still connected; a detached handle
@@ -6909,19 +6915,54 @@ _FIND_MENU_JS = (
if (!txt || txt.length > ROW_TEXT_MAX) return null;
return { r, txt };
};
+ // Role-less lists are grouped only on the trigger's own ARIA word: its aria-controls target, or —
+ // when it declares a listbox/combobox — the rows' nearest shared ancestor. A plain button gives
+ // no such word, and guessing would merge unrelated option sets on the page.
+ const trigDeclares = !!trigger && (/(^|\s)combobox(\s|$)/i.test(trigger.getAttribute('role') || '')
+ || (trigger.getAttribute('aria-haspopup') || '').toLowerCase() === 'listbox');
+ let controlled = null;
+ try {
+ // A plain toggle's aria-controls names a revealed panel, not a list: only a declared picker's
+ // aria-controls is read as its option container.
+ const cid = trigDeclares ? trigger.getAttribute('aria-controls') : null;
+ const root = trigger.getRootNode();
+ controlled = cid ? ((root && root.getElementById) ? root.getElementById(cid) : document.getElementById(cid)) : null;
+ if (controlled && controlled.getBoundingClientRect().height > 500) controlled = null;
+ } catch (e) { controlled = null; }
+ const controlledList = !!controlled && /^(|listbox|menu|tree|grid)$/.test(controlled.getAttribute('role') || '');
const rows = [];
+ // Clickable rows left out of the menu (too long to be undeclared, or over the row caps), so a list they sit
+ // in is not whole. Only an element's OWN pointer counts: a container's cursor is inherited by its paragraphs.
+ const unreadLong = [];
+ const ownClickable = (el) => {
+ if (el.tagName === 'BUTTON' || el.tagName === 'A') return true;
+ try {
+ const up = el.parentElement;
+ return getComputedStyle(el).cursor === 'pointer' && !(up && getComputedStyle(up).cursor === 'pointer');
+ } catch (e) { return false; }
+ };
for (const el of pScopeAll()) {
if (reuse !== 'any' && (preHas(el) || focusHas(el))) continue;
const tag = el.tagName;
const box = rowBox(el);
- if (!box) continue;
+ if (!box) {
+ if (arg.saltUnlisted && vis(el.getBoundingClientRect()) && (el.innerText || '').trim() && ownClickable(el)) unreadLong.push(el);
+ continue;
+ }
const { r, txt } = box;
- // A long row is an option only when the page declares one on it, in it or around it: an inherited
- // pointer cursor otherwise makes every paragraph of a clickable popover read as a menu row.
+ // A long row is an option only where the page declares one on it, in it, around it, or by its list (a
+ // listbox/menu, or the list a combobox controls): a pointer cursor alone makes every paragraph a row.
if (txt.length > UNDECLARED_ROW_TEXT_MAX) {
let declaresOption = false;
- try { declaresOption = el.matches(DECLARED_ROW_SEL) || !!el.querySelector(DECLARED_ROW_SEL) || !!composedClosest(el, DECLARED_ROW_SEL); } catch (e) { declaresOption = false; }
- if (!declaresOption) continue;
+ try {
+ declaresOption = el.matches(DECLARED_ROW_SEL) || !!el.querySelector(DECLARED_ROW_SEL) || !!composedClosest(el, DECLARED_ROW_SEL)
+ || !!composedClosest(el, '[role="listbox"],[role="menu"]')
+ || (controlledList && controlled !== el && pContains(controlled, el));
+ } catch (e) { declaresOption = false; }
+ if (!declaresOption) {
+ if (arg.saltUnlisted && ownClickable(el)) unreadLong.push(el);
+ continue;
+ }
}
// Options are individually actionable rows. Requiring it per-row keeps a dialog's title/body
// text from being listed as "options" (and a horizontal Confirm/Cancel button pair then fails
@@ -6942,20 +6983,6 @@ _FIND_MENU_JS = (
// Group by parent AND grandparent so both flat menus (card > button*N) and nested ones
// (ul > li > button) find their shared container.
const groups = new Map();
- // Role-less lists are grouped only on the trigger's own ARIA word: its aria-controls target, or —
- // when it declares a listbox/combobox — the rows' nearest shared ancestor. A plain button gives
- // no such word, and guessing would merge unrelated option sets on the page.
- const trigDeclares = !!trigger && (/(^|\s)combobox(\s|$)/i.test(trigger.getAttribute('role') || '')
- || (trigger.getAttribute('aria-haspopup') || '').toLowerCase() === 'listbox');
- let controlled = null;
- try {
- // A plain toggle's aria-controls names a revealed panel, not a list: only a declared picker's
- // aria-controls is read as its option container.
- const cid = trigDeclares ? trigger.getAttribute('aria-controls') : null;
- const root = trigger.getRootNode();
- controlled = cid ? ((root && root.getElementById) ? root.getElementById(cid) : document.getElementById(cid)) : null;
- if (controlled && controlled.getBoundingClientRect().height > 500) controlled = null;
- } catch (e) { controlled = null; }
// parentElement is null at a shadow boundary (a ShadowRoot is not an Element), so a menu whose
// rows are written straight into the root -- root.innerHTML = '<div role="option">...' -- would
// group under nothing and never be found. The host stands in for the boundary.
@@ -7147,15 +7174,21 @@ _FIND_MENU_JS = (
listed.length = 0;
for (const c of best.g) listed.push({ leaves: [c], reason: 'single_row_pieces' });
}
+ const longUnread = unreadLong.some((el) => pContains(best.p, el) && !best.g.some((c) => pContains(el, c.el) || pContains(c.el, el)));
+ const withheld = listed.find((e) => e.reason) || (longUnread ? { reason: 'long_row_unread' } : null);
pQSA('[data-tv3-menu]').forEach((e) => e.removeAttribute('data-tv3-menu'));
+ // A withheld note lists no rows, so a numbered tag could only be clicked by guessing a position. `saltUnlisted`
+ // callers salt those tags (still read by the click-commit checks; get_html strips them, _SALTED_MENU_ATTR_RE).
+ // One salt per document, so a rescan of an unchanged menu writes the same tags (the page fingerprint hashes them).
+ if (withheld && arg.saltUnlisted && !window.__tv3_menu_salt) window.__tv3_menu_salt = 'w' + Math.random().toString(36).slice(2, 8) + '-';
+ const salt = withheld && arg.saltUnlisted ? window.__tv3_menu_salt : '';
let n = 0;
const tagOf = new Map();
for (const c of best.g) {
n++;
- c.el.setAttribute('data-tv3-menu', String(n));
+ c.el.setAttribute('data-tv3-menu', salt + String(n));
tagOf.set(c, n);
}
- const withheld = listed.find((e) => e.reason);
const options = [];
for (const e of listed) {
if (options.length >= 15) break;
@@ -7327,9 +7360,11 @@ _MENU_OPTION_TEXTS_JS = (
} catch (e) { /* attributes unreadable: no veto values */ }
// The nearest tagged ancestor, so a caller can tell a node nested inside another candidate from a sibling.
const outer = el.parentElement ? composedClosest(el.parentElement, '[data-tv3-' + attr + ']') : null;
+ // A withheld menu's tags carry a random prefix (see _FIND_MENU_JS): the position is the trailing number.
+ const num = (e) => parseInt(String(e.getAttribute('data-tv3-' + attr)).split('-').pop(), 10);
return {
- n: parseInt(el.getAttribute('data-tv3-' + attr), 10),
- inside: outer ? parseInt(outer.getAttribute('data-tv3-' + attr), 10) : null,
+ n: num(el),
+ inside: outer ? num(outer) : null,
text: (el.innerText || el.textContent || '').trim(),
nav: nav,
setsize: Number.isFinite(setsize) && setsize > 0 ? setsize : 0,
@@ -10200,7 +10235,9 @@ def _menu_mark_parts(options: list[dict[str, Any]], cap: int, text_cap: int = _M
return parts
-_MENU_WITHHOLD_REASONS = frozenset({"declared_row_over_caps", "bare_text_beside", "single_row_pieces"})
+_MENU_WITHHOLD_REASONS = frozenset(
+ {"declared_row_over_caps", "bare_text_beside", "single_row_pieces", "long_row_unread"}
+)
def _menu_note_fields(found: dict[str, Any]) -> dict[str, Any]:
@@ -12985,6 +13022,7 @@ def build_browser_tools(
# token belongs to the element, not the number -- but it is ours, not the page's, and it
# costs truncation budget the model needs for real markup.
html = _ACT_ATTR_RE.sub("", html)
+ html = _SALTED_MENU_ATTR_RE.sub("", html)
html = _mask_refs(html)
scoped = bool(selector)
windowed = _window(html, offset, lambda end, total: _markup_cut(end, total, scoped=scoped))
@@ -13378,6 +13416,10 @@ def build_browser_tools(
menu_note_census.update(_menu_note_fields(found))
return _menu_open_note(found, selector, clicked_row=clicked_row)
+ async def _click_menu_scan(page: Any, arg: dict[str, Any]) -> Any:
+ # Every menu the click tool reports goes through here, so a withheld one never keeps guessable numbers.
+ return await page.evaluate(_FIND_MENU_JS, {**arg, "saltUnlisted": True})
+
async def _click_reaction(
page: Any, selector: str, pre: dict[str, Any], url_before: str, *, doc_planted: bool
) -> tuple[str | None, str | None]:
@@ -13464,7 +13506,7 @@ def build_browser_tools(
# "committed" from "expanded" and the child rows can -- a cascading click that opened
# them committed nothing yet.
try:
- found = await page.evaluate(_FIND_MENU_JS, await _probe_arg(page, selector))
+ found = await _click_menu_scan(page, await _probe_arg(page, selector))
except Exception:
return None
if isinstance(found, dict) and found.get("count"):
@@ -13481,7 +13523,7 @@ def build_browser_tools(
deadline = time.monotonic() + 2.4
while True:
try:
- found = await page.evaluate(_FIND_MENU_JS, {"sel": selector, "el": None, "cascade": True})
+ found = await _click_menu_scan(page, {"sel": selector, "el": None, "cascade": True})
except Exception:
return None
if isinstance(found, dict) and found.get("count"):
@@ -13591,7 +13633,7 @@ def build_browser_tools(
after_raw = await page.evaluate(_MENU_AFTER_JS, await _probe_arg(page, selector))
except Exception:
return None, None
- found = await page.evaluate(_FIND_MENU_JS, await _probe_arg(page, selector))
+ found = await _click_menu_scan(page, await _probe_arg(page, selector))
if isinstance(found, dict) and found.get("count"):
return _menu_note(found, selector), None
if isinstance(after_raw, dict) and not after_raw.get("stillOpen"):
@@ -13600,11 +13642,18 @@ def build_browser_tools(
'an option\'s [data-tv3-menu="N"] selector while the menu is open.'
), None
return None, None
- found = await page.evaluate(_FIND_MENU_JS, await _probe_arg(page, selector))
+ found = await _click_menu_scan(page, await _probe_arg(page, selector))
if isinstance(found, dict) and found.get("count"):
return _menu_note(found, selector), None
return None, None
+ async def _rows_unlisted(page: Any, selector: str) -> bool:
+ try:
+ return await _holders(page, selector) == 0 and await _holders(page, _SALTED_MENU_SELECTOR) > 0
+ except Exception:
+ # Unreadable: the click proceeds as it did before this check, through the marker path's own errors.
+ return False
+
async def click(args: dict[str, Any]) -> ToolResult:
page, error = await _resolve_page()
if error is not None:
@@ -13612,6 +13661,13 @@ def build_browser_tools(
selector = args.get("selector")
if not selector:
return ToolResult.error("click needs a selector, or mark=N from the last look().")
+ if _NUMBERED_MENU_SELECTOR_RE.match(selector.strip()) and await _rows_unlisted(page, selector):
+ # Not a page change: the menu is still open, its rows were simply never numbered.
+ raise ToolRefusal(
+ f"{selector} names no row: the open menu's options could not be listed, so its rows have no "
+ "numbers. Re-observe and click the option you want by its ref.",
+ error_class="rows_unlisted",
+ )
if _TV3_MARKER_SELECTOR_RE.match(selector.strip()):
matches = await _marker_matches(page, selector)
if matches == 0:
@@ -19280,8 +19336,16 @@ def build_browser_tools(
between two documents on the model's behalf is the wrong-element commit in a new costume.
"""
found: list[Any] = []
+ guessed_number = bool(_NUMBERED_MENU_SELECTOR_RE.match(selector.strip()))
+
+ async def _answers(realm: Any) -> bool:
+ if await _holders(realm, selector) > 0:
+ return True
+ # A realm whose menu was withheld salts its tags, but a guessed number still names one of its rows.
+ return guessed_number and await _holders(realm, _SALTED_MENU_SELECTOR) > 0
+
try:
- if await _holders(page, selector) > 0:
+ if await _answers(page):
found.append(page)
except Exception:
return page, None
@@ -19294,7 +19358,7 @@ def build_browser_tools(
unreadable = 0
for frame in frames:
try:
- if await _holders(frame, selector) > 0:
+ if await _answers(frame):
found.append(frame)
except Exception:
# The page's own query already answered, so the selector is well formed and it is this
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index 96e565897..7c8c3e5ea 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -30,6 +30,7 @@ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorRetryableTask
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.taskv3 import loop as loop_module
+from skyvern.forge.taskv3 import tools as taskv3_tools_module
from skyvern.forge.taskv3.auth_tools import _COMPLETION_BLOCKED, VerificationFailure, VerificationState
from skyvern.forge.taskv3.engine import MAX_TOKENS_CEILING, MAX_TOKENS_PER_ACTION_STEP, taskv3_runaway_backstops
from skyvern.forge.taskv3.goal_check import GoalVerdict
@@ -13195,3 +13196,9 @@ async def test_goal_check_enforce_asks_once_per_finish_and_logs_the_recheck_fiel
assert first["second_verdict"] is None
assert first["would_fail"] is None
assert first["second_latency_s"] is None
+
+
[email protected]("reason", sorted(taskv3_tools_module._MENU_WITHHOLD_REASONS))
+def test_every_menu_withhold_reason_reaches_the_call_record(reason: str) -> None:
+ result = ToolResult.ok("menu", data={"menu_note": "withheld", "menu_rows": 3, "withhold_reason": reason})
+ assert loop_module._menu_note_record_fields("click", result).get("withhold_reason") == reason
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 52986a936..94124220a 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -10,6 +10,7 @@ import ast
import asyncio
import contextlib
import html
+import html as html_lib
import inspect
import itertools
import json
@@ -9502,6 +9503,12 @@ _CUSTOM_OPTION_ROW_JS = """(txt, onPick) => {
pytest.param(_PLAIN_OPTION_ROW_JS, _LONG_MENU_OPTIONS, id="role-option-rows"),
pytest.param(_CUSTOM_OPTION_ROW_JS, _LONG_MENU_OPTIONS, id="custom-option-element"),
pytest.param(_PLAIN_OPTION_ROW_JS, _SHARED_PREFIX_MENU_OPTIONS, id="answers-alike-until-past-60-chars"),
+ # No role in the row at all: only the listbox around it declares what it is.
+ pytest.param(
+ _CUSTOM_OPTION_ROW_JS.replace("row.appendChild(aria);", ""),
+ _LONG_MENU_OPTIONS,
+ id="roleless-row-in-a-listbox",
+ ),
],
)
async def test_dom_a_menu_lists_an_option_whose_caption_is_long_and_it_commits(
@@ -9523,6 +9530,116 @@ async def test_dom_a_menu_lists_an_option_whose_caption_is_long_and_it_commits(
assert await page.evaluate("() => window.__picked") == options[0]
+# A combobox whose aria-controls list holds option components with no role anywhere in the row: each caption
+# sits a few wrappers deep and the row inherits a pointer cursor. Only the list declares what its rows are.
+_CONTROLLED_LIST_COMBOBOX_HTML = """
+<!doctype html><html><body style="margin:0;font:13px sans-serif">
+ <input id="combo" role="combobox" aria-haspopup="listbox" aria-controls="menu-q1" aria-expanded="false"
+ style="position:absolute;top:40px;left:40px;width:420px;height:28px">
+ <script>
+ window.__picked = null;
+ const OPTIONS = __OPTIONS__;
+ const SHADOW = __SHADOW__;
+ const combo = document.getElementById('combo');
+ combo.addEventListener('click', () => {
+ if (document.getElementById('menu-q1')) return;
+ const list = document.createElement('div');
+ list.id = 'menu-q1';
+ list.setAttribute('style', 'position:absolute;top:74px;left:40px;width:760px;background:#fff;border:1px solid #ccc');
+ for (const txt of OPTIONS) {
+ const row = document.createElement('x-select-option');
+ row.setAttribute('style', 'display:block;padding:4px 6px;cursor:pointer');
+ row.innerHTML = '<div><div><x-typo><x-truncate></x-truncate></x-typo></div></div>';
+ row.querySelector('x-truncate').textContent = txt;
+ if (SHADOW) row.attachShadow({ mode: 'open' }).innerHTML = '<slot></slot><div role="option"></div>';
+ row.addEventListener('click', () => { window.__picked = txt; combo.value = txt; list.remove(); });
+ list.appendChild(row);
+ }
+ if (SHADOW) list.setAttribute('role', 'listbox');
+ document.body.appendChild(list);
+ combo.setAttribute('aria-expanded', 'true');
+ });
+ </script>
+</body></html>
+"""
+
+
+def _numbered_menu_tags(page: Any) -> Any:
+ return page.evaluate(
+ "() => Array.from(document.querySelectorAll('[data-tv3-menu]'))"
+ ".filter((e) => /^\\d+$/.test(e.getAttribute('data-tv3-menu'))).length"
+ )
+
+
+@_skip_no_browser
+ ("options", "shadow"),
+ [
+ pytest.param(_LONG_MENU_OPTIONS, False, id="long-first-caption"),
+ # The production shape: a listbox whose option components draw an empty role=option in their own shadow
+ # root, beside the slotted caption rather than around it.
+ pytest.param(_LONG_MENU_OPTIONS, True, id="long-first-caption-beside-a-shadow-option"),
+ pytest.param(("Yes", "No", "Prefer not to say"), False, id="short-captions"),
+ ],
+)
+async def test_dom_a_long_caption_in_a_comboboxs_controlled_list_keeps_every_menu_number_in_page_order(
+ options: tuple[str, ...], shadow: bool
+) -> None:
+ # The long caption was dropped as an undeclared paragraph, so the rows after it took its numbers and
+ # [data-tv3-menu="1"] committed the second option.
+ html = _CONTROLLED_LIST_COMBOBOX_HTML.replace("__OPTIONS__", json.dumps(list(options))).replace(
+ "__SHADOW__", json.dumps(shadow)
+ )
+ async with _content_page(html) as page:
+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")
+ opened = await click.handler({"selector": "#combo"})
+ assert opened.status == "ok", opened.content
+ assert [text for _, text in _menu_note_entries(opened.content)] == list(options), opened.content
+ picked = await click.handler({"selector": '[data-tv3-menu="1"]'})
+ assert picked.status == "ok", picked.content
+ assert await page.evaluate("() => window.__picked") == options[0]
+
+
+@_skip_no_browser
+async def test_dom_a_long_paragraph_in_a_panel_a_combobox_controls_is_not_a_menu_row() -> None:
+ # Only a list the combobox controls declares its rows; a help panel's long clickable paragraph is not an option,
+ # and since it may be one the finder cannot read, the menu is withheld rather than listed without it.
+ html = (
+ _CONTROLLED_LIST_COMBOBOX_HTML.replace("__OPTIONS__", json.dumps(["Alpha", "Beta"]))
+ .replace("__SHADOW__", "false")
+ .replace(
+ "document.body.appendChild(list);",
+ "list.setAttribute('role', 'region'); const p = document.createElement('div');"
+ " p.setAttribute('style', 'cursor:pointer;padding:4px 6px');"
+ " p.textContent = 'Need help choosing? Read the guidance on how each of these answers is used later on.';"
+ " list.appendChild(p); document.body.appendChild(list);",
+ )
+ )
+ async with _content_page(html) as page:
+ opened = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#combo"})
+ assert _menu_note_entries(opened.content) == [], opened.content
+ assert (opened.data or {}).get("withhold_reason") == "long_row_unread", opened.data
+
+
+@_skip_no_browser
+async def test_dom_a_native_select_lists_and_commits_a_long_option_by_its_label() -> None:
+ html = (
+ '<!doctype html><html><body><select id="s"><option value="">Choose</option>'
+ + "".join(f'<option value="v{i}">{text}</option>' for i, text in enumerate(_LONG_MENU_OPTIONS))
+ + "</select></body></html>"
+ )
+ async with _content_page(html) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ observed = await _tool(tools, "observe").handler({})
+ assert _LONG_MENU_OPTIONS[0] in observed.content, observed.content
+ r = await _tool(tools, "select_option").handler({"selector": "#s", "label": _LONG_MENU_OPTIONS[0]})
+ assert r.status == "ok", r.content
+ assert await page.evaluate("() => document.getElementById('s').value") == "v0"
+
+
# Rows whose text starts with '~' are rendered hidden, as a filtering widget hides what it filtered out.
_DECLARED_COUNT_ROW_JS = """(txt, onPick) => {
const row = document.createElement('div');
@@ -9691,7 +9808,12 @@ async def test_dom_a_menu_note_lists_no_rows_it_cannot_read_whole() -> None:
# fragments as options. The note sends the model to observe instead.
html = _two_line_menu_html(_TWO_LINE_OPTIONS, declared=False)
async with _content_page(html) as page:
- opened = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#trigger"})
+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")
+ opened = await click.handler({"selector": "#trigger"})
+ # Nothing was listed, so no row may be reachable by guessing its position.
+ assert await _numbered_menu_tags(page) == 0
+ guessed = await click.handler({"selector": '[data-tv3-menu="1"]'})
+ assert guessed.status == "error" and await page.evaluate("() => window.__picked") is None, guessed.content
# The page-text delta after the note quotes the page itself; only the note must not list fragments.
note = opened.content[: (opened.data or {}).get("delta_at", len(opened.content))]
assert "could not be read as whole rows" in note, opened.content
@@ -9700,6 +9822,152 @@ async def test_dom_a_menu_note_lists_no_rows_it_cannot_read_whole() -> None:
assert (opened.data or {}).get("withhold_reason") == "bare_text_beside", opened.data
+# Every valid CSS spelling (an unquoted number is not a valid identifier, so the page refuses it as invalid).
+_GUESSED_MENU_NUMBER_SPELLINGS = (
+ '[data-tv3-menu="2"]',
+ "[data-tv3-menu='2']",
+ '[ data-tv3-menu = "2" ]',
+ '[DATA-TV3-MENU="2"]',
+)
+_MENU_TAGS_JS = (
+ "() => Array.from(document.querySelectorAll('[data-tv3-menu]')).map((e) => e.getAttribute('data-tv3-menu'))"
+)
+
+
+@_skip_no_browser
+async def test_dom_a_withheld_menus_rows_answer_only_to_their_own_tag_and_no_output_prints_it() -> None:
+ # A withheld menu's rows keep a tag so the click commit check still names the pick, but the tag is salted:
+ # a guessed number matches nothing, and no output the model reads may print the salted tag back.
+ html = _two_line_menu_html(_TWO_LINE_OPTIONS, declared=False)
+ async with _content_page(html) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ click = _tool(tools, "click")
+ opened = await click.handler({"selector": "#trigger"})
+ assert "could not be read as whole rows" in opened.content, opened.content
+ tag = await page.evaluate("() => document.querySelector('[data-tv3-menu]').getAttribute('data-tv3-menu')")
+ salt = tag.rsplit("-", 1)[0]
+ assert not tag.isdigit() and salt, tag
+ outputs = [opened.content]
+ for name, args in (
+ ("observe", {}),
+ ("look", {}),
+ ("get_html", {"format": "html"}),
+ ("get_html", {"selector": "#list", "format": "html"}),
+ ("get_html", {"format": "text"}),
+ ):
+ outputs.append(str((await _tool(tools, name).handler(args)).content))
+ for guess in _GUESSED_MENU_NUMBER_SPELLINGS:
+ guessed = await click.handler({"selector": guess})
+ # The menu is still open, so the refusal must not read as a page change (that resets the stall nudges).
+ assert guessed.error_class == "rows_unlisted", guessed.content
+ assert not (guessed.data or {}).get("page_state_changed"), guessed.data
+ outputs.append(guessed.content)
+ for out in outputs:
+ assert salt not in out, out
+ # A rescan of the unchanged menu writes the same tags, so the page fingerprint does not churn.
+ before = await page.evaluate(_MENU_TAGS_JS)
+ await page.evaluate(
+ taskv3_tools._FIND_MENU_JS, {"sel": "#trigger", "el": None, "reuse": "any", "saltUnlisted": True}
+ )
+ assert await page.evaluate(_MENU_TAGS_JS) == before
+ # A hidden menu's leftover salted rows are stale, not an open menu with unnumbered rows.
+ await page.evaluate("() => { document.getElementById('list').style.display = 'none'; }")
+ hidden_guess = await click.handler({"selector": '[data-tv3-menu="2"]'})
+ assert hidden_guess.error_class != "rows_unlisted", hidden_guess.content
+ await page.evaluate("() => { document.getElementById('list').style.display = ''; }")
+ picked = await click.handler({"selector": f'[data-tv3-menu="{tag}"]'})
+ assert "Selected option" in picked.content, picked.content
+ assert await page.evaluate("() => window.__picked") is not None
+
+
+def _pointer_rows(*texts: str) -> str:
+ return "".join(
+ f'<div style="cursor:pointer;padding:3px 6px" onclick="window.__picked = this.textContent">{t}</div>'
+ for t in texts
+ )
+
+
+@_skip_no_browser
+ ("body", "listed"),
+ [
+ # Nothing declares the long first row, so it cannot be listed; a note listing the other two as the whole
+ # menu would number them 1 and 2 and send a pick of the first row to the second.
+ pytest.param(_pointer_rows(_LONG_MENU_OPTIONS[0], "Visa required", "Visa later"), [], id="long-clickable-row"),
+ pytest.param(
+ '<p style="margin:2px 6px">'
+ + _LONG_MENU_OPTIONS[0]
+ + "</p>"
+ + _pointer_rows("Visa required", "Visa later"),
+ ["Visa required", "Visa later"],
+ id="long-paragraph-that-is-not-a-row",
+ ),
+ # Over the row caps, so dropped before the long-row check: still a row the list leaves out.
+ pytest.param(
+ _pointer_rows("Unrestricted " * 18, "Visa required", "Visa later"), [], id="row-over-the-row-caps"
+ ),
+ ],
+)
+async def test_dom_a_popover_with_a_long_undeclared_row_is_withheld_not_listed_short(
+ body: str, listed: list[str]
+) -> None:
+ async with _content_page(_popover_html(body)) as page:
+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")
+ opened = await click.handler({"selector": "#trigger"})
+ assert [text for _, text in _menu_note_entries(opened.content)] == listed, opened.content
+ if listed:
+ return
+ assert (opened.data or {}).get("withhold_reason") == "long_row_unread", opened.data
+ assert await _numbered_menu_tags(page) == 0
+ guessed = await click.handler({"selector": '[data-tv3-menu="1"]'})
+ assert guessed.error_class == "rows_unlisted", guessed.content
+ assert await page.evaluate("() => window.__picked") is None
+
+
+@_skip_no_browser
+async def test_dom_a_pointer_container_with_a_long_help_paragraph_still_lists_its_rows() -> None:
+ # The container's pointer cursor is inherited by its help paragraph; that alone must not withhold the menu.
+ body = (
+ '<p style="margin:2px 6px">'
+ + _LONG_MENU_OPTIONS[0]
+ + "</p>"
+ + "".join(
+ f'<div style="padding:3px 6px" onclick="window.__picked = this.textContent">{t}</div>'
+ for t in ("Alpha", "Beta", "Gamma")
+ )
+ )
+ async with _content_page(_popover_html(body, "cursor:pointer;")) as page:
+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")
+ opened = await click.handler({"selector": "#trigger"})
+ assert [text for _, text in _menu_note_entries(opened.content)] == ["Alpha", "Beta", "Gamma"], opened.content
+ await click.handler({"selector": '[data-tv3-menu="2"]'})
+ assert await page.evaluate("() => window.__picked") == "Beta"
+
+
+@_skip_no_browser
+async def test_dom_a_guessed_menu_number_stays_ambiguous_beside_a_child_frames_leftover_tag() -> None:
+ # The page's withheld rows are salted, so a guessed "2" no longer matches there; a leftover "2" in a child
+ # frame must not become the only match and take the click.
+ frame_row = '<div data-tv3-menu="2" onclick="parent.__frame_clicked = true">Leftover</div>'
+ html = _two_line_menu_html(_TWO_LINE_OPTIONS, declared=False).replace(
+ "<script>",
+ f'<iframe srcdoc="{html_lib.escape(frame_row)}" style="position:absolute;top:400px"></iframe><script>',
+ 1,
+ )
+ async with _content_page(html) as page:
+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")
+ opened = await click.handler({"selector": "#trigger"})
+ assert "could not be read as whole rows" in opened.content, opened.content
+ for guess in _GUESSED_MENU_NUMBER_SPELLINGS:
+ guessed = await click.handler({"selector": guess})
+ assert guessed.status == "error", (guess, guessed.content)
+ assert await page.evaluate("() => window.__frame_clicked === undefined && window.__picked === null"), guess
+
+
@_skip_no_browser
@pytest.mark.asyncio
async def test_dom_a_lone_option_of_two_pieces_is_still_tagged_leaf_by_leaf() -> None:
@@ -9715,6 +9983,7 @@ async def test_dom_a_lone_option_of_two_pieces_is_still_tagged_leaf_by_leaf() ->
assert (opened.data or {}).get("withhold_reason") == "single_row_pieces", opened.data
assert opened.data["menu_rows"] == 1, opened.data
assert await page.evaluate("() => document.querySelectorAll('[data-tv3-menu]').length") == 2
+ assert await _numbered_menu_tags(page) == 0
@_skip_no_browser
@@ -9810,11 +10079,20 @@ async def test_dom_a_roleless_popover_lists_its_clickable_rows_apart(
"<div><button>Copy</button> Ctrl+C</div><div><button>Paste</button> Ctrl+V</div>",
id="button-beside-a-shortcut-hint",
),
+ pytest.param(
+ "".join(
+ f'<div role="option" style="height:100px"><div style="cursor:pointer">{name}</div>'
+ f'<div style="cursor:pointer">{name} account</div></div>'
+ for name in ("Northwind", "Contoso")
+ ),
+ id="declared-row-over-the-caps",
+ ),
],
)
async def test_dom_a_menu_note_withholds_leaves_beside_bare_text_since_it_may_be_their_identifier(body: str) -> None:
async with _content_page(_popover_html(body)) as page:
opened = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#trigger"})
+ assert await _numbered_menu_tags(page) == 0
assert "could not be read as whole rows" in opened.content, opened.content
assert "opened a menu of" not in opened.content, opened.content