Skip to content

Automate browser based workflows with AI

List whole option rows when a Task V3 click opens a menu (#8716)

pedrohsdba0cf10c26e63
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index f74abb4a2..6e61dfe5a 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -1993,6 +1993,9 @@ _TOOL_CALL_RECORD_FIELDS = frozenset(
"nav_error_code",
"same_page",
"readiness_read_failed",
+ "menu_note",
+ "menu_rows",
+ "withhold_reason",
"text_delta_seconds",
"text_delta_lines",
"text_delta_chars",
@@ -2236,6 +2239,25 @@ def _navigate_record_fields(tool_name: str, args: dict[str, Any], result: ToolRe
return fields
+def _menu_note_record_fields(tool_name: str, result: ToolResult | None) -> dict[str, str | int]:
+ """Whether a click's menu-open note listed its rows or withheld them, and why, re-checked against the closed
+ vocabulary. Empty for every other tool and for clicks with no note."""
+ if tool_name != "click" or result is None:
+ return {}
+ data = result.data or {}
+ fields: dict[str, str | int] = {}
+ note = data.get("menu_note")
+ if note in ("listed", "withheld"):
+ fields["menu_note"] = note
+ rows = data.get("menu_rows")
+ if isinstance(rows, int) and not isinstance(rows, bool):
+ fields["menu_rows"] = rows
+ reason = data.get("withhold_reason")
+ if note == "withheld" and reason in ("declared_row_over_caps", "bare_text_beside", "single_row_pieces"):
+ fields["withhold_reason"] = reason
+ return fields
+
+
def _observe_summary_fields(result: ToolResult) -> dict[str, int]:
"""Counts only: the summary is built by the tool, but an indexed field is re-checked here."""
summary = (result.data or {}).get("summary")
@@ -4275,6 +4297,7 @@ async def run_agent_tool_loop(
# Which URL the call was about, so a navigate row is attributable to a target without the
# arguments themselves being logged.
navigate_fields = _navigate_record_fields(tool_name, args, result)
+ menu_note_fields = _menu_note_record_fields(tool_name, result)
# Conditional for the same reason observe's counters are: a record only carries a field
# the call actually produced, so an ok call's record keeps exactly the fields it has
# today and `resolve_seconds` is absent (not null) on tools with no address to resolve.
@@ -4424,6 +4447,7 @@ async def run_agent_tool_loop(
**cost_fields,
**observe_summary,
**navigate_fields,
+ **menu_note_fields,
**attribution,
)
if spec is not None and spec.billable:
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 72b010407..efc8d1189 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -726,17 +726,23 @@ def _exact_tier_key(text: str) -> str:
return normalize_option_label(_canon_label(text))
+def _row_identity(o: dict[str, Any]) -> str:
+ """The row's identity as `_MENU_OPTION_TEXTS_JS`'s `rowIdentity` derived it. A row with no `identity` key
+ is a note entry, whose `text` is already that identity; an empty one (a row nothing could read) is absent."""
+ identity = o.get("identity")
+ return identity if isinstance(identity, str) and identity else str(o.get("text") or "")
+
+
def _lone_duplicate_candidate(rows: list[dict[str, Any]]) -> int | None:
- """Whether ≥2 matched rows are the SAME candidate rendered more than once: canonical TEXT agreement
- across every row is the load-bearing check, and a present aria-label, `val`, or other declared value
- is a VETO on top of it — one disagreeing across otherwise-text-identical rows marks them distinct, an
+ """Whether ≥2 matched rows are the SAME candidate rendered more than once: canonical row IDENTITY
+ agreement across every row is the load-bearing check, and a present aria-label, `val`, or other declared
+ value is a VETO on top of it — one disagreeing across otherwise-identical rows marks them distinct, an
absent one never does. Returns the FIRST row's `n` when the whole set collapses to one candidate, else
None so the caller keeps refusing.
"""
if len(rows) < 2:
return None
- texts = {_canon_label(str(o.get("text") or "")) for o in rows}
- if len(texts) != 1:
+ if len({_canon_label(_row_identity(o)) for o in rows}) != 1:
return None
# The tagged leaf and its option ancestor are independent name surfaces: a shared leaf label
# ("Choose") must not mask ancestors that disagree, so each position vetoes on its own.
@@ -957,7 +963,7 @@ def _ambiguous_rows_error(
shown = rows[:15]
listing = "; ".join(
(f'[data-tv3-sugg="{o.get("n")}"][data-tv3-pick="{live_token}"] ' if live_token is not None else "")
- + repr(str(o.get("text") or "")[:60])
+ + repr(_model_text(o.get("text") or "", 60))
+ (_row_value_suffix(o, groups[_canon_label(str(o.get("text") or ""))]) if live_token is not None else "")
for o in shown
)
@@ -1000,6 +1006,21 @@ def _ambiguous_rows_error(
return ToolResult.error(message, data=data, error_class="no_matching_row")
+# The longest row text a menu listing shows. `_FIND_MENU_JS` reads rows no longer than this, but the cut
+# itself happens in `_model_text`, after masking.
+_MENU_ROW_TEXT_MAX = 200
+
+
+def _model_text(text: object, cap: int) -> str:
+ """`text` as the model may see it, at most `cap` chars. Masks hidden values BEFORE the cut, as `_window`
+ does: the loop's whole-substring mask cannot match a value the cut split, so its prefix would leak."""
+ shown = str(text)
+ ctx = skyvern_context.current()
+ if ctx is not None:
+ shown = ctx.hide_from_model(shown)
+ return shown[:cap]
+
+
def _row_value_suffix(o: dict[str, Any], rows: list[dict[str, Any]]) -> str:
"""Every present distinguishing surface for one row — value, accessible label, then any other
declared value (data-code, data-key, ...) — each truncated to 60 chars like the row text already is,
@@ -1011,16 +1032,19 @@ def _row_value_suffix(o: dict[str, Any], rows: list[dict[str, Any]]) -> str:
"""
parts = ""
if o.get("val") is not None:
- parts += f" (value {str(o.get('val'))[:60]!r})"
+ parts += f" (value {_model_text(o.get('val'), 60)!r})"
if o.get("label"):
- parts += f" (label {str(o.get('label'))[:60]!r})"
+ parts += f" (label {_model_text(o.get('label'), 60)!r})"
# The ancestor's name is its own surface: when it differs from the preferred display label (a
# shared leaf label masking distinct row names) AND disagrees across the rows, the disagreeing
# name is the one that converts — gated like the veto itself, on cross-row disagreement.
ancestor_name = (o.get("labels") or [None, None])[1]
ancestor_names = {str((p.get("labels") or [None, None])[1]) for p in rows if (p.get("labels") or [None, None])[1]}
if ancestor_name and str(ancestor_name) != str(o.get("label") or "") and len(ancestor_names) >= 2:
- parts += f" (row label {str(ancestor_name)[:60]!r})"
+ parts += f" (row label {_model_text(ancestor_name, 60)!r})"
+ # The row's identity, when it is what tells the rows apart (a name over two account numbers).
+ if len({_canon_label(_row_identity(p)) for p in rows}) >= 2:
+ parts += f" (row {_model_text(_row_identity(o), _MENU_ROW_TEXT_MAX)!r})"
# `vals` entries are attribute-keyed for comparison; render only the value part, and subtract
# what the value surface already showed so a row never prints one value twice.
shown_already = {str(o.get("val")).strip()} if o.get("val") is not None else set()
@@ -1033,7 +1057,7 @@ def _row_value_suffix(o: dict[str, Any], rows: list[dict[str, Any]]) -> str:
if p is not o:
common_to_all &= {str(v) for v in (p.get("vals") or [])}
vals = [
- bare[:60]
+ _model_text(bare, 60)
for v in sorted(raw_vals, key=lambda v: v in common_to_all)
if (bare := v.split("=", 1)[-1]) not in shown_already
]
@@ -1051,29 +1075,37 @@ def _identical_text_rows_error(
*,
note: str | None = None,
live_token: str | None = None,
+ menu_open: bool = False,
) -> ToolResult:
"""The refusal owed when ≥2 rows match the value at the exact tier and are not one duplicate-rendered
candidate: the exact tier folds case/apostrophes, so every row already IS the requested text and only
- a direct click on a named row can choose between them. With `live_token` the query stays typed and
- the rows' token-qualified selectors stay clickable; otherwise the field's prior value is restored
- and the refusal directs a re-open instead.
+ a direct click on a named row can choose between them. With `menu_open` the menu stays open and its
+ rows' menu tags are named; with `live_token` the query stays typed and the rows' token-qualified
+ selectors stay clickable; otherwise the field's prior value is restored and the refusal directs a re-open.
"""
shown = rows[:15]
def _sel(o: dict[str, Any]) -> str:
# A selector is only named while it can be honored: after a restore the list may have closed
# and the stale tags may re-land on different rows at the next scan.
+ if menu_open:
+ return f'[data-tv3-menu="{o.get("n")}"] '
return f'[data-tv3-sugg="{o.get("n")}"][data-tv3-pick="{live_token}"] ' if live_token is not None else ""
- listing = "; ".join(f"{_sel(o)}{str(o.get('text') or '')[:60]!r}{_row_value_suffix(o, rows)}" for o in shown)
+ listing = "; ".join(f"{_sel(o)}{_model_text(o.get('text') or '', 60)!r}{_row_value_suffix(o, rows)}" for o in shown)
more = len(rows) - len(shown)
- next_step = (
- "click the intended row directly by its listed selector; the typed query was left "
- "in the field to keep the list open for that click"
- if live_token is not None
- else "type the value to reopen the list, then click the intended row directly; the field's "
- "prior value was put back"
- )
+ if menu_open:
+ next_step = "click the intended row directly by its listed selector; the list was left open for that click"
+ elif live_token is not None:
+ next_step = (
+ "click the intended row directly by its listed selector; the typed query was left "
+ "in the field to keep the list open for that click"
+ )
+ else:
+ next_step = (
+ "type the value to reopen the list, then click the intended row directly; the field's "
+ "prior value was put back"
+ )
if live_token is not None and note:
note = note.removesuffix(" — type the option's full label")
tail = f" ({note})" if note else ""
@@ -1081,7 +1113,7 @@ def _identical_text_rows_error(
f"{value!r} matches {len(rows)} rows in {selector} whose labels the exact matcher cannot tell "
f"apart by text: {listing}{f'; +{more} more' if more > 0 else ''}{tail} — {next_step} — the field "
"is NOT filled",
- data={"stop_batch": True} if live_token is not None else {"release_own_list": True},
+ data=None if menu_open else {"stop_batch": True} if live_token is not None else {"release_own_list": True},
error_class="identical_rows",
)
@@ -1615,6 +1647,108 @@ _ROW_SEMANTICS_JS = r"""
const OPT_SEL = '[role="option"],[role="menuitemradio"],[role="menuitemcheckbox"],[role="treeitem"],[role="radio"]';
const NAV_SEL = 'a[href],button,[role="button"],[role="link"],[role="menuitem"],[role="tab"]';
const LIST_SEL = '[role="listbox"],[role="menu"],[role="tree"],[role="grid"],[role="radiogroup"],datalist';
+ const DECLARED_ROW_SEL = OPT_SEL + ',[role="menuitem"]';
+ // The text a row renders, read through the flat tree: innerText stops at a shadow host, so an option
+ // component that draws its label in its own root would read as ''. A row declared inside the row is its
+ // own row, so a category's text is its own label, not its children's.
+ const composedRowText = (top) => {
+ let out = '';
+ let budget = 2000;
+ const walk = (n, depth) => {
+ if (depth > 32) return;
+ let kids = null;
+ if (String(n.localName || '') === 'slot') {
+ try { kids = n.assignedNodes({ flatten: true }); } catch (e) { kids = null; }
+ }
+ if (!kids || !kids.length) {
+ let sr = null;
+ try { sr = n.nodeType === 1 ? n.shadowRoot : null; } catch (e) { sr = null; }
+ kids = sr && sr.nodeType === 11 ? sr.childNodes : n.childNodes;
+ }
+ for (const c of kids) {
+ if (budget-- <= 0) return;
+ if (c.nodeType === 3) {
+ const p = c.parentElement || (c.parentNode && c.parentNode.host) || null;
+ let shown = true;
+ try { shown = !p || getComputedStyle(p).visibility !== 'hidden'; } catch (e) { shown = true; }
+ if (shown) out += c.data;
+ continue;
+ }
+ if (c.nodeType !== 1) continue;
+ const name = String(c.localName || '');
+ if (name === 'script' || name === 'style' || name === 'template' || name === 'noscript') continue;
+ if (name === 'br') { out += ' '; continue; }
+ try { if (c.matches(DECLARED_ROW_SEL)) continue; } catch (e) { /* unmatchable: read it */ }
+ let display = '';
+ try { display = getComputedStyle(c).display; } catch (e) { display = ''; }
+ if (display === 'none') continue;
+ const block = display !== '' && display !== 'contents' && !/^inline/.test(display);
+ if (block) out += ' ';
+ walk(c, depth + 1);
+ if (block) out += ' ';
+ }
+ };
+ try { walk(top, 0); } catch (e) { /* a partial read is still the row's text so far */ }
+ return out.replace(/\s+/g, ' ').trim();
+ };
+ // An undeclared leaf's row box: its largest ancestor that holds no other of the list's `leaves`, so a label whose
+ // identifier sits in a sibling element reads as one row. A leaf no other leaf bounds (a lone filtered row) climbs
+ // only while everything else an ancestor renders sits on the leaf's own line, so a header or count line above or
+ // below the row, or the field around the list, never joins its identity.
+ const onLeafLine = (a, prev, lr) => {
+ let sr = null;
+ try { sr = a.shadowRoot; } catch (e) { sr = null; }
+ for (const n of [...a.childNodes, ...(sr ? sr.childNodes : [])]) {
+ if (n === prev) continue;
+ let r = null;
+ if (n.nodeType === 3) {
+ if (!/\S/.test(n.data)) continue;
+ const range = document.createRange();
+ range.selectNodeContents(n);
+ r = range.getBoundingClientRect();
+ } else if (n.nodeType === 1) {
+ r = n.getBoundingClientRect();
+ } else continue;
+ if (r.width === 0 && r.height === 0) continue;
+ if (r.bottom <= lr.top || r.top >= lr.bottom) return false;
+ }
+ return true;
+ };
+ const leafBoxes = (leaves) => {
+ const under = new Map();
+ for (const t of leaves) {
+ for (let a = t, h = 0; a && a.nodeType === 1 && h < 40; h++, a = composedParentElement(a)) {
+ under.set(a, (under.get(a) || 0) + 1);
+ }
+ }
+ const boxes = new Map();
+ for (const el of leaves) {
+ let box = el;
+ let lineBox = el;
+ let bounded = false;
+ let lr = null;
+ try { lr = el.getBoundingClientRect(); } catch (e) { lr = null; }
+ let online = !!lr && lr.height > 0;
+ for (let a = composedParentElement(el), h = 0; a && a.nodeType === 1 && h < 39; h++, a = composedParentElement(a)) {
+ if (under.get(a) !== under.get(el)) { bounded = true; break; }
+ if (online) {
+ try { online = onLeafLine(a, box, lr); } catch (e) { online = false; }
+ if (online) lineBox = a;
+ }
+ box = a;
+ }
+ boxes.set(el, bounded ? box : lineBox);
+ }
+ return boxes;
+ };
+ // A row's identity: the text inside its declared row, whitespace-collapsed. A node no row is declared
+ // around reads its row `box` (see leafBoxes). The note, the refusals and the duplicate collapse all read
+ // this one value. A closed root reads as '', so the leaf's own text stands in for it.
+ const rowIdentity = (el, box) => {
+ let row = null;
+ try { row = composedClosest(el, DECLARED_ROW_SEL); } catch (e) { row = null; }
+ return composedRowText(row || box || el) || (el.innerText || '').replace(/\s+/g, ' ').trim();
+ };
// Composed, not closest(): an option component declares the row's role on its HOST, outside the
// root the pointer leaf lives in, so a same-root lookup reads that row as a bare button.
const isNavRow = (el) => {
@@ -1985,7 +2119,7 @@ _FIND_SUGGESTION_JS = (
n++;
c.el.setAttribute('data-tv3-sugg', String(n));
if (c.bare) bareNs.push(n);
- if (options.length < 15) options.push({ n, text: (c.el.innerText || '').trim().slice(0, 60) });
+ if (options.length < 15) options.push({ n, text: (c.el.innerText || '').trim() });
}
return { count: n, options, declared: true, bare: bareNs };
}
@@ -2004,7 +2138,7 @@ _FIND_SUGGESTION_JS = (
for (const c of safe) {
n++;
c.el.setAttribute('data-tv3-sugg', String(n));
- if (options.length < 15) options.push({ n, text: (c.el.innerText || '').trim().slice(0, 60) });
+ if (options.length < 15) options.push({ n, text: (c.el.innerText || '').trim() });
}
return { count: n, options, declared: false };
}
@@ -2276,7 +2410,7 @@ _FIND_CATEGORIES_JS = (
}
if (!hasPopup && !hasExpanded && childCount < 2 && !sideCharm) continue;
const label = el.getAttribute('aria-label') || (el.innerText || '').trim().split('\n')[0];
- const text = label.trim().slice(0, 80);
+ const text = label.trim();
if (!text) continue;
cats.push({ el, text });
}
@@ -3152,7 +3286,7 @@ PENDING_MARKER_JS = (
" const isElementControl = ctl.tagName === 'BUTTON' || ctl.tagName === 'INPUT';"
" let inner = '';"
" try { inner = isElementControl ? (ctl.innerText || '') : '' } catch(e) {}"
- " const t = String(own.trim() || inner || (isButtonInput ? ctl.value : '') || '').trim().slice(0, 60);"
+ " const t = String(own.trim() || inner || (isButtonInput ? ctl.value : '') || '').trim();"
" if (!/^(submitting|processing|sending|uploading)\\b/i.test(t)) return null;"
" const r = ctl.getBoundingClientRect();"
" if (r.width < 8 || r.height < 8) return null;"
@@ -3174,6 +3308,9 @@ PENDING_MARKER_JS = (
PENDING_MARKER_UNKNOWN_FRAME = "a frame of this page did not respond in time to confirm it is not still submitting"
+_ARIA_BUSY_SUFFIX = " (aria-busy)"
+
+
async def pending_marker(page: Any, selector: str) -> str | None:
"""The text the page still shows `selector`'s control as in flight with, or None.
@@ -3235,7 +3372,13 @@ async def pending_marker(page: Any, selector: str) -> str | None:
LOG.warning("taskv3 pending-marker probe failed on the control", selector=selector, exc_info=True)
continue
if marker:
- return marker
+ # The finish gate quotes this text to the model, so it is cut after masking, like row text.
+ text, busy = (
+ (marker[: -len(_ARIA_BUSY_SUFFIX)], _ARIA_BUSY_SUFFIX)
+ if marker.endswith(_ARIA_BUSY_SUFFIX)
+ else (marker, "")
+ )
+ return _model_text(text, 60) + busy
return None
@@ -5724,7 +5867,7 @@ _CLICK_PRECHECK_JS = (
// an arbitrary row, that case is flagged so the handler makes no claims about it at all.
if (el === target || pContains(el, target)) {
isOption = true;
- optText = (el.innerText || '').trim().slice(0, 80);
+ optText = (el.innerText || '').trim();
optState = state(el);
optSel = selState(el);
optKids = el.children.length;
@@ -5880,8 +6023,9 @@ _FIND_MENU_JS = (
// 80 is the shared "row-sized text, not a paragraph" ceiling. A row the page declares an option or
// menu item may run to ROW_TEXT_MAX, so a long option caption is listed whole.
const UNDECLARED_ROW_TEXT_MAX = 80;
- const ROW_TEXT_MAX = 200;
- const DECLARED_ROW_SEL = OPT_SEL + ',[role="menuitem"]';
+ const ROW_TEXT_MAX = """
+ + str(_MENU_ROW_TEXT_MAX)
+ + r""";
// `cascade`: the caller just clicked a row that DETACHED (a category replacing the list with its
// children). The trigger is gone, so trigger-anchored geometry/ARIA is waived — new rows in a
// FLOATING container carry the claim instead (enforced below).
@@ -5908,16 +6052,24 @@ _FIND_MENU_JS = (
// A cascading widget may HIDE its old stage instead of detaching it: a connected trigger with a
// zeroed rect anchors geometry at the viewport origin and would reject legitimate children.
const tr = cascade && tr0 && !(tr0.width > 0 && tr0.height > 0) ? null : tr0;
+ // `composed`: a declared row may draw its text in its own shadow root, where innerText does not reach.
+ const rowBox = (el, composed) => {
+ const tag = el.tagName;
+ if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || tag === 'SCRIPT' || tag === 'STYLE' || tag === 'LABEL' || tag === 'FORM') return null;
+ if (el.children.length > 8) return null;
+ const r = el.getBoundingClientRect();
+ if (!vis(r) || r.height > 90) return null;
+ const txt = (el.innerText || '').trim() || (composed ? composedRowText(el) : '');
+ if (!txt || txt.length > ROW_TEXT_MAX) return null;
+ return { r, txt };
+ };
const rows = [];
for (const el of pScopeAll()) {
if (reuse !== 'any' && (preHas(el) || focusHas(el))) continue;
const tag = el.tagName;
- if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || tag === 'SCRIPT' || tag === 'STYLE' || tag === 'LABEL' || tag === 'FORM') continue;
- if (el.children.length > 8) continue;
- const r = el.getBoundingClientRect();
- if (!vis(r) || r.height > 90) continue;
- const txt = (el.innerText || '').trim();
- if (!txt || txt.length > ROW_TEXT_MAX) continue;
+ const box = rowBox(el);
+ if (!box) continue;
+ const { r, txt } = box;
// A long row is an option only when the page declares one on it, in it or around it: an inherited
// pointer cursor otherwise makes every paragraph of a clickable popover read as a menu row.
if (txt.length > UNDECLARED_ROW_TEXT_MAX) {
@@ -6060,15 +6212,112 @@ _FIND_MENU_JS = (
if (!best || g.length > best.g.length || (g.length === best.g.length && pr.height < best.h)) best = { p, g, h: pr.height };
}
if (!best) return null;
- pQSA('[data-tv3-menu]').forEach((e) => e.removeAttribute('data-tv3-menu'));
best.g.sort((a, b) => a.r.top - b.r.top || a.r.left - b.r.left);
- const options = [];
+ // The note lists one ENTRY per option while every leaf keeps its own tag (the click and match surfaces).
+ // The leaves of one declared row are one entry, shown with the row's text and pointing at the tag of its
+ // link, else its first leaf that is not a separate control; a row that nests declared rows, existed before the click,
+ // exceeds the row caps or holds only controls is not that unit.
+ const CONTROL_SEL = 'button,a[href],input,[role="button"],[role="checkbox"],[role="switch"],[role="radio"]';
+ const separateControl = (leaf, row) => {
+ const m = composedClosest(leaf, CONTROL_SEL);
+ if (!m || m === row) return false;
+ return pContains(row, m);
+ };
+ // An undeclared leaf is a fragment when a visible text node outside every leaf sits directly in the leaf's
+ // own wrapper below the container (a second line, a bare identifier beside a label), or when its row box
+ // renders text the leaf does not (an identifier in a sibling element). A header in its own element does not
+ // count unless it shares the leaf's box.
+ const boxes = leafBoxes(best.g.map((c) => c.el));
+ const bareTextBeside = (c) => {
+ let declared = null;
+ try { declared = composedClosest(c.el, DECLARED_ROW_SEL); } catch (e) { declared = null; }
+ const box = boxes.get(c.el);
+ if (!declared && box && box !== c.el && composedRowText(box) !== composedRowText(c.el)) return true;
+ let a = composedParentElement(c.el);
+ for (let hops = 0; a && a !== best.p && hops < 20; hops++, a = composedParentElement(a)) {
+ for (const t of a.childNodes) {
+ if (t.nodeType !== 3 || !(t.nodeValue || '').replace(/[\u200B-\u200D\uFEFF]/g, '').trim()) continue;
+ try { if (a.getClientRects().length > 0 && getComputedStyle(a).visibility !== 'hidden') return true; } catch (e) { return true; }
+ }
+ }
+ return a !== best.p;
+ };
+ const entries = [];
+ const byRow = new Map();
+ const leavesIn = new Map();
+ for (const c of best.g) {
+ const row = composedClosest(c.el, DECLARED_ROW_SEL);
+ if (row) leavesIn.set(row, (leavesIn.get(row) || 0) + 1);
+ }
+ for (const c of best.g) {
+ const row = composedClosest(c.el, DECLARED_ROW_SEL);
+ if (!row || row === c.el) {
+ entries.push({ leaves: [c], reason: row ? null : (bareTextBeside(c) ? 'bare_text_beside' : null) });
+ continue;
+ }
+ let nests = true;
+ try { nests = !!row.querySelector(DECLARED_ROW_SEL); } catch (e) { nests = true; }
+ const preexisting = reuse !== 'any' && preHas(row);
+ const box = nests || preexisting ? null : rowBox(row, true);
+ if (!box) {
+ // A row holding one leaf is that leaf's option whatever its size; only a multi-leaf row can be over the caps.
+ const reason = nests || preexisting || leavesIn.get(row) === 1
+ ? (bareTextBeside(c) ? 'bare_text_beside' : null)
+ : 'declared_row_over_caps';
+ entries.push({ leaves: [c], reason });
+ continue;
+ }
+ if (!byRow.has(row)) {
+ const entry = { leaves: [], row, reason: null };
+ byRow.set(row, entry);
+ entries.push(entry);
+ }
+ byRow.get(row).leaves.push(c);
+ }
+ const listed = [];
+ for (const e of entries) {
+ if (!e.row) { listed.push(e); continue; }
+ // A link inside the row is the row's own action (clicking a badge beside it does nothing). A button is
+ // not preferred: a row's button is as often a separate control (a favourite star) as its action.
+ const link = e.leaves.find((c) => {
+ const a = composedClosest(c.el, 'a[href]');
+ return a ? a !== e.row && pContains(e.row, a) : false;
+ });
+ const label = link || e.leaves.find((c) => !separateControl(c.el, e.row));
+ if (label) { listed.push({ leaves: e.leaves, head: label, reason: null }); continue; }
+ // A row of controls only: each control is its own entry, named by its own text (the row's would name
+ // every control alike). A row rendering text beyond its controls is named by the row's text when it holds
+ // one control, and is not listable when it holds several.
+ const own = e.leaves.map((c) => composedRowText(c.el)).join('').replace(/\s+/g, '');
+ if (composedRowText(e.row).replace(/\s+/g, '') !== own) {
+ if (e.leaves.length === 1) listed.push({ leaves: e.leaves, head: e.leaves[0], reason: null });
+ else for (const c of e.leaves) listed.push({ leaves: [c], control: true, reason: 'bare_text_beside' });
+ continue;
+ }
+ for (const c of e.leaves) listed.push({ leaves: [c], control: true, reason: bareTextBeside(c) ? 'bare_text_beside' : null });
+ }
+ // One row's own pieces are not a menu to list, but the click and match surfaces still read them leaf by leaf.
+ if (listed.length < 2) {
+ listed.length = 0;
+ for (const c of best.g) listed.push({ leaves: [c], reason: 'single_row_pieces' });
+ }
+ pQSA('[data-tv3-menu]').forEach((e) => e.removeAttribute('data-tv3-menu'));
let n = 0;
+ const tagOf = new Map();
for (const c of best.g) {
n++;
c.el.setAttribute('data-tv3-menu', String(n));
- if (options.length < 15) options.push({ n, text: c.txt.slice(0, ROW_TEXT_MAX) });
+ tagOf.set(c, n);
+ }
+ const withheld = listed.find((e) => e.reason);
+ const options = [];
+ for (const e of listed) {
+ if (options.length >= 15) break;
+ const head = e.head || e.leaves[0];
+ const text = e.control ? head.txt.replace(/\s+/g, ' ') : rowIdentity(head.el, boxes.get(head.el));
+ options.push({ n: tagOf.get(head), text });
}
+ const count = listed.length;
const partial = windowPartial(best.g, best.p);
// A listbox or menu declares its options even when a row renders nothing this finder can read. Only
// rendered, outermost options count (a filter hides the rest), unless the options state the set size.
@@ -6089,14 +6338,18 @@ _FIND_MENU_JS = (
}
}
} catch (e) { declared = 0; }
- return declared > n ? { count: n, options, partial, declared } : { count: n, options, partial };
+ // `count` is the note's rows; `tagged` is the leaves the full-text read should find, row pieces included;
+ // `rows` is the distinct rendered rows those leaves sit in, the unit aria-setsize counts.
+ const rendered = new Set(best.g.map((c) => composedClosest(c.el, DECLARED_ROW_SEL) || c.el)).size;
+ const out = { count, tagged: n, rows: rendered, options, partial, whole: !withheld, withheld_reason: withheld ? withheld.reason : null };
+ if (declared > count) out.declared = declared;
+ return out;
}"""
)
# Read the FULL (untruncated) label of every row tagged data-tv3-<attr>, across the same pierced reach
-# the tagger tags in. A tagger caps its returned `options` at 15 and truncates each to 60 chars for
-# payload size; the deterministic match must see the whole list at full length so a value beyond the
-# 15th row, or a label longer than 60 chars, is neither missed nor matched on a cut-off token. `nav`
+# the tagger tags in. A tagger caps its returned `options` at 15 for payload size; the deterministic match
+# must see the whole list so a value beyond the 15th row is not missed. `nav`
# marks a row this tool must not auto-click. `arg.attr` selects which tagger's rows to read ("menu" for
# _FIND_MENU_JS, "sugg" for _FIND_SUGGESTION_JS) so the same full-length read serves both.
# `val` and `label` are the identity a duplicate-rendered candidate carries when its TEXT does not: a
@@ -6135,7 +6388,9 @@ _MENU_OPTION_TEXTS_JS = (
const al = node.getAttribute('aria-label');
return al && al.trim() ? al.trim() : '';
};
- return Array.from(pQSA('[data-tv3-' + attr + ']')).map((el) => {
+ const tagged = Array.from(pQSA('[data-tv3-' + attr + ']'));
+ const boxes = leafBoxes(tagged);
+ return tagged.map((el) => {
// An option whose ancestor declares aria-setsize is a child that declares none, so read the
// closest declaring ancestor or the incomplete-list guard is bypassed.
const nav = isNavRow(el);
@@ -6215,6 +6470,7 @@ _MENU_OPTION_TEXTS_JS = (
// Leaf and ancestor names as separate veto surfaces: a shared leaf label ("Choose") must not
// mask option ancestors whose names disagree.
labels: [accessibleName(el) || null, opt ? accessibleName(opt) || null : null],
+ identity: rowIdentity(el, boxes.get(el)),
};
});
}"""
@@ -9012,15 +9268,33 @@ def observe_handles_js(retain_width: int = OBSERVE_RETAIN_WIDTH_MIN) -> str:
return _observe_js_returning("{ json: payload, els: outEls }", retain_width)
-def _menu_mark_parts(options: list[dict[str, Any]], cap: int) -> list[str]:
+def _menu_mark_parts(options: list[dict[str, Any]], cap: int, text_cap: int = _MENU_ROW_TEXT_MAX) -> list[str]:
parts = []
for o in (options or [])[:cap]:
# option texts are page-controlled and land in the LLM transcript — same sanitation as filenames
- text = _DOWNLOAD_NOTICE_SANITIZE_RE.sub("", str(o.get("text", "")))
+ text = _DOWNLOAD_NOTICE_SANITIZE_RE.sub("", _model_text(o.get("text", ""), text_cap))
parts.append(f'[data-tv3-menu="{o.get("n")}"] {text!r}')
return parts
+_MENU_WITHHOLD_REASONS = frozenset({"declared_row_over_caps", "bare_text_beside", "single_row_pieces"})
+
+
+def _menu_note_fields(found: dict[str, Any]) -> dict[str, Any]:
+ """Counts and closed-vocabulary labels only: these fields land on an indexed call record."""
+ withheld = found.get("whole") is False
+ fields: dict[str, Any] = {
+ "menu_note": "withheld" if withheld else "listed",
+ # Rendered rows, not note entries: a withheld note's entries are row pieces, so only this unit is the
+ # same whether the note was listed or withheld.
+ "menu_rows": int(found.get("rows") or found.get("count") or 0),
+ }
+ reason = found.get("withheld_reason")
+ if withheld and reason in _MENU_WITHHOLD_REASONS:
+ fields["withhold_reason"] = reason
+ return fields
+
+
def _menu_open_note(found: dict[str, Any], selector: str, *, clicked_row: bool = False) -> str:
count = int(found.get("count") or 0)
parts = _menu_mark_parts(found.get("options") or [], 15)
@@ -9030,13 +9304,20 @@ def _menu_open_note(found: dict[str, Any], selector: str, *, clicked_row: bool =
# of the ones it is about to declare stale.
closer = "the row you just clicked" if clicked_row else selector
declared = found.get("declared")
+ stale = "any data-tv3-menu selector from an earlier result now points at a different row or at nothing."
+ if found.get("whole") is False:
+ # A fragment listed as an option is a false list, and the count is the fragments', so neither is shown.
+ return (
+ "This click opened a menu, but its options could not be read as whole rows, so they are not "
+ f"listed here. Re-observe to read them before picking one; clicking {closer} again or elsewhere "
+ f"closes the menu, and {stale}"
+ )
if isinstance(declared, int) and not isinstance(declared, bool) and declared > count:
overflow += f" (the list declares {declared} options; {count} are listed)"
return (
f"This click opened a menu of {count} options: {'; '.join(parts)}{overflow}. To select one, click "
f'its [data-tv3-menu="N"] selector NOW — clicking {closer} again or elsewhere closes the menu '
- "and destroys these options. These numbers are freshly assigned: any data-tv3-menu selector "
- "from an earlier result now points at a different row or at nothing."
+ f"and destroys these options. These numbers are freshly assigned: {stale}"
)
@@ -9049,7 +9330,7 @@ async def _categories_note(page: Any, selector: str) -> str | None:
return None
if not found or not found.get("count"):
return None
- items = "; ".join(_menu_mark_parts(found.get("categories") or [], 8))
+ items = "; ".join(_menu_mark_parts(found.get("categories") or [], 8, 80))
return (
f"Some rows near this field carry an expand affordance and may be categories whose options are "
f"nested rather than shown in the flat list: {items}. If one could contain your value, click its "
@@ -12055,12 +12336,20 @@ def build_browser_tools(
except Exception:
return 1
+ # What the last menu-open note reported, for the click's call record; reset before each reaction read.
+ menu_note_census: dict[str, Any] = {}
+
+ def _menu_note(found: dict[str, Any], selector: str, *, clicked_row: bool = False) -> str:
+ menu_note_census.clear()
+ menu_note_census.update(_menu_note_fields(found))
+ return _menu_open_note(found, selector, clicked_row=clicked_row)
+
async def _click_reaction(
page: Any, selector: str, pre: dict[str, Any], url_before: str, *, doc_planted: bool
) -> tuple[str | None, str | None]:
# Returns (note, commit_error) — at most one set. Raises are the caller's to swallow (fail-open:
# a probe failure must degrade to the bare pre-feature ok, never fail the click).
- opt = _DOWNLOAD_NOTICE_SANITIZE_RE.sub("", str(pre.get("optText") or "")) or selector
+ opt = _DOWNLOAD_NOTICE_SANITIZE_RE.sub("", _model_text(pre.get("optText") or "", 80)) or selector
if pre.get("isOption"):
# Commit evidence, any one suffices: navigation, the menu closing, the option's own state
# changing vs the post-hover baseline (multi-select menus commit WITHOUT closing), or a
@@ -12145,7 +12434,7 @@ def build_browser_tools(
except Exception:
return None
if isinstance(found, dict) and found.get("count"):
- return _menu_open_note(found, selector, clicked_row=True)
+ return _menu_note(found, selector, clicked_row=True)
return None
async def _cascade_child_note() -> str | None:
@@ -12162,7 +12451,7 @@ def build_browser_tools(
except Exception:
return None
if isinstance(found, dict) and found.get("count"):
- return _menu_open_note(found, selector, clicked_row=True)
+ return _menu_note(found, selector, clicked_row=True)
now = time.monotonic()
if now < baseline:
# Children scheduled on a timer may announce nothing (no busy row) for a beat:
@@ -12270,7 +12559,7 @@ def build_browser_tools(
return None, None
found = await page.evaluate(_FIND_MENU_JS, await _probe_arg(page, selector))
if isinstance(found, dict) and found.get("count"):
- return _menu_open_note(found, selector), None
+ return _menu_note(found, selector), None
if isinstance(after_raw, dict) and not after_raw.get("stillOpen"):
return (
"Note: this click CLOSED the open menu — no option was selected. To select, click "
@@ -12279,7 +12568,7 @@ def build_browser_tools(
return None, None
found = await page.evaluate(_FIND_MENU_JS, await _probe_arg(page, selector))
if isinstance(found, dict) and found.get("count"):
- return _menu_open_note(found, selector), None
+ return _menu_note(found, selector), None
return None, None
async def click(args: dict[str, Any]) -> ToolResult:
@@ -12726,12 +13015,14 @@ def build_browser_tools(
# The click may have done something other than toggle (opened a menu, selected an
# option); the menu reaction is the authority on that before the toggle verdict stands.
if pre is not None:
+ menu_note_census.clear()
try:
note, commit_error = await _click_reaction(
page, selector, pre, url_before, doc_planted=doc_planted
)
except Exception:
note, commit_error = None, None
+ transition_data.update(menu_note_census)
if commit_error is not None:
return ToolResult.error(commit_error, data=transition_data)
if note:
@@ -12744,11 +13035,13 @@ def build_browser_tools(
if pre is None:
return ToolResult.ok(base, data=transition_data)
+ menu_note_census.clear()
try:
note, commit_error = await _click_reaction(page, selector, pre, url_before, doc_planted=doc_planted)
except Exception:
LOG.debug("taskv3 click reaction probe failed", selector=selector, exc_info=True)
return ToolResult.ok(base, data=transition_data)
+ transition_data.update(menu_note_census)
if commit_error is not None:
return ToolResult.error(commit_error, data=transition_data)
return ToolResult.ok(base + "\n" + note if note else base, data=transition_data)
@@ -13313,10 +13606,8 @@ def build_browser_tools(
return await _find_suggestion_rows(page, selector, probe or value, match=value if probe else None)
async def _full_rows() -> list[dict[str, Any]]:
- # The tagger truncates each label to 60 chars for payload size; the match must see the whole
- # text so a value that differs only past char 60, or a >60-char row, is not mismatched. With
- # no full-length read there is no list to match against -- the truncated labels would feed
- # both the uniqueness matcher and a whole-text click guard -- so the caller refuses instead.
+ # The tagger returns at most 15 rows; the match must see every tagged row, and with no
+ # full-length read there is no list to match against, so the caller refuses instead.
try:
raw = await page.evaluate(_MENU_OPTION_TEXTS_JS, {"attr": "sugg"})
except Exception as e:
@@ -13389,7 +13680,7 @@ def build_browser_tools(
idx = survivors[0].get("n")
return True, rows, idx, overflow
if exact_only:
- # The tagged labels are cut at 60 chars; an exact match has to see the whole text.
+ # The tagger returns at most 15 rows; an exact match has to see every one.
rows = await _full_rows()
return False, rows, _match_option_exact(value, _without_nested_copies(value, rows)), 0
return False, tagged, 1, 0
@@ -14031,7 +14322,7 @@ def build_browser_tools(
if len(rows) == 1 and not pick.overflow:
# A lone row is either the site's empty-list text ("No Items.") or a single near match; only the
# page's wording can tell them apart, so name the row and both next steps.
- only = str(rows[0].get("text") or "")[:80]
+ only = _model_text(rows[0].get("text") or "", 80)
return _settled(
ToolResult.error(
f"the site's search for {typed!r} returned one row, {only!r}, which is not {value!r} — the "
@@ -15098,6 +15389,7 @@ def build_browser_tools(
live_token=live_token,
)
if pick.suggestion:
+ suggestion = _model_text(pick.suggestion, 60)
verdict, matches = await _typeahead_commit_verdict(page, selector, pick.committed, pick.readable)
if verdict is CommitStatus.OK:
if pick.declared:
@@ -15107,14 +15399,14 @@ def build_browser_tools(
if closed is not None:
return closed
return ToolResult.ok(
- f"typed into {selector}; it is a typeahead — selected {pick.suggestion!r} "
+ f"typed into {selector}; it is a typeahead — selected {suggestion!r} "
f"(committed value: {pick.committed!r})"
)
if not pick.committed and pick.shared_surface:
# The widget renders only a short form of the label ("+1"), and either another row visible
# at the click showed it too or the field was unreadable before the click.
return ToolResult.ok(
- f"typed into {selector}; it is a typeahead — selected {pick.suggestion!r}, and the "
+ f"typed into {selector}; it is a typeahead — selected {suggestion!r}, and the "
f"field now shows {pick.shared_surface!r}, which cannot be tied to this pick alone, "
"so the commit could not be verified — re-observe to confirm the value before "
"relying on it"
@@ -15123,7 +15415,7 @@ def build_browser_tools(
# INV-1: the field re-resolved to n≠1 after the click (remounted or now ambiguous), so
# there is no stable element to read the commit off — soft, not a false did-not-commit.
return ToolResult.ok(
- f"clicked suggestion {pick.suggestion!r} for {selector}, but it re-resolved to {matches} "
+ f"clicked suggestion {suggestion!r} for {selector}, but it re-resolved to {matches} "
"elements so the commit could not be verified — re-observe to confirm the value "
"before relying on it"
)
@@ -15136,18 +15428,18 @@ def build_browser_tools(
why = await _unverifiable_because(page, selector)
if why:
return ToolResult.ok(
- f"clicked suggestion {pick.suggestion!r} for {selector}; {why}, so the commit could not "
+ f"clicked suggestion {suggestion!r} for {selector}; {why}, so the commit could not "
"be verified — re-observe to confirm the value before relying on it"
)
return ToolResult.error(
- f"clicked suggestion {pick.suggestion!r} for {selector} but it did not commit — the field is "
+ f"clicked suggestion {suggestion!r} for {selector} but it did not commit — the field is "
"NOT filled; re-observe and retry, do not proceed"
)
# DID_NOT_COMMIT: the field is NOT filled. The loop then skips any later click or Enter
# in the same batch -- it may be an unvalidated submit, and no production submit guard
# exists yet.
return ToolResult.error(
- f"clicked suggestion {pick.suggestion!r} for {selector} but it did not commit — the field is NOT "
+ f"clicked suggestion {suggestion!r} for {selector} but it did not commit — the field is NOT "
"filled; re-observe and retry, do not proceed"
)
# No suggestion list surfaced. The finder pierces open shadow roots, so it can see a list
@@ -15287,7 +15579,7 @@ def build_browser_tools(
# Read the whole tagged list at full length so the match is neither missed on a >60-char label
# nor computed over a truncated ≤15 slice (which would let "unique in the first 15" stand in for
# "unique in the menu").
- count = int(found.get("count") or 0)
+ tagged = int(found.get("tagged") or 0)
read: list[dict[str, Any]] = []
try:
full_rows = await page.evaluate(_MENU_OPTION_TEXTS_JS, {"attr": "menu"})
@@ -15302,11 +15594,17 @@ def build_browser_tools(
# `overflowed` = the enumerated set is not the whole list, so uniqueness cannot be established and
# ALL auto-commit is refused. That is true when the full read failed, when `_FIND_MENU_JS` tagged
- # more rows than the read returned, OR when a row's `aria-setsize` declares more options than were
- # rendered (a virtualised list whose window is all that is in the DOM — count == len(read) there).
+ # more leaves than the read returned (a row remounted between the two calls), OR when a row's `aria-setsize`
+ # declares more options than there are rendered rows (a virtualised list whose window is all that is in
+ # the DOM). aria-setsize counts rows, not leaves: two-leaf rows read back more leaves than rows.
declared = max((int(o.get("setsize") or 0) for o in read), default=0)
+ rendered_rows = found.get("rows")
+ if not isinstance(rendered_rows, int) or isinstance(rendered_rows, bool):
+ rendered_rows = len(read)
size_unknown = any(o.get("setsize_unknown") for o in read)
- overflowed = not read or count > len(read) or declared > len(read) or size_unknown or bool(found.get("partial"))
+ overflowed = (
+ not read or tagged > len(read) or declared > rendered_rows or size_unknown or bool(found.get("partial"))
+ )
rows = read or (found.get("options") or [])
rows.sort(key=_n_order)
# Never auto-click a navigational row (`<a href>`/`<button>`/menuitem): `_FIND_MENU_JS` enumerates
@@ -15316,7 +15614,7 @@ def build_browser_tools(
# Refusing to auto-click is not the same as having nothing to show: hand the model the marks
# so a button-built select is one deliberate click away instead of an empty listing.
shown_nav = "; ".join(
- f'[data-tv3-menu="{o.get("n")}"] {str(o.get("text") or "")[:60]!r}' for o in rows[:15]
+ f'[data-tv3-menu="{o.get("n")}"] {_model_text(o.get("text") or "", 60)!r}' for o in rows[:15]
)
return ToolResult.error(
f"opened {selector} but every row is a link/button this tool will not auto-click ({shown_nav}"
@@ -15903,18 +16201,21 @@ def build_browser_tools(
if len(contenders) >= 2 or exact_twins or (contenders and walk_complete):
return _ambiguous_error()
if contenders:
+ only = _model_text(contenders[0], 60)
return ToolResult.error(
f"{value!r} matched no option in the {len(options)} rows {selector}'s list renders — it "
- f"declares more, and the only row naming it is {contenders[0][:60]!r}; the field is NOT "
+ f"declares more, and the only row naming it is {only!r}; the field is NOT "
"filled — pass one option's full text, or look() and click it"
)
- vocab = "; ".join(repr(str(o.get("text") or "")[:60]) for o in shown)
+ vocab = "; ".join(repr(_model_text(o.get("text") or "", 60)) for o in shown)
more = f"; +{len(options) - len(shown)} more" if len(options) > len(shown) else ""
return ToolResult.error(
f"{value!r} matched no option in {selector}'s list — scrolled through all "
f"{len(options)} options ({vocab}{more}); pass one option's exact text"
)
- listing = "; ".join(f'[data-tv3-menu="{o.get("n")}"] {str(o.get("text") or "")[:60]!r}' for o in shown)
+ listing = "; ".join(
+ f'[data-tv3-menu="{o.get("n")}"] {_model_text(o.get("text") or "", 60)!r}' for o in shown
+ )
if overflowed:
return ToolResult.error(
f"{value!r} matched no option in the part of {selector}'s list we could read "
@@ -15932,12 +16233,15 @@ def build_browser_tools(
text_counts[t] = text_counts.get(t, 0) + 1
def _listed_row(o: dict[str, Any], canon_text: str) -> str:
- entry = f'[data-tv3-menu="{o.get("n")}"] {str(o.get("text") or "")[:60]!r}'
+ entry = f'[data-tv3-menu="{o.get("n")}"] {_model_text(o.get("text") or "", 60)!r}'
if text_counts[canon_text] >= 2:
same_text = [p for p, pt in zip(shown, shown_canon_texts) if pt == canon_text]
entry += _row_value_suffix(o, same_text)
return entry
+ same_as_value = [o for o in options if _exact_tier_key(str(o.get("text") or "")) == _exact_tier_key(value)]
+ if len(same_as_value) >= 2:
+ return _identical_text_rows_error(selector, value, same_as_value, menu_open=True)
listing = "; ".join(_listed_row(o, t) for o, t in zip(shown, shown_canon_texts))
not_shown = len(options) - len(shown)
if not_shown > 0:
@@ -16130,6 +16434,7 @@ def build_browser_tools(
surface_vouched_pre_click: bool = False,
shared_surface: str | None = None,
) -> ToolResult:
+ opt_txt = _model_text(opt_txt, 60)
verdict, matches = await _typeahead_commit_verdict(page, selector, committed, readable)
if verdict is CommitStatus.OK:
if declared:
@@ -16460,7 +16765,7 @@ def build_browser_tools(
if offered and offered_total > len(offered):
offered_note = (
f". The list offers {offered_total} rows; the first {len(offered)}: "
- + "; ".join(repr(t[:60]) for t in offered)
+ + "; ".join(repr(_model_text(t, 60)) for t in offered)
+ (
search_retry + "; if the label you want is not among them, a longer search narrows it"
if search_retry
@@ -16471,7 +16776,7 @@ def build_browser_tools(
elif offered:
offered_note = (
". The list offers: "
- + "; ".join(repr(t[:60]) for t in offered)
+ + "; ".join(repr(_model_text(t, 60)) for t in offered)
+ (search_retry or " — call select_combobox again with one of these exact labels")
)
else:
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 017ad4af0..4f3bcf920 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -65,6 +65,7 @@ from skyvern.forge.taskv3.loop import (
_navigate_record_fields,
)
from skyvern.forge.taskv3.tools import (
+ _MENU_OPTION_TEXTS_JS,
_OPAQUE_ID_RUN_RE,
_SEMANTIC_COMMIT_STATE_JS,
NAVIGATION_DEAD_END_STATUSES,
@@ -9455,6 +9456,505 @@ async def test_dom_a_menu_of_menuitems_lists_a_long_item_and_it_commits() -> Non
assert await page.evaluate("() => window.__picked") == _LONG_MENU_OPTIONS[0]
+# A declared option row drawn from several elements under an inherited pointer cursor: a name line, then
+# a line whose identifier is a bare text node beside a <b> label. Entries are "name|number". The trigger
+# shows the name only after a pick, as such pickers do.
+_TWO_LINE_OPTION_ROW_JS = """(txt, onPick) => {
+ const [name, num] = txt.split('|');
+ const row = document.createElement('div');
+ row.setAttribute('role', 'option');
+ row.setAttribute('style', 'padding:2px 6px;cursor:pointer;white-space:normal');
+ row.innerHTML = '<div><div><i title="favourite"></i><span class="nm"></span>'
+ + '<span class="id"><br><b>Account: </b></span></div></div>';
+ row.querySelector('.nm').textContent = name;
+ row.querySelector('.id').appendChild(document.createTextNode(num));
+ row.addEventListener('click', () => {
+ onPick();
+ document.getElementById('trigger').textContent = name;
+ });
+ return row;
+}"""
+
+# Row 2 and row 4 share a name, so only the number tells them apart.
+_TWO_LINE_OPTIONS = [
+ "Northwind Ltd|100200300",
+ "Contoso Group|200300400",
+ "Fabrikam Inc|300400500",
+ "Contoso Group|400500600",
+ "Tailspin Co|500600700",
+ "Litware Ltd|600700800",
+]
+
+
+def _two_line_menu_html(options: list[str], *, list_style: str = "", declared: bool = True) -> str:
+ row_js = (
+ _TWO_LINE_OPTION_ROW_JS
+ if declared
+ else _TWO_LINE_OPTION_ROW_JS.replace("row.setAttribute('role', 'option');", "")
+ )
+ html = _LONG_OPTION_MENU_FIXTURE_HTML.replace("__OPTIONS__", json.dumps(options)).replace("__ROWS__", row_js)
+ return html.replace("'border:1px solid #ccc;", f"'{list_style}border:1px solid #ccc;")
+
+
+def _menu_note_entries(content: str) -> list[tuple[str, str]]:
+ return re.findall(r"\[data-tv3-menu=\"(\d+)\"\] '([^']*)'", content)
+
+
+@_skip_no_browser
+async def test_dom_a_menu_note_lists_each_multi_element_option_row_whole() -> None:
+ # One entry per declared row, carrying the row's full text (the number is a bare text node) and the tag of
+ # its name leaf; every leaf keeps its own tag, so clicks and select_combobox see what they always saw.
+ html = _two_line_menu_html(_TWO_LINE_OPTIONS)
+ async with _content_page(html) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ opened = await _tool(tools, "click").handler({"selector": "#trigger"})
+ assert "opened a menu of 6 options" in opened.content, opened.content
+ assert (opened.data or {}).get("menu_note") == "listed" and opened.data["menu_rows"] == 6, opened.data
+ entries = _menu_note_entries(opened.content)
+ assert [text for _, text in entries] == [
+ f"{name} Account: {num}" for name, num in (e.split("|") for e in _TWO_LINE_OPTIONS)
+ ], opened.content
+ picked = await _tool(tools, "click").handler({"selector": f'[data-tv3-menu="{entries[3][0]}"]'})
+ assert picked.status == "ok", picked.content
+ assert await page.evaluate("() => window.__picked") == _TWO_LINE_OPTIONS[3]
+
+ async with _content_page(html) as page:
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "select_combobox").handler(
+ {"selector": "#trigger", "value": "Fabrikam Inc"}
+ )
+ assert r.status == "ok", r.content
+ assert await page.evaluate("() => window.__picked") == _TWO_LINE_OPTIONS[2]
+
+ # Rows 2 and 4 share the name; their full row texts differ, so they are two options, not one rendered twice,
+ # and the refusal shows what tells them apart.
+ async with _content_page(html) as page:
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "select_combobox").handler(
+ {"selector": "#trigger", "value": "Contoso Group"}
+ )
+ assert r.status == "error" and r.error_class == "identical_rows", r.content
+ assert "'Contoso Group' matches 2 rows" in r.content, r.content
+ assert "200300400" in r.content and "400500600" in r.content, r.content
+ assert await page.evaluate("() => window.__picked") is None
+
+
+@_skip_no_browser
+async def test_dom_a_menu_note_lists_no_rows_it_cannot_read_whole() -> None:
+ # With no declared row the leaves cannot be joined into options, so listing them would present
+ # fragments as options. The note sends the model to observe instead.
+ html = _two_line_menu_html(_TWO_LINE_OPTIONS, declared=False)
+ async with _content_page(html) as page:
+ opened = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#trigger"})
+ # The page-text delta after the note quotes the page itself; only the note must not list fragments.
+ note = opened.content[: (opened.data or {}).get("delta_at", len(opened.content))]
+ assert "could not be read as whole rows" in note, opened.content
+ assert "opened a menu of" not in note, opened.content
+ assert "Account:" not in note, opened.content
+ assert (opened.data or {}).get("withhold_reason") == "bare_text_beside", opened.data
+
+
+@_skip_no_browser
+async def test_dom_a_lone_option_of_two_pieces_is_still_tagged_leaf_by_leaf() -> None:
+ # Not listed as a menu (one row's pieces), but tagged as main tags it, so select_combobox still finds the list.
+ body = (
+ '<div role="listbox"><div role="option">'
+ '<div style="cursor:pointer;height:22px">Jane Doe</div><div style="cursor:pointer;height:22px">[email protected]</div>'
+ "</div></div>"
+ )
+ async with _content_page(_popover_html(body)) as page:
+ opened = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#trigger"})
+ assert "opened a menu of" not in opened.content, opened.content
+ assert (opened.data or {}).get("withhold_reason") == "single_row_pieces", opened.data
+ assert opened.data["menu_rows"] == 1, opened.data
+ assert await page.evaluate("() => document.querySelectorAll('[data-tv3-menu]').length") == 2
+
+
+@_skip_no_browser
+ ("list_style", "expected"),
+ [
+ pytest.param("max-height:400px;overflow-y:auto;", ["18"], id="scrolling-list"),
+ # An unclipped list taller than the finder's 500px container cap is never grouped, so the full list
+ # cannot be listed; the contract is that nothing smaller (one row's pieces) is listed instead.
+ pytest.param("", [], id="unclipped-list-over-the-container-cap"),
+ ],
+)
+async def test_dom_a_long_menu_is_listed_whole_or_not_at_all(list_style: str, expected: list[str]) -> None:
+ html = _two_line_menu_html([f"Account holder {i}|{100200300 + i}" for i in range(18)], list_style=list_style)
+ async with _content_page(html) as page:
+ opened = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#trigger"})
+ assert re.findall(r"opened a menu of (\d+) options", opened.content) == expected, opened.content
+ assert "'Account:'" not in opened.content, opened.content
+
+
+def _popover_html(body: str, pop_style: str = "") -> str:
+ return f"""<!doctype html><html><body style="margin:0">
+ <button id="trigger" style="position:absolute;top:40px;left:40px;width:160px;height:28px">More</button>
+ <script>
+ window.__picked = null;
+ window.__fav = [];
+ document.getElementById('trigger').addEventListener('click', () => {{
+ const pop = document.createElement('div');
+ pop.setAttribute('style', 'position:absolute;top:74px;left:40px;width:260px;{pop_style}'
+ + 'background:#fff;border:1px solid #ccc;font:13px sans-serif');
+ pop.innerHTML = {json.dumps(body)};
+ document.body.appendChild(pop);
+ }});
+ </script>
+ </body></html>"""
+
+
+_SECTION_BODY = (
+ '<div>{hdr}<div style="cursor:pointer;height:22px">Alpha</div><div style="cursor:pointer;height:22px">Beta</div></div>'
+ '<div>{hdr2}<div style="cursor:pointer;height:22px">Gamma</div><div style="cursor:pointer;height:22px">Delta</div></div>'
+)
+
+
+@_skip_no_browser
+ ("body", "pop_style", "labels"),
+ [
+ # Nothing declares a row: two one-line actions in one pointer container look like one two-line row.
+ pytest.param(
+ '<div style="height:24px">Keep draft</div><div style="height:24px">Discard draft</div>',
+ "cursor:pointer;",
+ ["Keep draft", "Discard draft"],
+ id="two-actions-in-one-pointer-container",
+ ),
+ pytest.param(
+ _SECTION_BODY.format(hdr="<div>Recent</div>", hdr2="<div>All</div>"),
+ "",
+ ["Alpha", "Beta", "Gamma", "Delta"],
+ id="sections-with-headers",
+ ),
+ pytest.param(
+ _SECTION_BODY.format(hdr="", hdr2=""),
+ "",
+ ["Alpha", "Beta", "Gamma", "Delta"],
+ id="sections-without-headers",
+ ),
+ ],
+)
+async def test_dom_a_roleless_popover_lists_its_clickable_rows_apart(
+ body: str, pop_style: str, labels: list[str]
+) -> None:
+ async with _content_page(_popover_html(body, pop_style)) as page:
+ opened = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#trigger"})
+ assert f"opened a menu of {len(labels)} options" in opened.content, opened.content
+ for label in labels:
+ assert repr(label) in opened.content, opened.content
+
+
+@_skip_no_browser
+ "body",
+ [
+ pytest.param(
+ '<div><a href="#">Contoso Group</a> Account: 400500600</div>'
+ '<div><a href="#">Fabrikam Inc</a> Account: 300400500</div>'
+ '<div><a href="#">Contoso Group</a> Account: 200300400</div>',
+ id="link-beside-its-account-number",
+ ),
+ pytest.param(
+ "<div><button>Copy</button> Ctrl+C</div><div><button>Paste</button> Ctrl+V</div>",
+ id="button-beside-a-shortcut-hint",
+ ),
+ ],
+)
+async def test_dom_a_menu_note_withholds_leaves_beside_bare_text_since_it_may_be_their_identifier(body: str) -> None:
+ async with _content_page(_popover_html(body)) as page:
+ opened = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#trigger"})
+ assert "could not be read as whole rows" in opened.content, opened.content
+ assert "opened a menu of" not in opened.content, opened.content
+
+
+@_skip_no_browser
+async def test_dom_a_menu_row_is_tagged_on_its_label_not_on_a_control_beside_it() -> None:
+ body = "".join(
+ f'<div role="option" style="cursor:pointer;height:24px" onclick="window.__picked={name!r}; this.parentNode.remove()">'
+ f'<button onclick="event.stopPropagation(); window.__fav.push({name!r})">*</button><span>{name}</span></div>'
+ for name in ("Red", "Green", "Blue")
+ )
+ async with _content_page(_popover_html(body)) as page:
+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")
+ opened = await click.handler({"selector": "#trigger"})
+ assert "opened a menu of 3 options" in opened.content, opened.content
+ tag = next(n for n, text in _menu_note_entries(opened.content) if text.endswith("Green"))
+ picked = await click.handler({"selector": f'[data-tv3-menu="{tag}"]'})
+ assert picked.status == "ok", picked.content
+ assert await page.evaluate("() => [window.__picked, window.__fav]") == ["Green", []]
+
+
+@_skip_no_browser
+async def test_dom_a_menu_row_whose_action_is_a_link_is_listed_on_the_link_not_its_badge() -> None:
+ body = "".join(
+ f'<div role="option" style="cursor:pointer;height:24px"><a href="#" onclick="event.preventDefault();'
+ f' window.__picked={name!r}">{name}</a> <span>{badge}</span></div>'
+ for name, badge in (("Inbox", 3), ("Sent", 12), ("Drafts", 1))
+ )
+ async with _content_page(_popover_html(body)) as page:
+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")
+ opened = await click.handler({"selector": "#trigger"})
+ assert "opened a menu of 3 options" in opened.content, opened.content
+ tag = next(n for n, text in _menu_note_entries(opened.content) if text.startswith("Sent"))
+ await click.handler({"selector": f'[data-tv3-menu="{tag}"]'})
+ assert await page.evaluate("() => window.__picked") == "Sent"
+
+
+# A declared row whose label follows a short leading piece: avatar initials, or a flag glyph.
+_LEADING_PIECE_ROW_JS = """(txt, onPick) => {
+ const [lead, name] = txt.split('|');
+ const row = document.createElement('div');
+ row.setAttribute('role', 'option');
+ row.setAttribute('style', 'height:26px;padding:2px 6px;cursor:pointer');
+ row.innerHTML = '<span class="ld" style="margin-right:6px"></span><span class="nm"></span>';
+ row.querySelector('.ld').textContent = lead;
+ row.querySelector('.nm').textContent = name;
+ row.addEventListener('click', () => {
+ onPick();
+ document.getElementById('trigger').textContent = name;
+ });
+ return row;
+}"""
+
+
+@_skip_no_browser
+ ("options", "value"),
+ [
+ pytest.param(["JD|John Doe", "AS|Ann Smith", "BL|Bo Lee"], "Ann Smith", id="avatar-initials"),
+ pytest.param(
+ ["\U0001f1e8\U0001f1e6|Canada", "\U0001f1f2\U0001f1fd|Mexico", "\U0001f1fa\U0001f1f8|United States"],
+ "Mexico",
+ id="flag-glyph",
+ ),
+ ],
+)
+async def test_dom_a_menu_row_with_a_leading_piece_commits_by_name_and_by_its_listed_tag(
+ options: list[str], value: str
+) -> None:
+ html = _LONG_OPTION_MENU_FIXTURE_HTML.replace("__OPTIONS__", json.dumps(options)).replace(
+ "__ROWS__", _LEADING_PIECE_ROW_JS
+ )
+ wanted = next(o for o in options if o.endswith("|" + value))
+ async with _content_page(html) as page:
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "select_combobox").handler(
+ {"selector": "#trigger", "value": value}
+ )
+ assert r.status == "ok", r.content
+ assert await page.evaluate("() => window.__picked") == wanted
+
+ async with _content_page(html) as page:
+ click = _tool(build_browser_tools(_fixed_page_provider(page)), "click")
+ opened = await click.handler({"selector": "#trigger"})
+ assert f"opened a menu of {len(options)} options" in opened.content, opened.content
+ tag = next(n for n, text in _menu_note_entries(opened.content) if text.endswith(value))
+ picked = await click.handler({"selector": f'[data-tv3-menu="{tag}"]'})
+ assert picked.status == "ok", picked.content
+ assert await page.evaluate("() => window.__picked") == wanted
+
+
+# The second "Jane Doe" row remounts, untagged, as soon as a tag lands on it: the full-text read then
+# misses its leaves, and the remaining "Jane Doe" looks unique.
+_REMOUNTING_ROW_JS = (
+ """(txt, onPick) => {
+ const make = """
+ + _LEADING_PIECE_ROW_JS
+ + """;
+ const mount = () => {
+ const row = make(txt, onPick);
+ if (txt.startsWith('J2')) {
+ new MutationObserver(() => {
+ if (row.isConnected && row.querySelector('[data-tv3-menu]')) row.replaceWith(mount());
+ }).observe(row, { subtree: true, attributes: true, attributeFilter: ['data-tv3-menu'] });
+ }
+ return row;
+ };
+ return mount();
+}"""
+)
+
+
+@_skip_no_browser
+async def test_dom_select_combobox_does_not_commit_a_name_whose_twin_row_was_tagged_but_not_read() -> None:
+ options = ["J1|Jane Doe", "AS|Ann Smith", "J2|Jane Doe"]
+ html = _LONG_OPTION_MENU_FIXTURE_HTML.replace("__OPTIONS__", json.dumps(options)).replace(
+ "__ROWS__", _REMOUNTING_ROW_JS
+ )
+ async with _content_page(html) as page:
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "select_combobox").handler(
+ {"selector": "#trigger", "value": "Jane Doe"}
+ )
+ assert r.status == "error", r.content
+ assert await page.evaluate("() => window.__picked") is None
+
+
+# Two "Jane Doe" rows told apart only by an identifier elsewhere in the row, in each shape a row's text can
+# take. `toggled` rows sit inside the trigger and are visible before the click that reveals their text.
+# `shadow` draws the whole row in the option's own open root; `closed` slots the row's light text through a
+# closed root, which no read can enter. `roleless` declares no row: the name is the clickable leaf and the
+# identifier sits in a sibling element. `control` declares the row but its only clickable leaf is a nested button.
+# The open list sits in the field's wrapper, beside its label and a text input that are not part of any row.
+_ROW_SHAPE_PAGE = """<!doctype html><html><head><style>
+ #list:not(.open) .tg { display: none; }
+</style></head><body style="margin:0;font:13px sans-serif">
+ <div id="field"><label>State</label> <input type="text" style="width:80px"></div>
+ <div id="trigger" style="position:absolute;top:40px;left:40px;width:420px;cursor:pointer">
+ <div style="height:120px">Select...</div>
+ </div>
+ <script>
+ window.__picked = null;
+ const SHAPE = __SHAPE__;
+ const ROWS = __ROWS__;
+ const makeRow = ([ident, val]) => {
+ const row = document.createElement('div');
+ if (SHAPE === 'roleless') {
+ row.style.cssText = 'padding:2px 6px';
+ row.innerHTML = '<a class="nm" style="cursor:pointer">Jane Doe</a> <span>' + ident + '</span>';
+ if (val) row.firstChild.setAttribute('data-value', val);
+ row.addEventListener('click', (e) => { e.stopPropagation(); window.__picked = ident; });
+ return row;
+ }
+ row.setAttribute('role', 'option');
+ if (val) row.setAttribute('data-value', val);
+ row.style.cssText = 'padding:2px 6px;cursor:pointer;' + (SHAPE === 'tall' ? 'height:100px' : '');
+ const name = '<span class="nm tg">Jane Doe</span> ';
+ const plain = (html) => '<span class="tg" style="cursor:default">' + html + '</span>';
+ if (SHAPE === 'shadow') {
+ row.attachShadow({ mode: 'open' }).innerHTML = name + '<span>' + ident + '</span>';
+ } else if (SHAPE === 'closed') {
+ row.attachShadow({ mode: 'closed' }).innerHTML = '<b style="font-weight:normal"><slot></slot></b>';
+ row.innerHTML = name + '<span>' + ident + '</span>';
+ } else row.innerHTML = {
+ multi: name + '<span class="tg">' + ident + '</span>',
+ nested: name + plain('<span><b>' + ident + '</b></span>'),
+ bare: name + ident,
+ tall: name + plain(ident),
+ toggled: name + plain(ident),
+ control: '<button type="button" class="nm">Jane Doe</button> ' + plain(ident),
+ }[SHAPE];
+ row.addEventListener('click', (e) => { e.stopPropagation(); window.__picked = ident; });
+ return row;
+ };
+ const build = () => {
+ const list = document.createElement('div');
+ list.id = 'list';
+ if (SHAPE !== 'roleless') list.setAttribute('role', 'listbox');
+ for (const r of ROWS) list.appendChild(makeRow(r));
+ return list;
+ };
+ const trigger = document.getElementById('trigger');
+ if (SHAPE === 'toggled') trigger.appendChild(build());
+ trigger.addEventListener('click', () => {
+ if (SHAPE === 'toggled') { document.getElementById('list').classList.toggle('open'); return; }
+ const ex = document.getElementById('list');
+ if (ex) { ex.remove(); return; }
+ const list = build();
+ list.classList.add('open');
+ list.style.cssText = 'position:absolute;top:170px;left:40px;width:420px;background:#fff';
+ document.getElementById('field').appendChild(list);
+ });
+ </script>
+</body></html>"""
+
+
+@_skip_no_browser
+ "shape", ["multi", "nested", "bare", "tall", "toggled", "shadow", "closed", "roleless", "control"]
+)
+async def test_dom_a_rows_identity_is_all_its_text_whatever_the_rows_shape(shape: str) -> None:
+ # Rows whose full text differs are listed apart and never collapsed; rows whose full text is identical
+ # are refused when a value tells them apart, and otherwise are copies of one row (a re-render duplicate,
+ # pinned here so a change to that is explicit) and the first is clicked.
+ def page_for(rows: list[list[str]]) -> str:
+ return _ROW_SHAPE_PAGE.replace("__SHAPE__", json.dumps(shape)).replace("__ROWS__", json.dumps(rows))
+
+ distinct = [["ID-4471", ""], ["ID-9083", ""]]
+ async with _content_page(page_for(distinct)) as page:
+ note = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#trigger"})
+ listed = re.findall(r"\[data-tv3-menu=\"\d+\"\] '([^']*)'", note.content)
+ # No row is declared around a roleless leaf, so its identifier in a sibling element withholds the note.
+ assert sorted(listed) == ([] if shape == "roleless" else ["Jane Doe ID-4471", "Jane Doe ID-9083"]), note.content
+ # A row's identity is the same whether the list renders it among others or alone, as after a filter.
+ among = await page.evaluate(_MENU_OPTION_TEXTS_JS, {"attr": "menu"})
+ await page.evaluate(
+ "() => { const l = document.getElementById('list'); while (l.children.length > 1) l.lastChild.remove(); }"
+ )
+ alone = await page.evaluate(_MENU_OPTION_TEXTS_JS, {"attr": "menu"})
+ first = {o["n"]: o["identity"] for o in among}
+ assert alone and all(o["identity"] == first[o["n"]] for o in alone), (among, alone)
+ for rows, value in ((distinct, "Jane Doe"), (distinct, "Jane"), ([["ID-4471", "a"], ["ID-4471", "b"]], "Jane Doe")):
+ async with _content_page(page_for(rows)) as page:
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "select_combobox").handler(
+ {"selector": "#trigger", "value": value}
+ )
+ assert r.status == "error", (rows, value, r.content)
+ assert await page.evaluate("() => window.__picked") is None, (rows, value, r.content)
+ if rows is distinct:
+ assert "ID-4471" in r.content and "ID-9083" in r.content, r.content
+ async with _content_page(page_for([["ID-4471", ""], ["ID-4471", ""]])) as page:
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "select_combobox").handler(
+ {"selector": "#trigger", "value": "Jane Doe"}
+ )
+ assert "ID-4471" in str(await page.evaluate("() => window.__picked")), r.content
+
+
+_CATEGORY_TREE_PAGE = """<!doctype html><html><body style="margin:0;font:13px sans-serif">
+ <div id="trigger" style="position:absolute;top:40px;left:40px;width:300px;height:30px;cursor:pointer">Pick...</div>
+ <script>
+ document.getElementById('trigger').addEventListener('click', () => {
+ const tree = document.createElement('div');
+ tree.setAttribute('role', 'tree');
+ tree.style.cssText = 'position:absolute;top:80px;left:40px;width:300px;background:#fff';
+ const item = (label, kids) => '<div role="treeitem" style="cursor:pointer;padding:2px"'
+ + (kids ? ' aria-expanded="true"' : '') + '><span>' + label + '</span>'
+ + (kids ? '<div role="group" style="padding-left:12px">' + kids.map((k) => item(k)).join('') + '</div>' : '')
+ + '</div>';
+ tree.innerHTML = item('Fruits', ['Apple', 'Banana', 'Cherry']) + item('Vegetables', ['Leek', 'Onion']);
+ document.body.appendChild(tree);
+ });
+ </script></body></html>"""
+
+
+@_skip_no_browser
+async def test_dom_a_category_row_is_listed_by_its_own_label_not_its_child_rows() -> None:
+ async with _content_page(_CATEGORY_TREE_PAGE) as page:
+ note = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#trigger"})
+ listed = re.findall(r"\[data-tv3-menu=\"\d+\"\] '([^']*)'", note.content)
+ assert listed == ["Fruits", "Apple", "Banana", "Cherry", "Vegetables", "Leek", "Onion"], note.content
+
+
+@_skip_no_browser
+async def test_dom_select_combobox_weighs_a_declared_set_size_against_rows_not_leaves() -> None:
+ # 4 two-leaf rows of a list declaring 6: the leaves read back (8) outnumber the declared rows, but the
+ # exact "United States" row is not rendered, so the prefix row must not be committed in its place.
+ options = ["US|United States Minor Outlying Islands", "CA|Canada", "MX|Mexico", "GB|United Kingdom"]
+ rows_js = (
+ "(txt, onPick) => { const row = ("
+ + _LEADING_PIECE_ROW_JS
+ + ")(txt, onPick); row.setAttribute('aria-setsize', '6'); return row; }"
+ )
+ html = _LONG_OPTION_MENU_FIXTURE_HTML.replace("__OPTIONS__", json.dumps(options)).replace("__ROWS__", rows_js)
+ async with _content_page(html) as page:
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "select_combobox").handler(
+ {"selector": "#trigger", "value": "United States"}
+ )
+ assert r.status == "error", r.content
+ assert await page.evaluate("() => window.__picked") is None
+
+
@_skip_no_browser
@pytest.mark.asyncio
async def test_dom_a_menu_note_counts_the_items_a_menu_declares() -> None:
@@ -17222,6 +17722,27 @@ async def test_pending_marker_reads_a_frozen_submit_control() -> None:
assert await _pending_marker_of(page, "#done") is None
+@_skip_no_browser
+async def test_pending_marker_masks_a_hidden_value_straddling_its_cap() -> None:
+ # The finish gate quotes the marker to the model: a cut before the mask would leave a fragment in clear.
+ secret = "Qz7Kp4Wm9Xr2Vt6Ny3Lb8Hc5"
+ fragments = {secret[:k] for k in range(4, len(secret) + 1)}
+ ctx = SkyvernContext(organization_id="o_1")
+ ctx.register_secret_value(secret, hide_from_model=True)
+ skyvern_context.set(ctx)
+ try:
+ for start in (60 - len(secret) + 4, 60 - 4):
+ label = "Processing " + "." * (start - len("Processing ")) + secret
+ async with _live_page(f'<button id="busy" aria-busy="true">{label}</button>') as page:
+ marker = await _pending_marker_of(page, "#busy")
+ assert marker and marker.endswith(" (aria-busy)"), (start, marker)
+ leaked = sorted(f for f in fragments if f in ctx.hide_from_model(marker))
+ assert not leaked, (start, leaked, marker)
+ finally:
+ skyvern_context.reset()
+
+
@_skip_no_browser
@pytest.mark.asyncio
async def test_pending_marker_never_reads_a_text_inputs_typed_value() -> None:
@@ -23661,6 +24182,8 @@ def _popup_search_combobox_html(
filtered_tag: str = "",
row_class: str = "",
spinner_class: str = "spinner",
+ roleless: bool = False,
+ bare_div_note: str = "",
) -> str:
# A click-to-open anchor (not typeable) whose popup renders only the first `window` matching rows and,
# optionally, its own filter input. The form also holds an unrelated text input, plus one the open-click
@@ -23678,6 +24201,9 @@ def _popup_search_combobox_html(
# read's own microtask. `page_busy` adds a visible "progressbar" or "spinner" beside the field, outside the popup.
# `spinner_until_twin` shows a class-only spinner inside the popup until the late twin lands; `filtered_tag`
# builds the filtered rows from that element instead of <li>; `row_class` sets every row's class attribute.
+ # `roleless` declares no role on the list or its rows. `bare_div_note` ("header" or "footer") builds the list
+ # and its rows from <div>s that do not scroll, with a match count inside the list: a <p> above the rows, or a bare
+ # text node below them.
states = json.dumps(items or _POPUP_SEARCH_STATES[:rows])
return f"""
<!doctype html><html><body style="margin:0">
@@ -23720,6 +24246,8 @@ def _popup_search_combobox_html(
var FILTERED_TAG = {json.dumps(filtered_tag)};
var ROW_CLASS = {json.dumps(row_class)};
var SPINNER_CLASS = {json.dumps(spinner_class)};
+ var ROLELESS = {"true" if roleless else "false"};
+ var BARE_NOTE = {json.dumps(bare_div_note)};
var pending = null, twin = null, queryAt = 0, tripped = false;
var anchor = document.getElementById('state');
var hidden = document.getElementById('state-value');
@@ -23745,9 +24273,22 @@ def _popup_search_combobox_html(
if (!q) return true;
return BY_CODE ? s[1].toLowerCase().indexOf(q) === 0 : s[0].toLowerCase().indexOf(q) === 0;
}});
- matched.slice(0, q && FILTERED_WINDOW ? FILTERED_WINDOW : WINDOW).forEach(function (s) {{
- var li = document.createElement(q && FILTERED_TAG ? FILTERED_TAG : 'li');
- li.setAttribute('role', 'option');
+ var shown = matched.slice(0, q && FILTERED_WINDOW ? FILTERED_WINDOW : WINDOW);
+ var note = null;
+ if (BARE_NOTE) {{
+ var count = shown.length + (shown.length === 1 ? ' result' : ' results');
+ if (BARE_NOTE === 'header') {{
+ note = document.createElement('p');
+ note.style.cssText = 'margin:0;height:18px;color:#666';
+ note.textContent = count;
+ list.appendChild(note);
+ }} else {{
+ note = document.createTextNode(count);
+ }}
+ }}
+ shown.forEach(function (s) {{
+ var li = document.createElement(q && FILTERED_TAG ? FILTERED_TAG : BARE_NOTE ? 'div' : 'li');
+ if (!ROLELESS) li.setAttribute('role', 'option');
if (ROW_CLASS) li.className = ROW_CLASS;
if (SETSIZE) li.setAttribute('aria-setsize', String(matched.length));
if (UNKNOWN_SIZE) li.setAttribute('aria-setsize', '-1');
@@ -23763,6 +24304,7 @@ def _popup_search_combobox_html(
}});
list.appendChild(li);
}});
+ if (note && BARE_NOTE === 'footer') list.appendChild(note);
if (q && VIRTUAL_PX) {{
var spacer = document.createElement('li');
spacer.setAttribute('aria-hidden', 'true');
@@ -23814,9 +24356,10 @@ def _popup_search_combobox_html(
if (FLAT) box.style.cssText = 'position:absolute;left:0;top:32px;width:236px;z-index:5';
popup.appendChild(box);
}}
- list = document.createElement('ul');
- list.setAttribute('role', 'listbox');
- list.style.cssText = 'list-style:none;margin:0;padding:0;max-height:240px;overflow-y:auto;background:#fff'
+ list = document.createElement(BARE_NOTE ? 'div' : 'ul');
+ if (!ROLELESS) list.setAttribute('role', 'listbox');
+ list.style.cssText = (BARE_NOTE ? 'margin:0;padding:0;background:#fff'
+ : 'list-style:none;margin:0;padding:0;max-height:240px;overflow-y:auto;background:#fff')
+ (FLAT ? ';position:absolute;left:0;top:58px;width:240px;z-index:5' : '');
if (TWIN_ON_READ) new MutationObserver(function () {{
if (tripped || !queryAt || performance.now() - queryAt < 3500) return;
@@ -23952,6 +24495,29 @@ async def test_select_combobox_popup_filter_ignores_a_busy_indicator_outside_its
assert await page.eval_on_selector("#state-value", "el => el.value") == "Texas", r.content
+@_skip_no_browser
[email protected]("bare_div_note", ["", "header", "footer"], ids=["ul", "div_header", "div_footer"])
+async def test_select_combobox_popup_filter_commits_the_lone_row_of_a_roleless_list(bare_div_note: str) -> None:
+ # A roleless window of a longer list, which its search box filters to one row: that row's identity is its own
+ # text, not the text of the list or popup around it (a match count, the search box, the trigger), so it is the
+ # same row the open window showed. The div shapes have no list element and no scroller to stop at.
+ items = [("North Carolina", "NC"), *(s for s in _POPUP_SEARCH_STATES[:31] if s[0] != "North Carolina")]
+ html = _popup_search_combobox_html(
+ items=items,
+ declared_rows=60,
+ filtered_window=1,
+ roleless=True,
+ bare_div_note=bare_div_note,
+ window=8 if bare_div_note else 25,
+ )
+ async with _content_page(html) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "select_combobox").handler({"selector": "#state", "value": "North Carolina"})
+ assert r.status == "ok", r.content
+ assert await page.eval_on_selector("#state-value", "el => el.value") == "North Carolina", r.content
+
+
@_skip_no_browser
@pytest.mark.asyncio
async def test_select_combobox_popup_filter_refuses_a_prefix_hit_the_open_list_showed_two_of() -> None:
@@ -34286,6 +34852,145 @@ async def test_get_html_redacts_a_hidden_secret_before_a_window_boundary_can_spl
skyvern_context.reset()
+def _straddle_renders() -> list[tuple[str, int, Callable[[str], str]]]:
+ def rows_error(rows: list[dict[str, Any]]) -> str:
+ return taskv3_tools._identical_text_rows_error("#who", "Jane Doe", rows, menu_open=True).content
+
+ return [
+ (
+ "menu-note",
+ taskv3_tools._MENU_ROW_TEXT_MAX,
+ lambda t: taskv3_tools._menu_open_note(
+ {"count": 2, "options": [{"n": 1, "text": t}, {"n": 2, "text": "B"}]}, "#m"
+ ),
+ ),
+ ("identical-rows-text", 60, lambda t: rows_error([{"n": 1, "text": t}, {"n": 2, "text": t}])),
+ (
+ "ambiguous-rows-text",
+ 60,
+ lambda t: taskv3_tools._ambiguous_rows_error(
+ "#who", "John", [{"n": 1, "text": t}, {"n": 2, "text": "B"}], next_step="pass the full text"
+ ).content,
+ ),
+ (
+ "row-identity",
+ taskv3_tools._MENU_ROW_TEXT_MAX,
+ lambda t: rows_error(
+ [{"n": 1, "text": "Jane Doe", "identity": t}, {"n": 2, "text": "Jane Doe", "identity": "B"}]
+ ),
+ ),
+ (
+ "value",
+ 60,
+ lambda t: rows_error([{"n": 1, "text": "Jane Doe", "val": t}, {"n": 2, "text": "Jane Doe", "val": "B"}]),
+ ),
+ ("label", 60, lambda t: rows_error([{"n": 1, "text": "Jane Doe", "label": t}, {"n": 2, "text": "Jane Doe"}])),
+ (
+ "declared-values",
+ 60,
+ lambda t: rows_error(
+ [{"n": 1, "text": "Jane Doe", "vals": [f"data-code={t}"]}, {"n": 2, "text": "Jane Doe"}]
+ ),
+ ),
+ ]
+
+
[email protected](("cap", "render"), [pytest.param(c, r, id=i) for i, c, r in _straddle_renders()])
+def test_a_row_listing_masks_a_hidden_value_before_any_cut_so_no_fragment_of_it_survives(
+ cap: int, render: Callable[[str], str]
+) -> None:
+ # The loop hides values by whole-substring replacement AFTER the tool returns, so a value a cut split
+ # matches nothing there: every render must mask before it cuts, at every offset around its cap.
+ from skyvern.forge.sdk.core import skyvern_context
+ from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+
+ secret = "Qz7Kp4Wm9Xr2Vt6Ny3Lb8Hc5"
+ fragments = {secret[:k] for k in range(4, len(secret) + 1)} | {secret[-k:] for k in range(4, len(secret) + 1)}
+ ctx = SkyvernContext(organization_id="o_1")
+ ctx.register_secret_value(secret, hide_from_model=True)
+ skyvern_context.set(ctx)
+ try:
+ for start in range(cap - len(secret) - 1, cap + 2):
+ shown = ctx.hide_from_model(render("." * start + secret))
+ leaked = sorted(f for f in fragments if f in shown)
+ assert not leaked, (start, leaked, shown)
+ finally:
+ skyvern_context.reset()
+
+
+@_skip_no_browser
+async def test_dom_a_menu_note_masks_a_hidden_value_straddling_the_row_cap() -> None:
+ # The same property as above, fed by the finder's real output: a cut of the row text inside the page
+ # script would split the value before any mask could see it whole.
+ secret = "Qz7Kp4Wm9Xr2Vt6Ny3Lb8Hc5"
+ cap = taskv3_tools._MENU_ROW_TEXT_MAX
+ fragments = {secret[:k] for k in range(4, len(secret) + 1)}
+ ctx = SkyvernContext(organization_id="o_1")
+ ctx.register_secret_value(secret, hide_from_model=True)
+ skyvern_context.set(ctx)
+ try:
+ for start in (cap - len(secret) + 4, cap - len(secret) // 2, cap - 4):
+ ident = "." * (start - len("Jane Doe ")) + secret
+ rows = [[ident, ""], ["ID-9083", ""]]
+ html = _ROW_SHAPE_PAGE.replace("__SHAPE__", json.dumps("tall")).replace("__ROWS__", json.dumps(rows))
+ async with _content_page(html) as page:
+ opened = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler(
+ {"selector": "#trigger"}
+ )
+ note = ctx.hide_from_model(opened.content[: (opened.data or {}).get("delta_at", len(opened.content))])
+ assert "'Jane Doe ID-9083'" in note, (start, note)
+ leaked = sorted(f for f in fragments if f in note)
+ assert not leaked, (start, leaked, note)
+ finally:
+ skyvern_context.reset()
+
+
+_UNDECLARED_SUGGESTION_PAGE = """<!doctype html><html><body style="margin:0;font:13px sans-serif">
+ <input id="city" type="text" autocomplete="off" style="position:absolute;top:40px;left:40px;width:300px;height:24px">
+ <div id="list" style="position:absolute;top:70px;left:40px;width:600px;background:#fff"></div>
+ <script>
+ const city = document.getElementById('city'), list = document.getElementById('list');
+ city.addEventListener('input', () => {
+ list.innerHTML = '';
+ if (city.value.trim().length < 3) return;
+ const row = document.createElement('div');
+ row.style.cssText = 'cursor:pointer;height:22px;white-space:nowrap';
+ row.textContent = window.__label;
+ row.addEventListener('mousedown', (e) => e.preventDefault());
+ row.addEventListener('click', () => { city.value = window.__label; list.innerHTML = ''; });
+ list.appendChild(row);
+ });
+ </script></body></html>"""
+
+
+@_skip_no_browser
+async def test_dom_a_typeahead_suggestion_masks_a_hidden_value_straddling_its_display_cap() -> None:
+ # The picked row's text reaches the model in the typeahead's verdict, cut to 60 for display: a cut
+ # before the mask would split the value so the loop's whole-substring mask could not see it.
+ secret = "Qz7Kp4Wm9Xr2Vt6Ny3Lb8Hc5"
+ fragments = {secret[:k] for k in range(4, len(secret) + 1)}
+ ctx = SkyvernContext(organization_id="o_1")
+ ctx.register_secret_value(secret, hide_from_model=True)
+ skyvern_context.set(ctx)
+ try:
+ for start in (60 - len(secret) + 4, 60 - len(secret) // 2, 60 - 4):
+ label = "San Diego " + "." * (start - len("San Diego ")) + secret
+ for tool, args in (("type", {"text": "San Diego"}), ("select_combobox", {"value": "San Diego"})):
+ async with _content_page(_UNDECLARED_SUGGESTION_PAGE) as page:
+ await page.evaluate("(l) => { window.__label = l; }", label)
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), tool).handler(
+ {"selector": "#city", **args}
+ )
+ shown = ctx.hide_from_model(r.content)
+ assert "selected" in shown, (tool, start, shown)
+ leaked = sorted(f for f in fragments if f in shown)
+ assert not leaked, (tool, start, leaked, shown)
+ finally:
+ skyvern_context.reset()
+
+
@pytest.mark.asyncio
async def test_get_html_offset_normalization_survives_the_selector_guards_arg_copy() -> None:
# `_with_selector_guard` hands the handler a COPY of args whenever a selector is present, and the