Skip to content

Automate browser based workflows with AI

fix(SKY-11933): harden custom-select is_choice_input classification (#7287)

Marc Kelechavab72395646304
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 2acc3be50..b9a2dfd7b 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -10,7 +10,7 @@ import uuid
from collections import deque
from datetime import datetime, timedelta, timezone
from pathlib import Path
-from typing import Any, Awaitable, Callable, List, NotRequired, TypedDict
+from typing import Any, Awaitable, Callable, List, TypedDict
import structlog
from fuzzysearch import find_near_matches
@@ -5963,8 +5963,9 @@ def _collect_option_texts(elements: list[dict]) -> list[str]:
return out
-def _custom_select_choice_input_ids(node: dict) -> set[str]:
+def _custom_select_descendant_choice_inputs(node: dict) -> tuple[set[str], bool]:
input_ids: set[str] = set()
+ contains_choice_input = False
queue: deque[dict] = deque(node.get("children") or [])
while queue:
child = queue.popleft()
@@ -5974,11 +5975,13 @@ def _custom_select_choice_input_ids(node: dict) -> set[str]:
attrs = child.get("attributes") or {}
input_type = str(attrs.get("type") or "").lower()
element_id = str(child.get("id") or "")
- if tag == "input" and input_type in ("checkbox", "radio") and element_id:
- input_ids.add(element_id)
+ if tag == "input" and input_type in ("checkbox", "radio"):
+ contains_choice_input = True
+ if element_id:
+ input_ids.add(element_id)
for grandchild in child.get("children") or []:
queue.append(grandchild)
- return input_ids
+ return input_ids, contains_choice_input
def _custom_select_choice_value(node: dict) -> str | None:
@@ -6013,17 +6016,17 @@ class _CustomSelectCandidate(TypedDict):
label: str | None
element_id: str | None
value: str | None
- is_choice_input: NotRequired[bool]
+ is_choice_input: bool
def _custom_select_candidates_from_elements(elements: list[dict]) -> list[_CustomSelectCandidate]:
- queue: deque[tuple[dict, bool]] = deque((element, False) for element in elements)
+ queue: deque[tuple[dict, bool, bool]] = deque((element, False, False) for element in elements)
candidates: list[_CustomSelectCandidate] = []
seen: set[tuple[str | None, str | None, str | None]] = set()
covered_choice_input_ids: set[str] = set()
while queue:
- node, in_choice_surface = queue.popleft()
+ node, in_choice_surface, in_multiselectable = queue.popleft()
if not isinstance(node, dict):
continue
@@ -6034,15 +6037,18 @@ def _custom_select_candidates_from_elements(elements: list[dict]) -> list[_Custo
element_id = str(node.get("id") or "") or None
value = _custom_select_choice_value(node)
label = _select_shadow_label_from_node(node) or value
- choice_input_ids = _custom_select_choice_input_ids(node)
+ choice_input_ids, contains_choice_input = _custom_select_descendant_choice_inputs(node)
is_choice_input = tag == "input" and input_type in ("checkbox", "radio")
is_option_node = role in _CUSTOM_SELECT_CHOICE_ROLES or tag == "option" or (tag == "li" and in_choice_surface)
- is_label_choice = tag == "label" and bool(choice_input_ids)
- # Whether the candidate is checkbox/radio-shaped, computed here (rather than re-derived
- # from the resolved element later) because only the full tree walk sees an option/label
- # wrapper around a checkbox/radio: the emitted candidate is the wrapper, not the covered
- # <input> itself.
- is_choice_input_shape = is_choice_input or bool(choice_input_ids) or role in _CUSTOM_SELECT_CHOICE_INPUT_ROLES
+ is_label_choice = tag == "label" and contains_choice_input
+ # Only the full tree walk can see toggle semantics inherited from wrappers or an ancestor
+ # multiselect container; the resolved candidate element alone does not carry that context.
+ is_choice_input_shape = (
+ is_choice_input
+ or contains_choice_input
+ or role in _CUSTOM_SELECT_CHOICE_INPUT_ROLES
+ or (in_multiselectable and role in {"option", "treeitem"})
+ )
has_choice_state = "aria-selected" in attrs or "aria-checked" in attrs
is_clickable_choice = (
bool(node.get("interactable"))
@@ -6075,8 +6081,12 @@ def _custom_select_candidates_from_elements(elements: list[dict]) -> list[_Custo
covered_choice_input_ids.update(choice_input_ids)
child_in_choice_surface = in_choice_surface or _is_custom_select_choice_surface(role)
+ aria_multiselectable = attrs.get("aria-multiselectable")
+ child_in_multiselectable = in_multiselectable or (
+ isinstance(aria_multiselectable, str) and aria_multiselectable.lower() == "true"
+ )
for child in node.get("children") or []:
- queue.append((child, child_in_choice_surface))
+ queue.append((child, child_in_choice_surface, child_in_multiselectable))
return candidates
@@ -6108,6 +6118,8 @@ _CUSTOM_SELECT_MATCHED_STATE_JS = r"""
].map(normalize).find(Boolean) || "";
const role = normalize(el.getAttribute("role"));
const nestedChoice = el.querySelector?.("input[type='checkbox'], input[type='radio']");
+ const multiselectable = el.closest?.("[aria-multiselectable]");
+ const inMultiselectable = normalize(multiselectable?.getAttribute("aria-multiselectable")) === "true";
const ariaSelected = el.getAttribute("aria-selected") === "true";
const ariaChecked = el.getAttribute("aria-checked") === "true";
const selectedAttr = el.hasAttribute("selected") || el.selected === true;
@@ -6115,7 +6127,16 @@ _CUSTOM_SELECT_MATCHED_STATE_JS = r"""
(el.matches?.("input[type='checkbox'], input[type='radio']") && el.checked)
|| nestedChoice?.checked
);
- return { label, role, ariaSelected, ariaChecked, selectedAttr, checked };
+ return {
+ label,
+ role,
+ nestedChoice: nestedChoice != null,
+ inMultiselectable,
+ ariaSelected,
+ ariaChecked,
+ selectedAttr,
+ checked
+ };
}
"""
@@ -6262,7 +6283,9 @@ def _custom_select_matched_state_confirms_pre_click(state: dict | None, expected
return False
if any(bool(state.get(field)) for field in ("ariaChecked", "selectedAttr", "checked")):
return True
- if str(state.get("role") or "").lower() == "option":
+ # In an aria-multiselectable container aria-selected IS the committed state (clicking would
+ # toggle it off); only single-select options treat bare aria-selected as a keyboard highlight.
+ if str(state.get("role") or "").lower() == "option" and not bool(state.get("inMultiselectable")):
return False
return bool(state.get("ariaSelected"))
@@ -6437,9 +6460,9 @@ async def _select_deterministic_custom_option(
matched_candidate = option_candidates[resolution.matched_index]
element_id = matched_candidate.get("element_id")
matched_label = resolution.matched_label
- # Computed by the tree walk in _custom_select_candidates_from_elements, which sees the
- # label-wraps-checkbox case; the resolved element below may just be the <label>.
- matched_option_is_choice_input = bool(matched_candidate.get("is_choice_input"))
+ # Computed by the tree walk, which sees wrapper and multiselect-container toggle semantics
+ # that are not necessarily present on the resolved element itself.
+ matched_option_is_choice_input = matched_candidate["is_choice_input"]
if not element_id:
return None
@@ -6450,6 +6473,15 @@ async def _select_deterministic_custom_option(
if await selected_element.get_attr("role") == "listbox":
return None
+ matched_state = await _read_custom_select_matched_state(selected_element)
+ live_role = str((matched_state or {}).get("role") or "").lower()
+ live_toggle_shaped = (
+ (bool((matched_state or {}).get("inMultiselectable")) and live_role in {"option", "treeitem"})
+ or live_role in _CUSTOM_SELECT_CHOICE_INPUT_ROLES
+ or bool((matched_state or {}).get("nestedChoice"))
+ )
+ matched_option_is_choice_input = matched_option_is_choice_input or live_toggle_shaped
+
readback_scope_element = await _resolve_custom_select_readback_scope_element(
get_readback_scope_element=get_readback_scope_element,
target_value=target_value,
@@ -6460,7 +6492,6 @@ async def _select_deterministic_custom_option(
expected_label = _normalize_select_shadow_text(matched_label)
if expected_label:
- matched_state = await _read_custom_select_matched_state(selected_element)
if _custom_select_matched_state_confirms_pre_click(matched_state, expected_label):
return ActionSuccess(), matched_label
if await _custom_select_scope_confirms_committed(
@@ -6507,9 +6538,8 @@ async def _select_deterministic_custom_option(
return None
if not matched_option_is_choice_input:
- # A non-checkbox/radio option (e.g. a button/div-anchored single-select listbox) can be
- # safely replayed by the LLM mini-agent; only checkbox/radio panels risk toggling into an
- # unintended state on a retry, so only those hard-fail below.
+ # A non-toggle option (e.g. a button/div-anchored single-select listbox) can be safely
+ # replayed by the LLM mini-agent. Toggle-shaped options hard-fail below instead.
LOG.info(
"Deterministic custom-select read-back inconclusive on non-choice-input option; routing to LLM fallback",
target_value=target_value,
diff --git a/tests/unit/test_no_available_option.py b/tests/unit/test_no_available_option.py
index c353c1aa6..890391513 100644
--- a/tests/unit/test_no_available_option.py
+++ b/tests/unit/test_no_available_option.py
@@ -79,11 +79,16 @@ def _stub_evaluate(
"""
matched_states = list(matched_state) if isinstance(matched_state, list) else None
+ def _matched_state_with_defaults(state: dict | None) -> dict | None:
+ if state is None:
+ return None
+ return {"role": "", "nestedChoice": False, "inMultiselectable": False, **state}
+
async def _evaluate(*, frame: object, expression: str, arg: object = None) -> object:
- if "return { label," in expression:
+ if "const nestedChoice =" in expression:
if matched_states is not None:
- return matched_states.pop(0) if matched_states else None
- return matched_state
+ return _matched_state_with_defaults(matched_states.pop(0)) if matched_states else None
+ return _matched_state_with_defaults(matched_state)
if "anchorIsComboboxInput" in expression:
return committed
return None
@@ -389,6 +394,97 @@ class TestCustomSelectCandidates:
{"label": "Choice", "element_id": "opt-choice", "value": None, "is_choice_input": True}
]
+ @pytest.mark.parametrize(
+ ("role", "aria_multiselectable", "expected"),
+ [
+ ("option", "true", True),
+ ("treeitem", "TRUE", True),
+ ("option", None, False),
+ ("treeitem", "false", False),
+ ("option", True, False),
+ ],
+ )
+ def test_multiselectable_container_marks_option_shapes_as_choice_inputs(
+ self,
+ role: str,
+ aria_multiselectable: str | bool | None,
+ expected: bool,
+ ) -> None:
+ container_attributes: dict[str, object] = {"role": "tree" if role == "treeitem" else "listbox"}
+ if aria_multiselectable is not None:
+ container_attributes["aria-multiselectable"] = aria_multiselectable
+ tree = [
+ {
+ "id": "choice-panel",
+ "tagName": "div",
+ "attributes": container_attributes,
+ "children": [
+ {
+ "id": "choice-alpha",
+ "tagName": "div",
+ "attributes": {"role": role, "aria-selected": "false"},
+ "text": "Alpha",
+ "interactable": True,
+ }
+ ],
+ }
+ ]
+
+ assert _custom_select_candidates_from_elements(tree) == [
+ {"label": "Alpha", "element_id": "choice-alpha", "value": None, "is_choice_input": expected}
+ ]
+
+ def test_label_wrapping_idless_checkbox_is_choice_input_shaped(self) -> None:
+ tree = [
+ {
+ "id": "choice-label",
+ "tagName": "label",
+ "text": "Alpha",
+ "interactable": True,
+ "children": [
+ {
+ "tagName": "input",
+ "attributes": {"type": "checkbox"},
+ "interactable": True,
+ }
+ ],
+ }
+ ]
+
+ assert _custom_select_candidates_from_elements(tree) == [
+ {"label": "Alpha", "element_id": "choice-label", "value": None, "is_choice_input": True}
+ ]
+
+ def test_every_candidate_includes_is_choice_input(self) -> None:
+ candidates = _custom_select_candidates_from_elements(
+ [
+ {
+ "id": "choice-panel",
+ "tagName": "div",
+ "attributes": {"role": "listbox"},
+ "children": [
+ {
+ "id": "choice-alpha",
+ "tagName": "div",
+ "attributes": {"role": "option"},
+ "text": "Alpha",
+ "interactable": True,
+ }
+ ],
+ },
+ {
+ "id": "choice-radio",
+ "tagName": "div",
+ "attributes": {"role": "radio"},
+ "text": "Beta",
+ "interactable": True,
+ },
+ ]
+ )
+
+ assert candidates
+ assert all(isinstance(candidate["is_choice_input"], bool) for candidate in candidates)
+
class TestDeterministicCustomSelect:
@pytest.mark.asyncio
@@ -405,8 +501,8 @@ class TestDeterministicCustomSelect:
result = await _select_deterministic_custom_option(
target_value="United States",
get_option_candidates=lambda: [
- {"label": "United States", "element_id": "us-1", "value": "US"},
- {"label": "United States", "element_id": "us-2", "value": "USA"},
+ {"label": "United States", "element_id": "us-1", "value": "US", "is_choice_input": False},
+ {"label": "United States", "element_id": "us-2", "value": "USA", "is_choice_input": False},
],
field_context={},
page=MagicMock(),
@@ -431,7 +527,9 @@ class TestDeterministicCustomSelect:
result = await _select_deterministic_custom_option(
target_value="Job Board",
- get_option_candidates=lambda: [{"label": "Job Board", "element_id": "source-job-board", "value": None}],
+ get_option_candidates=lambda: [
+ {"label": "Job Board", "element_id": "source-job-board", "value": None, "is_choice_input": False}
+ ],
field_context={},
page=MagicMock(),
get_skyvern_element=get_skyvern_element,
@@ -449,7 +547,9 @@ class TestDeterministicCustomSelect:
result = await _select_deterministic_custom_option(
target_value="15",
- get_option_candidates=lambda: [{"label": "15", "element_id": "day-15", "value": None}],
+ get_option_candidates=lambda: [
+ {"label": "15", "element_id": "day-15", "value": None, "is_choice_input": False}
+ ],
field_context={"is_date_related": True},
page=MagicMock(),
get_skyvern_element=get_skyvern_element,
@@ -473,7 +573,9 @@ class TestDeterministicCustomSelect:
result = await _select_deterministic_custom_option(
target_value="Job Board",
- get_option_candidates=lambda: [{"label": "Job Board", "element_id": "source-panel", "value": None}],
+ get_option_candidates=lambda: [
+ {"label": "Job Board", "element_id": "source-panel", "value": None, "is_choice_input": False}
+ ],
field_context={},
page=MagicMock(),
get_skyvern_element=AsyncMock(return_value=selected_element),
@@ -513,7 +615,55 @@ class TestDeterministicCustomSelect:
result = await _select_deterministic_custom_option(
target_value="Choice",
- get_option_candidates=lambda: [{"label": "Choice", "element_id": "choice-radio", "value": None}],
+ get_option_candidates=lambda: [
+ {"label": "Choice", "element_id": "choice-radio", "value": None, "is_choice_input": True}
+ ],
+ field_context={},
+ page=MagicMock(),
+ get_skyvern_element=AsyncMock(return_value=selected_element),
+ task=_task(), # type: ignore[arg-type]
+ )
+
+ assert result is not None
+ action_result, matched_label = result
+ assert action_result.success
+ assert matched_label == "Choice"
+ selected_element.click.assert_not_awaited()
+ selected_element.scroll_into_view.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_already_selected_multiselectable_option_returns_success_without_clicking(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(
+ handler.app,
+ "EXPERIMENTATION_PROVIDER",
+ SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ )
+ monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
+ monkeypatch.setattr(
+ handler.SkyvernFrame,
+ "evaluate",
+ _stub_evaluate(
+ matched_state={
+ "label": "Choice",
+ "role": "option",
+ "inMultiselectable": True,
+ "ariaSelected": True,
+ "ariaChecked": False,
+ "selectedAttr": False,
+ "checked": False,
+ }
+ ),
+ )
+ selected_element = _FakeCustomElement(role="option")
+ selected_element.get_locator().count = AsyncMock(return_value=1)
+
+ result = await _select_deterministic_custom_option(
+ target_value="Choice",
+ get_option_candidates=lambda: [
+ {"label": "Choice", "element_id": "choice-option", "value": None, "is_choice_input": True}
+ ],
field_context={},
page=MagicMock(),
get_skyvern_element=AsyncMock(return_value=selected_element),
@@ -549,7 +699,9 @@ class TestDeterministicCustomSelect:
result = await _select_deterministic_custom_option(
target_value="WAKE",
- get_option_candidates=lambda: [{"label": "WAKE", "element_id": "county-wake", "value": "WAKE"}],
+ get_option_candidates=lambda: [
+ {"label": "WAKE", "element_id": "county-wake", "value": "WAKE", "is_choice_input": True}
+ ],
field_context={},
page=MagicMock(),
get_skyvern_element=AsyncMock(return_value=selected_element),
@@ -577,6 +729,8 @@ class TestDeterministicCustomSelect:
{
"label": "Job Board",
"role": "option",
+ "nestedChoice": False,
+ "inMultiselectable": False,
"ariaSelected": True,
"ariaChecked": False,
"selectedAttr": False,
@@ -585,6 +739,8 @@ class TestDeterministicCustomSelect:
{
"label": "Job Board",
"role": "option",
+ "nestedChoice": False,
+ "inMultiselectable": False,
"ariaSelected": True,
"ariaChecked": False,
"selectedAttr": False,
@@ -594,7 +750,7 @@ class TestDeterministicCustomSelect:
scope_args: list[dict[str, object]] = []
async def evaluate(*, frame: object, expression: str, arg: object = None) -> object:
- if "return { label," in expression:
+ if "const nestedChoice =" in expression:
return matched_states.pop(0)
if "anchorIsComboboxInput" in expression:
assert isinstance(arg, list)
@@ -610,7 +766,9 @@ class TestDeterministicCustomSelect:
result = await _select_deterministic_custom_option(
target_value="Job Board",
- get_option_candidates=lambda: [{"label": "Job Board", "element_id": "source-job-board", "value": None}],
+ get_option_candidates=lambda: [
+ {"label": "Job Board", "element_id": "source-job-board", "value": None, "is_choice_input": False}
+ ],
field_context={},
page=MagicMock(),
get_skyvern_element=AsyncMock(return_value=selected_element),
@@ -648,7 +806,9 @@ class TestDeterministicCustomSelect:
result = await _select_deterministic_custom_option(
target_value="WAKE",
- get_option_candidates=lambda: [{"label": "WAKE", "element_id": "county-wake", "value": "WAKE"}],
+ get_option_candidates=lambda: [
+ {"label": "WAKE", "element_id": "county-wake", "value": "WAKE", "is_choice_input": True}
+ ],
field_context={},
page=MagicMock(),
get_skyvern_element=AsyncMock(return_value=selected_element),
@@ -686,7 +846,9 @@ class TestDeterministicCustomSelect:
result = await _select_deterministic_custom_option(
target_value="Job Board",
- get_option_candidates=lambda: [{"label": "Job Board", "element_id": "source-job-board", "value": None}],
+ get_option_candidates=lambda: [
+ {"label": "Job Board", "element_id": "source-job-board", "value": None, "is_choice_input": False}
+ ],
field_context={},
page=MagicMock(),
get_skyvern_element=AsyncMock(return_value=selected_element),
@@ -744,6 +906,118 @@ class TestDeterministicCustomSelect:
assert "could not be verified" in action_result.exception_message
selected_element.click.assert_awaited_once()
+ @pytest.mark.asyncio
+ async def test_live_multiselectable_option_hard_fails_when_trimmed_tree_loses_ancestor_state(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(
+ handler.app,
+ "EXPERIMENTATION_PROVIDER",
+ SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ )
+ monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
+ monkeypatch.setattr(
+ handler.SkyvernFrame,
+ "evaluate",
+ _stub_evaluate(
+ matched_state={
+ "label": "Alpha",
+ "role": "option",
+ "inMultiselectable": True,
+ "ariaSelected": False,
+ "selectedAttr": False,
+ "checked": False,
+ },
+ committed=False,
+ ),
+ )
+ selected_element = _FakeCustomElement(role="option")
+ selected_element.get_locator().count = AsyncMock(return_value=1)
+ readback_scope_element = _FakeAnchorElement(tag_name="button")
+ candidates = _custom_select_candidates_from_elements(
+ [
+ {
+ "id": "choice-panel",
+ "tagName": "div",
+ "children": [
+ {
+ "id": "choice-alpha",
+ "tagName": "div",
+ "attributes": {"role": "option", "aria-selected": "false"},
+ "text": "Alpha",
+ "interactable": True,
+ }
+ ],
+ }
+ ]
+ )
+
+ result = await _select_deterministic_custom_option(
+ target_value="Alpha",
+ get_option_candidates=lambda: candidates,
+ field_context={},
+ page=MagicMock(),
+ get_skyvern_element=AsyncMock(return_value=selected_element),
+ get_readback_scope_element=AsyncMock(return_value=readback_scope_element),
+ task=_task(), # type: ignore[arg-type]
+ )
+
+ assert candidates[0]["is_choice_input"] is False
+ assert result is not None
+ action_result, matched_label = result
+ assert not action_result.success
+ assert action_result.skip_remaining_actions is True
+ assert matched_label == "Alpha"
+ selected_element.click.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("walk_is_choice_input", [True, False])
+ async def test_missing_live_matched_state_falls_back_to_walk_classification(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ walk_is_choice_input: bool,
+ ) -> None:
+ monkeypatch.setattr(
+ handler.app,
+ "EXPERIMENTATION_PROVIDER",
+ SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ )
+ monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
+ monkeypatch.setattr(
+ handler.SkyvernFrame,
+ "evaluate",
+ _stub_evaluate(matched_state=None, committed=False),
+ )
+ selected_element = _FakeCustomElement()
+ selected_element.get_locator().count = AsyncMock(return_value=1)
+
+ result = await _select_deterministic_custom_option(
+ target_value="Alpha",
+ get_option_candidates=lambda: [
+ {
+ "label": "Alpha",
+ "element_id": "choice-alpha",
+ "value": None,
+ "is_choice_input": walk_is_choice_input,
+ }
+ ],
+ field_context={},
+ page=MagicMock(),
+ get_skyvern_element=AsyncMock(return_value=selected_element),
+ get_readback_scope_element=AsyncMock(return_value=_FakeAnchorElement()),
+ task=_task(), # type: ignore[arg-type]
+ )
+
+ if walk_is_choice_input:
+ assert result is not None
+ action_result, matched_label = result
+ assert not action_result.success
+ assert action_result.skip_remaining_actions is True
+ assert matched_label == "Alpha"
+ else:
+ assert result is None
+ selected_element.click.assert_awaited_once()
+
@pytest.mark.asyncio
async def test_clicked_but_unverified_non_choice_option_soft_fails_to_llm(
self, monkeypatch: pytest.MonkeyPatch
@@ -808,7 +1082,9 @@ class TestDeterministicCustomSelect:
result = await _select_deterministic_custom_option(
target_value="WAKE",
- get_option_candidates=lambda: [{"label": "WAKE", "element_id": "source-wake", "value": "WAKE"}],
+ get_option_candidates=lambda: [
+ {"label": "WAKE", "element_id": "source-wake", "value": "WAKE", "is_choice_input": False}
+ ],
field_context={},
page=MagicMock(),
get_skyvern_element=AsyncMock(return_value=selected_element),
@@ -861,7 +1137,7 @@ class TestDeterministicCustomSelect:
async def evaluate(*, frame: object, expression: str, arg: object = None) -> object:
captured["expression"] = expression
captured["arg"] = arg
- if "return { label," in expression:
+ if "const nestedChoice =" in expression:
return None
return True
@@ -991,6 +1267,92 @@ class TestDeterministicCustomSelect:
selected_element.click.assert_awaited_once()
select_from_dropdown_by_value.assert_not_awaited()
+ @pytest.mark.asyncio
+ async def test_handle_select_option_action_multiselectable_option_hard_fails_via_live_dom(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(
+ handler.app,
+ "EXPERIMENTATION_PROVIDER",
+ SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True)),
+ )
+ monkeypatch.setattr(handler.app, "AGENT_FUNCTION", AgentFunction())
+ monkeypatch.setattr(
+ handler.SkyvernFrame,
+ "evaluate",
+ _stub_evaluate(
+ matched_state={
+ "label": "Choice",
+ "role": "option",
+ "inMultiselectable": True,
+ "ariaSelected": False,
+ "selectedAttr": False,
+ "checked": False,
+ },
+ committed=False,
+ ),
+ )
+
+ anchor_element = _FakeAnchorElement()
+ selected_element = _FakeCustomElement(role="option")
+ selected_element.get_locator().count = AsyncMock(return_value=1)
+ fake_frame = MagicMock()
+ fake_frame.safe_wait_for_animation_end = AsyncMock()
+ # Trimmed-shape option node: role=option survives trimming, but the container's
+ # aria-multiselectable does not — the walk classifies False and only the live-DOM
+ # pre-click read (stubbed above) can catch the toggle hazard.
+ fake_incremental = _FakeIncrementalScrapePage(
+ [
+ [{"id": "opened-option"}],
+ [
+ {
+ "id": "choice-option",
+ "tagName": "div",
+ "attributes": {"role": "option"},
+ "text": "Choice",
+ "interactable": True,
+ }
+ ],
+ ]
+ )
+
+ class FakeDomUtil:
+ def __init__(self, *_args: object, **_kwargs: object) -> None:
+ pass
+
+ async def get_skyvern_element_by_id(self, _element_id: str) -> _FakeAnchorElement:
+ return anchor_element
+
+ select_from_dropdown_by_value = AsyncMock(return_value=handler.ActionSuccess())
+ monkeypatch.setattr(handler, "DomUtil", FakeDomUtil)
+ monkeypatch.setattr(handler.SkyvernFrame, "create_instance", AsyncMock(return_value=fake_frame))
+ monkeypatch.setattr(handler, "IncrementalScrapePage", MagicMock(return_value=fake_incremental))
+ monkeypatch.setattr(
+ handler,
+ "_get_input_or_select_context",
+ AsyncMock(return_value=InputOrSelectContext(field="Field", is_required=True)),
+ )
+ monkeypatch.setattr(handler, "locate_dropdown_menu", AsyncMock(return_value=None))
+ monkeypatch.setattr(handler.SkyvernElement, "create_from_incremental", AsyncMock(return_value=selected_element))
+ monkeypatch.setattr(handler, "select_from_dropdown_by_value", select_from_dropdown_by_value)
+
+ results = await handler.handle_select_option_action(
+ action=_select_action(),
+ page=MagicMock(),
+ scraped_page=SimpleNamespace(
+ id_to_element_dict={"field-control": {"id": "field-control"}},
+ id_to_css_dict={},
+ ),
+ task=_task(), # type: ignore[arg-type]
+ step=MagicMock(),
+ )
+
+ assert len(results) == 1
+ assert not results[0].success
+ assert results[0].skip_remaining_actions is True
+ selected_element.click.assert_awaited_once()
+ select_from_dropdown_by_value.assert_not_awaited()
+
@pytest.mark.asyncio
async def test_select_from_dropdown_deterministic_success_skips_custom_select_prompt_and_llm(
self, monkeypatch: pytest.MonkeyPatch
@@ -1063,7 +1425,7 @@ class TestDeterministicCustomSelect:
async def evaluate(*, frame: object, expression: str, arg: object = None) -> object:
if "anchorIsComboboxInput" in expression:
captured["expression"] = expression
- if "return { label," in expression:
+ if "const nestedChoice =" in expression:
return None
return False