Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-12383): commit invalid combobox selection after typing surfaces its options (#7480)
LawyZhengac8d6583cd3f
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex bafe29c64..a5709f41f 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -3110,6 +3110,40 @@ def _incremental_tree_contains_target_value(elements: list[dict], target_value:return False+def _incremental_tree_contains_option_with_target_value(elements: list[dict], target_value: str) -> bool:+ # Match the target only against real option candidates (what the selector would click), unlike the+ # broad search-bar helper above, so a "No results for <target>" banner cannot admit a selection.+ normalized_target = _normalize_dropdown_match_text(target_value)+ if not normalized_target:+ return False+ for candidate in _custom_select_candidates_from_elements(elements):+ label = candidate.get("label")+ if isinstance(label, str) and normalized_target in _normalize_dropdown_match_text(label):+ return True+ return False+++def _attr_indicates_aria_invalid(raw: object) -> bool:+ # Compare as a normalized string so a literal False (bool or "false") reads as valid, not truthy;+ # aria-invalid is "true"/"grammar"/"spelling" when rejected, "false"/absent when accepted.+ if raw is None:+ return False+ return str(raw).strip().casefold() not in ("", "false")+++async def _is_commit_required_combobox(skyvern_element: SkyvernElement) -> bool:+ role = await skyvern_element.get_attr("role")+ aria_autocomplete = await skyvern_element.get_attr("aria-autocomplete")+ is_combobox = str(role or "").strip().casefold() == "combobox" or str(+ aria_autocomplete or ""+ ).strip().casefold() in ("list", "both")+ if not is_combobox:+ return False+ # aria-invalid is read live (dynamic) because it reflects post-input state, not the pre-input scrape.+ aria_invalid = await skyvern_element.get_attr("aria-invalid", mode="dynamic")+ return _attr_indicates_aria_invalid(aria_invalid)++@traced(name="skyvern.agent.action.input_text")async def handle_input_text_action(action: actions.InputTextAction,@@ -3677,6 +3711,42 @@ async def handle_input_text_action(if action.stop_batch_after_dropdown_select:select_result.action_result.skip_remaining_actions = Truereturn [select_result.action_result]+ elif (+ input_or_select_context is not None+ and not input_or_select_context.is_search_bar+ and not input_or_select_context.is_location_input+ and not is_secret_value+ and _incremental_tree_contains_option_with_target_value(incremental_element, text)+ and await _is_commit_required_combobox(skyvern_element)+ ):+ # A role=combobox / aria-autocomplete field that is still aria-invalid after typing+ # only commits by picking a rendered option; the Tab hack below won't do that. Force one+ # deterministic selection against the surfaced option. This does not touch+ # is_auto_completion_input() or the speculative pre-input fanout.+ LOG.info(+ "Detected target-matching option after typing into an invalid combobox; committing selection",+ element_id=skyvern_element.get_id(),+ target_value=text,+ )+ action.set_has_mini_agent()+ select_result = await sequentially_select_from_dropdown(+ action=select_action,+ input_or_select_context=input_or_select_context,+ page=page,+ dom=dom,+ skyvern_element=skyvern_element,+ skyvern_frame=skyvern_frame,+ incremental_scraped=incremental_scraped,+ step=step,+ task=task,+ force_select=True,+ target_value=text,+ )+ if select_result and select_result.action_result and select_result.action_result.success:+ auto_complete_hacky_flag = False+ if action.stop_batch_after_dropdown_select:+ select_result.action_result.skip_remaining_actions = True+ return [select_result.action_result]except PlaywrightError as inc_error:# Handle Playwright-specific errors during incremental element processing# (e.g., TOTP form auto-submit, or search-dropdown selection triggering navigation)diff --git a/tests/unit/conftest.py b/tests/unit/conftest.pyindex b263a0075..7878007db 100644--- a/tests/unit/conftest.py+++ b/tests/unit/conftest.py@@ -217,3 +217,43 @@ async def sqlite_engine_factory(@pytest_asyncio.fixtureasync def sqlite_engine(sqlite_engine_factory: Callable[[], AsyncEngine]) -> AsyncEngine:return sqlite_engine_factory()+++def make_input_element_mock(*, element_id: str = "AADC", attrs: dict[str, object] | None = None) -> MagicMock:+ # SkyvernElement double for handle_input_text_action tests. attrs=None makes every get_attr return+ # None (plain search-bar case); pass a dict to drive specific attrs (e.g. a combobox's role /+ # aria-autocomplete / aria-invalid).+ el = MagicMock()+ el.get_id.return_value = element_id+ el.get_tag_name.return_value = "input"+ el.get_frame.return_value = MagicMock()+ locator = MagicMock()+ locator.focus = AsyncMock()+ el.get_locator.return_value = locator+ el.is_disabled = AsyncMock(return_value=False)+ el.get_selectable = AsyncMock(return_value=False)+ el.has_hidden_attr = AsyncMock(return_value=False)+ el.is_readonly = AsyncMock(return_value=False)+ el.has_attr = AsyncMock(return_value=False)+ el.is_spinbtn_input = AsyncMock(return_value=False)+ el.is_editable = AsyncMock(return_value=True)+ el.supports_text_input = AsyncMock(return_value=True)+ el.is_visible = AsyncMock(return_value=True)+ el.is_raw_input = AsyncMock(return_value=False)+ el.is_auto_completion_input = AsyncMock(return_value=False)+ el.find_blocking_element = AsyncMock(return_value=(None, False))+ el.get_element_handler = AsyncMock(return_value=MagicMock())+ el.input_sequentially = AsyncMock()+ el.input_clear = AsyncMock()+ el.scroll_into_view = AsyncMock()+ el.press_key = AsyncMock()+ el.blur = AsyncMock()+ if attrs is None:+ el.get_attr = AsyncMock(return_value=None)+ else:++ def _get_attr(name: str, *args: object, **kwargs: object) -> object:+ return attrs.get(name)++ el.get_attr = AsyncMock(side_effect=_get_attr)+ return eldiff --git a/tests/unit/test_input_text_combobox_commit.py b/tests/unit/test_input_text_combobox_commit.pynew file mode 100644index 000000000..15d6a0c97--- /dev/null+++ b/tests/unit/test_input_text_combobox_commit.py@@ -0,0 +1,321 @@+"""Tests for the post-input combobox-commit gate in ``handle_input_text_action``.++An ``aria-autocomplete`` combobox (e.g. ``role=combobox aria-autocomplete=both``) is not an+``is_auto_completion_input()`` (that predicate matches only ``list``), so typing into it never+runs the deterministic type-then-select flow. When such a field stays ``aria-invalid`` after+typing — because a value is only committed by picking a rendered option — the old code fell to+the blind Tab hack, which does not commit the option, so the planner looped clear/retype until+``REACH_MAX_STEPS``.++The gate here reuses the existing post-input incremental-DOM block: only when a genuine option+node exposes the typed value AND the source input is a still-invalid combobox does it force one+deterministic selection. It deliberately does NOT touch ``is_auto_completion_input()`` or the+speculative pre-input fanout, and it matches option-like nodes only (not arbitrary tree text) so+a "No results for <x>" banner cannot admit a selection attempt.+"""++from __future__ import annotations++from datetime import UTC, datetime+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from skyvern.forge.sdk.models import StepStatus+from skyvern.webeye.actions import handler+from skyvern.webeye.actions.actions import InputOrSelectContext, InputTextAction+from skyvern.webeye.actions.handler import handle_input_text_action+from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess+from tests.unit.conftest import make_input_element_mock+from tests.unit.helpers import make_organization, make_step, make_task++_NOW = datetime.now(UTC)+_ORG = make_organization(_NOW)+_TASK = make_task(_NOW, _ORG, navigation_payload={}, navigation_goal="Fill the job title")+_STEP = make_step(_NOW, _TASK, step_id="stp-1", status=StepStatus.created, order=0, output=None)++_TARGET = "Backend Engineer"+++def _listbox_with_option(label: str) -> list[dict]:+ return [+ {+ "tagName": "ul",+ "attributes": {"role": "listbox"},+ "children": [+ {"tagName": "li", "attributes": {"role": "option"}, "id": "OPT1", "text": label},+ ],+ }+ ]+++# --------------------------------------------------------------------------- #+# _attr_indicates_aria_invalid — string-normalized truthiness (never bare bool)+# --------------------------------------------------------------------------- #+ "raw,expected",+ [+ ("true", True),+ ("True", True),+ ("grammar", True),+ ("spelling", True),+ (True, True),+ ("false", False),+ ("False", False),+ (False, False),+ ("", False),+ (None, False),+ ],+)+def test_attr_indicates_aria_invalid(raw: object, expected: bool) -> None:+ assert handler._attr_indicates_aria_invalid(raw) is expected+++# --------------------------------------------------------------------------- #+# _incremental_tree_contains_option_with_target_value — option-only matching+# --------------------------------------------------------------------------- #+def test_option_helper_matches_option_label() -> None:+ assert handler._incremental_tree_contains_option_with_target_value(_listbox_with_option(_TARGET), _TARGET) is True+++def test_option_helper_ignores_no_results_banner() -> None:+ banner = [{"tagName": "div", "attributes": {"role": "status"}, "text": f"No results for {_TARGET}"}]+ assert handler._incremental_tree_contains_option_with_target_value(banner, _TARGET) is False+++def test_option_helper_ignores_bare_li_outside_listbox() -> None:+ # A bare <li> with no listbox/menu ancestor is not an option candidate, so a+ # "No results" line rendered as a stray <li> must not false-trigger.+ stray = [{"tagName": "li", "id": "X", "text": f"No results for {_TARGET}"}]+ assert handler._incremental_tree_contains_option_with_target_value(stray, _TARGET) is False+++def test_option_helper_no_match_when_options_lack_target() -> None:+ assert (+ handler._incremental_tree_contains_option_with_target_value(_listbox_with_option("Frontend Developer"), _TARGET)+ is False+ )+++def test_option_helper_empty_target_is_false() -> None:+ assert handler._incremental_tree_contains_option_with_target_value(_listbox_with_option(_TARGET), "") is False+++# --------------------------------------------------------------------------- #+# _is_commit_required_combobox — combobox (role/aria-autocomplete) AND aria-invalid+# --------------------------------------------------------------------------- #+def _element_with_attrs(attrs: dict[str, object]) -> MagicMock:+ el = MagicMock()+ el.get_id.return_value = "CBX"++ def _get_attr(name: str, *args: object, **kwargs: object) -> object:+ return attrs.get(name)++ el.get_attr = AsyncMock(side_effect=_get_attr)+ return el+++async def test_commit_required_true_for_invalid_both_combobox() -> None:+ el = _element_with_attrs({"role": "combobox", "aria-autocomplete": "both", "aria-invalid": "true"})+ assert await handler._is_commit_required_combobox(el) is True+++async def test_commit_required_true_via_aria_autocomplete_list() -> None:+ el = _element_with_attrs({"role": None, "aria-autocomplete": "list", "aria-invalid": "true"})+ assert await handler._is_commit_required_combobox(el) is True+++async def test_commit_required_false_when_valid() -> None:+ el = _element_with_attrs({"role": "combobox", "aria-autocomplete": "both", "aria-invalid": "false"})+ assert await handler._is_commit_required_combobox(el) is False+++async def test_commit_required_false_when_aria_invalid_absent() -> None:+ el = _element_with_attrs({"role": "combobox", "aria-autocomplete": "both", "aria-invalid": None})+ assert await handler._is_commit_required_combobox(el) is False+++async def test_commit_required_false_for_non_combobox() -> None:+ el = _element_with_attrs({"role": "textbox", "aria-autocomplete": None, "aria-invalid": "true"})+ assert await handler._is_commit_required_combobox(el) is False+++# --------------------------------------------------------------------------- #+# handle_input_text_action — end-to-end wiring of the combobox-commit branch+# --------------------------------------------------------------------------- #+def _pressed_keys(el: MagicMock) -> list[str]:+ return [call.args[0] for call in el.press_key.call_args_list if call.args]+++async def _run_combobox_input(+ *,+ attrs: dict[str, object],+ options: list[dict],+ select_success: bool,+ stop_flag: bool,+ is_search_bar: bool = False,+ is_location_input: bool = False,+ is_secret: bool = False,+) -> tuple[list, MagicMock, MagicMock]:+ skyvern_el = make_input_element_mock(element_id="CBX", attrs=attrs)+ dom_instance = MagicMock()+ dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=skyvern_el)++ inc = MagicMock()+ inc.start_listen_dom_increment = AsyncMock()+ inc.stop_listen_dom_increment = AsyncMock()+ if is_secret:+ # Secret-valued params skip Block A's ArrowDown probe (its guard excludes secrets),+ # so Block B is the first and only incremental read.+ inc.get_incremental_element_tree = AsyncMock(return_value=options)+ else:+ # Block A (ArrowDown probe on the empty field) surfaces nothing; Block B (after typing) surfaces options.+ inc.get_incremental_element_tree = AsyncMock(side_effect=[[], options])++ skyvern_frame = MagicMock()+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()++ scraped_page = MagicMock()+ scraped_page.id_to_element_dict = {"CBX": {"tagName": "input"}}++ context = InputOrSelectContext(field="Title", is_search_bar=is_search_bar, is_location_input=is_location_input)++ select_result = MagicMock()+ select_result.action_result = ActionSuccess() if select_success else ActionFailure(Exception("not committed"))++ # A secret makes the resolved text differ from action.text, so is_secret_value becomes True.+ action_text = "{{secret_param}}" if is_secret else _TARGET+ action = InputTextAction(element_id="CBX", text=action_text, reasoning="type the job title")+ action.stop_batch_after_dropdown_select = stop_flag++ select_mock = AsyncMock(return_value=select_result)++ with (+ patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),+ patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance", new=AsyncMock(return_value=skyvern_frame)),+ patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),+ patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(return_value="")),+ patch(+ "skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task",+ return_value=_TARGET,+ ),+ patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),+ patch("skyvern.webeye.actions.handler.sequentially_select_from_dropdown", new=select_mock),+ ):+ results = await handle_input_text_action(+ action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP+ )+ return results, skyvern_el, select_mock+++_INVALID_BOTH = {"role": "combobox", "aria-autocomplete": "both", "aria-invalid": "true"}+++async def test_invalid_combobox_commits_and_suppresses_tab() -> None:+ """Invalid combobox + option matching the typed value -> one forced selection, no Tab hack."""+ results, el, select_mock = await _run_combobox_input(+ attrs=_INVALID_BOTH, options=_listbox_with_option(_TARGET), select_success=True, stop_flag=False+ )+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)+ select_mock.assert_awaited_once()+ assert select_mock.await_args.kwargs["force_select"] is True+ assert select_mock.await_args.kwargs["target_value"] == _TARGET+ assert "Tab" not in _pressed_keys(el)+ assert not results[0].skip_remaining_actions+++async def test_invalid_combobox_commit_stops_batch_when_flagged() -> None:+ """A trailing clobbering action (flag set) -> skip_remaining_actions=True, mirroring search-bar semantics."""+ results, _el, select_mock = await _run_combobox_input(+ attrs=_INVALID_BOTH, options=_listbox_with_option(_TARGET), select_success=True, stop_flag=True+ )+ select_mock.assert_awaited_once()+ assert len(results) == 1 and results[0].skip_remaining_actions is True+++async def test_non_combobox_does_not_trigger_select() -> None:+ """A plain textbox (not a combobox) must never enter the deterministic selection path."""+ results, el, select_mock = await _run_combobox_input(+ attrs={"role": "textbox", "aria-autocomplete": None, "aria-invalid": "true"},+ options=_listbox_with_option(_TARGET),+ select_success=True,+ stop_flag=True,+ )+ select_mock.assert_not_awaited()+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)+++async def test_valid_combobox_does_not_trigger_select() -> None:+ """A combobox already reporting aria-invalid=false has committed; no selection needed."""+ results, _el, select_mock = await _run_combobox_input(+ attrs={"role": "combobox", "aria-autocomplete": "both", "aria-invalid": "false"},+ options=_listbox_with_option(_TARGET),+ select_success=True,+ stop_flag=True,+ )+ select_mock.assert_not_awaited()+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)+++async def test_no_matching_option_does_not_trigger_select() -> None:+ """Invalid combobox but the dropdown has no option matching the typed value -> no selection."""+ results, _el, select_mock = await _run_combobox_input(+ attrs=_INVALID_BOTH,+ options=_listbox_with_option("Frontend Developer"),+ select_success=True,+ stop_flag=True,+ )+ select_mock.assert_not_awaited()+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)+++async def test_selection_failure_falls_back_to_tab() -> None:+ """When the forced selection fails to commit, behavior degrades to today's Tab hack + ActionSuccess."""+ results, el, select_mock = await _run_combobox_input(+ attrs=_INVALID_BOTH, options=_listbox_with_option(_TARGET), select_success=False, stop_flag=False+ )+ select_mock.assert_awaited_once()+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)+ assert "Tab" in _pressed_keys(el)+++async def test_search_bar_does_not_use_combobox_branch() -> None:+ """Search bars keep their own path; the combobox branch (which force-selects) must not fire for them."""+ # A search bar whose surfaced tree has no target match: the search-bar branch is skipped, and the+ # combobox branch must not pick it up either (guardrail: search-bar behavior unchanged).+ results, _el, select_mock = await _run_combobox_input(+ attrs=_INVALID_BOTH,+ options=_listbox_with_option("Frontend Developer"),+ select_success=True,+ stop_flag=True,+ is_search_bar=True,+ )+ select_mock.assert_not_awaited()+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)+++async def test_secret_valued_action_does_not_trigger_select() -> None:+ """A secret-valued parameter must never enter the selection path: its value would otherwise be+ logged (target_value=...) and sent into the custom-select LLM prompt via target_value=text."""+ results, _el, select_mock = await _run_combobox_input(+ attrs=_INVALID_BOTH,+ options=_listbox_with_option(_TARGET),+ select_success=True,+ stop_flag=True,+ is_secret=True,+ )+ select_mock.assert_not_awaited()+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)diff --git a/tests/unit/test_input_text_search_bar_skip.py b/tests/unit/test_input_text_search_bar_skip.pyindex 94e23af5d..c666663d9 100644--- a/tests/unit/test_input_text_search_bar_skip.py+++ b/tests/unit/test_input_text_search_bar_skip.py@@ -19,6 +19,7 @@ from skyvern.webeye.actions.actions import InputOrSelectContext, InputTextActionfrom skyvern.webeye.actions.handler import handle_input_text_actionfrom skyvern.webeye.actions.handler_utils import keys_include_enter, should_stop_batch_after_dropdown_selectfrom skyvern.webeye.actions.responses import ActionSuccess+from tests.unit.conftest import make_input_element_mockfrom tests.unit.helpers import make_organization, make_step, make_task_NOW = datetime.now(UTC)@@ -80,36 +81,8 @@ def test_flag_is_transient_not_serialized() -> None:# --------------------------------------------------------------------------- ## handle_input_text_action — gated batch-stop behavior# --------------------------------------------------------------------------- #-def _mock_search_input() -> MagicMock:- el = MagicMock()- el.get_id.return_value = "AADC"- el.get_tag_name.return_value = "input"- el.get_frame.return_value = MagicMock()- locator = MagicMock()- locator.focus = AsyncMock()- el.get_locator.return_value = locator- el.is_disabled = AsyncMock(return_value=False)- el.get_selectable = AsyncMock(return_value=False)- el.has_hidden_attr = AsyncMock(return_value=False)- el.is_readonly = AsyncMock(return_value=False)- el.get_attr = AsyncMock(return_value=None)- el.is_spinbtn_input = AsyncMock(return_value=False)- el.is_editable = AsyncMock(return_value=True)- el.supports_text_input = AsyncMock(return_value=True)- el.is_visible = AsyncMock(return_value=True)- el.is_raw_input = AsyncMock(return_value=False)- el.is_auto_completion_input = AsyncMock(return_value=False)- el.find_blocking_element = AsyncMock(return_value=(None, False))- el.get_element_handler = AsyncMock(return_value=MagicMock())- el.input_sequentially = AsyncMock()- el.input_clear = AsyncMock()- el.scroll_into_view = AsyncMock()- el.press_key = AsyncMock()- return el--async def _run_search_bar_input(stop_flag: bool, incremental: list[dict]) -> list:- skyvern_el = _mock_search_input()+ skyvern_el = make_input_element_mock(element_id="AADC")dom_instance = MagicMock()dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=skyvern_el)