Skip to content

Automate browser based workflows with AI

fix(SKY-12383): commit invalid combobox selection after typing surfaces its options (#7480)

LawyZhengac8d6583cd3f
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index bafe29c64..a5709f41f 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -3110,6 +3110,40 @@ def _incremental_tree_contains_target_value(elements: list[dict], target_value:
return False
+def _incremental_tree_contains_option_with_target_value(elements: list[dict], target_value: str) -> bool:
+ # Match the target only against real option candidates (what the selector would click), unlike the
+ # broad search-bar helper above, so a "No results for <target>" banner cannot admit a selection.
+ normalized_target = _normalize_dropdown_match_text(target_value)
+ if not normalized_target:
+ return False
+ for candidate in _custom_select_candidates_from_elements(elements):
+ label = candidate.get("label")
+ if isinstance(label, str) and normalized_target in _normalize_dropdown_match_text(label):
+ return True
+ return False
+
+
+def _attr_indicates_aria_invalid(raw: object) -> bool:
+ # Compare as a normalized string so a literal False (bool or "false") reads as valid, not truthy;
+ # aria-invalid is "true"/"grammar"/"spelling" when rejected, "false"/absent when accepted.
+ if raw is None:
+ return False
+ return str(raw).strip().casefold() not in ("", "false")
+
+
+async def _is_commit_required_combobox(skyvern_element: SkyvernElement) -> bool:
+ role = await skyvern_element.get_attr("role")
+ aria_autocomplete = await skyvern_element.get_attr("aria-autocomplete")
+ is_combobox = str(role or "").strip().casefold() == "combobox" or str(
+ aria_autocomplete or ""
+ ).strip().casefold() in ("list", "both")
+ if not is_combobox:
+ return False
+ # aria-invalid is read live (dynamic) because it reflects post-input state, not the pre-input scrape.
+ aria_invalid = await skyvern_element.get_attr("aria-invalid", mode="dynamic")
+ return _attr_indicates_aria_invalid(aria_invalid)
+
+
@traced(name="skyvern.agent.action.input_text")
async def handle_input_text_action(
action: actions.InputTextAction,
@@ -3677,6 +3711,42 @@ async def handle_input_text_action(
if action.stop_batch_after_dropdown_select:
select_result.action_result.skip_remaining_actions = True
return [select_result.action_result]
+ elif (
+ input_or_select_context is not None
+ and not input_or_select_context.is_search_bar
+ and not input_or_select_context.is_location_input
+ and not is_secret_value
+ and _incremental_tree_contains_option_with_target_value(incremental_element, text)
+ and await _is_commit_required_combobox(skyvern_element)
+ ):
+ # A role=combobox / aria-autocomplete field that is still aria-invalid after typing
+ # only commits by picking a rendered option; the Tab hack below won't do that. Force one
+ # deterministic selection against the surfaced option. This does not touch
+ # is_auto_completion_input() or the speculative pre-input fanout.
+ LOG.info(
+ "Detected target-matching option after typing into an invalid combobox; committing selection",
+ element_id=skyvern_element.get_id(),
+ target_value=text,
+ )
+ action.set_has_mini_agent()
+ select_result = await sequentially_select_from_dropdown(
+ action=select_action,
+ input_or_select_context=input_or_select_context,
+ page=page,
+ dom=dom,
+ skyvern_element=skyvern_element,
+ skyvern_frame=skyvern_frame,
+ incremental_scraped=incremental_scraped,
+ step=step,
+ task=task,
+ force_select=True,
+ target_value=text,
+ )
+ if select_result and select_result.action_result and select_result.action_result.success:
+ auto_complete_hacky_flag = False
+ if action.stop_batch_after_dropdown_select:
+ select_result.action_result.skip_remaining_actions = True
+ return [select_result.action_result]
except PlaywrightError as inc_error:
# Handle Playwright-specific errors during incremental element processing
# (e.g., TOTP form auto-submit, or search-dropdown selection triggering navigation)
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
index b263a0075..7878007db 100644
--- a/tests/unit/conftest.py
+++ b/tests/unit/conftest.py
@@ -217,3 +217,43 @@ async def sqlite_engine_factory(
@pytest_asyncio.fixture
async def sqlite_engine(sqlite_engine_factory: Callable[[], AsyncEngine]) -> AsyncEngine:
return sqlite_engine_factory()
+
+
+def make_input_element_mock(*, element_id: str = "AADC", attrs: dict[str, object] | None = None) -> MagicMock:
+ # SkyvernElement double for handle_input_text_action tests. attrs=None makes every get_attr return
+ # None (plain search-bar case); pass a dict to drive specific attrs (e.g. a combobox's role /
+ # aria-autocomplete / aria-invalid).
+ el = MagicMock()
+ el.get_id.return_value = element_id
+ el.get_tag_name.return_value = "input"
+ el.get_frame.return_value = MagicMock()
+ locator = MagicMock()
+ locator.focus = AsyncMock()
+ el.get_locator.return_value = locator
+ el.is_disabled = AsyncMock(return_value=False)
+ el.get_selectable = AsyncMock(return_value=False)
+ el.has_hidden_attr = AsyncMock(return_value=False)
+ el.is_readonly = AsyncMock(return_value=False)
+ el.has_attr = AsyncMock(return_value=False)
+ el.is_spinbtn_input = AsyncMock(return_value=False)
+ el.is_editable = AsyncMock(return_value=True)
+ el.supports_text_input = AsyncMock(return_value=True)
+ el.is_visible = AsyncMock(return_value=True)
+ el.is_raw_input = AsyncMock(return_value=False)
+ el.is_auto_completion_input = AsyncMock(return_value=False)
+ el.find_blocking_element = AsyncMock(return_value=(None, False))
+ el.get_element_handler = AsyncMock(return_value=MagicMock())
+ el.input_sequentially = AsyncMock()
+ el.input_clear = AsyncMock()
+ el.scroll_into_view = AsyncMock()
+ el.press_key = AsyncMock()
+ el.blur = AsyncMock()
+ if attrs is None:
+ el.get_attr = AsyncMock(return_value=None)
+ else:
+
+ def _get_attr(name: str, *args: object, **kwargs: object) -> object:
+ return attrs.get(name)
+
+ el.get_attr = AsyncMock(side_effect=_get_attr)
+ return el
diff --git a/tests/unit/test_input_text_combobox_commit.py b/tests/unit/test_input_text_combobox_commit.py
new file mode 100644
index 000000000..15d6a0c97
--- /dev/null
+++ b/tests/unit/test_input_text_combobox_commit.py
@@ -0,0 +1,321 @@
+"""Tests for the post-input combobox-commit gate in ``handle_input_text_action``.
+
+An ``aria-autocomplete`` combobox (e.g. ``role=combobox aria-autocomplete=both``) is not an
+``is_auto_completion_input()`` (that predicate matches only ``list``), so typing into it never
+runs the deterministic type-then-select flow. When such a field stays ``aria-invalid`` after
+typing — because a value is only committed by picking a rendered option — the old code fell to
+the blind Tab hack, which does not commit the option, so the planner looped clear/retype until
+``REACH_MAX_STEPS``.
+
+The gate here reuses the existing post-input incremental-DOM block: only when a genuine option
+node exposes the typed value AND the source input is a still-invalid combobox does it force one
+deterministic selection. It deliberately does NOT touch ``is_auto_completion_input()`` or the
+speculative pre-input fanout, and it matches option-like nodes only (not arbitrary tree text) so
+a "No results for <x>" banner cannot admit a selection attempt.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from skyvern.forge.sdk.models import StepStatus
+from skyvern.webeye.actions import handler
+from skyvern.webeye.actions.actions import InputOrSelectContext, InputTextAction
+from skyvern.webeye.actions.handler import handle_input_text_action
+from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess
+from tests.unit.conftest import make_input_element_mock
+from tests.unit.helpers import make_organization, make_step, make_task
+
+_NOW = datetime.now(UTC)
+_ORG = make_organization(_NOW)
+_TASK = make_task(_NOW, _ORG, navigation_payload={}, navigation_goal="Fill the job title")
+_STEP = make_step(_NOW, _TASK, step_id="stp-1", status=StepStatus.created, order=0, output=None)
+
+_TARGET = "Backend Engineer"
+
+
+def _listbox_with_option(label: str) -> list[dict]:
+ return [
+ {
+ "tagName": "ul",
+ "attributes": {"role": "listbox"},
+ "children": [
+ {"tagName": "li", "attributes": {"role": "option"}, "id": "OPT1", "text": label},
+ ],
+ }
+ ]
+
+
+# --------------------------------------------------------------------------- #
+# _attr_indicates_aria_invalid — string-normalized truthiness (never bare bool)
+# --------------------------------------------------------------------------- #
+ "raw,expected",
+ [
+ ("true", True),
+ ("True", True),
+ ("grammar", True),
+ ("spelling", True),
+ (True, True),
+ ("false", False),
+ ("False", False),
+ (False, False),
+ ("", False),
+ (None, False),
+ ],
+)
+def test_attr_indicates_aria_invalid(raw: object, expected: bool) -> None:
+ assert handler._attr_indicates_aria_invalid(raw) is expected
+
+
+# --------------------------------------------------------------------------- #
+# _incremental_tree_contains_option_with_target_value — option-only matching
+# --------------------------------------------------------------------------- #
+def test_option_helper_matches_option_label() -> None:
+ assert handler._incremental_tree_contains_option_with_target_value(_listbox_with_option(_TARGET), _TARGET) is True
+
+
+def test_option_helper_ignores_no_results_banner() -> None:
+ banner = [{"tagName": "div", "attributes": {"role": "status"}, "text": f"No results for {_TARGET}"}]
+ assert handler._incremental_tree_contains_option_with_target_value(banner, _TARGET) is False
+
+
+def test_option_helper_ignores_bare_li_outside_listbox() -> None:
+ # A bare <li> with no listbox/menu ancestor is not an option candidate, so a
+ # "No results" line rendered as a stray <li> must not false-trigger.
+ stray = [{"tagName": "li", "id": "X", "text": f"No results for {_TARGET}"}]
+ assert handler._incremental_tree_contains_option_with_target_value(stray, _TARGET) is False
+
+
+def test_option_helper_no_match_when_options_lack_target() -> None:
+ assert (
+ handler._incremental_tree_contains_option_with_target_value(_listbox_with_option("Frontend Developer"), _TARGET)
+ is False
+ )
+
+
+def test_option_helper_empty_target_is_false() -> None:
+ assert handler._incremental_tree_contains_option_with_target_value(_listbox_with_option(_TARGET), "") is False
+
+
+# --------------------------------------------------------------------------- #
+# _is_commit_required_combobox — combobox (role/aria-autocomplete) AND aria-invalid
+# --------------------------------------------------------------------------- #
+def _element_with_attrs(attrs: dict[str, object]) -> MagicMock:
+ el = MagicMock()
+ el.get_id.return_value = "CBX"
+
+ def _get_attr(name: str, *args: object, **kwargs: object) -> object:
+ return attrs.get(name)
+
+ el.get_attr = AsyncMock(side_effect=_get_attr)
+ return el
+
+
+async def test_commit_required_true_for_invalid_both_combobox() -> None:
+ el = _element_with_attrs({"role": "combobox", "aria-autocomplete": "both", "aria-invalid": "true"})
+ assert await handler._is_commit_required_combobox(el) is True
+
+
+async def test_commit_required_true_via_aria_autocomplete_list() -> None:
+ el = _element_with_attrs({"role": None, "aria-autocomplete": "list", "aria-invalid": "true"})
+ assert await handler._is_commit_required_combobox(el) is True
+
+
+async def test_commit_required_false_when_valid() -> None:
+ el = _element_with_attrs({"role": "combobox", "aria-autocomplete": "both", "aria-invalid": "false"})
+ assert await handler._is_commit_required_combobox(el) is False
+
+
+async def test_commit_required_false_when_aria_invalid_absent() -> None:
+ el = _element_with_attrs({"role": "combobox", "aria-autocomplete": "both", "aria-invalid": None})
+ assert await handler._is_commit_required_combobox(el) is False
+
+
+async def test_commit_required_false_for_non_combobox() -> None:
+ el = _element_with_attrs({"role": "textbox", "aria-autocomplete": None, "aria-invalid": "true"})
+ assert await handler._is_commit_required_combobox(el) is False
+
+
+# --------------------------------------------------------------------------- #
+# handle_input_text_action — end-to-end wiring of the combobox-commit branch
+# --------------------------------------------------------------------------- #
+def _pressed_keys(el: MagicMock) -> list[str]:
+ return [call.args[0] for call in el.press_key.call_args_list if call.args]
+
+
+async def _run_combobox_input(
+ *,
+ attrs: dict[str, object],
+ options: list[dict],
+ select_success: bool,
+ stop_flag: bool,
+ is_search_bar: bool = False,
+ is_location_input: bool = False,
+ is_secret: bool = False,
+) -> tuple[list, MagicMock, MagicMock]:
+ skyvern_el = make_input_element_mock(element_id="CBX", attrs=attrs)
+ dom_instance = MagicMock()
+ dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=skyvern_el)
+
+ inc = MagicMock()
+ inc.start_listen_dom_increment = AsyncMock()
+ inc.stop_listen_dom_increment = AsyncMock()
+ if is_secret:
+ # Secret-valued params skip Block A's ArrowDown probe (its guard excludes secrets),
+ # so Block B is the first and only incremental read.
+ inc.get_incremental_element_tree = AsyncMock(return_value=options)
+ else:
+ # Block A (ArrowDown probe on the empty field) surfaces nothing; Block B (after typing) surfaces options.
+ inc.get_incremental_element_tree = AsyncMock(side_effect=[[], options])
+
+ skyvern_frame = MagicMock()
+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()
+
+ scraped_page = MagicMock()
+ scraped_page.id_to_element_dict = {"CBX": {"tagName": "input"}}
+
+ context = InputOrSelectContext(field="Title", is_search_bar=is_search_bar, is_location_input=is_location_input)
+
+ select_result = MagicMock()
+ select_result.action_result = ActionSuccess() if select_success else ActionFailure(Exception("not committed"))
+
+ # A secret makes the resolved text differ from action.text, so is_secret_value becomes True.
+ action_text = "{{secret_param}}" if is_secret else _TARGET
+ action = InputTextAction(element_id="CBX", text=action_text, reasoning="type the job title")
+ action.stop_batch_after_dropdown_select = stop_flag
+
+ select_mock = AsyncMock(return_value=select_result)
+
+ with (
+ patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),
+ patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance", new=AsyncMock(return_value=skyvern_frame)),
+ patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),
+ patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(return_value="")),
+ patch(
+ "skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task",
+ return_value=_TARGET,
+ ),
+ patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),
+ patch("skyvern.webeye.actions.handler.sequentially_select_from_dropdown", new=select_mock),
+ ):
+ results = await handle_input_text_action(
+ action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP
+ )
+ return results, skyvern_el, select_mock
+
+
+_INVALID_BOTH = {"role": "combobox", "aria-autocomplete": "both", "aria-invalid": "true"}
+
+
+async def test_invalid_combobox_commits_and_suppresses_tab() -> None:
+ """Invalid combobox + option matching the typed value -> one forced selection, no Tab hack."""
+ results, el, select_mock = await _run_combobox_input(
+ attrs=_INVALID_BOTH, options=_listbox_with_option(_TARGET), select_success=True, stop_flag=False
+ )
+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)
+ select_mock.assert_awaited_once()
+ assert select_mock.await_args.kwargs["force_select"] is True
+ assert select_mock.await_args.kwargs["target_value"] == _TARGET
+ assert "Tab" not in _pressed_keys(el)
+ assert not results[0].skip_remaining_actions
+
+
+async def test_invalid_combobox_commit_stops_batch_when_flagged() -> None:
+ """A trailing clobbering action (flag set) -> skip_remaining_actions=True, mirroring search-bar semantics."""
+ results, _el, select_mock = await _run_combobox_input(
+ attrs=_INVALID_BOTH, options=_listbox_with_option(_TARGET), select_success=True, stop_flag=True
+ )
+ select_mock.assert_awaited_once()
+ assert len(results) == 1 and results[0].skip_remaining_actions is True
+
+
+async def test_non_combobox_does_not_trigger_select() -> None:
+ """A plain textbox (not a combobox) must never enter the deterministic selection path."""
+ results, el, select_mock = await _run_combobox_input(
+ attrs={"role": "textbox", "aria-autocomplete": None, "aria-invalid": "true"},
+ options=_listbox_with_option(_TARGET),
+ select_success=True,
+ stop_flag=True,
+ )
+ select_mock.assert_not_awaited()
+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)
+
+
+async def test_valid_combobox_does_not_trigger_select() -> None:
+ """A combobox already reporting aria-invalid=false has committed; no selection needed."""
+ results, _el, select_mock = await _run_combobox_input(
+ attrs={"role": "combobox", "aria-autocomplete": "both", "aria-invalid": "false"},
+ options=_listbox_with_option(_TARGET),
+ select_success=True,
+ stop_flag=True,
+ )
+ select_mock.assert_not_awaited()
+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)
+
+
+async def test_no_matching_option_does_not_trigger_select() -> None:
+ """Invalid combobox but the dropdown has no option matching the typed value -> no selection."""
+ results, _el, select_mock = await _run_combobox_input(
+ attrs=_INVALID_BOTH,
+ options=_listbox_with_option("Frontend Developer"),
+ select_success=True,
+ stop_flag=True,
+ )
+ select_mock.assert_not_awaited()
+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)
+
+
+async def test_selection_failure_falls_back_to_tab() -> None:
+ """When the forced selection fails to commit, behavior degrades to today's Tab hack + ActionSuccess."""
+ results, el, select_mock = await _run_combobox_input(
+ attrs=_INVALID_BOTH, options=_listbox_with_option(_TARGET), select_success=False, stop_flag=False
+ )
+ select_mock.assert_awaited_once()
+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)
+ assert "Tab" in _pressed_keys(el)
+
+
+async def test_search_bar_does_not_use_combobox_branch() -> None:
+ """Search bars keep their own path; the combobox branch (which force-selects) must not fire for them."""
+ # A search bar whose surfaced tree has no target match: the search-bar branch is skipped, and the
+ # combobox branch must not pick it up either (guardrail: search-bar behavior unchanged).
+ results, _el, select_mock = await _run_combobox_input(
+ attrs=_INVALID_BOTH,
+ options=_listbox_with_option("Frontend Developer"),
+ select_success=True,
+ stop_flag=True,
+ is_search_bar=True,
+ )
+ select_mock.assert_not_awaited()
+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)
+
+
+async def test_secret_valued_action_does_not_trigger_select() -> None:
+ """A secret-valued parameter must never enter the selection path: its value would otherwise be
+ logged (target_value=...) and sent into the custom-select LLM prompt via target_value=text."""
+ results, _el, select_mock = await _run_combobox_input(
+ attrs=_INVALID_BOTH,
+ options=_listbox_with_option(_TARGET),
+ select_success=True,
+ stop_flag=True,
+ is_secret=True,
+ )
+ select_mock.assert_not_awaited()
+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)
diff --git a/tests/unit/test_input_text_search_bar_skip.py b/tests/unit/test_input_text_search_bar_skip.py
index 94e23af5d..c666663d9 100644
--- a/tests/unit/test_input_text_search_bar_skip.py
+++ b/tests/unit/test_input_text_search_bar_skip.py
@@ -19,6 +19,7 @@ from skyvern.webeye.actions.actions import InputOrSelectContext, InputTextAction
from skyvern.webeye.actions.handler import handle_input_text_action
from skyvern.webeye.actions.handler_utils import keys_include_enter, should_stop_batch_after_dropdown_select
from skyvern.webeye.actions.responses import ActionSuccess
+from tests.unit.conftest import make_input_element_mock
from tests.unit.helpers import make_organization, make_step, make_task
_NOW = datetime.now(UTC)
@@ -80,36 +81,8 @@ def test_flag_is_transient_not_serialized() -> None:
# --------------------------------------------------------------------------- #
# handle_input_text_action — gated batch-stop behavior
# --------------------------------------------------------------------------- #
-def _mock_search_input() -> MagicMock:
- el = MagicMock()
- el.get_id.return_value = "AADC"
- el.get_tag_name.return_value = "input"
- el.get_frame.return_value = MagicMock()
- locator = MagicMock()
- locator.focus = AsyncMock()
- el.get_locator.return_value = locator
- el.is_disabled = AsyncMock(return_value=False)
- el.get_selectable = AsyncMock(return_value=False)
- el.has_hidden_attr = AsyncMock(return_value=False)
- el.is_readonly = AsyncMock(return_value=False)
- el.get_attr = AsyncMock(return_value=None)
- el.is_spinbtn_input = AsyncMock(return_value=False)
- el.is_editable = AsyncMock(return_value=True)
- el.supports_text_input = AsyncMock(return_value=True)
- el.is_visible = AsyncMock(return_value=True)
- el.is_raw_input = AsyncMock(return_value=False)
- el.is_auto_completion_input = AsyncMock(return_value=False)
- el.find_blocking_element = AsyncMock(return_value=(None, False))
- el.get_element_handler = AsyncMock(return_value=MagicMock())
- el.input_sequentially = AsyncMock()
- el.input_clear = AsyncMock()
- el.scroll_into_view = AsyncMock()
- el.press_key = AsyncMock()
- return el
-
-
async def _run_search_bar_input(stop_flag: bool, incremental: list[dict]) -> list:
- skyvern_el = _mock_search_input()
+ skyvern_el = make_input_element_mock(element_id="AADC")
dom_instance = MagicMock()
dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=skyvern_el)