Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Stop deterministic normal-select from falling back to the LLM on false negatives (#7464)
Marc Kelechavaeb6f4aa47471
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 9f19f2ba0..d4b0f94fe 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -1558,13 +1558,21 @@ async def _select_deterministic_normal_option(return action_result-async def _verify_normal_select_option(+async def _normal_select_readback_contradicts(*,locator: Locator,matched_index: int,matched_label: str | None,matched_value: str | None,) -> bool:+ """True only when a readable read-back disagrees with the deterministic selection.++ An unreadable read-back is not a contradiction. select_option has already reported the value+ committed, and a select's own change event can rerender or navigate the page away, detaching the+ element; re-running the LLM fallback against that stale locator only fails again by value, label,+ and index. Same reasoning as `is_post_dispatch_click_timeout`: the side effect landed, so a+ timed-out post-action read must not trigger a duplicating fallback chain.+ """try:selection = await locator.evaluate(r"""@@ -1580,11 +1588,12 @@ async def _verify_normal_select_option(value: option ? normalize(option.value) : normalize(select.value),};}- """+ """,+ timeout=settings.BROWSER_ACTION_TIMEOUT_MS,)except Exception:LOG.info(- "Failed to read normal select option after deterministic selection",+ "Failed to read normal select option after deterministic selection; keeping select_option success",expected_index=matched_index,expected_label=matched_label,expected_value=matched_value,@@ -1594,7 +1603,7 @@ async def _verify_normal_select_option(if not isinstance(selection, dict):LOG.info(- "Normal select read-back returned unexpected payload",+ "Normal select read-back returned unexpected payload; keeping select_option success",expected_index=matched_index,expected_label=matched_label,expected_value=matched_value,@@ -1602,26 +1611,32 @@ async def _verify_normal_select_option()return False+ # The read-back JS trims, but scraped option text does not: domUtils.removeMultipleSpaces()+ # collapses whitespace runs without trimming, and returns early when there is no double+ # space/tab/newline, so a single edge space survives verbatim into matched_label/matched_value.+ expected_label = " ".join(matched_label.split()) if matched_label is not None else None+ expected_value = " ".join(matched_value.split()) if matched_value is not None else None+actual_index = selection.get("index")actual_value = selection.get("value")actual_label = selection.get("label") or actual_valueif (actual_index == matched_index- and actual_label == matched_label- and (matched_value is None or actual_value == matched_value)+ and actual_label == expected_label+ and (expected_value is None or actual_value == expected_value)):- return True+ return FalseLOG.info("Normal select read-back did not match deterministic option",expected_index=matched_index,- expected_label=matched_label,- expected_value=matched_value,+ expected_label=expected_label,+ expected_value=expected_value,actual_index=actual_index,actual_label=actual_label,actual_value=actual_value,)- return False+ return Trueasync def check_date_format(@@ -7639,7 +7654,7 @@ async def normal_select(matched_value=resolution.matched_value,matched_index=resolution.matched_index,)- if _normal_select_successful(deterministic_result) and await _verify_normal_select_option(+ if _normal_select_successful(deterministic_result) and not await _normal_select_readback_contradicts(locator=locator,matched_index=resolution.matched_index,matched_label=resolution.matched_label,diff --git a/tests/unit/test_normal_select_deterministic.py b/tests/unit/test_normal_select_deterministic.pyindex 1c74806bc..6b16bf878 100644--- a/tests/unit/test_normal_select_deterministic.py+++ b/tests/unit/test_normal_select_deterministic.py@@ -5,6 +5,7 @@ from types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, callimport pytest+from playwright.async_api import TimeoutError as PlaywrightTimeoutErrorfrom skyvern.forge.agent_functions import AgentFunctionfrom skyvern.webeye.actions import handler@@ -66,19 +67,23 @@ def _locator(*,selected_label: str | None = None,selected_index: int | None = None,+ evaluate_side_effect: BaseException | None = None,) -> MagicMock:locator = MagicMock()locator.click = AsyncMock()locator.select_option = AsyncMock()locator.input_value = AsyncMock(return_value=selected_value)- locator.evaluate = AsyncMock(- return_value=_selected_option_readback(- options,- selected_value,- selected_label=selected_label,- selected_index=selected_index,+ if evaluate_side_effect is not None:+ locator.evaluate = AsyncMock(side_effect=evaluate_side_effect)+ else:+ locator.evaluate = AsyncMock(+ return_value=_selected_option_readback(+ options,+ selected_value,+ selected_label=selected_label,+ selected_index=selected_index,+ ))- )return locator@@ -93,6 +98,7 @@ async def _run_normal_select(selected_label: str | None = None,selected_index: int | None = None,llm_response: dict | None = None,+ evaluate_side_effect: BaseException | None = None,) -> tuple[list, MagicMock, AsyncMock]:provider = SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=feature_enabled),@@ -106,7 +112,13 @@ async def _run_normal_select(monkeypatch.setattr(handler.prompt_engine, "load_prompt", MagicMock(return_value="prompt"))monkeypatch.setattr(handler.skyvern_context, "ensure_context", MagicMock(return_value=SimpleNamespace(tz_info=UTC)))- locator = _locator(options, selected_value, selected_label=selected_label, selected_index=selected_index)+ locator = _locator(+ options,+ selected_value,+ selected_label=selected_label,+ selected_index=selected_index,+ evaluate_side_effect=evaluate_side_effect,+ )result = await handler.normal_select(action=action,skyvern_element=_FakeSelectElement(options, locator, selected_attr), # type: ignore[arg-type]@@ -260,7 +272,7 @@ async def test_normal_select_unsafe_match_falls_back_to_llm(@pytest.mark.asyncio-async def test_normal_select_readback_failure_falls_back_to_llm(monkeypatch: pytest.MonkeyPatch) -> None:+async def test_normal_select_readback_value_mismatch_falls_back_to_llm(monkeypatch: pytest.MonkeyPatch) -> None:action = _select_action("United States")result, locator, normal_select_llm = await _run_normal_select(monkeypatch,@@ -285,6 +297,116 @@ async def test_normal_select_readback_failure_falls_back_to_llm(monkeypatch: pytnormal_select_llm.assert_awaited_once()[email protected]("scraped_label", [" United States", "United States "])+async def test_normal_select_edge_whitespace_label_verifies_without_llm(+ monkeypatch: pytest.MonkeyPatch,+ scraped_label: str,+) -> None:+ """Read-back trims the label but the scraped option text does not (domUtils.removeMultipleSpaces+ collapses runs without trimming), so an edge-space-only difference must not read as a mismatch."""+ result, locator, normal_select_llm = await _run_normal_select(+ monkeypatch,+ feature_enabled=True,+ action=_select_action("United States"),+ options=[+ {"optionIndex": 0, "text": "Canada", "value": "CA"},+ {"optionIndex": 1, "text": scraped_label, "value": "US"},+ ],+ selected_value="US",+ selected_label="United States",+ selected_index=1,+ )++ assert handler._normal_select_successful(result)+ normal_select_llm.assert_not_awaited()+ locator.select_option.assert_awaited_once_with(+ value="US",+ timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS,+ )++[email protected]("scraped_value", [" US", "US "])+async def test_normal_select_edge_whitespace_value_verifies_without_llm(+ monkeypatch: pytest.MonkeyPatch,+ scraped_value: str,+) -> None:+ """domUtils.removeMultipleSpaces() leaves option.value untrimmed too, so the value arm needs the+ same normalization as the label arm."""+ result, locator, normal_select_llm = await _run_normal_select(+ monkeypatch,+ feature_enabled=True,+ action=_select_action("United States"),+ options=[+ {"optionIndex": 0, "text": "Canada", "value": "CA"},+ {"optionIndex": 1, "text": "United States", "value": scraped_value},+ ],+ selected_value="US",+ selected_label="United States",+ selected_index=1,+ )++ assert handler._normal_select_successful(result)+ normal_select_llm.assert_not_awaited()+ locator.select_option.assert_awaited_once_with(+ value=scraped_value,+ timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS,+ )+++async def test_normal_select_readback_timeout_keeps_success_and_skips_stale_locator(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """select_option already committed the value; a select-triggered rerender detaches the element so+ the read-back times out. That must not spend the Playwright default 30s, and must not drive the LLM+ fallback through the now-stale locator."""+ action = _select_action("United States")+ result, locator, normal_select_llm = await _run_normal_select(+ monkeypatch,+ feature_enabled=True,+ action=action,+ options=[+ {"optionIndex": 0, "text": "Canada", "value": "CA"},+ {"optionIndex": 1, "text": "United States", "value": "US"},+ ],+ selected_value="US",+ evaluate_side_effect=PlaywrightTimeoutError("Timeout 30000ms exceeded waiting for locator"),+ llm_response={"value": "US", "index": None},+ )++ assert handler._normal_select_successful(result)+ normal_select_llm.assert_not_awaited()+ assert not action.has_mini_agent+ locator.select_option.assert_awaited_once_with(+ value="US",+ timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS,+ )+ assert locator.evaluate.await_args.kwargs["timeout"] == handler.settings.BROWSER_ACTION_TIMEOUT_MS+++async def test_normal_select_readback_index_mismatch_falls_back_to_llm(monkeypatch: pytest.MonkeyPatch) -> None:+ action = _select_action("United States")+ result, locator, normal_select_llm = await _run_normal_select(+ monkeypatch,+ feature_enabled=True,+ action=action,+ options=[+ {"optionIndex": 0, "text": "Canada", "value": "CA"},+ {"optionIndex": 1, "text": "United States", "value": "US"},+ ],+ selected_value="US",+ selected_index=0,+ llm_response={"value": "US", "index": None},+ )++ assert handler._normal_select_successful(result)+ assert action.has_mini_agent is True+ normal_select_llm.assert_awaited_once()++@pytest.mark.asyncioasync def test_normal_select_flag_off_uses_llm_path(monkeypatch: pytest.MonkeyPatch) -> None:action = _select_action("United States")