Skip to content

Automate browser based workflows with AI

Stop deterministic normal-select from falling back to the LLM on false negatives (#7464)

Marc Kelechavaeb6f4aa47471
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 9f19f2ba0..d4b0f94fe 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -1558,13 +1558,21 @@ async def _select_deterministic_normal_option(
return action_result
-async def _verify_normal_select_option(
+async def _normal_select_readback_contradicts(
*,
locator: Locator,
matched_index: int,
matched_label: str | None,
matched_value: str | None,
) -> bool:
+ """True only when a readable read-back disagrees with the deterministic selection.
+
+ An unreadable read-back is not a contradiction. select_option has already reported the value
+ committed, and a select's own change event can rerender or navigate the page away, detaching the
+ element; re-running the LLM fallback against that stale locator only fails again by value, label,
+ and index. Same reasoning as `is_post_dispatch_click_timeout`: the side effect landed, so a
+ timed-out post-action read must not trigger a duplicating fallback chain.
+ """
try:
selection = await locator.evaluate(
r"""
@@ -1580,11 +1588,12 @@ async def _verify_normal_select_option(
value: option ? normalize(option.value) : normalize(select.value),
};
}
- """
+ """,
+ timeout=settings.BROWSER_ACTION_TIMEOUT_MS,
)
except Exception:
LOG.info(
- "Failed to read normal select option after deterministic selection",
+ "Failed to read normal select option after deterministic selection; keeping select_option success",
expected_index=matched_index,
expected_label=matched_label,
expected_value=matched_value,
@@ -1594,7 +1603,7 @@ async def _verify_normal_select_option(
if not isinstance(selection, dict):
LOG.info(
- "Normal select read-back returned unexpected payload",
+ "Normal select read-back returned unexpected payload; keeping select_option success",
expected_index=matched_index,
expected_label=matched_label,
expected_value=matched_value,
@@ -1602,26 +1611,32 @@ async def _verify_normal_select_option(
)
return False
+ # The read-back JS trims, but scraped option text does not: domUtils.removeMultipleSpaces()
+ # collapses whitespace runs without trimming, and returns early when there is no double
+ # space/tab/newline, so a single edge space survives verbatim into matched_label/matched_value.
+ expected_label = " ".join(matched_label.split()) if matched_label is not None else None
+ expected_value = " ".join(matched_value.split()) if matched_value is not None else None
+
actual_index = selection.get("index")
actual_value = selection.get("value")
actual_label = selection.get("label") or actual_value
if (
actual_index == matched_index
- and actual_label == matched_label
- and (matched_value is None or actual_value == matched_value)
+ and actual_label == expected_label
+ and (expected_value is None or actual_value == expected_value)
):
- return True
+ return False
LOG.info(
"Normal select read-back did not match deterministic option",
expected_index=matched_index,
- expected_label=matched_label,
- expected_value=matched_value,
+ expected_label=expected_label,
+ expected_value=expected_value,
actual_index=actual_index,
actual_label=actual_label,
actual_value=actual_value,
)
- return False
+ return True
async def check_date_format(
@@ -7639,7 +7654,7 @@ async def normal_select(
matched_value=resolution.matched_value,
matched_index=resolution.matched_index,
)
- if _normal_select_successful(deterministic_result) and await _verify_normal_select_option(
+ if _normal_select_successful(deterministic_result) and not await _normal_select_readback_contradicts(
locator=locator,
matched_index=resolution.matched_index,
matched_label=resolution.matched_label,
diff --git a/tests/unit/test_normal_select_deterministic.py b/tests/unit/test_normal_select_deterministic.py
index 1c74806bc..6b16bf878 100644
--- a/tests/unit/test_normal_select_deterministic.py
+++ b/tests/unit/test_normal_select_deterministic.py
@@ -5,6 +5,7 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, call
import pytest
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
from skyvern.forge.agent_functions import AgentFunction
from skyvern.webeye.actions import handler
@@ -66,19 +67,23 @@ def _locator(
*,
selected_label: str | None = None,
selected_index: int | None = None,
+ evaluate_side_effect: BaseException | None = None,
) -> MagicMock:
locator = MagicMock()
locator.click = AsyncMock()
locator.select_option = AsyncMock()
locator.input_value = AsyncMock(return_value=selected_value)
- locator.evaluate = AsyncMock(
- return_value=_selected_option_readback(
- options,
- selected_value,
- selected_label=selected_label,
- selected_index=selected_index,
+ if evaluate_side_effect is not None:
+ locator.evaluate = AsyncMock(side_effect=evaluate_side_effect)
+ else:
+ locator.evaluate = AsyncMock(
+ return_value=_selected_option_readback(
+ options,
+ selected_value,
+ selected_label=selected_label,
+ selected_index=selected_index,
+ )
)
- )
return locator
@@ -93,6 +98,7 @@ async def _run_normal_select(
selected_label: str | None = None,
selected_index: int | None = None,
llm_response: dict | None = None,
+ evaluate_side_effect: BaseException | None = None,
) -> tuple[list, MagicMock, AsyncMock]:
provider = SimpleNamespace(
is_feature_enabled_cached=AsyncMock(return_value=feature_enabled),
@@ -106,7 +112,13 @@ async def _run_normal_select(
monkeypatch.setattr(handler.prompt_engine, "load_prompt", MagicMock(return_value="prompt"))
monkeypatch.setattr(handler.skyvern_context, "ensure_context", MagicMock(return_value=SimpleNamespace(tz_info=UTC)))
- locator = _locator(options, selected_value, selected_label=selected_label, selected_index=selected_index)
+ locator = _locator(
+ options,
+ selected_value,
+ selected_label=selected_label,
+ selected_index=selected_index,
+ evaluate_side_effect=evaluate_side_effect,
+ )
result = await handler.normal_select(
action=action,
skyvern_element=_FakeSelectElement(options, locator, selected_attr), # type: ignore[arg-type]
@@ -260,7 +272,7 @@ async def test_normal_select_unsafe_match_falls_back_to_llm(
@pytest.mark.asyncio
-async def test_normal_select_readback_failure_falls_back_to_llm(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_normal_select_readback_value_mismatch_falls_back_to_llm(monkeypatch: pytest.MonkeyPatch) -> None:
action = _select_action("United States")
result, locator, normal_select_llm = await _run_normal_select(
monkeypatch,
@@ -285,6 +297,116 @@ async def test_normal_select_readback_failure_falls_back_to_llm(monkeypatch: pyt
normal_select_llm.assert_awaited_once()
[email protected]("scraped_label", [" United States", "United States "])
+async def test_normal_select_edge_whitespace_label_verifies_without_llm(
+ monkeypatch: pytest.MonkeyPatch,
+ scraped_label: str,
+) -> None:
+ """Read-back trims the label but the scraped option text does not (domUtils.removeMultipleSpaces
+ collapses runs without trimming), so an edge-space-only difference must not read as a mismatch."""
+ result, locator, normal_select_llm = await _run_normal_select(
+ monkeypatch,
+ feature_enabled=True,
+ action=_select_action("United States"),
+ options=[
+ {"optionIndex": 0, "text": "Canada", "value": "CA"},
+ {"optionIndex": 1, "text": scraped_label, "value": "US"},
+ ],
+ selected_value="US",
+ selected_label="United States",
+ selected_index=1,
+ )
+
+ assert handler._normal_select_successful(result)
+ normal_select_llm.assert_not_awaited()
+ locator.select_option.assert_awaited_once_with(
+ value="US",
+ timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS,
+ )
+
+
[email protected]("scraped_value", [" US", "US "])
+async def test_normal_select_edge_whitespace_value_verifies_without_llm(
+ monkeypatch: pytest.MonkeyPatch,
+ scraped_value: str,
+) -> None:
+ """domUtils.removeMultipleSpaces() leaves option.value untrimmed too, so the value arm needs the
+ same normalization as the label arm."""
+ result, locator, normal_select_llm = await _run_normal_select(
+ monkeypatch,
+ feature_enabled=True,
+ action=_select_action("United States"),
+ options=[
+ {"optionIndex": 0, "text": "Canada", "value": "CA"},
+ {"optionIndex": 1, "text": "United States", "value": scraped_value},
+ ],
+ selected_value="US",
+ selected_label="United States",
+ selected_index=1,
+ )
+
+ assert handler._normal_select_successful(result)
+ normal_select_llm.assert_not_awaited()
+ locator.select_option.assert_awaited_once_with(
+ value=scraped_value,
+ timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS,
+ )
+
+
+async def test_normal_select_readback_timeout_keeps_success_and_skips_stale_locator(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """select_option already committed the value; a select-triggered rerender detaches the element so
+ the read-back times out. That must not spend the Playwright default 30s, and must not drive the LLM
+ fallback through the now-stale locator."""
+ action = _select_action("United States")
+ result, locator, normal_select_llm = await _run_normal_select(
+ monkeypatch,
+ feature_enabled=True,
+ action=action,
+ options=[
+ {"optionIndex": 0, "text": "Canada", "value": "CA"},
+ {"optionIndex": 1, "text": "United States", "value": "US"},
+ ],
+ selected_value="US",
+ evaluate_side_effect=PlaywrightTimeoutError("Timeout 30000ms exceeded waiting for locator"),
+ llm_response={"value": "US", "index": None},
+ )
+
+ assert handler._normal_select_successful(result)
+ normal_select_llm.assert_not_awaited()
+ assert not action.has_mini_agent
+ locator.select_option.assert_awaited_once_with(
+ value="US",
+ timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS,
+ )
+ assert locator.evaluate.await_args.kwargs["timeout"] == handler.settings.BROWSER_ACTION_TIMEOUT_MS
+
+
+async def test_normal_select_readback_index_mismatch_falls_back_to_llm(monkeypatch: pytest.MonkeyPatch) -> None:
+ action = _select_action("United States")
+ result, locator, normal_select_llm = await _run_normal_select(
+ monkeypatch,
+ feature_enabled=True,
+ action=action,
+ options=[
+ {"optionIndex": 0, "text": "Canada", "value": "CA"},
+ {"optionIndex": 1, "text": "United States", "value": "US"},
+ ],
+ selected_value="US",
+ selected_index=0,
+ llm_response={"value": "US", "index": None},
+ )
+
+ assert handler._normal_select_successful(result)
+ assert action.has_mini_agent is True
+ normal_select_llm.assert_awaited_once()
+
+
@pytest.mark.asyncio
async def test_normal_select_flag_off_uses_llm_path(monkeypatch: pytest.MonkeyPatch) -> None:
action = _select_action("United States")