Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11946, SKY-11945): compute select shadow agreement by normalized committed value (#7286)
Marc Kelechava96d070896b14
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 5bf18746c..2acc3be50 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -339,6 +339,12 @@ def _truncate_select_shadow_field(text: str | None) -> str | None:return text[:SELECT_SHADOW_MATCH_FIELD_MAX_CHARS] + "…"+def _normalized_select_shadow_field(text: str | None) -> str | None:+ if text is None:+ return None+ return _truncate_select_shadow_field(_normalize_select_shadow_text(text))++class SelectShadowAgreement(BaseModel):agrees: bool | Nonellm_index: int | None = None@@ -531,18 +537,17 @@ def _select_shadow_agrees_with_native_choice(if matched_index is None:agreement.agrees = Falsereturn agreement- if llm_index is not None:- agreement.agrees = matched_index == llm_index- return agreement- if llm_value is None or matched_index >= len(candidates):- return agreement- matched_candidate = candidates[matched_index]llm_value_norm = _normalize_select_shadow_text(llm_value)- agreement.agrees = llm_value_norm in {- _normalize_select_shadow_text(matched_candidate.get("label")),- _normalize_select_shadow_text(matched_candidate.get("value")),- }+ if llm_value_norm:+ if matched_index < len(candidates):+ matched_candidate = candidates[matched_index]+ agreement.agrees = llm_value_norm in {+ _normalize_select_shadow_text(matched_candidate.get("label")),+ _normalize_select_shadow_text(matched_candidate.get("value")),+ }+ elif llm_index is not None:+ agreement.agrees = matched_index == llm_indexreturn agreement@@ -561,15 +566,14 @@ def _select_shadow_agrees_with_element_choice(return agreementmatched_candidate = candidates[matched_index]- matched_element_id = matched_candidate.get("element_id")- if matched_element_id and llm_element_id:- agreement.agrees = matched_element_id == llm_element_id- return agreement- if llm_value is None:- return agreement- agreement.agrees = _normalize_select_shadow_text(matched_candidate.get("label")) == _normalize_select_shadow_text(- llm_value- )+ llm_value_norm = _normalize_select_shadow_text(llm_value)+ # Element ids are unstable across incremental scrapes, so id equality never decides+ # agreement — ids stay in the logged detail fields as metadata only.+ if llm_value_norm:+ agreement.agrees = llm_value_norm in {+ _normalize_select_shadow_text(matched_candidate.get("label")),+ _normalize_select_shadow_text(matched_candidate.get("value")),+ }return agreement@@ -600,6 +604,10 @@ def _log_select_shadow_match("llm_index": result.llm_index,"llm_value": _truncate_select_shadow_field(result.llm_value),"llm_element_id": result.llm_element_id,+ "normalized_target_value": _normalized_select_shadow_field(target_value),+ "normalized_matched_label": _normalized_select_shadow_field(matched_candidate.get("label")),+ "normalized_matched_value": _normalized_select_shadow_field(matched_candidate.get("value")),+ "normalized_llm_value": _normalized_select_shadow_field(result.llm_value),}disagreement_fields = {key: value for key, value in disagreement_fields.items() if value is not None}LOG.info(diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.pyindex 2629148cb..87d9e06ba 100644--- a/tests/unit/test_select_shadow_match.py+++ b/tests/unit/test_select_shadow_match.py@@ -1,5 +1,5 @@from types import SimpleNamespace-from unittest.mock import AsyncMock, Mock+from unittest.mock import AsyncMock, MagicMock, Mockfrom zoneinfo import ZoneInfoimport pytest@@ -10,6 +10,18 @@ from skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.webeye.actions import handlerfrom skyvern.webeye.actions.actions import InputOrSelectContext, SelectOption, SelectOptionAction+_SHADOW_OPTION = {+ "id": "matched-id",+ "tagName": "div",+ "attributes": {"role": "option"},+ "text": "Committed Value",+ "interactable": True,+}+++def _shadow_task() -> SimpleNamespace:+ return SimpleNamespace(navigation_goal="goal", navigation_payload={}, llm_key=None)+class _FakeSelectLocator:def __init__(self) -> None:@@ -124,6 +136,10 @@ async def test_normal_select_shadow_disagreement_emits_rich_fields(assert event["matched_value"] == "years"assert event["llm_index"] == 1assert event["llm_value"] == "months"+ assert event["normalized_target_value"] == "years"+ assert event["normalized_matched_label"] == "years"+ assert event["normalized_matched_value"] == "years"+ assert event["normalized_llm_value"] == "months"assert "matched_element_id" not in eventassert "llm_element_id" not in event@@ -151,6 +167,10 @@ async def test_normal_select_agreement_stays_lean("llm_index","llm_value","llm_element_id",+ "normalized_target_value",+ "normalized_matched_label",+ "normalized_matched_value",+ "normalized_llm_value",):assert lean_only not in event@@ -159,9 +179,9 @@ async def test_normal_select_agreement_stays_lean(async def test_normal_select_disagreement_truncates_free_text_fields(monkeypatch: pytest.MonkeyPatch,) -> None:- long_target = "A" * 200- long_option = "B" * 200- long_llm_value = "C" * 200+ long_target = " " + "A’ B " * 40+ long_option = " " + "C’ D " * 40+ long_llm_value = " " + "E’ F " * 40logs = await _run_normal_select_with_shadow_log(monkeypatch,target_label=long_target,@@ -177,8 +197,17 @@ async def test_normal_select_disagreement_truncates_free_text_fields(bound = handler.SELECT_SHADOW_MATCH_FIELD_MAX_CHARSassert event["match_agrees_with_llm"] is Falsefor field in ("target_value", "matched_label", "matched_value", "llm_value"):- assert len(event[field]) == bound + 1- assert event[field].endswith("…")+ value = event[field]+ assert isinstance(value, str)+ assert len(value) == bound + 1+ assert value.endswith("…")+ for field, raw_value in (+ ("normalized_target_value", long_target),+ ("normalized_matched_label", long_target),+ ("normalized_matched_value", long_option),+ ("normalized_llm_value", long_llm_value),+ ):+ assert event[field] == handler._truncate_select_shadow_field(handler._normalize_select_shadow_text(raw_value))@pytest.mark.asyncio@@ -272,3 +301,267 @@ def test_select_shadow_match_normalizes_curly_apostrophes() -> None:assert matched_index == 0assert tier == "exact"+++ ("matched_index", "llm_element_id", "llm_value", "expected_agrees"),+ [+ pytest.param(0, "other-id", " COMMITTED VALUE ", True, id="value-match-overrides-different-id"),+ pytest.param(0, "matched-id", "paraphrased value", False, id="value-decides-despite-matching-id"),+ pytest.param(0, "other-id", "different value", False, id="different-value-and-id"),+ pytest.param(0, "matched-id", None, None, id="id-never-decides-without-value"),+ pytest.param(0, "other-id", None, None, id="differing-ids-unjudgeable-without-value"),+ pytest.param(0, None, None, None, id="no-signals"),+ pytest.param(0, None, " ‘ ` ’ ", None, id="normalized-empty-value-is-unavailable"),+ pytest.param(None, "matched-id", "committed value", False, id="no-matcher-choice"),+ pytest.param(1, "matched-id", "committed value", None, id="matcher-index-out-of-range"),+ ],+)+def test_element_choice_shadow_agreement_uses_available_value_and_id_signals(+ matched_index: int | None,+ llm_element_id: str | None,+ llm_value: str | None,+ expected_agrees: bool | None,+) -> None:+ candidates: list[dict[str, str | None]] = [+ {"label": "Display Label", "value": "committed value", "element_id": "matched-id"}+ ]++ agreement = handler._select_shadow_agrees_with_element_choice(+ candidates,+ matched_index,+ llm_element_id=llm_element_id,+ llm_value=llm_value,+ )++ assert agreement.agrees is expected_agrees+++def test_element_choice_shadow_agreement_uses_exact_normalizer() -> None:+ agreement = handler._select_shadow_agrees_with_element_choice(+ [{"label": "Worker’s Compensation", "value": None, "element_id": "matched-id"}],+ 0,+ llm_element_id="other-id",+ llm_value=" WORKER'S\tCOMPENSATION ",+ )++ assert agreement.agrees is True+++ ("matched_index", "llm_index", "llm_value", "expected_agrees"),+ [+ pytest.param(0, 1, "same value", True, id="value-match-overrides-different-index"),+ pytest.param(0, 0, "paraphrased value", False, id="value-decides-despite-matching-index"),+ pytest.param(0, 1, "different value", False, id="different-value-and-index"),+ pytest.param(0, None, None, None, id="no-signals"),+ pytest.param(0, -1, None, False, id="llm-abstained"),+ pytest.param(None, 0, "same value", False, id="no-matcher-choice"),+ pytest.param(2, 2, "same value", None, id="invalid-candidate-index-with-value-stays-unjudged"),+ ],+)+def test_native_choice_shadow_agreement_uses_available_value_and_index_signals(+ matched_index: int | None,+ llm_index: int | None,+ llm_value: str | None,+ expected_agrees: bool | None,+) -> None:+ candidates: list[dict[str, str | None]] = [+ {"label": "Duplicate", "value": "same value"},+ {"label": "Duplicate", "value": "same value"},+ ]++ agreement = handler._select_shadow_agrees_with_native_choice(+ candidates,+ matched_index,+ llm_index=llm_index,+ llm_value=llm_value,+ )++ assert agreement.agrees is expected_agrees+++def test_custom_select_same_value_different_element_id_logs_agreement(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)++ with structlog.testing.capture_logs() as logs:+ handler._log_select_shadow_match(+ prompt_name="custom-select/dropdown",+ target_value="Workers' Compensation",+ get_candidates=lambda: [{"label": "Worker’s Compensation", "value": None, "element_id": "el-A"}],+ agreement=lambda candidates, matched_index: handler._select_shadow_agrees_with_element_choice(+ candidates,+ matched_index,+ llm_element_id="el-B",+ llm_value="workers compensation",+ ),+ )++ (event,) = (entry for entry in logs if entry.get("event") == "select_shadow_match")+ assert event["match_tier"] == "exact"+ assert event["match_found"] is True+ assert event["match_agrees_with_llm"] is True+ assert "matched_element_id" not in event+++ ("llm_value", "expected_agrees"),+ [+ pytest.param(" committed value ", True, id="normalized-value-match"),+ pytest.param("Different Value", False, id="value-mismatch"),+ ],+)+async def test_auto_completion_call_path_forwards_committed_value_to_shadow_match(+ monkeypatch: pytest.MonkeyPatch,+ llm_value: str,+ expected_agrees: bool,+) -> None:+ frame = MagicMock()+ skyvern_frame = MagicMock(safe_wait_for_animation_end=AsyncMock())+ incremental_scraped = MagicMock(+ start_listen_dom_increment=AsyncMock(),+ stop_listen_dom_increment=AsyncMock(),+ get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]),+ )+ incremental_scraped.build_html_tree.return_value = "<div>Committed Value</div>"+ skyvern_element = MagicMock(+ get_frame=Mock(return_value=frame),+ get_element_handler=AsyncMock(return_value=MagicMock()),+ press_fill=AsyncMock(),+ press_key=AsyncMock(),+ is_visible=AsyncMock(return_value=True),+ input_clear=AsyncMock(),+ )+ monkeypatch.setattr(handler.SkyvernFrame, "create_instance", AsyncMock(return_value=skyvern_frame))+ monkeypatch.setattr(handler, "IncrementalScrapePage", Mock(return_value=incremental_scraped))+ monkeypatch.setattr(handler, "get_slim_output_template_value", AsyncMock(return_value=""))+ monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))+ monkeypatch.setattr(+ handler.app,+ "AUTO_COMPLETION_LLM_API_HANDLER",+ AsyncMock(+ return_value={+ "id": "different-id",+ "value": llm_value,+ "direct_searching": True,+ }+ ),+ )+ monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)++ with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):+ with structlog.testing.capture_logs() as logs:+ await handler.choose_auto_completion_dropdown(+ context=InputOrSelectContext(field="Field", is_search_bar=False),+ page=MagicMock(),+ scraped_page=MagicMock(),+ dom=MagicMock(),+ text="Committed Value",+ skyvern_element=skyvern_element,+ step=SimpleNamespace(step_id="step-1"),+ task=_shadow_task(),+ )++ events = [log for log in logs if log.get("event") == "select_shadow_match"]+ assert len(events) == 1+ event = events[0]+ assert event["prompt_name"] == "auto-completion-choose-option"+ assert event["match_agrees_with_llm"] is expected_agrees+ if not expected_agrees:+ assert event["normalized_matched_label"] == "committed value"+ assert event["normalized_llm_value"] == "different value"+++async def test_emerging_select_call_path_forwards_committed_value_to_shadow_match(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ selected_element = MagicMock(+ get_attr=AsyncMock(return_value=None),+ scroll_into_view=AsyncMock(),+ click=AsyncMock(),+ )+ dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=selected_element))+ llm_handler = AsyncMock(+ return_value={"id": "different-id", "value": " committed value ", "action_type": "click"}+ )+ monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))+ monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div>Committed Value</div>"))+ monkeypatch.setattr(handler, "_select_deterministic_custom_option", AsyncMock(return_value=None))+ monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))+ monkeypatch.setattr(+ handler.LLMAPIHandlerFactory,+ "get_override_llm_api_handler",+ Mock(return_value=llm_handler),+ )+ monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)++ scraped_page = SimpleNamespace(id_to_css_dict={})+ scraped_page_after_open = SimpleNamespace(+ id_to_css_dict={"matched-id": "[data-id=matched-id]"},+ element_tree_trimmed=[_SHADOW_OPTION],+ )+ with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):+ with structlog.testing.capture_logs() as logs:+ await handler.select_from_emerging_elements(+ current_element_id="field-id",+ options=handler.CustomSelectPromptOptions(target_value="Committed Value"),+ page=MagicMock(),+ scraped_page=scraped_page,+ scraped_page_after_open=scraped_page_after_open,+ new_interactable_element_ids=["matched-id"],+ step=SimpleNamespace(step_id="step-1"),+ task=_shadow_task(),+ )++ events = [log for log in logs if log.get("event") == "select_shadow_match"]+ assert len(events) == 1+ assert events[0]["prompt_name"] == "custom-select/emerging"+ assert events[0]["match_agrees_with_llm"] is True+++async def test_dropdown_select_call_path_forwards_committed_value_to_shadow_match(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ incremental_scraped = MagicMock(+ get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]),+ )+ incremental_scraped.build_element_tree.return_value = "<div>Committed Value</div>"+ selected_element = MagicMock(+ get_tag_name=Mock(return_value="div"),+ get_attr=AsyncMock(return_value=None),+ scroll_into_view=AsyncMock(),+ click=AsyncMock(),+ )+ monkeypatch.setattr(handler, "locate_dropdown_menu", AsyncMock(return_value=None))+ monkeypatch.setattr(handler, "_select_deterministic_custom_option", AsyncMock(return_value=None))+ monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))+ monkeypatch.setattr(+ handler.app,+ "CUSTOM_SELECT_AGENT_LLM_API_HANDLER",+ AsyncMock(return_value={"id": "different-id", "value": " committed value ", "action_type": "click"}),+ )+ monkeypatch.setattr(handler.SkyvernElement, "create_from_incremental", AsyncMock(return_value=selected_element))+ monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)++ with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):+ with structlog.testing.capture_logs() as logs:+ await handler.select_from_dropdown(+ context=InputOrSelectContext(field="Field", is_required=True),+ page=MagicMock(),+ skyvern_element=MagicMock(get_id=Mock(return_value="field-id")),+ skyvern_frame=MagicMock(),+ incremental_scraped=incremental_scraped,+ check_filter_funcs=[],+ step=SimpleNamespace(step_id="step-1"),+ task=_shadow_task(),+ force_select=True,+ target_value="Committed Value",+ )++ events = [log for log in logs if log.get("event") == "select_shadow_match"]+ assert len(events) == 1+ assert events[0]["prompt_name"] == "custom-select/dropdown"+ assert events[0]["match_agrees_with_llm"] is True