Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11570 log shadow-match disagreement details (#7027)
Marc Kelechavaef22a35461d3
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 2d33c164b..7337517b1 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -17,7 +17,7 @@ from fuzzysearch import find_near_matchesfrom opentelemetry import trace as otel_tracefrom playwright._impl._errors import Error as PlaywrightErrorfrom playwright.async_api import Download, FileChooser, Frame, Locator, Page, Response, TimeoutError-from pydantic import BaseModel+from pydantic import BaseModel, field_validatorfrom skyvern.config import settingsfrom skyvern.constants import (@@ -322,26 +322,64 @@ def _select_shadow_candidates_from_elements(elements: list[dict]) -> list[dict[sreturn candidates+SELECT_SHADOW_MATCH_FIELD_MAX_CHARS = 120+++def _truncate_select_shadow_field(text: str | None) -> str | None:+ if text is None:+ return None+ if len(text) <= SELECT_SHADOW_MATCH_FIELD_MAX_CHARS:+ return text+ return text[:SELECT_SHADOW_MATCH_FIELD_MAX_CHARS] + "…"+++class SelectShadowAgreement(BaseModel):+ agrees: bool | None+ llm_index: int | None = None+ llm_value: str | None = None+ llm_element_id: str | None = None++ # Fields come straight from LLM JSON; malformed values must never drop the shadow event.+ @field_validator("llm_value", "llm_element_id", mode="before")+ @classmethod+ def _coerce_llm_text(cls, value: Any) -> str | None:+ return None if value is None else str(value)++ @field_validator("llm_index", mode="before")+ @classmethod+ def _coerce_llm_index(cls, value: Any) -> int | None:+ if value is None or isinstance(value, int):+ return value+ try:+ return int(str(value).strip())+ except ValueError:+ return None++def _select_shadow_agrees_with_native_choice(candidates: list[dict[str, str | None]],matched_index: int | None,*,llm_index: int | None,llm_value: str | None,-) -> bool | None:+) -> SelectShadowAgreement:+ agreement = SelectShadowAgreement(agrees=None, llm_index=llm_index, llm_value=llm_value)if matched_index is None:- return False+ agreement.agrees = False+ return agreementif llm_index is not None:- return matched_index == llm_index+ agreement.agrees = matched_index == llm_index+ return agreementif llm_value is None or matched_index >= len(candidates):- return None+ return agreementmatched_candidate = candidates[matched_index]llm_value_norm = _normalize_select_shadow_text(llm_value)- return llm_value_norm in {+ agreement.agrees = llm_value_norm in {_normalize_select_shadow_text(matched_candidate.get("label")),_normalize_select_shadow_text(matched_candidate.get("value")),}+ return agreementdef _select_shadow_agrees_with_element_choice(@@ -350,19 +388,25 @@ def _select_shadow_agrees_with_element_choice(*,llm_element_id: str | None,llm_value: str | None,-) -> bool | None:+) -> SelectShadowAgreement:+ agreement = SelectShadowAgreement(agrees=None, llm_value=llm_value, llm_element_id=llm_element_id)if matched_index is None:- return False+ agreement.agrees = False+ return agreementif matched_index >= len(candidates):- return None+ return agreementmatched_candidate = candidates[matched_index]matched_element_id = matched_candidate.get("element_id")if matched_element_id and llm_element_id:- return matched_element_id == llm_element_id+ agreement.agrees = matched_element_id == llm_element_id+ return agreementif llm_value is None:- return None- return _normalize_select_shadow_text(matched_candidate.get("label")) == _normalize_select_shadow_text(llm_value)+ return agreement+ agreement.agrees = _normalize_select_shadow_text(matched_candidate.get("label")) == _normalize_select_shadow_text(+ llm_value+ )+ return agreementdef _log_select_shadow_match(@@ -370,7 +414,7 @@ def _log_select_shadow_match(prompt_name: str,target_value: str | None,get_candidates: Callable[[], list[dict[str, str | None]]],- agreement: Callable[[list[dict[str, str | None]], int | None], bool | None],+ agreement: Callable[[list[dict[str, str | None]], int | None], SelectShadowAgreement],) -> None:if not _select_shadow_match_enabled():return@@ -379,13 +423,29 @@ def _log_select_shadow_match(candidates = get_candidates()option_labels = [candidate["label"] or "" for candidate in candidates]matched_index, tier = classify_option_match(target_value, option_labels)+ result = agreement(candidates, matched_index)+ disagreement_fields: dict[str, Any] = {}+ if matched_index is not None and result.agrees is not True:+ matched_candidate = candidates[matched_index] if matched_index < len(candidates) else {}+ disagreement_fields = {+ "target_value": _truncate_select_shadow_field(target_value),+ "matched_index": matched_index,+ "matched_label": _truncate_select_shadow_field(matched_candidate.get("label")),+ "matched_value": _truncate_select_shadow_field(matched_candidate.get("value")),+ "matched_element_id": matched_candidate.get("element_id"),+ "llm_index": result.llm_index,+ "llm_value": _truncate_select_shadow_field(result.llm_value),+ "llm_element_id": result.llm_element_id,+ }+ disagreement_fields = {key: value for key, value in disagreement_fields.items() if value is not None}LOG.info("select_shadow_match",prompt_name=prompt_name,option_count=len(option_labels),match_tier=tier,match_found=matched_index is not None,- match_agrees_with_llm=agreement(candidates, matched_index),+ match_agrees_with_llm=result.agrees,+ **disagreement_fields,)except Exception:LOG.debug("select_shadow_match failed", exc_info=True)diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.pyindex 3f6f80e39..2629148cb 100644--- a/tests/unit/test_select_shadow_match.py+++ b/tests/unit/test_select_shadow_match.py@@ -104,6 +104,83 @@ async def test_normal_select_logs_shadow_match_tier_and_llm_agreement(assert logs[0]["match_agrees_with_llm"] is expected_agrees+async def test_normal_select_shadow_disagreement_emits_rich_fields(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ logs = await _run_normal_select_with_shadow_log(+ monkeypatch,+ target_label="Years",+ llm_response={"index": 1, "value": "months"},+ )++ assert len(logs) == 1+ event = logs[0]+ assert event["match_found"] is True+ assert event["match_agrees_with_llm"] is False+ assert event["target_value"] == "Years"+ assert event["matched_index"] == 0+ assert event["matched_label"] == "Years"+ assert event["matched_value"] == "years"+ assert event["llm_index"] == 1+ assert event["llm_value"] == "months"+ assert "matched_element_id" not in event+ assert "llm_element_id" not in event+++async def test_normal_select_agreement_stays_lean(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ logs = await _run_normal_select_with_shadow_log(+ monkeypatch,+ target_label="Years",+ llm_response={"index": 0, "value": "years"},+ )++ assert len(logs) == 1+ event = logs[0]+ assert event["match_found"] is True+ assert event["match_agrees_with_llm"] is True+ for lean_only in (+ "target_value",+ "matched_index",+ "matched_label",+ "matched_value",+ "matched_element_id",+ "llm_index",+ "llm_value",+ "llm_element_id",+ ):+ assert lean_only not in event+++async def test_normal_select_disagreement_truncates_free_text_fields(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ long_target = "A" * 200+ long_option = "B" * 200+ long_llm_value = "C" * 200+ logs = await _run_normal_select_with_shadow_log(+ monkeypatch,+ target_label=long_target,+ llm_response={"index": 1, "value": long_llm_value},+ options=[+ {"optionIndex": 0, "text": long_target, "value": long_option},+ {"optionIndex": 1, "text": "Other", "value": "other"},+ ],+ )++ assert len(logs) == 1+ event = logs[0]+ bound = handler.SELECT_SHADOW_MATCH_FIELD_MAX_CHARS+ assert event["match_agrees_with_llm"] is False+ for field in ("target_value", "matched_label", "matched_value", "llm_value"):+ assert len(event[field]) == bound + 1+ assert event[field].endswith("…")++@pytest.mark.asyncioasync def test_normal_select_shadow_match_keeps_blank_placeholder_index_alignment(monkeypatch: pytest.MonkeyPatch,@@ -159,6 +236,37 @@ def test_shadow_match_candidate_errors_do_not_escape_live_path(monkeypatch: pyteagreement.assert_not_called()+async def test_non_string_llm_values_are_coerced_not_dropped(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ logs = await _run_normal_select_with_shadow_log(+ monkeypatch,+ target_label="Years",+ llm_response={"index": 1, "value": 2024},+ )++ assert len(logs) == 1+ event = logs[0]+ assert event["match_agrees_with_llm"] is False+ assert event["llm_value"] == "2024"+++def test_select_shadow_agreement_coerces_malformed_llm_fields() -> None:+ agreement = handler.SelectShadowAgreement(+ agrees=None,+ llm_index="2",+ llm_value=2024,+ llm_element_id=97,+ )+ assert agreement.llm_index == 2+ assert agreement.llm_value == "2024"+ assert agreement.llm_element_id == "97"++ non_numeric_index = handler.SelectShadowAgreement(agrees=None, llm_index="not-a-number")+ assert non_numeric_index.llm_index is None++def test_select_shadow_match_normalizes_curly_apostrophes() -> None:matched_index, tier = handler.classify_option_match("Worker’s Compensation", ["Workers Compensation"])