Skip to content

Automate browser based workflows with AI

62.7 KB
Raw Blame
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, Mock
from zoneinfo import ZoneInfo

import pytest
import structlog.testing

from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.webeye.actions import handler
from skyvern.webeye.actions.actions import ClickAction, InputOrSelectContext, SelectOption, SelectOptionAction

_SHADOW_OPTION = {
    "id": "matched-id",
    "tagName": "div",
    "attributes": {"role": "option"},
    "text": "Committed Value",
    "interactable": True,
}


def _shadow_task() -> SimpleNamespace:
    return SimpleNamespace(
        task_id="task-1",
        workflow_run_id=None,
        navigation_goal="goal",
        navigation_payload={},
        llm_key=None,
        organization_id=None,
        workflow_permanent_id=None,
    )


class _FakeSelectLocator:
    def __init__(self) -> None:
        self.click = AsyncMock()
        self.select_option = AsyncMock()


class _FakeSelectElement:
    def __init__(self, options: list[dict[str, object]]) -> None:
        self._options = options
        self._locator = _FakeSelectLocator()

    async def get_attr(self, *_args: object, **_kwargs: object) -> str | None:
        return None

    async def refresh_select_options(self) -> tuple[list[dict[str, object]], str]:
        return self._options, ""

    def build_HTML(self) -> str:
        return "<select><option>stub</option></select>"

    def get_locator(self) -> _FakeSelectLocator:
        return self._locator

    def get_options(self) -> list[dict[str, object]]:
        return self._options


async def _run_normal_select_with_shadow_log(
    monkeypatch: pytest.MonkeyPatch,
    *,
    target_label: str,
    llm_response: dict[str, object],
    options: list[dict[str, object]] | None = None,
) -> list[dict[str, object]]:
    options = options or [
        {"optionIndex": 0, "text": "Years", "value": "years"},
        {"optionIndex": 1, "text": "Months", "value": "months"},
    ]
    action = SelectOptionAction(
        element_id="select-1",
        reasoning="choose duration",
        intention="duration",
        option=SelectOption(label=target_label),
        input_or_select_context=InputOrSelectContext(field="Duration", is_required=True),
    )
    monkeypatch.setattr(handler.prompt_engine, "load_prompt", lambda *_args, **_kwargs: "prompt")
    monkeypatch.setattr(handler.app, "NORMAL_SELECT_AGENT_LLM_API_HANDLER", AsyncMock(return_value=llm_response))
    monkeypatch.setattr(handler, "read_run_cancellation", AsyncMock(return_value=None))
    monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)

    context = SkyvernContext(tz_info=ZoneInfo("UTC"))
    with skyvern_context.scoped(context):
        with structlog.testing.capture_logs() as logs:
            results = await handler.normal_select(
                action=action,
                skyvern_element=_FakeSelectElement(options),
                task=_shadow_task(),
                step=SimpleNamespace(step_id="step-1"),
                builder=SimpleNamespace(),
            )

    assert len(results) == 1
    return [log for log in logs if log.get("event") == "select_shadow_match"]


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("target_label", "llm_response", "expected_tier", "expected_agrees"),
    [
        ("Years", {"index": 0, "value": "years"}, "exact", True),
        ("Year", {"index": 1, "value": "months"}, "stem", False),
    ],
)
async def test_normal_select_logs_shadow_match_tier_and_llm_agreement(
    monkeypatch: pytest.MonkeyPatch,
    target_label: str,
    llm_response: dict[str, object],
    expected_tier: str,
    expected_agrees: bool,
) -> None:
    logs = await _run_normal_select_with_shadow_log(
        monkeypatch,
        target_label=target_label,
        llm_response=llm_response,
    )

    assert len(logs) == 1
    assert logs[0]["prompt_name"] == "normal-select"
    assert logs[0]["option_count"] == 2
    assert logs[0]["match_tier"] == expected_tier
    assert logs[0]["match_found"] is True
    assert logs[0]["match_agrees_with_llm"] is expected_agrees


@pytest.mark.asyncio
async def test_normal_select_shadow_disagreement_emits_rich_fields(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logs = await _run_normal_select_with_shadow_log(
        monkeypatch,
        target_label="Years",
        llm_response={"index": 1, "value": "months"},
    )

    assert len(logs) == 1
    event = logs[0]
    assert event["match_found"] is True
    assert event["match_agrees_with_llm"] is False
    assert event["target_value"] == "Years"
    assert event["matched_index"] == 0
    assert event["matched_label"] == "Years"
    assert event["matched_value"] == "years"
    assert event["llm_index"] == 1
    assert event["llm_value"] == "months"
    assert event["normalized_target_value"] == "years"
    assert event["normalized_matched_label"] == "years"
    assert event["normalized_matched_value"] == "years"
    assert event["normalized_llm_value"] == "months"
    assert "matched_element_id" not in event
    assert "llm_element_id" not in event


@pytest.mark.asyncio
async def test_normal_select_agreement_stays_lean(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logs = await _run_normal_select_with_shadow_log(
        monkeypatch,
        target_label="Years",
        llm_response={"index": 0, "value": "years"},
    )

    assert len(logs) == 1
    event = logs[0]
    assert event["match_found"] is True
    assert event["match_agrees_with_llm"] is True
    for lean_only in (
        "target_value",
        "matched_index",
        "matched_label",
        "matched_value",
        "matched_element_id",
        "llm_index",
        "llm_value",
        "llm_element_id",
        "normalized_target_value",
        "normalized_matched_label",
        "normalized_matched_value",
        "normalized_llm_value",
    ):
        assert lean_only not in event


@pytest.mark.asyncio
async def test_normal_select_disagreement_truncates_free_text_fields(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    long_target = "  " + "A’   B " * 40
    long_option = "  " + "C’   D " * 40
    long_llm_value = "  " + "E’   F " * 40
    logs = await _run_normal_select_with_shadow_log(
        monkeypatch,
        target_label=long_target,
        llm_response={"index": 1, "value": long_llm_value},
        options=[
            {"optionIndex": 0, "text": long_target, "value": long_option},
            {"optionIndex": 1, "text": "Other", "value": "other"},
        ],
    )

    assert len(logs) == 1
    event = logs[0]
    bound = handler.SELECT_SHADOW_MATCH_FIELD_MAX_CHARS
    assert event["match_agrees_with_llm"] is False
    for field in ("target_value", "matched_label", "matched_value", "llm_value"):
        value = event[field]
        assert isinstance(value, str)
        assert len(value) == bound + 1
        assert value.endswith("…")
    for field, raw_value in (
        ("normalized_target_value", long_target),
        ("normalized_matched_label", long_target),
        ("normalized_matched_value", long_option),
        ("normalized_llm_value", long_llm_value),
    ):
        assert event[field] == handler._truncate_select_shadow_field(handler._normalize_select_shadow_text(raw_value))


@pytest.mark.asyncio
async def test_normal_select_shadow_match_keeps_blank_placeholder_index_alignment(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logs = await _run_normal_select_with_shadow_log(
        monkeypatch,
        target_label="Years",
        llm_response={"index": 1, "value": "years"},
        options=[
            {"optionIndex": 0, "text": "", "value": ""},
            {"optionIndex": 1, "text": "Years", "value": "years"},
        ],
    )

    assert len(logs) == 1
    assert logs[0]["option_count"] == 2
    assert logs[0]["match_tier"] == "exact"
    assert logs[0]["match_found"] is True
    assert logs[0]["match_agrees_with_llm"] is True


def test_shadow_match_disabled_skips_candidate_work(monkeypatch: pytest.MonkeyPatch) -> None:
    get_candidates = Mock(side_effect=AssertionError("should not build shadow candidates"))
    agreement = Mock()
    monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", False)

    with structlog.testing.capture_logs() as logs:
        handler._log_select_shadow_match(
            prompt_name="normal-select",
            target_value="Years",
            get_candidates=get_candidates,
            agreement=agreement,
        )

    get_candidates.assert_not_called()
    agreement.assert_not_called()
    assert [log for log in logs if log.get("event") == "select_shadow_match"] == []


def test_shadow_match_candidate_errors_do_not_escape_live_path(monkeypatch: pytest.MonkeyPatch) -> None:
    get_candidates = Mock(side_effect=RuntimeError("extractor failed"))
    agreement = Mock()
    monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)

    handler._log_select_shadow_match(
        prompt_name="normal-select",
        target_value="Years",
        get_candidates=get_candidates,
        agreement=agreement,
    )

    get_candidates.assert_called_once()
    agreement.assert_not_called()


@pytest.mark.asyncio
async def test_non_string_llm_values_are_coerced_not_dropped(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    logs = await _run_normal_select_with_shadow_log(
        monkeypatch,
        target_label="Years",
        llm_response={"index": 1, "value": 2024},
    )

    assert len(logs) == 1
    event = logs[0]
    assert event["match_agrees_with_llm"] is False
    assert event["llm_value"] == "2024"


def test_select_shadow_agreement_coerces_malformed_llm_fields() -> None:
    agreement = handler.SelectShadowAgreement(
        agrees=None,
        llm_index="2",
        llm_value=2024,
        llm_element_id=97,
    )
    assert agreement.llm_index == 2
    assert agreement.llm_value == "2024"
    assert agreement.llm_element_id == "97"

    non_numeric_index = handler.SelectShadowAgreement(agrees=None, llm_index="not-a-number")
    assert non_numeric_index.llm_index is None


def test_select_shadow_match_normalizes_curly_apostrophes() -> None:
    matched_index, tier = handler.classify_option_match("Worker’s Compensation", ["Workers Compensation"])

    assert matched_index == 0
    assert tier == "exact"


@pytest.mark.parametrize(
    ("matched_index", "llm_element_id", "llm_value", "expected_agrees"),
    [
        pytest.param(0, "other-id", "  COMMITTED   VALUE  ", True, id="value-match-overrides-different-id"),
        pytest.param(0, "matched-id", "paraphrased value", False, id="value-decides-despite-matching-id"),
        pytest.param(0, "other-id", "different value", False, id="different-value-and-id"),
        pytest.param(0, "matched-id", None, None, id="id-never-decides-without-value"),
        pytest.param(0, "other-id", None, None, id="differing-ids-unjudgeable-without-value"),
        pytest.param(0, None, None, None, id="no-signals"),
        pytest.param(0, None, " ‘ ` ’ ", None, id="normalized-empty-value-is-unavailable"),
        pytest.param(None, "matched-id", "committed value", False, id="no-matcher-choice"),
        pytest.param(1, "matched-id", "committed value", None, id="matcher-index-out-of-range"),
    ],
)
def test_element_choice_shadow_agreement_uses_available_value_and_id_signals(
    matched_index: int | None,
    llm_element_id: str | None,
    llm_value: str | None,
    expected_agrees: bool | None,
) -> None:
    candidates: list[dict[str, str | None]] = [
        {"label": "Display Label", "value": "committed value", "element_id": "matched-id"}
    ]

    agreement = handler._select_shadow_agrees_with_element_choice(
        candidates,
        matched_index,
        llm_element_id=llm_element_id,
        llm_value=llm_value,
    )

    assert agreement.agrees is expected_agrees


def test_element_choice_shadow_agreement_uses_exact_normalizer() -> None:
    agreement = handler._select_shadow_agrees_with_element_choice(
        [{"label": "Worker’s   Compensation", "value": None, "element_id": "matched-id"}],
        0,
        llm_element_id="other-id",
        llm_value="  WORKER'S\tCOMPENSATION ",
    )

    assert agreement.agrees is True


@pytest.mark.parametrize(
    ("matched_index", "llm_index", "llm_value", "expected_agrees"),
    [
        pytest.param(0, 1, "same value", True, id="value-match-overrides-different-index"),
        pytest.param(0, 0, "paraphrased value", False, id="value-decides-despite-matching-index"),
        pytest.param(0, 1, "different value", False, id="different-value-and-index"),
        pytest.param(0, None, None, None, id="no-signals"),
        pytest.param(0, -1, None, False, id="llm-abstained"),
        pytest.param(None, 0, "same value", False, id="no-matcher-choice"),
        pytest.param(2, 2, "same value", None, id="invalid-candidate-index-with-value-stays-unjudged"),
    ],
)
def test_native_choice_shadow_agreement_uses_available_value_and_index_signals(
    matched_index: int | None,
    llm_index: int | None,
    llm_value: str | None,
    expected_agrees: bool | None,
) -> None:
    candidates: list[dict[str, str | None]] = [
        {"label": "Duplicate", "value": "same value"},
        {"label": "Duplicate", "value": "same value"},
    ]

    agreement = handler._select_shadow_agrees_with_native_choice(
        candidates,
        matched_index,
        llm_index=llm_index,
        llm_value=llm_value,
    )

    assert agreement.agrees is expected_agrees


def test_custom_select_same_value_different_element_id_logs_agreement(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)

    with structlog.testing.capture_logs() as logs:
        handler._log_select_shadow_match(
            prompt_name="custom-select/dropdown",
            target_value="Workers' Compensation",
            get_candidates=lambda: [{"label": "Worker’s   Compensation", "value": None, "element_id": "el-A"}],
            agreement=lambda candidates, matched_index: handler._select_shadow_agrees_with_element_choice(
                candidates,
                matched_index,
                llm_element_id="el-B",
                llm_value="workers compensation",
            ),
        )

    (event,) = (entry for entry in logs if entry.get("event") == "select_shadow_match")
    assert event["match_tier"] == "exact"
    assert event["match_found"] is True
    assert event["match_agrees_with_llm"] is True
    assert "matched_element_id" not in event


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("llm_value", "expected_agrees"),
    [
        pytest.param("  committed   value  ", True, id="normalized-value-match"),
        pytest.param("Different Value", False, id="value-mismatch"),
    ],
)
async def test_auto_completion_call_path_forwards_committed_value_to_shadow_match(
    monkeypatch: pytest.MonkeyPatch,
    llm_value: str,
    expected_agrees: bool,
) -> None:
    frame = MagicMock()
    skyvern_frame = MagicMock(safe_wait_for_animation_end=AsyncMock())
    incremental_scraped = MagicMock(
        start_listen_dom_increment=AsyncMock(),
        stop_listen_dom_increment=AsyncMock(),
        get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]),
    )
    incremental_scraped.build_html_tree.return_value = "<div>Committed Value</div>"
    skyvern_element = MagicMock(
        get_frame=Mock(return_value=frame),
        get_element_handler=AsyncMock(return_value=MagicMock()),
        press_fill=AsyncMock(),
        press_key=AsyncMock(),
        is_visible=AsyncMock(return_value=True),
        input_clear=AsyncMock(),
    )
    monkeypatch.setattr(handler.SkyvernFrame, "create_instance", AsyncMock(return_value=skyvern_frame))
    monkeypatch.setattr(handler, "IncrementalScrapePage", Mock(return_value=incremental_scraped))
    monkeypatch.setattr(handler, "get_slim_output_template_value", AsyncMock(return_value=""))
    monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
    monkeypatch.setattr(
        handler.app,
        "AUTO_COMPLETION_LLM_API_HANDLER",
        AsyncMock(
            return_value={
                "id": "different-id",
                "value": llm_value,
                "direct_searching": True,
            }
        ),
    )
    monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)

    with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
        with structlog.testing.capture_logs() as logs:
            await handler.choose_auto_completion_dropdown(
                context=InputOrSelectContext(field="Field", is_search_bar=False),
                page=MagicMock(),
                scraped_page=MagicMock(),
                dom=MagicMock(),
                text="Committed Value",
                skyvern_element=skyvern_element,
                step=SimpleNamespace(step_id="step-1"),
                task=_shadow_task(),
                is_secret_value=False,
            )

    events = [log for log in logs if log.get("event") == "select_shadow_match"]
    assert len(events) == 1
    event = events[0]
    assert event["prompt_name"] == "auto-completion-choose-option"
    assert event["match_agrees_with_llm"] is expected_agrees
    if not expected_agrees:
        assert event["normalized_matched_label"] == "committed value"
        assert event["normalized_llm_value"] == "different value"


@pytest.mark.asyncio
async def test_emerging_select_call_path_forwards_committed_value_to_shadow_match(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    selected_element = MagicMock(
        get_attr=AsyncMock(return_value=None),
        scroll_into_view=AsyncMock(),
        click=AsyncMock(),
    )
    dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=selected_element))
    llm_handler = AsyncMock(
        return_value={"id": "different-id", "value": "  committed   value  ", "action_type": "click"}
    )
    monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))
    monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div>Committed Value</div>"))
    deterministic_select = AsyncMock(return_value=None)
    monkeypatch.setattr(handler, "_select_deterministic_custom_option", deterministic_select)
    monkeypatch.setattr(handler, "_verify_custom_select_option_with_settle", AsyncMock(return_value=(True, "scope")))
    monkeypatch.setattr(handler, "_verify_custom_select_option", AsyncMock(return_value=(True, "scope")))
    monkeypatch.setattr(handler, "_wait_custom_select_render_settle", AsyncMock())
    monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
    monkeypatch.setattr(
        handler.LLMAPIHandlerFactory,
        "get_override_llm_api_handler",
        Mock(return_value=llm_handler),
    )
    monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
    monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)

    scraped_page = SimpleNamespace(id_to_css_dict={})
    scraped_page_after_open = SimpleNamespace(
        id_to_css_dict={"matched-id": "[data-id=matched-id]"},
        id_to_element_dict={"field-id": {"id": "field-id"}},
        element_tree_trimmed=[_SHADOW_OPTION],
    )
    with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
        with structlog.testing.capture_logs() as logs:
            await handler.select_from_emerging_elements(
                current_element_id="field-id",
                options=handler.CustomSelectPromptOptions(target_value="Committed Value"),
                page=MagicMock(),
                scraped_page=scraped_page,
                scraped_page_after_open=scraped_page_after_open,
                new_interactable_element_ids=["matched-id"],
                step=SimpleNamespace(step_id="step-1"),
                task=_shadow_task(),
            )

    events = [log for log in logs if log.get("event") == "select_shadow_match"]
    assert len(events) == 1
    assert events[0]["prompt_name"] == "custom-select/emerging"
    assert events[0]["match_agrees_with_llm"] is True
    assert deterministic_select.await_args.kwargs["execute"] is False


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("entry_action_type", "expected_execute"),
    [("select_option", True), ("input_text", True), ("input_text_converted", False), ("click", False)],
)
async def test_dropdown_select_call_path_forwards_committed_value_and_scopes_execution(
    monkeypatch: pytest.MonkeyPatch, entry_action_type: str, expected_execute: bool
) -> None:
    incremental_scraped = MagicMock(
        get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]),
    )
    incremental_scraped.build_element_tree.return_value = "<div>Committed Value</div>"
    selected_element = MagicMock(
        get_tag_name=Mock(return_value="div"),
        get_attr=AsyncMock(return_value=None),
        scroll_into_view=AsyncMock(),
        click=AsyncMock(),
    )
    monkeypatch.setattr(handler, "locate_dropdown_menu", AsyncMock(return_value=None))
    deterministic_select = AsyncMock(return_value=None)
    monkeypatch.setattr(handler, "_select_deterministic_custom_option", deterministic_select)
    monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
    monkeypatch.setattr(
        handler.app,
        "CUSTOM_SELECT_AGENT_LLM_API_HANDLER",
        AsyncMock(return_value={"id": "different-id", "value": "  committed   value  ", "action_type": "click"}),
    )
    monkeypatch.setattr(handler.SkyvernElement, "create_from_incremental", AsyncMock(return_value=selected_element))
    monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)

    with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
        with structlog.testing.capture_logs() as logs:
            await handler.select_from_dropdown(
                context=InputOrSelectContext(field="Field", is_required=True),
                page=MagicMock(),
                skyvern_element=MagicMock(get_id=Mock(return_value="field-id")),
                skyvern_frame=MagicMock(),
                incremental_scraped=incremental_scraped,
                check_filter_funcs=[],
                step=SimpleNamespace(step_id="step-1"),
                task=_shadow_task(),
                force_select=True,
                target_value="Committed Value",
                entry_action_type=entry_action_type,
            )

    events = [log for log in logs if log.get("event") == "select_shadow_match"]
    assert len(events) == 1
    assert events[0]["prompt_name"] == "custom-select/dropdown"
    assert events[0]["match_agrees_with_llm"] is True
    assert deterministic_select.await_args.kwargs["execute"] is expected_execute


@pytest.mark.asyncio
async def test_dropdown_terminal_result_never_reaches_custom_select_llm(monkeypatch: pytest.MonkeyPatch) -> None:
    incremental_scraped = MagicMock(get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]))
    incremental_scraped.build_element_tree.return_value = "<div>Committed Value</div>"
    failure = handler.ActionFailure(Exception("unverified deterministic click"))
    failure.skip_remaining_actions = True
    monkeypatch.setattr(handler, "locate_dropdown_menu", AsyncMock(return_value=None))
    monkeypatch.setattr(
        handler,
        "_select_deterministic_custom_option",
        AsyncMock(return_value=(failure, "Committed Value")),
    )
    llm_handler = AsyncMock()
    monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)

    result = await handler.select_from_dropdown(
        context=InputOrSelectContext(field="Field", is_required=True),
        page=MagicMock(),
        skyvern_element=MagicMock(get_id=Mock(return_value="field-id")),
        skyvern_frame=MagicMock(),
        incremental_scraped=incremental_scraped,
        check_filter_funcs=[],
        step=SimpleNamespace(step_id="step-1"),
        task=_shadow_task(),
        force_select=True,
        target_value="Committed Value",
    )

    assert result.action_result is failure
    llm_handler.assert_not_awaited()


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("use_strict_verification", "expected"),
    [(False, (True, "matched_state")), (True, (False, "none"))],
)
async def test_custom_select_verify_strict_posture_is_entry_scoped(
    monkeypatch: pytest.MonkeyPatch,
    use_strict_verification: bool,
    expected: tuple[bool, str],
) -> None:
    monkeypatch.setattr(
        handler,
        "_read_custom_select_matched_state",
        AsyncMock(
            return_value={
                "label": "Choice",
                "role": "option",
                "inMultiselectable": False,
                "ariaSelected": True,
            }
        ),
    )
    monkeypatch.setattr(
        handler,
        "_custom_select_scope_confirms_committed",
        AsyncMock(return_value=(False, "none")),
    )

    result = await handler._verify_custom_select_option(
        matched_element=MagicMock(),
        readback_scope_element=MagicMock(),
        anchor_is_combobox_input=True,
        matched_element_id="choice-1",
        matched_label="Choice",
        use_strict_verification=use_strict_verification,
    )

    assert result == expected


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("use_strict_verification", "expected_allow_tokens", "expected_allow_single_value_scope"),
    [(False, True, False), (True, False, True)],
)
async def test_custom_select_verify_strict_posture_controls_aria_selected_tokens(
    monkeypatch: pytest.MonkeyPatch,
    use_strict_verification: bool,
    expected_allow_tokens: bool,
    expected_allow_single_value_scope: bool,
) -> None:
    monkeypatch.setattr(handler, "_read_custom_select_matched_state", AsyncMock(return_value=None))
    scope_confirms = AsyncMock(return_value=(False, "none"))
    monkeypatch.setattr(handler, "_custom_select_scope_confirms_committed", scope_confirms)

    await handler._verify_custom_select_option(
        matched_element=MagicMock(),
        readback_scope_element=MagicMock(),
        anchor_is_combobox_input=True,
        matched_element_id="choice-1",
        matched_label="Choice",
        use_strict_verification=use_strict_verification,
    )

    assert scope_confirms.await_args.kwargs["allow_aria_selected_option_tokens"] is expected_allow_tokens
    assert scope_confirms.await_args.kwargs["allow_single_value_scope"] is expected_allow_single_value_scope


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("committed_state", "expected"),
    [
        ({"matched": True, "branch": "scope_input_value"}, (True, "scope_input_value")),
        ({"matched": True, "branch": "scope_single_value"}, (True, "scope_single_value")),
        ({"matched": False, "branch": "none"}, (False, "none")),
    ],
)
async def test_custom_select_verify_preserves_scope_branch(
    monkeypatch: pytest.MonkeyPatch,
    committed_state: dict[str, object],
    expected: tuple[bool, str],
) -> None:
    matched_element = MagicMock()
    matched_element.get_locator.return_value.count = AsyncMock(return_value=1)

    async def evaluate_element_scoped(_element: object, expression: str, _arg: object = None) -> object:
        if expression == handler._CUSTOM_SELECT_MATCHED_STATE_JS:
            return None
        return committed_state

    monkeypatch.setattr(handler, "_evaluate_element_scoped", AsyncMock(side_effect=evaluate_element_scoped))

    result = await handler._verify_custom_select_option(
        matched_element=matched_element,
        readback_scope_element=MagicMock(),
        anchor_is_combobox_input=True,
        matched_element_id="choice-1",
        matched_label="Choice",
        use_strict_verification=True,
    )

    assert result == expected


_OUTCOME_TASK = SimpleNamespace(
    workflow_run_id="wr",
    task_id="task",
    organization_id="org",
    url="https://test",
    navigation_goal="goal",
    navigation_payload={},
    llm_key=None,
    workflow_permanent_id=None,
)


async def _run_outcome_case(
    monkeypatch: pytest.MonkeyPatch,
    case: str,
    *,
    family_enabled: bool = True,
    assigned: bool = False,
    execute: bool = True,
    entry_action_type: str = "select_option",
    anchor_is_input: bool = False,
    selection_group_id: str = "group-1",
    select_depth: int = 0,
) -> tuple[object, dict[str, object] | None, SimpleNamespace, AsyncMock, AsyncMock]:
    handler._COLLAPSE_XP_ASSIGNMENT_MEMO.clear()
    family = (
        AsyncMock(side_effect=RuntimeError("gate")) if case == "gate_error" else AsyncMock(return_value=family_enabled)
    )
    provider = SimpleNamespace(
        is_feature_enabled_cached=family,
        resolve_feature_enabled_unrecorded=AsyncMock(return_value=assigned),
    )
    resolution = SimpleNamespace(
        fallback_to_llm=case == "no_match",
        matched_index=None if case == "no_match" else 2 if case == "bad_index" else 0,
        matched_label="Choice",
        matched_tier=None if case == "no_match" else "stem" if case == "stem" else "exact",
    )
    resolver = AsyncMock(side_effect=RuntimeError("matcher") if case == "matcher_error" else None)
    resolver.return_value = resolution
    click_error = (
        handler.InteractWithDisabledElement(element_id="choice-1")
        if case == "disabled_click"
        else RuntimeError("after dispatch")
        if case == "post_click"
        else None
    )
    selected = SimpleNamespace(
        get_attr=AsyncMock(return_value="listbox" if case == "listbox" else None),
        scroll_into_view=AsyncMock(),
        click=AsyncMock(side_effect=click_error),
    )
    get_skyvern_element = AsyncMock(return_value=selected)
    monkeypatch.setattr(handler.app, "EXPERIMENTATION_PROVIDER", provider)
    monkeypatch.setattr(handler.app, "AGENT_FUNCTION", SimpleNamespace(resolve_field_option=resolver))
    monkeypatch.setattr(
        handler,
        "_read_custom_select_matched_state",
        AsyncMock(side_effect=RuntimeError("before click") if case == "pre_click" else None, return_value=None),
    )
    monkeypatch.setattr(handler, "_resolve_custom_select_readback_scope_element", AsyncMock(return_value=MagicMock()))
    monkeypatch.setattr(
        handler,
        "_anchor_is_combobox_input",
        AsyncMock(return_value=anchor_is_input or case.startswith("reset")),
    )
    monkeypatch.setattr(
        handler,
        "_custom_select_matched_state_confirms_pre_click",
        Mock(return_value=case == "precommit_matched"),
    )

    async def scope_confirms_committed(**kwargs: object) -> tuple[bool, str]:
        committed = case == "precommit_scope" or (
            case == "precommit_single_value" and kwargs["allow_single_value_scope"] is True
        )
        return committed, "scope_single_value" if committed else "none"

    monkeypatch.setattr(
        handler, "_custom_select_scope_confirms_committed", AsyncMock(side_effect=scope_confirms_committed)
    )
    monkeypatch.setattr(
        handler,
        "_verify_custom_select_option_with_settle",
        AsyncMock(return_value=(case == "verified", "matched_state" if case == "verified" else "none")),
    )
    monkeypatch.setattr(handler, "_reset_custom_select_combobox_input", AsyncMock(return_value=case == "reset_ok"))
    monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value="typed"))
    candidates = (
        Mock(side_effect=RuntimeError("walker"))
        if case == "walker_error"
        else Mock(
            return_value=[]
            if case == "no_options"
            else [
                {
                    "label": "Choice",
                    "element_id": None if case in {"no_element_id", "stem"} else "choice-1",
                    "value": "choice",
                    "is_choice_input": case == "toggle",
                }
            ],
        )
    )
    with structlog.testing.capture_logs() as logs:
        result = await handler._select_deterministic_custom_option(
            target_value=None if case == "no_target" else "Choice",
            get_option_candidates=candidates,
            field_context={"is_date_related": True} if case == "date_related" else {},
            page=MagicMock(),
            get_skyvern_element=get_skyvern_element,
            task=_OUTCOME_TASK,
            execute=False if case == "execution_disabled" else execute,
            step=SimpleNamespace(step_id="step"),
            entry_action_type=entry_action_type,
            selection_group_id=selection_group_id,
            select_depth=select_depth,
        )
    events = [log for log in logs if log.get("event") == "custom_select_family_outcome"]
    assert len(events) == (0 if case in {"no_target", "date_related", "no_options"} else 1)
    return result, events[0] if events else None, provider, resolver, get_skyvern_element


@pytest.mark.asyncio
@pytest.mark.parametrize("entry_action_type", ["select_option", "input_text", "input_text_converted"])
async def test_outcome_event_emitted_once_per_opportunity(
    monkeypatch: pytest.MonkeyPatch, entry_action_type: str
) -> None:
    _, event, _, _, _ = await _run_outcome_case(monkeypatch, "control", entry_action_type=entry_action_type)
    assert event is not None
    assert event["outcome"] == "llm_fallback_control"
    assert event["entry_action_type"] == entry_action_type


@pytest.mark.asyncio
@pytest.mark.parametrize(
    (
        "entry_action_type",
        "anchor_is_input",
        "expected_use_strict_verification",
        "expected_allow_single_value_scope",
    ),
    [
        ("select_option", True, False, False),
        ("select_option", False, False, False),
        ("input_text", True, True, True),
        ("input_text_converted", True, False, False),
    ],
)
async def test_deterministic_custom_select_verification_is_entry_scoped(
    monkeypatch: pytest.MonkeyPatch,
    entry_action_type: str,
    anchor_is_input: bool,
    expected_use_strict_verification: bool,
    expected_allow_single_value_scope: bool,
) -> None:
    await _run_outcome_case(
        monkeypatch,
        "verified",
        assigned=True,
        entry_action_type=entry_action_type,
        anchor_is_input=anchor_is_input,
    )

    verify = handler._verify_custom_select_option_with_settle
    assert isinstance(verify, AsyncMock)
    assert verify.await_args.kwargs["use_strict_verification"] is expected_use_strict_verification
    scope_confirms = handler._custom_select_scope_confirms_committed
    assert isinstance(scope_confirms, AsyncMock)
    assert scope_confirms.await_args.kwargs["allow_single_value_scope"] is expected_allow_single_value_scope


@pytest.mark.asyncio
async def test_executable_entry_constant_disables_input_text_single_value_scope(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(handler, "_EXECUTABLE_CUSTOM_SELECT_ENTRIES", ("select_option",))

    await _run_outcome_case(
        monkeypatch,
        "verified",
        assigned=True,
        entry_action_type="input_text",
        anchor_is_input=True,
    )

    scope_confirms = handler._custom_select_scope_confirms_committed
    assert isinstance(scope_confirms, AsyncMock)
    assert scope_confirms.await_args.kwargs["allow_single_value_scope"] is False
    verify = handler._verify_custom_select_option_with_settle
    assert isinstance(verify, AsyncMock)
    assert verify.await_args.kwargs["use_strict_verification"] is False


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("case", "expected_outcome"),
    [
        ("precommit_matched", "success_precommit"),
        ("precommit_scope", "success_precommit"),
        ("verified", "llm_fallback_execution_disabled"),
    ],
)
async def test_input_text_non_input_anchor_keeps_precommit_but_never_clicks(
    monkeypatch: pytest.MonkeyPatch,
    case: str,
    expected_outcome: str,
) -> None:
    result, event, _, _, get_skyvern_element = await _run_outcome_case(
        monkeypatch,
        case,
        assigned=True,
        entry_action_type="input_text",
        anchor_is_input=False,
    )

    if expected_outcome == "success_precommit":
        assert result is not None
        assert isinstance(result[0], handler.ActionSuccess)
    else:
        assert result is None
    assert event is not None
    assert event["outcome"] == expected_outcome
    get_skyvern_element.return_value.click.assert_not_awaited()


@pytest.mark.asyncio
@pytest.mark.parametrize("entry_action_type", ["click", "input_text_converted"])
async def test_nonexecutable_entry_does_not_single_value_preconfirm(
    monkeypatch: pytest.MonkeyPatch,
    entry_action_type: str,
) -> None:
    result, event, _, _, get_skyvern_element = await _run_outcome_case(
        monkeypatch,
        "precommit_single_value",
        assigned=True,
        execute=False,
        entry_action_type=entry_action_type,
        anchor_is_input=True,
    )

    assert result is None
    assert event is not None
    assert event["outcome"] == "llm_fallback_execution_disabled"
    get_skyvern_element.return_value.click.assert_not_awaited()


@pytest.mark.asyncio
async def test_executable_entry_constant_disables_input_text_single_value_preconfirm(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(handler, "_EXECUTABLE_CUSTOM_SELECT_ENTRIES", ("select_option",))

    result, event, _, _, get_skyvern_element = await _run_outcome_case(
        monkeypatch,
        "precommit_single_value",
        assigned=True,
        execute=False,
        entry_action_type="input_text",
        anchor_is_input=True,
    )

    assert result is None
    assert event is not None
    assert event["outcome"] == "llm_fallback_execution_disabled"
    get_skyvern_element.return_value.click.assert_not_awaited()


@pytest.mark.asyncio
async def test_click_route_without_target_emits_nothing_and_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
    gate = AsyncMock()
    monkeypatch.setattr(handler, "_resolve_collapse_gate", gate)

    with structlog.testing.capture_logs() as logs:
        result = await handler._select_deterministic_custom_option(
            target_value=None,
            get_option_candidates=Mock(),
            field_context={},
            page=MagicMock(),
            get_skyvern_element=AsyncMock(),
            task=_OUTCOME_TASK,
            execute=True,
            entry_action_type="click",
        )

    assert result is None
    assert [log for log in logs if log.get("event") == "custom_select_family_outcome"] == []
    gate.assert_not_awaited()


@pytest.mark.asyncio
async def test_click_ingress_routes_no_target_and_emits_nothing(monkeypatch: pytest.MonkeyPatch) -> None:
    select_from_emerging_elements = AsyncMock(return_value=MagicMock())
    monkeypatch.setattr(handler, "select_from_emerging_elements", select_from_emerging_elements)
    monkeypatch.setattr(handler, "_build_after_click_verify_prompt", AsyncMock(return_value="prompt"))
    monkeypatch.setattr(
        handler,
        "resolve_check_user_goal_handler",
        AsyncMock(
            return_value=AsyncMock(
                return_value={"thoughts": "continue", "user_goal_achieved": False, "should_terminate": False}
            )
        ),
    )
    monkeypatch.setattr(handler, "locate_dropdown_menu", AsyncMock(return_value=MagicMock()))
    monkeypatch.setattr(
        handler,
        "_get_input_or_select_context",
        AsyncMock(return_value=InputOrSelectContext(field="Field", is_required=True, is_date_related=False)),
    )
    monkeypatch.setattr(
        handler,
        "DomUtil",
        Mock(
            return_value=SimpleNamespace(
                get_skyvern_element_by_id=AsyncMock(
                    return_value=SimpleNamespace(is_interactable=Mock(return_value=True))
                )
            )
        ),
    )

    scraped_page_after_open = SimpleNamespace(id_to_css_dict={"choice-id": "[data-id=choice-id]"})
    scraped_page = SimpleNamespace(
        url="https://test",
        id_to_css_dict={},
        generate_scraped_page_without_screenshots=AsyncMock(return_value=scraped_page_after_open),
    )
    incremental_scraped = SimpleNamespace(
        get_incremental_elements_num=AsyncMock(return_value=1),
        get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]),
    )

    with structlog.testing.capture_logs() as logs:
        await handler.handle_sequential_click_for_dropdown(
            action=ClickAction(element_id="field-id", reasoning="click", intention="choose"),
            action_history=[],
            anchor_element=MagicMock(get_id=Mock(return_value="field-id")),
            dom=MagicMock(),
            page=SimpleNamespace(url="https://test"),
            skyvern_frame=SimpleNamespace(safe_wait_for_animation_end=AsyncMock(), engine_selection=object()),
            scraped_page=scraped_page,
            incremental_scraped=incremental_scraped,
            task=_OUTCOME_TASK,
            step=SimpleNamespace(step_id="step"),
        )

    options = select_from_emerging_elements.await_args.kwargs["options"]
    assert options.target_value is None
    assert [log for log in logs if log.get("event") == "custom_select_family_outcome"] == []


@pytest.mark.asyncio
async def test_outcome_event_once_per_dropdown_level_with_group_id(monkeypatch: pytest.MonkeyPatch) -> None:
    handler._COLLAPSE_XP_ASSIGNMENT_MEMO.clear()
    monkeypatch.setattr(
        handler.app,
        "EXPERIMENTATION_PROVIDER",
        SimpleNamespace(
            is_feature_enabled_cached=AsyncMock(return_value=True),
            resolve_feature_enabled_unrecorded=AsyncMock(return_value=False),
        ),
    )
    monkeypatch.setattr(
        handler.app,
        "AGENT_FUNCTION",
        SimpleNamespace(
            resolve_field_option=AsyncMock(
                return_value=SimpleNamespace(
                    fallback_to_llm=False,
                    matched_index=0,
                    matched_label="Choice",
                    matched_tier="exact",
                )
            )
        ),
    )
    monkeypatch.setattr(
        handler.app,
        "CUSTOM_SELECT_AGENT_LLM_API_HANDLER",
        AsyncMock(return_value={"id": "matched-id", "value": "Choice", "action_type": "input_text"}),
    )
    monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
    monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value=""))
    monkeypatch.setattr(
        handler, "get_actual_value_of_parameter_if_secret_with_task", Mock(side_effect=lambda _, value: value)
    )

    dropdown = MagicMock(
        get_element_handler=AsyncMock(return_value=MagicMock()),
        get_locator=Mock(return_value=MagicMock(count=AsyncMock(return_value=1))),
    )
    input_element = MagicMock(
        get_tag_name=Mock(return_value="input"),
        get_locator=Mock(return_value=MagicMock()),
        scroll_into_view=AsyncMock(),
        is_readonly=AsyncMock(return_value=False),
        input_clear=AsyncMock(),
        input_sequentially=AsyncMock(),
    )
    skyvern_frame = MagicMock(
        safe_wait_for_animation_end=AsyncMock(),
        get_element_scrollable=AsyncMock(return_value=False),
        get_element_visible=AsyncMock(return_value=True),
    )
    incremental_scraped = MagicMock(get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]))
    incremental_scraped.build_element_tree.return_value = "<div>Choice</div>"
    monkeypatch.setattr(handler, "try_to_find_potential_scrollable_element", AsyncMock(return_value=dropdown))
    monkeypatch.setattr(handler.SkyvernElement, "create_from_incremental", AsyncMock(return_value=input_element))

    action = SelectOptionAction(
        element_id="field-id",
        option=SelectOption(label="Choice"),
        input_or_select_context=InputOrSelectContext(field="Field"),
    )
    with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
        with structlog.testing.capture_logs() as logs:
            await handler.sequentially_select_from_dropdown(
                action=action,
                input_or_select_context=InputOrSelectContext(field="Field"),
                page=MagicMock(),
                dom=MagicMock(),
                skyvern_element=MagicMock(),
                skyvern_frame=skyvern_frame,
                incremental_scraped=incremental_scraped,
                step=SimpleNamespace(step_id="step"),
                task=_OUTCOME_TASK,
                dropdown_menu_element=dropdown,
                force_select=True,
                target_value="Choice",
            )

    events = [log for log in logs if log.get("event") == "custom_select_family_outcome"]
    assert len(events) == 3
    assert [event["outcome"] for event in events] == ["llm_fallback_control"] * 3
    assert events[0]["selection_group_id"]
    assert len({event["selection_group_id"] for event in events}) == 1
    assert [event["select_depth"] for event in events] == [0, 1, 2]


async def _run_post_reset_fallback_case(
    monkeypatch: pytest.MonkeyPatch,
    *,
    llm_response: object,
    entry_action_type: str = "input_text",
    input_value_reads: list[str | None] | None = None,
) -> tuple[object, list[dict[str, object]], MagicMock, MagicMock, BaseException | None, AsyncMock]:
    handler._COLLAPSE_XP_ASSIGNMENT_MEMO.clear()
    monkeypatch.setattr(
        handler.app,
        "EXPERIMENTATION_PROVIDER",
        SimpleNamespace(
            is_feature_enabled_cached=AsyncMock(return_value=True),
            resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
        ),
    )
    monkeypatch.setattr(
        handler.app,
        "AGENT_FUNCTION",
        SimpleNamespace(
            resolve_field_option=AsyncMock(
                return_value=SimpleNamespace(
                    fallback_to_llm=False,
                    matched_index=0,
                    matched_label="Committed Value",
                    matched_tier="exact",
                )
            )
        ),
    )
    llm_handler = (
        AsyncMock(side_effect=llm_response)
        if isinstance(llm_response, Exception)
        else AsyncMock(return_value=llm_response)
    )
    monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
    monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
    monkeypatch.setattr(handler, "_read_custom_select_matched_state", AsyncMock(return_value=None))
    monkeypatch.setattr(handler, "_custom_select_scope_confirms_committed", AsyncMock(return_value=(False, "none")))
    monkeypatch.setattr(handler, "_verify_custom_select_option_with_settle", AsyncMock(return_value=(False, "none")))
    if input_value_reads is None:
        input_value_reads = (
            [""]
            if entry_action_type == "select_option"
            else ["Committed Value", "", "Committed Value", "Committed Value"]
        )
    get_input_value_mock = AsyncMock(side_effect=input_value_reads)
    monkeypatch.setattr(handler, "get_input_value", get_input_value_mock)
    monkeypatch.setattr(
        handler, "get_actual_value_of_parameter_if_secret_with_task", Mock(side_effect=lambda _, value: value)
    )

    reset_locator = MagicMock(fill=AsyncMock())
    anchor = MagicMock(
        get_id=Mock(return_value="field-id"),
        get_tag_name=Mock(return_value="input"),
        get_locator=Mock(return_value=reset_locator),
        click=AsyncMock(),
    )
    matched_element = MagicMock(
        get_attr=AsyncMock(return_value=None),
        scroll_into_view=AsyncMock(),
        click=AsyncMock(),
    )
    input_element = MagicMock(
        get_tag_name=Mock(return_value="input"),
        get_locator=Mock(return_value=MagicMock()),
        scroll_into_view=AsyncMock(),
        is_readonly=AsyncMock(return_value=False),
        input_clear=AsyncMock(),
        input_sequentially=AsyncMock(),
    )
    monkeypatch.setattr(
        handler.SkyvernElement,
        "create_from_incremental",
        AsyncMock(side_effect=[matched_element, input_element]),
    )

    dropdown = MagicMock(get_element_handler=AsyncMock(return_value=MagicMock()))
    skyvern_frame = MagicMock(get_element_scrollable=AsyncMock(return_value=False), engine_selection=object())
    incremental_scraped = MagicMock(get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]))
    incremental_scraped.build_element_tree.return_value = "<div>Committed Value</div>"
    monkeypatch.setattr(handler, "try_to_find_potential_scrollable_element", AsyncMock(return_value=dropdown))

    result: object = None
    raised: BaseException | None = None
    with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
        with structlog.testing.capture_logs() as logs:
            try:
                result = await handler.select_from_dropdown(
                    context=InputOrSelectContext(field="Field"),
                    page=MagicMock(),
                    skyvern_element=anchor,
                    skyvern_frame=skyvern_frame,
                    incremental_scraped=incremental_scraped,
                    check_filter_funcs=[],
                    step=SimpleNamespace(step_id="step"),
                    task=_OUTCOME_TASK,
                    dropdown_menu_element=dropdown,
                    force_select=True,
                    target_value="Committed Value",
                    entry_action_type=entry_action_type,
                    selection_group_id="group-1",
                )
            except Exception as exc:
                raised = exc

    events = [log for log in logs if log.get("event") == "custom_select_family_outcome"]
    return result, events, input_element, reset_locator, raised, get_input_value_mock


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "llm_response",
    [
        RuntimeError("fallback failed"),
        {"id": "", "action_type": "", "reasoning": "no match"},
        {"id": "matched-id", "value": "Committed Value", "action_type": {"x": 1}},
        {"id": "matched-id", "action_type": "input_text", "reasoning": "no value"},
        {"id": "matched-id", "value": "", "action_type": "input_text", "reasoning": "empty value"},
    ],
)
async def test_post_reset_llm_failure_emits_single_terminal_outcome(
    monkeypatch: pytest.MonkeyPatch,
    llm_response: object,
) -> None:
    result, events, _, _, raised, _ = await _run_post_reset_fallback_case(monkeypatch, llm_response=llm_response)

    assert raised is None
    assert isinstance(result.action_result, handler.ActionFailure)
    assert result.action_result.skip_remaining_actions is True
    assert [event["outcome"] for event in events] == ["terminal_llm_fallback_exception"]
    assert events[0]["entry_action_type"] == "input_text"
    assert events[0]["selection_group_id"] == "group-1"
    assert events[0]["select_depth"] == 0
    assert events[0]["family_gate_enabled"] is True
    assert events[0]["assigned"] is True
    assert events[0]["llm_fallback_requested"] is True


@pytest.mark.asyncio
async def test_post_reset_restored_text_is_retyped_and_emits_single_proceeded_outcome(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    result, events, input_element, reset_locator, raised, _ = await _run_post_reset_fallback_case(
        monkeypatch,
        llm_response={"id": "matched-id", "value": "Committed Value", "action_type": "input_text"},
    )

    assert raised is None
    assert isinstance(result.action_result, handler.ActionSuccess)
    input_element.input_clear.assert_awaited_once()
    input_element.input_sequentially.assert_awaited_once_with("Committed Value")
    assert [c.args for c in reset_locator.fill.await_args_list] == [("",), ("Committed Value",)]
    assert [event["outcome"] for event in events] == ["llm_fallback_reset_verified"]
    assert events[0]["entry_action_type"] == "input_text"
    assert events[0]["selection_group_id"] == "group-1"
    assert events[0]["select_depth"] == 0
    assert events[0]["family_gate_enabled"] is True
    assert events[0]["assigned"] is True
    assert events[0]["llm_fallback_requested"] is True


@pytest.mark.asyncio
async def test_select_option_reset_path_matches_main(monkeypatch: pytest.MonkeyPatch) -> None:
    llm_error = RuntimeError("fallback failed")
    result, events, _, reset_locator, raised, get_input_value_mock = await _run_post_reset_fallback_case(
        monkeypatch,
        llm_response=llm_error,
        entry_action_type="select_option",
    )

    assert raised is llm_error
    assert result is None
    assert [event["outcome"] for event in events] == ["llm_fallback_reset_verified"]
    assert events[0]["entry_action_type"] == "select_option"
    assert [c.args for c in reset_locator.fill.await_args_list] == [("",)]
    assert get_input_value_mock.await_count == 1


@pytest.mark.asyncio
async def test_failed_restore_readback_is_terminal_not_reset_verified(monkeypatch: pytest.MonkeyPatch) -> None:
    llm_handler_never_called = RuntimeError("llm should not run")
    result, events, _, reset_locator, raised, _ = await _run_post_reset_fallback_case(
        monkeypatch,
        llm_response=llm_handler_never_called,
        input_value_reads=["Committed Value", "", "typed over"],
    )

    assert raised is None
    assert isinstance(result.action_result, handler.ActionFailure)
    assert result.action_result.skip_remaining_actions is True
    assert [event["outcome"] for event in events] == ["terminal_unverified_reset"]
    assert [c.args for c in reset_locator.fill.await_args_list] == [("",), ("Committed Value",)]


@pytest.mark.asyncio
async def test_unreadable_anchor_before_click_falls_back_without_clicking(monkeypatch: pytest.MonkeyPatch) -> None:
    llm_error = RuntimeError("llm path")
    result, events, _, reset_locator, raised, _ = await _run_post_reset_fallback_case(
        monkeypatch,
        llm_response=llm_error,
        input_value_reads=[None],
    )

    assert raised is llm_error
    assert result is None
    assert [event["outcome"] for event in events] == ["llm_fallback_pre_click_error"]
    reset_locator.fill.assert_not_awaited()


@pytest.mark.asyncio
async def test_outcome_event_family_off_emits_without_assignment_resolution(monkeypatch: pytest.MonkeyPatch) -> None:
    _, event, provider, resolver, _ = await _run_outcome_case(monkeypatch, "family_off", family_enabled=False)
    assert event is not None
    assert event["outcome"] == "llm_fallback_family_off"
    assert (event["assigned"], event["eligible"], event["match_tier"]) == (None, True, "exact")
    provider.resolve_feature_enabled_unrecorded.assert_not_awaited()
    resolver.assert_awaited_once()


@pytest.mark.asyncio
async def test_outcome_event_key_allowlist_and_enum(monkeypatch: pytest.MonkeyPatch) -> None:
    _, event, _, _, _ = await _run_outcome_case(monkeypatch, "control")
    assert event is not None
    expected_keys = set(
        "event log_level family workflow_run_id task_id organization_id step_id entry_action_type selection_group_id "
        "select_depth script_mode family_gate_enabled assigned gate_error encountered eligible match_tier "
        "option_count attempted click_attempted anchor_is_combobox_input verify_branch verified_success outcome "
        "llm_fallback_requested duration_ms".split()
    )
    expected_outcomes = set(
        "llm_fallback_gate_error llm_fallback_eval_error llm_fallback_family_off llm_fallback_control "
        "llm_fallback_no_match llm_fallback_match_unactionable llm_fallback_pre_click_error "
        "llm_fallback_reset_verified llm_fallback_post_click_unverified llm_fallback_tier_excluded "
        "llm_fallback_execution_disabled success_precommit success_verified terminal_post_click_exception "
        "terminal_llm_fallback_exception "
        "terminal_unverified_reset terminal_unverified_click "
        "terminal_unverified_toggle".split()
    )
    assert event["outcome"] == "llm_fallback_control"
    assert (event["anchor_is_combobox_input"], event["verify_branch"]) == (False, None)
    assert set(event) == expected_keys
    assert {outcome.value for outcome in handler.CustomSelectFamilyOutcome} == expected_outcomes


@pytest.mark.asyncio
async def test_outcome_event_gate_error_fields(monkeypatch: pytest.MonkeyPatch) -> None:
    _, event, _, _, _ = await _run_outcome_case(monkeypatch, "gate_error")
    assert event is not None
    assert event["outcome"] == "llm_fallback_gate_error"
    assert (event["option_count"], event["eligible"], event["match_tier"]) == (None, False, None)


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("case", "family_enabled", "assigned"),
    [
        (case, family, assigned)
        for case in ("walker_error", "matcher_error")
        for family, assigned in ((False, False), (True, False), (True, True))
    ],
)
async def test_outcome_event_eval_error_emits_on_all_arms(
    monkeypatch: pytest.MonkeyPatch, case: str, family_enabled: bool, assigned: bool
) -> None:
    result, event, _, _, _ = await _run_outcome_case(
        monkeypatch, case, family_enabled=family_enabled, assigned=assigned
    )
    assert event is not None
    assert result is None
    assert event["outcome"] == "llm_fallback_eval_error"


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("case", "family_enabled", "assigned", "outcome", "click_attempted", "return_shape"),
    [
        ("no_target", True, True, None, False, "none"),
        ("date_related", True, True, None, False, "none"),
        ("no_options", True, True, None, False, "none"),
        ("gate_error", True, True, "llm_fallback_gate_error", False, "none"),
        ("walker_error", True, True, "llm_fallback_eval_error", False, "none"),
        ("matcher_error", True, True, "llm_fallback_eval_error", False, "none"),
        ("family_off", False, True, "llm_fallback_family_off", False, "none"),
        ("control", True, False, "llm_fallback_control", False, "none"),
        ("no_match", True, True, "llm_fallback_no_match", False, "none"),
        ("bad_index", True, True, "llm_fallback_match_unactionable", False, "none"),
        ("stem", True, True, "llm_fallback_tier_excluded", False, "none"),
        ("execution_disabled", True, True, "llm_fallback_execution_disabled", False, "none"),
        ("no_element_id", True, True, "llm_fallback_match_unactionable", False, "none"),
        ("listbox", True, True, "llm_fallback_match_unactionable", False, "none"),
        ("pre_click", True, True, "llm_fallback_pre_click_error", False, "none"),
        ("precommit_matched", True, True, "success_precommit", False, "success"),
        ("precommit_scope", True, True, "success_precommit", False, "success"),
        ("verified", True, True, "success_verified", True, "success"),
        ("disabled_click", True, True, "llm_fallback_pre_click_error", True, "none"),
        ("post_click", True, True, "terminal_post_click_exception", True, "failure"),
        ("reset_ok", True, True, "llm_fallback_reset_verified", True, "none"),
        ("reset_failed", True, True, "terminal_unverified_reset", True, "failure"),
        ("nonchoice", True, True, "terminal_unverified_click", True, "failure"),
        ("toggle", True, True, "terminal_unverified_toggle", True, "failure"),
    ],
)
async def test_deterministic_custom_select_exit_matrix(
    monkeypatch: pytest.MonkeyPatch,
    case: str,
    family_enabled: bool,
    assigned: bool,
    outcome: str | None,
    click_attempted: bool,
    return_shape: str,
) -> None:
    result, event, _, _, get_skyvern_element = await _run_outcome_case(
        monkeypatch,
        case,
        family_enabled=family_enabled,
        assigned=assigned,
    )

    if outcome is None:
        assert event is None
    else:
        assert event is not None
        assert event["outcome"] == outcome
        assert event["click_attempted"] is click_attempted

    if return_shape == "none":
        assert result is None
    else:
        assert result is not None
        action_result, _matched_label = result
        expected_type = handler.ActionSuccess if return_shape == "success" else handler.ActionFailure
        assert isinstance(action_result, expected_type)
        if return_shape == "failure":
            assert action_result.skip_remaining_actions is True

    if event is not None and event["click_attempted"] is True and result is None:
        assert (case, event["outcome"]) in {
            ("disabled_click", "llm_fallback_pre_click_error"),
            ("reset_ok", "llm_fallback_reset_verified"),
        }
    if case == "stem":
        get_skyvern_element.assert_not_awaited()


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("case", "outcome", "attempted", "return_shape"),
    [
        ("execution_disabled", "llm_fallback_execution_disabled", True, "none"),
        ("precommit_matched", "success_precommit", True, "success"),
        ("listbox", "llm_fallback_match_unactionable", False, "none"),
    ],
)
async def test_execution_disabled_reads_precommit_state_without_clicking(
    monkeypatch: pytest.MonkeyPatch,
    case: str,
    outcome: str,
    attempted: bool,
    return_shape: str,
) -> None:
    result, event, _, _, get_skyvern_element = await _run_outcome_case(
        monkeypatch,
        case,
        assigned=True,
        execute=False,
    )

    assert event is not None
    assert event["outcome"] == outcome
    assert event["attempted"] is attempted
    assert event["click_attempted"] is False
    get_skyvern_element.assert_awaited_once_with("choice-1")
    selected = get_skyvern_element.return_value
    selected.scroll_into_view.assert_not_awaited()
    selected.click.assert_not_awaited()
    if return_shape == "none":
        assert result is None
    else:
        assert result is not None
        action_result, matched_label = result
        assert isinstance(action_result, handler.ActionSuccess)
        assert matched_label == "Choice"


@pytest.mark.asyncio
async def test_converted_route_reports_input_text_converted(monkeypatch: pytest.MonkeyPatch) -> None:
    element = MagicMock(
        is_disabled=AsyncMock(return_value=False),
        get_selectable=AsyncMock(return_value=True),
        supports_text_input=AsyncMock(return_value=True),
    )
    element.get_attr = AsyncMock(return_value=None)
    element.get_tag_name.return_value, element.get_id.return_value = "input", "field"
    monkeypatch.setattr(
        handler, "DomUtil", Mock(return_value=MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=element)))
    )
    monkeypatch.setattr(handler.SkyvernFrame, "create_instance", AsyncMock(return_value=MagicMock()))
    monkeypatch.setattr(handler, "IncrementalScrapePage", MagicMock())
    monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value=""))
    select = AsyncMock(return_value=[handler.ActionSuccess()])
    monkeypatch.setattr(handler, "handle_select_option_action", select)
    await handler.handle_input_text_action(
        handler.InputTextAction(element_id="field", text="Choice"),
        MagicMock(),
        SimpleNamespace(id_to_element_dict={"field": {"tagName": "input"}}),
        SimpleNamespace(workflow_run_id=None, task_id="task-1"),
        MagicMock(),
    )
    assert select.await_args.kwargs["entry_action_type"] == "input_text_converted"


@pytest.mark.asyncio
async def test_autocomplete_readback_threads_engine_selection(monkeypatch: pytest.MonkeyPatch) -> None:
    selection = object()
    element = MagicMock()
    element.get_tag_name.return_value = "input"
    element.get_locator.return_value = MagicMock()
    get_input_value = AsyncMock(return_value="Choice")
    monkeypatch.setattr(handler, "get_input_value", get_input_value)

    assert await handler._verify_autocomplete_input_readback(
        skyvern_element=element,
        matched_index=0,
        matched_label="Choice",
        engine_selection=selection,
    )
    get_input_value.assert_awaited_once_with("input", element.get_locator(), engine_selection=selection)