Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, Mock
from zoneinfo import ZoneInfo
import pytest
import structlog.testing
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.webeye.actions import handler
from skyvern.webeye.actions.actions import ClickAction, InputOrSelectContext, SelectOption, SelectOptionAction
_SHADOW_OPTION = {
"id": "matched-id",
"tagName": "div",
"attributes": {"role": "option"},
"text": "Committed Value",
"interactable": True,
}
def _shadow_task() -> SimpleNamespace:
return SimpleNamespace(
task_id="task-1",
workflow_run_id=None,
navigation_goal="goal",
navigation_payload={},
llm_key=None,
organization_id=None,
workflow_permanent_id=None,
)
class _FakeSelectLocator:
def __init__(self) -> None:
self.click = AsyncMock()
self.select_option = AsyncMock()
class _FakeSelectElement:
def __init__(self, options: list[dict[str, object]]) -> None:
self._options = options
self._locator = _FakeSelectLocator()
async def get_attr(self, *_args: object, **_kwargs: object) -> str | None:
return None
async def refresh_select_options(self) -> tuple[list[dict[str, object]], str]:
return self._options, ""
def build_HTML(self) -> str:
return "<select><option>stub</option></select>"
def get_locator(self) -> _FakeSelectLocator:
return self._locator
def get_options(self) -> list[dict[str, object]]:
return self._options
async def _run_normal_select_with_shadow_log(
monkeypatch: pytest.MonkeyPatch,
*,
target_label: str,
llm_response: dict[str, object],
options: list[dict[str, object]] | None = None,
) -> list[dict[str, object]]:
options = options or [
{"optionIndex": 0, "text": "Years", "value": "years"},
{"optionIndex": 1, "text": "Months", "value": "months"},
]
action = SelectOptionAction(
element_id="select-1",
reasoning="choose duration",
intention="duration",
option=SelectOption(label=target_label),
input_or_select_context=InputOrSelectContext(field="Duration", is_required=True),
)
monkeypatch.setattr(handler.prompt_engine, "load_prompt", lambda *_args, **_kwargs: "prompt")
monkeypatch.setattr(handler.app, "NORMAL_SELECT_AGENT_LLM_API_HANDLER", AsyncMock(return_value=llm_response))
monkeypatch.setattr(handler, "read_run_cancellation", AsyncMock(return_value=None))
monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)
context = SkyvernContext(tz_info=ZoneInfo("UTC"))
with skyvern_context.scoped(context):
with structlog.testing.capture_logs() as logs:
results = await handler.normal_select(
action=action,
skyvern_element=_FakeSelectElement(options),
task=_shadow_task(),
step=SimpleNamespace(step_id="step-1"),
builder=SimpleNamespace(),
)
assert len(results) == 1
return [log for log in logs if log.get("event") == "select_shadow_match"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("target_label", "llm_response", "expected_tier", "expected_agrees"),
[
("Years", {"index": 0, "value": "years"}, "exact", True),
("Year", {"index": 1, "value": "months"}, "stem", False),
],
)
async def test_normal_select_logs_shadow_match_tier_and_llm_agreement(
monkeypatch: pytest.MonkeyPatch,
target_label: str,
llm_response: dict[str, object],
expected_tier: str,
expected_agrees: bool,
) -> None:
logs = await _run_normal_select_with_shadow_log(
monkeypatch,
target_label=target_label,
llm_response=llm_response,
)
assert len(logs) == 1
assert logs[0]["prompt_name"] == "normal-select"
assert logs[0]["option_count"] == 2
assert logs[0]["match_tier"] == expected_tier
assert logs[0]["match_found"] is True
assert logs[0]["match_agrees_with_llm"] is expected_agrees
@pytest.mark.asyncio
async def test_normal_select_shadow_disagreement_emits_rich_fields(
monkeypatch: pytest.MonkeyPatch,
) -> None:
logs = await _run_normal_select_with_shadow_log(
monkeypatch,
target_label="Years",
llm_response={"index": 1, "value": "months"},
)
assert len(logs) == 1
event = logs[0]
assert event["match_found"] is True
assert event["match_agrees_with_llm"] is False
assert event["target_value"] == "Years"
assert event["matched_index"] == 0
assert event["matched_label"] == "Years"
assert event["matched_value"] == "years"
assert event["llm_index"] == 1
assert event["llm_value"] == "months"
assert event["normalized_target_value"] == "years"
assert event["normalized_matched_label"] == "years"
assert event["normalized_matched_value"] == "years"
assert event["normalized_llm_value"] == "months"
assert "matched_element_id" not in event
assert "llm_element_id" not in event
@pytest.mark.asyncio
async def test_normal_select_agreement_stays_lean(
monkeypatch: pytest.MonkeyPatch,
) -> None:
logs = await _run_normal_select_with_shadow_log(
monkeypatch,
target_label="Years",
llm_response={"index": 0, "value": "years"},
)
assert len(logs) == 1
event = logs[0]
assert event["match_found"] is True
assert event["match_agrees_with_llm"] is True
for lean_only in (
"target_value",
"matched_index",
"matched_label",
"matched_value",
"matched_element_id",
"llm_index",
"llm_value",
"llm_element_id",
"normalized_target_value",
"normalized_matched_label",
"normalized_matched_value",
"normalized_llm_value",
):
assert lean_only not in event
@pytest.mark.asyncio
async def test_normal_select_disagreement_truncates_free_text_fields(
monkeypatch: pytest.MonkeyPatch,
) -> None:
long_target = " " + "A’ B " * 40
long_option = " " + "C’ D " * 40
long_llm_value = " " + "E’ F " * 40
logs = await _run_normal_select_with_shadow_log(
monkeypatch,
target_label=long_target,
llm_response={"index": 1, "value": long_llm_value},
options=[
{"optionIndex": 0, "text": long_target, "value": long_option},
{"optionIndex": 1, "text": "Other", "value": "other"},
],
)
assert len(logs) == 1
event = logs[0]
bound = handler.SELECT_SHADOW_MATCH_FIELD_MAX_CHARS
assert event["match_agrees_with_llm"] is False
for field in ("target_value", "matched_label", "matched_value", "llm_value"):
value = event[field]
assert isinstance(value, str)
assert len(value) == bound + 1
assert value.endswith("…")
for field, raw_value in (
("normalized_target_value", long_target),
("normalized_matched_label", long_target),
("normalized_matched_value", long_option),
("normalized_llm_value", long_llm_value),
):
assert event[field] == handler._truncate_select_shadow_field(handler._normalize_select_shadow_text(raw_value))
@pytest.mark.asyncio
async def test_normal_select_shadow_match_keeps_blank_placeholder_index_alignment(
monkeypatch: pytest.MonkeyPatch,
) -> None:
logs = await _run_normal_select_with_shadow_log(
monkeypatch,
target_label="Years",
llm_response={"index": 1, "value": "years"},
options=[
{"optionIndex": 0, "text": "", "value": ""},
{"optionIndex": 1, "text": "Years", "value": "years"},
],
)
assert len(logs) == 1
assert logs[0]["option_count"] == 2
assert logs[0]["match_tier"] == "exact"
assert logs[0]["match_found"] is True
assert logs[0]["match_agrees_with_llm"] is True
def test_shadow_match_disabled_skips_candidate_work(monkeypatch: pytest.MonkeyPatch) -> None:
get_candidates = Mock(side_effect=AssertionError("should not build shadow candidates"))
agreement = Mock()
monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", False)
with structlog.testing.capture_logs() as logs:
handler._log_select_shadow_match(
prompt_name="normal-select",
target_value="Years",
get_candidates=get_candidates,
agreement=agreement,
)
get_candidates.assert_not_called()
agreement.assert_not_called()
assert [log for log in logs if log.get("event") == "select_shadow_match"] == []
def test_shadow_match_candidate_errors_do_not_escape_live_path(monkeypatch: pytest.MonkeyPatch) -> None:
get_candidates = Mock(side_effect=RuntimeError("extractor failed"))
agreement = Mock()
monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)
handler._log_select_shadow_match(
prompt_name="normal-select",
target_value="Years",
get_candidates=get_candidates,
agreement=agreement,
)
get_candidates.assert_called_once()
agreement.assert_not_called()
@pytest.mark.asyncio
async def test_non_string_llm_values_are_coerced_not_dropped(
monkeypatch: pytest.MonkeyPatch,
) -> None:
logs = await _run_normal_select_with_shadow_log(
monkeypatch,
target_label="Years",
llm_response={"index": 1, "value": 2024},
)
assert len(logs) == 1
event = logs[0]
assert event["match_agrees_with_llm"] is False
assert event["llm_value"] == "2024"
def test_select_shadow_agreement_coerces_malformed_llm_fields() -> None:
agreement = handler.SelectShadowAgreement(
agrees=None,
llm_index="2",
llm_value=2024,
llm_element_id=97,
)
assert agreement.llm_index == 2
assert agreement.llm_value == "2024"
assert agreement.llm_element_id == "97"
non_numeric_index = handler.SelectShadowAgreement(agrees=None, llm_index="not-a-number")
assert non_numeric_index.llm_index is None
def test_select_shadow_match_normalizes_curly_apostrophes() -> None:
matched_index, tier = handler.classify_option_match("Worker’s Compensation", ["Workers Compensation"])
assert matched_index == 0
assert tier == "exact"
@pytest.mark.parametrize(
("matched_index", "llm_element_id", "llm_value", "expected_agrees"),
[
pytest.param(0, "other-id", " COMMITTED VALUE ", True, id="value-match-overrides-different-id"),
pytest.param(0, "matched-id", "paraphrased value", False, id="value-decides-despite-matching-id"),
pytest.param(0, "other-id", "different value", False, id="different-value-and-id"),
pytest.param(0, "matched-id", None, None, id="id-never-decides-without-value"),
pytest.param(0, "other-id", None, None, id="differing-ids-unjudgeable-without-value"),
pytest.param(0, None, None, None, id="no-signals"),
pytest.param(0, None, " ‘ ` ’ ", None, id="normalized-empty-value-is-unavailable"),
pytest.param(None, "matched-id", "committed value", False, id="no-matcher-choice"),
pytest.param(1, "matched-id", "committed value", None, id="matcher-index-out-of-range"),
],
)
def test_element_choice_shadow_agreement_uses_available_value_and_id_signals(
matched_index: int | None,
llm_element_id: str | None,
llm_value: str | None,
expected_agrees: bool | None,
) -> None:
candidates: list[dict[str, str | None]] = [
{"label": "Display Label", "value": "committed value", "element_id": "matched-id"}
]
agreement = handler._select_shadow_agrees_with_element_choice(
candidates,
matched_index,
llm_element_id=llm_element_id,
llm_value=llm_value,
)
assert agreement.agrees is expected_agrees
def test_element_choice_shadow_agreement_uses_exact_normalizer() -> None:
agreement = handler._select_shadow_agrees_with_element_choice(
[{"label": "Worker’s Compensation", "value": None, "element_id": "matched-id"}],
0,
llm_element_id="other-id",
llm_value=" WORKER'S\tCOMPENSATION ",
)
assert agreement.agrees is True
@pytest.mark.parametrize(
("matched_index", "llm_index", "llm_value", "expected_agrees"),
[
pytest.param(0, 1, "same value", True, id="value-match-overrides-different-index"),
pytest.param(0, 0, "paraphrased value", False, id="value-decides-despite-matching-index"),
pytest.param(0, 1, "different value", False, id="different-value-and-index"),
pytest.param(0, None, None, None, id="no-signals"),
pytest.param(0, -1, None, False, id="llm-abstained"),
pytest.param(None, 0, "same value", False, id="no-matcher-choice"),
pytest.param(2, 2, "same value", None, id="invalid-candidate-index-with-value-stays-unjudged"),
],
)
def test_native_choice_shadow_agreement_uses_available_value_and_index_signals(
matched_index: int | None,
llm_index: int | None,
llm_value: str | None,
expected_agrees: bool | None,
) -> None:
candidates: list[dict[str, str | None]] = [
{"label": "Duplicate", "value": "same value"},
{"label": "Duplicate", "value": "same value"},
]
agreement = handler._select_shadow_agrees_with_native_choice(
candidates,
matched_index,
llm_index=llm_index,
llm_value=llm_value,
)
assert agreement.agrees is expected_agrees
def test_custom_select_same_value_different_element_id_logs_agreement(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)
with structlog.testing.capture_logs() as logs:
handler._log_select_shadow_match(
prompt_name="custom-select/dropdown",
target_value="Workers' Compensation",
get_candidates=lambda: [{"label": "Worker’s Compensation", "value": None, "element_id": "el-A"}],
agreement=lambda candidates, matched_index: handler._select_shadow_agrees_with_element_choice(
candidates,
matched_index,
llm_element_id="el-B",
llm_value="workers compensation",
),
)
(event,) = (entry for entry in logs if entry.get("event") == "select_shadow_match")
assert event["match_tier"] == "exact"
assert event["match_found"] is True
assert event["match_agrees_with_llm"] is True
assert "matched_element_id" not in event
@pytest.mark.asyncio
@pytest.mark.parametrize(
("llm_value", "expected_agrees"),
[
pytest.param(" committed value ", True, id="normalized-value-match"),
pytest.param("Different Value", False, id="value-mismatch"),
],
)
async def test_auto_completion_call_path_forwards_committed_value_to_shadow_match(
monkeypatch: pytest.MonkeyPatch,
llm_value: str,
expected_agrees: bool,
) -> None:
frame = MagicMock()
skyvern_frame = MagicMock(safe_wait_for_animation_end=AsyncMock())
incremental_scraped = MagicMock(
start_listen_dom_increment=AsyncMock(),
stop_listen_dom_increment=AsyncMock(),
get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]),
)
incremental_scraped.build_html_tree.return_value = "<div>Committed Value</div>"
skyvern_element = MagicMock(
get_frame=Mock(return_value=frame),
get_element_handler=AsyncMock(return_value=MagicMock()),
press_fill=AsyncMock(),
press_key=AsyncMock(),
is_visible=AsyncMock(return_value=True),
input_clear=AsyncMock(),
)
monkeypatch.setattr(handler.SkyvernFrame, "create_instance", AsyncMock(return_value=skyvern_frame))
monkeypatch.setattr(handler, "IncrementalScrapePage", Mock(return_value=incremental_scraped))
monkeypatch.setattr(handler, "get_slim_output_template_value", AsyncMock(return_value=""))
monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
monkeypatch.setattr(
handler.app,
"AUTO_COMPLETION_LLM_API_HANDLER",
AsyncMock(
return_value={
"id": "different-id",
"value": llm_value,
"direct_searching": True,
}
),
)
monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)
with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
with structlog.testing.capture_logs() as logs:
await handler.choose_auto_completion_dropdown(
context=InputOrSelectContext(field="Field", is_search_bar=False),
page=MagicMock(),
scraped_page=MagicMock(),
dom=MagicMock(),
text="Committed Value",
skyvern_element=skyvern_element,
step=SimpleNamespace(step_id="step-1"),
task=_shadow_task(),
is_secret_value=False,
)
events = [log for log in logs if log.get("event") == "select_shadow_match"]
assert len(events) == 1
event = events[0]
assert event["prompt_name"] == "auto-completion-choose-option"
assert event["match_agrees_with_llm"] is expected_agrees
if not expected_agrees:
assert event["normalized_matched_label"] == "committed value"
assert event["normalized_llm_value"] == "different value"
@pytest.mark.asyncio
async def test_emerging_select_call_path_forwards_committed_value_to_shadow_match(
monkeypatch: pytest.MonkeyPatch,
) -> None:
selected_element = MagicMock(
get_attr=AsyncMock(return_value=None),
scroll_into_view=AsyncMock(),
click=AsyncMock(),
)
dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=selected_element))
llm_handler = AsyncMock(
return_value={"id": "different-id", "value": " committed value ", "action_type": "click"}
)
monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))
monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div>Committed Value</div>"))
deterministic_select = AsyncMock(return_value=None)
monkeypatch.setattr(handler, "_select_deterministic_custom_option", deterministic_select)
monkeypatch.setattr(handler, "_verify_custom_select_option_with_settle", AsyncMock(return_value=(True, "scope")))
monkeypatch.setattr(handler, "_verify_custom_select_option", AsyncMock(return_value=(True, "scope")))
monkeypatch.setattr(handler, "_wait_custom_select_render_settle", AsyncMock())
monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
monkeypatch.setattr(
handler.LLMAPIHandlerFactory,
"get_override_llm_api_handler",
Mock(return_value=llm_handler),
)
monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)
scraped_page = SimpleNamespace(id_to_css_dict={})
scraped_page_after_open = SimpleNamespace(
id_to_css_dict={"matched-id": "[data-id=matched-id]"},
id_to_element_dict={"field-id": {"id": "field-id"}},
element_tree_trimmed=[_SHADOW_OPTION],
)
with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
with structlog.testing.capture_logs() as logs:
await handler.select_from_emerging_elements(
current_element_id="field-id",
options=handler.CustomSelectPromptOptions(target_value="Committed Value"),
page=MagicMock(),
scraped_page=scraped_page,
scraped_page_after_open=scraped_page_after_open,
new_interactable_element_ids=["matched-id"],
step=SimpleNamespace(step_id="step-1"),
task=_shadow_task(),
)
events = [log for log in logs if log.get("event") == "select_shadow_match"]
assert len(events) == 1
assert events[0]["prompt_name"] == "custom-select/emerging"
assert events[0]["match_agrees_with_llm"] is True
assert deterministic_select.await_args.kwargs["execute"] is False
@pytest.mark.asyncio
@pytest.mark.parametrize(
("entry_action_type", "expected_execute"),
[("select_option", True), ("input_text", True), ("input_text_converted", False), ("click", False)],
)
async def test_dropdown_select_call_path_forwards_committed_value_and_scopes_execution(
monkeypatch: pytest.MonkeyPatch, entry_action_type: str, expected_execute: bool
) -> None:
incremental_scraped = MagicMock(
get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]),
)
incremental_scraped.build_element_tree.return_value = "<div>Committed Value</div>"
selected_element = MagicMock(
get_tag_name=Mock(return_value="div"),
get_attr=AsyncMock(return_value=None),
scroll_into_view=AsyncMock(),
click=AsyncMock(),
)
monkeypatch.setattr(handler, "locate_dropdown_menu", AsyncMock(return_value=None))
deterministic_select = AsyncMock(return_value=None)
monkeypatch.setattr(handler, "_select_deterministic_custom_option", deterministic_select)
monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
monkeypatch.setattr(
handler.app,
"CUSTOM_SELECT_AGENT_LLM_API_HANDLER",
AsyncMock(return_value={"id": "different-id", "value": " committed value ", "action_type": "click"}),
)
monkeypatch.setattr(handler.SkyvernElement, "create_from_incremental", AsyncMock(return_value=selected_element))
monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)
with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
with structlog.testing.capture_logs() as logs:
await handler.select_from_dropdown(
context=InputOrSelectContext(field="Field", is_required=True),
page=MagicMock(),
skyvern_element=MagicMock(get_id=Mock(return_value="field-id")),
skyvern_frame=MagicMock(),
incremental_scraped=incremental_scraped,
check_filter_funcs=[],
step=SimpleNamespace(step_id="step-1"),
task=_shadow_task(),
force_select=True,
target_value="Committed Value",
entry_action_type=entry_action_type,
)
events = [log for log in logs if log.get("event") == "select_shadow_match"]
assert len(events) == 1
assert events[0]["prompt_name"] == "custom-select/dropdown"
assert events[0]["match_agrees_with_llm"] is True
assert deterministic_select.await_args.kwargs["execute"] is expected_execute
@pytest.mark.asyncio
async def test_dropdown_terminal_result_never_reaches_custom_select_llm(monkeypatch: pytest.MonkeyPatch) -> None:
incremental_scraped = MagicMock(get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]))
incremental_scraped.build_element_tree.return_value = "<div>Committed Value</div>"
failure = handler.ActionFailure(Exception("unverified deterministic click"))
failure.skip_remaining_actions = True
monkeypatch.setattr(handler, "locate_dropdown_menu", AsyncMock(return_value=None))
monkeypatch.setattr(
handler,
"_select_deterministic_custom_option",
AsyncMock(return_value=(failure, "Committed Value")),
)
llm_handler = AsyncMock()
monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
result = await handler.select_from_dropdown(
context=InputOrSelectContext(field="Field", is_required=True),
page=MagicMock(),
skyvern_element=MagicMock(get_id=Mock(return_value="field-id")),
skyvern_frame=MagicMock(),
incremental_scraped=incremental_scraped,
check_filter_funcs=[],
step=SimpleNamespace(step_id="step-1"),
task=_shadow_task(),
force_select=True,
target_value="Committed Value",
)
assert result.action_result is failure
llm_handler.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("use_strict_verification", "expected"),
[(False, (True, "matched_state")), (True, (False, "none"))],
)
async def test_custom_select_verify_strict_posture_is_entry_scoped(
monkeypatch: pytest.MonkeyPatch,
use_strict_verification: bool,
expected: tuple[bool, str],
) -> None:
monkeypatch.setattr(
handler,
"_read_custom_select_matched_state",
AsyncMock(
return_value={
"label": "Choice",
"role": "option",
"inMultiselectable": False,
"ariaSelected": True,
}
),
)
monkeypatch.setattr(
handler,
"_custom_select_scope_confirms_committed",
AsyncMock(return_value=(False, "none")),
)
result = await handler._verify_custom_select_option(
matched_element=MagicMock(),
readback_scope_element=MagicMock(),
anchor_is_combobox_input=True,
matched_element_id="choice-1",
matched_label="Choice",
use_strict_verification=use_strict_verification,
)
assert result == expected
@pytest.mark.asyncio
@pytest.mark.parametrize(
("use_strict_verification", "expected_allow_tokens", "expected_allow_single_value_scope"),
[(False, True, False), (True, False, True)],
)
async def test_custom_select_verify_strict_posture_controls_aria_selected_tokens(
monkeypatch: pytest.MonkeyPatch,
use_strict_verification: bool,
expected_allow_tokens: bool,
expected_allow_single_value_scope: bool,
) -> None:
monkeypatch.setattr(handler, "_read_custom_select_matched_state", AsyncMock(return_value=None))
scope_confirms = AsyncMock(return_value=(False, "none"))
monkeypatch.setattr(handler, "_custom_select_scope_confirms_committed", scope_confirms)
await handler._verify_custom_select_option(
matched_element=MagicMock(),
readback_scope_element=MagicMock(),
anchor_is_combobox_input=True,
matched_element_id="choice-1",
matched_label="Choice",
use_strict_verification=use_strict_verification,
)
assert scope_confirms.await_args.kwargs["allow_aria_selected_option_tokens"] is expected_allow_tokens
assert scope_confirms.await_args.kwargs["allow_single_value_scope"] is expected_allow_single_value_scope
@pytest.mark.asyncio
@pytest.mark.parametrize(
("committed_state", "expected"),
[
({"matched": True, "branch": "scope_input_value"}, (True, "scope_input_value")),
({"matched": True, "branch": "scope_single_value"}, (True, "scope_single_value")),
({"matched": False, "branch": "none"}, (False, "none")),
],
)
async def test_custom_select_verify_preserves_scope_branch(
monkeypatch: pytest.MonkeyPatch,
committed_state: dict[str, object],
expected: tuple[bool, str],
) -> None:
matched_element = MagicMock()
matched_element.get_locator.return_value.count = AsyncMock(return_value=1)
async def evaluate_element_scoped(_element: object, expression: str, _arg: object = None) -> object:
if expression == handler._CUSTOM_SELECT_MATCHED_STATE_JS:
return None
return committed_state
monkeypatch.setattr(handler, "_evaluate_element_scoped", AsyncMock(side_effect=evaluate_element_scoped))
result = await handler._verify_custom_select_option(
matched_element=matched_element,
readback_scope_element=MagicMock(),
anchor_is_combobox_input=True,
matched_element_id="choice-1",
matched_label="Choice",
use_strict_verification=True,
)
assert result == expected
_OUTCOME_TASK = SimpleNamespace(
workflow_run_id="wr",
task_id="task",
organization_id="org",
url="https://test",
navigation_goal="goal",
navigation_payload={},
llm_key=None,
workflow_permanent_id=None,
)
async def _run_outcome_case(
monkeypatch: pytest.MonkeyPatch,
case: str,
*,
family_enabled: bool = True,
assigned: bool = False,
execute: bool = True,
entry_action_type: str = "select_option",
anchor_is_input: bool = False,
selection_group_id: str = "group-1",
select_depth: int = 0,
) -> tuple[object, dict[str, object] | None, SimpleNamespace, AsyncMock, AsyncMock]:
handler._COLLAPSE_XP_ASSIGNMENT_MEMO.clear()
family = (
AsyncMock(side_effect=RuntimeError("gate")) if case == "gate_error" else AsyncMock(return_value=family_enabled)
)
provider = SimpleNamespace(
is_feature_enabled_cached=family,
resolve_feature_enabled_unrecorded=AsyncMock(return_value=assigned),
)
resolution = SimpleNamespace(
fallback_to_llm=case == "no_match",
matched_index=None if case == "no_match" else 2 if case == "bad_index" else 0,
matched_label="Choice",
matched_tier=None if case == "no_match" else "stem" if case == "stem" else "exact",
)
resolver = AsyncMock(side_effect=RuntimeError("matcher") if case == "matcher_error" else None)
resolver.return_value = resolution
click_error = (
handler.InteractWithDisabledElement(element_id="choice-1")
if case == "disabled_click"
else RuntimeError("after dispatch")
if case == "post_click"
else None
)
selected = SimpleNamespace(
get_attr=AsyncMock(return_value="listbox" if case == "listbox" else None),
scroll_into_view=AsyncMock(),
click=AsyncMock(side_effect=click_error),
)
get_skyvern_element = AsyncMock(return_value=selected)
monkeypatch.setattr(handler.app, "EXPERIMENTATION_PROVIDER", provider)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", SimpleNamespace(resolve_field_option=resolver))
monkeypatch.setattr(
handler,
"_read_custom_select_matched_state",
AsyncMock(side_effect=RuntimeError("before click") if case == "pre_click" else None, return_value=None),
)
monkeypatch.setattr(handler, "_resolve_custom_select_readback_scope_element", AsyncMock(return_value=MagicMock()))
monkeypatch.setattr(
handler,
"_anchor_is_combobox_input",
AsyncMock(return_value=anchor_is_input or case.startswith("reset")),
)
monkeypatch.setattr(
handler,
"_custom_select_matched_state_confirms_pre_click",
Mock(return_value=case == "precommit_matched"),
)
async def scope_confirms_committed(**kwargs: object) -> tuple[bool, str]:
committed = case == "precommit_scope" or (
case == "precommit_single_value" and kwargs["allow_single_value_scope"] is True
)
return committed, "scope_single_value" if committed else "none"
monkeypatch.setattr(
handler, "_custom_select_scope_confirms_committed", AsyncMock(side_effect=scope_confirms_committed)
)
monkeypatch.setattr(
handler,
"_verify_custom_select_option_with_settle",
AsyncMock(return_value=(case == "verified", "matched_state" if case == "verified" else "none")),
)
monkeypatch.setattr(handler, "_reset_custom_select_combobox_input", AsyncMock(return_value=case == "reset_ok"))
monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value="typed"))
candidates = (
Mock(side_effect=RuntimeError("walker"))
if case == "walker_error"
else Mock(
return_value=[]
if case == "no_options"
else [
{
"label": "Choice",
"element_id": None if case in {"no_element_id", "stem"} else "choice-1",
"value": "choice",
"is_choice_input": case == "toggle",
}
],
)
)
with structlog.testing.capture_logs() as logs:
result = await handler._select_deterministic_custom_option(
target_value=None if case == "no_target" else "Choice",
get_option_candidates=candidates,
field_context={"is_date_related": True} if case == "date_related" else {},
page=MagicMock(),
get_skyvern_element=get_skyvern_element,
task=_OUTCOME_TASK,
execute=False if case == "execution_disabled" else execute,
step=SimpleNamespace(step_id="step"),
entry_action_type=entry_action_type,
selection_group_id=selection_group_id,
select_depth=select_depth,
)
events = [log for log in logs if log.get("event") == "custom_select_family_outcome"]
assert len(events) == (0 if case in {"no_target", "date_related", "no_options"} else 1)
return result, events[0] if events else None, provider, resolver, get_skyvern_element
@pytest.mark.asyncio
@pytest.mark.parametrize("entry_action_type", ["select_option", "input_text", "input_text_converted"])
async def test_outcome_event_emitted_once_per_opportunity(
monkeypatch: pytest.MonkeyPatch, entry_action_type: str
) -> None:
_, event, _, _, _ = await _run_outcome_case(monkeypatch, "control", entry_action_type=entry_action_type)
assert event is not None
assert event["outcome"] == "llm_fallback_control"
assert event["entry_action_type"] == entry_action_type
@pytest.mark.asyncio
@pytest.mark.parametrize(
(
"entry_action_type",
"anchor_is_input",
"expected_use_strict_verification",
"expected_allow_single_value_scope",
),
[
("select_option", True, False, False),
("select_option", False, False, False),
("input_text", True, True, True),
("input_text_converted", True, False, False),
],
)
async def test_deterministic_custom_select_verification_is_entry_scoped(
monkeypatch: pytest.MonkeyPatch,
entry_action_type: str,
anchor_is_input: bool,
expected_use_strict_verification: bool,
expected_allow_single_value_scope: bool,
) -> None:
await _run_outcome_case(
monkeypatch,
"verified",
assigned=True,
entry_action_type=entry_action_type,
anchor_is_input=anchor_is_input,
)
verify = handler._verify_custom_select_option_with_settle
assert isinstance(verify, AsyncMock)
assert verify.await_args.kwargs["use_strict_verification"] is expected_use_strict_verification
scope_confirms = handler._custom_select_scope_confirms_committed
assert isinstance(scope_confirms, AsyncMock)
assert scope_confirms.await_args.kwargs["allow_single_value_scope"] is expected_allow_single_value_scope
@pytest.mark.asyncio
async def test_executable_entry_constant_disables_input_text_single_value_scope(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(handler, "_EXECUTABLE_CUSTOM_SELECT_ENTRIES", ("select_option",))
await _run_outcome_case(
monkeypatch,
"verified",
assigned=True,
entry_action_type="input_text",
anchor_is_input=True,
)
scope_confirms = handler._custom_select_scope_confirms_committed
assert isinstance(scope_confirms, AsyncMock)
assert scope_confirms.await_args.kwargs["allow_single_value_scope"] is False
verify = handler._verify_custom_select_option_with_settle
assert isinstance(verify, AsyncMock)
assert verify.await_args.kwargs["use_strict_verification"] is False
@pytest.mark.asyncio
@pytest.mark.parametrize(
("case", "expected_outcome"),
[
("precommit_matched", "success_precommit"),
("precommit_scope", "success_precommit"),
("verified", "llm_fallback_execution_disabled"),
],
)
async def test_input_text_non_input_anchor_keeps_precommit_but_never_clicks(
monkeypatch: pytest.MonkeyPatch,
case: str,
expected_outcome: str,
) -> None:
result, event, _, _, get_skyvern_element = await _run_outcome_case(
monkeypatch,
case,
assigned=True,
entry_action_type="input_text",
anchor_is_input=False,
)
if expected_outcome == "success_precommit":
assert result is not None
assert isinstance(result[0], handler.ActionSuccess)
else:
assert result is None
assert event is not None
assert event["outcome"] == expected_outcome
get_skyvern_element.return_value.click.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("entry_action_type", ["click", "input_text_converted"])
async def test_nonexecutable_entry_does_not_single_value_preconfirm(
monkeypatch: pytest.MonkeyPatch,
entry_action_type: str,
) -> None:
result, event, _, _, get_skyvern_element = await _run_outcome_case(
monkeypatch,
"precommit_single_value",
assigned=True,
execute=False,
entry_action_type=entry_action_type,
anchor_is_input=True,
)
assert result is None
assert event is not None
assert event["outcome"] == "llm_fallback_execution_disabled"
get_skyvern_element.return_value.click.assert_not_awaited()
@pytest.mark.asyncio
async def test_executable_entry_constant_disables_input_text_single_value_preconfirm(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(handler, "_EXECUTABLE_CUSTOM_SELECT_ENTRIES", ("select_option",))
result, event, _, _, get_skyvern_element = await _run_outcome_case(
monkeypatch,
"precommit_single_value",
assigned=True,
execute=False,
entry_action_type="input_text",
anchor_is_input=True,
)
assert result is None
assert event is not None
assert event["outcome"] == "llm_fallback_execution_disabled"
get_skyvern_element.return_value.click.assert_not_awaited()
@pytest.mark.asyncio
async def test_click_route_without_target_emits_nothing_and_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
gate = AsyncMock()
monkeypatch.setattr(handler, "_resolve_collapse_gate", gate)
with structlog.testing.capture_logs() as logs:
result = await handler._select_deterministic_custom_option(
target_value=None,
get_option_candidates=Mock(),
field_context={},
page=MagicMock(),
get_skyvern_element=AsyncMock(),
task=_OUTCOME_TASK,
execute=True,
entry_action_type="click",
)
assert result is None
assert [log for log in logs if log.get("event") == "custom_select_family_outcome"] == []
gate.assert_not_awaited()
@pytest.mark.asyncio
async def test_click_ingress_routes_no_target_and_emits_nothing(monkeypatch: pytest.MonkeyPatch) -> None:
select_from_emerging_elements = AsyncMock(return_value=MagicMock())
monkeypatch.setattr(handler, "select_from_emerging_elements", select_from_emerging_elements)
monkeypatch.setattr(handler, "_build_after_click_verify_prompt", AsyncMock(return_value="prompt"))
monkeypatch.setattr(
handler,
"resolve_check_user_goal_handler",
AsyncMock(
return_value=AsyncMock(
return_value={"thoughts": "continue", "user_goal_achieved": False, "should_terminate": False}
)
),
)
monkeypatch.setattr(handler, "locate_dropdown_menu", AsyncMock(return_value=MagicMock()))
monkeypatch.setattr(
handler,
"_get_input_or_select_context",
AsyncMock(return_value=InputOrSelectContext(field="Field", is_required=True, is_date_related=False)),
)
monkeypatch.setattr(
handler,
"DomUtil",
Mock(
return_value=SimpleNamespace(
get_skyvern_element_by_id=AsyncMock(
return_value=SimpleNamespace(is_interactable=Mock(return_value=True))
)
)
),
)
scraped_page_after_open = SimpleNamespace(id_to_css_dict={"choice-id": "[data-id=choice-id]"})
scraped_page = SimpleNamespace(
url="https://test",
id_to_css_dict={},
generate_scraped_page_without_screenshots=AsyncMock(return_value=scraped_page_after_open),
)
incremental_scraped = SimpleNamespace(
get_incremental_elements_num=AsyncMock(return_value=1),
get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]),
)
with structlog.testing.capture_logs() as logs:
await handler.handle_sequential_click_for_dropdown(
action=ClickAction(element_id="field-id", reasoning="click", intention="choose"),
action_history=[],
anchor_element=MagicMock(get_id=Mock(return_value="field-id")),
dom=MagicMock(),
page=SimpleNamespace(url="https://test"),
skyvern_frame=SimpleNamespace(safe_wait_for_animation_end=AsyncMock(), engine_selection=object()),
scraped_page=scraped_page,
incremental_scraped=incremental_scraped,
task=_OUTCOME_TASK,
step=SimpleNamespace(step_id="step"),
)
options = select_from_emerging_elements.await_args.kwargs["options"]
assert options.target_value is None
assert [log for log in logs if log.get("event") == "custom_select_family_outcome"] == []
@pytest.mark.asyncio
async def test_outcome_event_once_per_dropdown_level_with_group_id(monkeypatch: pytest.MonkeyPatch) -> None:
handler._COLLAPSE_XP_ASSIGNMENT_MEMO.clear()
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
SimpleNamespace(
is_feature_enabled_cached=AsyncMock(return_value=True),
resolve_feature_enabled_unrecorded=AsyncMock(return_value=False),
),
)
monkeypatch.setattr(
handler.app,
"AGENT_FUNCTION",
SimpleNamespace(
resolve_field_option=AsyncMock(
return_value=SimpleNamespace(
fallback_to_llm=False,
matched_index=0,
matched_label="Choice",
matched_tier="exact",
)
)
),
)
monkeypatch.setattr(
handler.app,
"CUSTOM_SELECT_AGENT_LLM_API_HANDLER",
AsyncMock(return_value={"id": "matched-id", "value": "Choice", "action_type": "input_text"}),
)
monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value=""))
monkeypatch.setattr(
handler, "get_actual_value_of_parameter_if_secret_with_task", Mock(side_effect=lambda _, value: value)
)
dropdown = MagicMock(
get_element_handler=AsyncMock(return_value=MagicMock()),
get_locator=Mock(return_value=MagicMock(count=AsyncMock(return_value=1))),
)
input_element = MagicMock(
get_tag_name=Mock(return_value="input"),
get_locator=Mock(return_value=MagicMock()),
scroll_into_view=AsyncMock(),
is_readonly=AsyncMock(return_value=False),
input_clear=AsyncMock(),
input_sequentially=AsyncMock(),
)
skyvern_frame = MagicMock(
safe_wait_for_animation_end=AsyncMock(),
get_element_scrollable=AsyncMock(return_value=False),
get_element_visible=AsyncMock(return_value=True),
)
incremental_scraped = MagicMock(get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]))
incremental_scraped.build_element_tree.return_value = "<div>Choice</div>"
monkeypatch.setattr(handler, "try_to_find_potential_scrollable_element", AsyncMock(return_value=dropdown))
monkeypatch.setattr(handler.SkyvernElement, "create_from_incremental", AsyncMock(return_value=input_element))
action = SelectOptionAction(
element_id="field-id",
option=SelectOption(label="Choice"),
input_or_select_context=InputOrSelectContext(field="Field"),
)
with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
with structlog.testing.capture_logs() as logs:
await handler.sequentially_select_from_dropdown(
action=action,
input_or_select_context=InputOrSelectContext(field="Field"),
page=MagicMock(),
dom=MagicMock(),
skyvern_element=MagicMock(),
skyvern_frame=skyvern_frame,
incremental_scraped=incremental_scraped,
step=SimpleNamespace(step_id="step"),
task=_OUTCOME_TASK,
dropdown_menu_element=dropdown,
force_select=True,
target_value="Choice",
)
events = [log for log in logs if log.get("event") == "custom_select_family_outcome"]
assert len(events) == 3
assert [event["outcome"] for event in events] == ["llm_fallback_control"] * 3
assert events[0]["selection_group_id"]
assert len({event["selection_group_id"] for event in events}) == 1
assert [event["select_depth"] for event in events] == [0, 1, 2]
async def _run_post_reset_fallback_case(
monkeypatch: pytest.MonkeyPatch,
*,
llm_response: object,
entry_action_type: str = "input_text",
input_value_reads: list[str | None] | None = None,
) -> tuple[object, list[dict[str, object]], MagicMock, MagicMock, BaseException | None, AsyncMock]:
handler._COLLAPSE_XP_ASSIGNMENT_MEMO.clear()
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
SimpleNamespace(
is_feature_enabled_cached=AsyncMock(return_value=True),
resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
),
)
monkeypatch.setattr(
handler.app,
"AGENT_FUNCTION",
SimpleNamespace(
resolve_field_option=AsyncMock(
return_value=SimpleNamespace(
fallback_to_llm=False,
matched_index=0,
matched_label="Committed Value",
matched_tier="exact",
)
)
),
)
llm_handler = (
AsyncMock(side_effect=llm_response)
if isinstance(llm_response, Exception)
else AsyncMock(return_value=llm_response)
)
monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
monkeypatch.setattr(handler, "_read_custom_select_matched_state", AsyncMock(return_value=None))
monkeypatch.setattr(handler, "_custom_select_scope_confirms_committed", AsyncMock(return_value=(False, "none")))
monkeypatch.setattr(handler, "_verify_custom_select_option_with_settle", AsyncMock(return_value=(False, "none")))
if input_value_reads is None:
input_value_reads = (
[""]
if entry_action_type == "select_option"
else ["Committed Value", "", "Committed Value", "Committed Value"]
)
get_input_value_mock = AsyncMock(side_effect=input_value_reads)
monkeypatch.setattr(handler, "get_input_value", get_input_value_mock)
monkeypatch.setattr(
handler, "get_actual_value_of_parameter_if_secret_with_task", Mock(side_effect=lambda _, value: value)
)
reset_locator = MagicMock(fill=AsyncMock())
anchor = MagicMock(
get_id=Mock(return_value="field-id"),
get_tag_name=Mock(return_value="input"),
get_locator=Mock(return_value=reset_locator),
click=AsyncMock(),
)
matched_element = MagicMock(
get_attr=AsyncMock(return_value=None),
scroll_into_view=AsyncMock(),
click=AsyncMock(),
)
input_element = MagicMock(
get_tag_name=Mock(return_value="input"),
get_locator=Mock(return_value=MagicMock()),
scroll_into_view=AsyncMock(),
is_readonly=AsyncMock(return_value=False),
input_clear=AsyncMock(),
input_sequentially=AsyncMock(),
)
monkeypatch.setattr(
handler.SkyvernElement,
"create_from_incremental",
AsyncMock(side_effect=[matched_element, input_element]),
)
dropdown = MagicMock(get_element_handler=AsyncMock(return_value=MagicMock()))
skyvern_frame = MagicMock(get_element_scrollable=AsyncMock(return_value=False), engine_selection=object())
incremental_scraped = MagicMock(get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]))
incremental_scraped.build_element_tree.return_value = "<div>Committed Value</div>"
monkeypatch.setattr(handler, "try_to_find_potential_scrollable_element", AsyncMock(return_value=dropdown))
result: object = None
raised: BaseException | None = None
with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
with structlog.testing.capture_logs() as logs:
try:
result = await handler.select_from_dropdown(
context=InputOrSelectContext(field="Field"),
page=MagicMock(),
skyvern_element=anchor,
skyvern_frame=skyvern_frame,
incremental_scraped=incremental_scraped,
check_filter_funcs=[],
step=SimpleNamespace(step_id="step"),
task=_OUTCOME_TASK,
dropdown_menu_element=dropdown,
force_select=True,
target_value="Committed Value",
entry_action_type=entry_action_type,
selection_group_id="group-1",
)
except Exception as exc:
raised = exc
events = [log for log in logs if log.get("event") == "custom_select_family_outcome"]
return result, events, input_element, reset_locator, raised, get_input_value_mock
@pytest.mark.asyncio
@pytest.mark.parametrize(
"llm_response",
[
RuntimeError("fallback failed"),
{"id": "", "action_type": "", "reasoning": "no match"},
{"id": "matched-id", "value": "Committed Value", "action_type": {"x": 1}},
{"id": "matched-id", "action_type": "input_text", "reasoning": "no value"},
{"id": "matched-id", "value": "", "action_type": "input_text", "reasoning": "empty value"},
],
)
async def test_post_reset_llm_failure_emits_single_terminal_outcome(
monkeypatch: pytest.MonkeyPatch,
llm_response: object,
) -> None:
result, events, _, _, raised, _ = await _run_post_reset_fallback_case(monkeypatch, llm_response=llm_response)
assert raised is None
assert isinstance(result.action_result, handler.ActionFailure)
assert result.action_result.skip_remaining_actions is True
assert [event["outcome"] for event in events] == ["terminal_llm_fallback_exception"]
assert events[0]["entry_action_type"] == "input_text"
assert events[0]["selection_group_id"] == "group-1"
assert events[0]["select_depth"] == 0
assert events[0]["family_gate_enabled"] is True
assert events[0]["assigned"] is True
assert events[0]["llm_fallback_requested"] is True
@pytest.mark.asyncio
async def test_post_reset_restored_text_is_retyped_and_emits_single_proceeded_outcome(
monkeypatch: pytest.MonkeyPatch,
) -> None:
result, events, input_element, reset_locator, raised, _ = await _run_post_reset_fallback_case(
monkeypatch,
llm_response={"id": "matched-id", "value": "Committed Value", "action_type": "input_text"},
)
assert raised is None
assert isinstance(result.action_result, handler.ActionSuccess)
input_element.input_clear.assert_awaited_once()
input_element.input_sequentially.assert_awaited_once_with("Committed Value")
assert [c.args for c in reset_locator.fill.await_args_list] == [("",), ("Committed Value",)]
assert [event["outcome"] for event in events] == ["llm_fallback_reset_verified"]
assert events[0]["entry_action_type"] == "input_text"
assert events[0]["selection_group_id"] == "group-1"
assert events[0]["select_depth"] == 0
assert events[0]["family_gate_enabled"] is True
assert events[0]["assigned"] is True
assert events[0]["llm_fallback_requested"] is True
@pytest.mark.asyncio
async def test_select_option_reset_path_matches_main(monkeypatch: pytest.MonkeyPatch) -> None:
llm_error = RuntimeError("fallback failed")
result, events, _, reset_locator, raised, get_input_value_mock = await _run_post_reset_fallback_case(
monkeypatch,
llm_response=llm_error,
entry_action_type="select_option",
)
assert raised is llm_error
assert result is None
assert [event["outcome"] for event in events] == ["llm_fallback_reset_verified"]
assert events[0]["entry_action_type"] == "select_option"
assert [c.args for c in reset_locator.fill.await_args_list] == [("",)]
assert get_input_value_mock.await_count == 1
@pytest.mark.asyncio
async def test_failed_restore_readback_is_terminal_not_reset_verified(monkeypatch: pytest.MonkeyPatch) -> None:
llm_handler_never_called = RuntimeError("llm should not run")
result, events, _, reset_locator, raised, _ = await _run_post_reset_fallback_case(
monkeypatch,
llm_response=llm_handler_never_called,
input_value_reads=["Committed Value", "", "typed over"],
)
assert raised is None
assert isinstance(result.action_result, handler.ActionFailure)
assert result.action_result.skip_remaining_actions is True
assert [event["outcome"] for event in events] == ["terminal_unverified_reset"]
assert [c.args for c in reset_locator.fill.await_args_list] == [("",), ("Committed Value",)]
@pytest.mark.asyncio
async def test_unreadable_anchor_before_click_falls_back_without_clicking(monkeypatch: pytest.MonkeyPatch) -> None:
llm_error = RuntimeError("llm path")
result, events, _, reset_locator, raised, _ = await _run_post_reset_fallback_case(
monkeypatch,
llm_response=llm_error,
input_value_reads=[None],
)
assert raised is llm_error
assert result is None
assert [event["outcome"] for event in events] == ["llm_fallback_pre_click_error"]
reset_locator.fill.assert_not_awaited()
@pytest.mark.asyncio
async def test_outcome_event_family_off_emits_without_assignment_resolution(monkeypatch: pytest.MonkeyPatch) -> None:
_, event, provider, resolver, _ = await _run_outcome_case(monkeypatch, "family_off", family_enabled=False)
assert event is not None
assert event["outcome"] == "llm_fallback_family_off"
assert (event["assigned"], event["eligible"], event["match_tier"]) == (None, True, "exact")
provider.resolve_feature_enabled_unrecorded.assert_not_awaited()
resolver.assert_awaited_once()
@pytest.mark.asyncio
async def test_outcome_event_key_allowlist_and_enum(monkeypatch: pytest.MonkeyPatch) -> None:
_, event, _, _, _ = await _run_outcome_case(monkeypatch, "control")
assert event is not None
expected_keys = set(
"event log_level family workflow_run_id task_id organization_id step_id entry_action_type selection_group_id "
"select_depth script_mode family_gate_enabled assigned gate_error encountered eligible match_tier "
"option_count attempted click_attempted anchor_is_combobox_input verify_branch verified_success outcome "
"llm_fallback_requested duration_ms".split()
)
expected_outcomes = set(
"llm_fallback_gate_error llm_fallback_eval_error llm_fallback_family_off llm_fallback_control "
"llm_fallback_no_match llm_fallback_match_unactionable llm_fallback_pre_click_error "
"llm_fallback_reset_verified llm_fallback_post_click_unverified llm_fallback_tier_excluded "
"llm_fallback_execution_disabled success_precommit success_verified terminal_post_click_exception "
"terminal_llm_fallback_exception "
"terminal_unverified_reset terminal_unverified_click "
"terminal_unverified_toggle".split()
)
assert event["outcome"] == "llm_fallback_control"
assert (event["anchor_is_combobox_input"], event["verify_branch"]) == (False, None)
assert set(event) == expected_keys
assert {outcome.value for outcome in handler.CustomSelectFamilyOutcome} == expected_outcomes
@pytest.mark.asyncio
async def test_outcome_event_gate_error_fields(monkeypatch: pytest.MonkeyPatch) -> None:
_, event, _, _, _ = await _run_outcome_case(monkeypatch, "gate_error")
assert event is not None
assert event["outcome"] == "llm_fallback_gate_error"
assert (event["option_count"], event["eligible"], event["match_tier"]) == (None, False, None)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("case", "family_enabled", "assigned"),
[
(case, family, assigned)
for case in ("walker_error", "matcher_error")
for family, assigned in ((False, False), (True, False), (True, True))
],
)
async def test_outcome_event_eval_error_emits_on_all_arms(
monkeypatch: pytest.MonkeyPatch, case: str, family_enabled: bool, assigned: bool
) -> None:
result, event, _, _, _ = await _run_outcome_case(
monkeypatch, case, family_enabled=family_enabled, assigned=assigned
)
assert event is not None
assert result is None
assert event["outcome"] == "llm_fallback_eval_error"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("case", "family_enabled", "assigned", "outcome", "click_attempted", "return_shape"),
[
("no_target", True, True, None, False, "none"),
("date_related", True, True, None, False, "none"),
("no_options", True, True, None, False, "none"),
("gate_error", True, True, "llm_fallback_gate_error", False, "none"),
("walker_error", True, True, "llm_fallback_eval_error", False, "none"),
("matcher_error", True, True, "llm_fallback_eval_error", False, "none"),
("family_off", False, True, "llm_fallback_family_off", False, "none"),
("control", True, False, "llm_fallback_control", False, "none"),
("no_match", True, True, "llm_fallback_no_match", False, "none"),
("bad_index", True, True, "llm_fallback_match_unactionable", False, "none"),
("stem", True, True, "llm_fallback_tier_excluded", False, "none"),
("execution_disabled", True, True, "llm_fallback_execution_disabled", False, "none"),
("no_element_id", True, True, "llm_fallback_match_unactionable", False, "none"),
("listbox", True, True, "llm_fallback_match_unactionable", False, "none"),
("pre_click", True, True, "llm_fallback_pre_click_error", False, "none"),
("precommit_matched", True, True, "success_precommit", False, "success"),
("precommit_scope", True, True, "success_precommit", False, "success"),
("verified", True, True, "success_verified", True, "success"),
("disabled_click", True, True, "llm_fallback_pre_click_error", True, "none"),
("post_click", True, True, "terminal_post_click_exception", True, "failure"),
("reset_ok", True, True, "llm_fallback_reset_verified", True, "none"),
("reset_failed", True, True, "terminal_unverified_reset", True, "failure"),
("nonchoice", True, True, "terminal_unverified_click", True, "failure"),
("toggle", True, True, "terminal_unverified_toggle", True, "failure"),
],
)
async def test_deterministic_custom_select_exit_matrix(
monkeypatch: pytest.MonkeyPatch,
case: str,
family_enabled: bool,
assigned: bool,
outcome: str | None,
click_attempted: bool,
return_shape: str,
) -> None:
result, event, _, _, get_skyvern_element = await _run_outcome_case(
monkeypatch,
case,
family_enabled=family_enabled,
assigned=assigned,
)
if outcome is None:
assert event is None
else:
assert event is not None
assert event["outcome"] == outcome
assert event["click_attempted"] is click_attempted
if return_shape == "none":
assert result is None
else:
assert result is not None
action_result, _matched_label = result
expected_type = handler.ActionSuccess if return_shape == "success" else handler.ActionFailure
assert isinstance(action_result, expected_type)
if return_shape == "failure":
assert action_result.skip_remaining_actions is True
if event is not None and event["click_attempted"] is True and result is None:
assert (case, event["outcome"]) in {
("disabled_click", "llm_fallback_pre_click_error"),
("reset_ok", "llm_fallback_reset_verified"),
}
if case == "stem":
get_skyvern_element.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("case", "outcome", "attempted", "return_shape"),
[
("execution_disabled", "llm_fallback_execution_disabled", True, "none"),
("precommit_matched", "success_precommit", True, "success"),
("listbox", "llm_fallback_match_unactionable", False, "none"),
],
)
async def test_execution_disabled_reads_precommit_state_without_clicking(
monkeypatch: pytest.MonkeyPatch,
case: str,
outcome: str,
attempted: bool,
return_shape: str,
) -> None:
result, event, _, _, get_skyvern_element = await _run_outcome_case(
monkeypatch,
case,
assigned=True,
execute=False,
)
assert event is not None
assert event["outcome"] == outcome
assert event["attempted"] is attempted
assert event["click_attempted"] is False
get_skyvern_element.assert_awaited_once_with("choice-1")
selected = get_skyvern_element.return_value
selected.scroll_into_view.assert_not_awaited()
selected.click.assert_not_awaited()
if return_shape == "none":
assert result is None
else:
assert result is not None
action_result, matched_label = result
assert isinstance(action_result, handler.ActionSuccess)
assert matched_label == "Choice"
@pytest.mark.asyncio
async def test_converted_route_reports_input_text_converted(monkeypatch: pytest.MonkeyPatch) -> None:
element = MagicMock(
is_disabled=AsyncMock(return_value=False),
get_selectable=AsyncMock(return_value=True),
supports_text_input=AsyncMock(return_value=True),
)
element.get_attr = AsyncMock(return_value=None)
element.get_tag_name.return_value, element.get_id.return_value = "input", "field"
monkeypatch.setattr(
handler, "DomUtil", Mock(return_value=MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=element)))
)
monkeypatch.setattr(handler.SkyvernFrame, "create_instance", AsyncMock(return_value=MagicMock()))
monkeypatch.setattr(handler, "IncrementalScrapePage", MagicMock())
monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value=""))
select = AsyncMock(return_value=[handler.ActionSuccess()])
monkeypatch.setattr(handler, "handle_select_option_action", select)
await handler.handle_input_text_action(
handler.InputTextAction(element_id="field", text="Choice"),
MagicMock(),
SimpleNamespace(id_to_element_dict={"field": {"tagName": "input"}}),
SimpleNamespace(workflow_run_id=None, task_id="task-1"),
MagicMock(),
)
assert select.await_args.kwargs["entry_action_type"] == "input_text_converted"
@pytest.mark.asyncio
async def test_autocomplete_readback_threads_engine_selection(monkeypatch: pytest.MonkeyPatch) -> None:
selection = object()
element = MagicMock()
element.get_tag_name.return_value = "input"
element.get_locator.return_value = MagicMock()
get_input_value = AsyncMock(return_value="Choice")
monkeypatch.setattr(handler, "get_input_value", get_input_value)
assert await handler._verify_autocomplete_input_readback(
skyvern_element=element,
matched_index=0,
matched_label="Choice",
engine_selection=selection,
)
get_input_value.assert_awaited_once_with("input", element.get_locator(), engine_selection=selection)