Skip to content

Automate browser based workflows with AI

Stop replaying custom-select clicks that may have already changed the page (#7696)

Marc Kelechava09d2ff445cdb
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 9523c9fc2..058a581bc 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -1998,7 +1998,9 @@ class ForgeAgent:
action_result=results,
)
else:
- if action_node.next is not None:
+ # A failure that set skip_remaining_actions must not fall through to the duplicate
+ # element id, or the action it was protecting against gets replayed anyway.
+ if action_node.next is not None and (not results or not results[-1].skip_remaining_actions):
LOG.warning(
"Action failed, but have duplicated element id in the action list. Continue excuting.",
step_order=step.order,
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 9152c731f..71c1be186 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -208,6 +208,8 @@ class CustomSelectFamilyOutcome(StrEnum):
llm_fallback_control = "llm_fallback_control"
llm_fallback_no_match = "llm_fallback_no_match"
llm_fallback_match_unactionable = "llm_fallback_match_unactionable"
+ llm_fallback_tier_excluded = "llm_fallback_tier_excluded"
+ llm_fallback_execution_disabled = "llm_fallback_execution_disabled"
llm_fallback_pre_click_error = "llm_fallback_pre_click_error"
llm_fallback_reset_verified = "llm_fallback_reset_verified"
llm_fallback_post_click_unverified = "llm_fallback_post_click_unverified"
@@ -7631,6 +7633,17 @@ async def _anchor_is_combobox_input(element: SkyvernElement | None) -> bool:
return False
+def _terminal_custom_select_failure(
+ *, target_value: str, matched_label: str | None
+) -> tuple[ActionFailure, str | None]:
+ action_failure = _no_element_matched_failure(
+ target_value,
+ "Deterministic custom-select click could not be verified by matched element read-back",
+ )
+ action_failure.skip_remaining_actions = True
+ return action_failure, matched_label
+
+
async def _select_deterministic_custom_option(
*,
target_value: str | None,
@@ -7640,6 +7653,7 @@ async def _select_deterministic_custom_option(
get_skyvern_element: Callable[[str], Awaitable[SkyvernElement]],
get_readback_scope_element: Callable[[], Awaitable[SkyvernElement | None]] | None = None,
task: Task,
+ execute: bool,
step: Step | None = None,
entry_action_type: str = "select_option",
selection_group_id: str | None = None,
@@ -7718,7 +7732,7 @@ async def _select_deterministic_custom_option(
option_count = len(option_candidates)
eligible = not resolution.fallback_to_llm and resolution.matched_index is not None
- match_tier = getattr(resolution, "matched_tier", None)
+ match_tier = resolution.matched_tier
if not gate.family_enabled:
emit(CustomSelectFamilyOutcome.llm_fallback_family_off)
return None
@@ -7731,6 +7745,9 @@ async def _select_deterministic_custom_option(
if resolution.matched_index >= len(option_candidates):
emit(CustomSelectFamilyOutcome.llm_fallback_match_unactionable)
return None
+ if resolution.matched_tier != "exact":
+ emit(CustomSelectFamilyOutcome.llm_fallback_tier_excluded)
+ return None
matched_candidate = option_candidates[resolution.matched_index]
element_id = matched_candidate.get("element_id")
@@ -7784,6 +7801,10 @@ async def _select_deterministic_custom_option(
emit(CustomSelectFamilyOutcome.success_precommit)
return ActionSuccess(), matched_label
+ if not execute:
+ emit(CustomSelectFamilyOutcome.llm_fallback_execution_disabled)
+ return None
+
await selected_element.scroll_into_view()
click_attempted = True
if on_click_attempted is not None:
@@ -7799,49 +7820,58 @@ async def _select_deterministic_custom_option(
if verified:
emit(CustomSelectFamilyOutcome.success_verified)
return ActionSuccess(), matched_label
- except Exception:
+ except Exception as exc:
+ if not click_attempted or isinstance(exc, InteractWithDisabledElement):
+ LOG.info(
+ "Deterministic custom-select failed; falling back to LLM path",
+ target_value=target_value,
+ matched_element_id=element_id,
+ matched_label=matched_label,
+ exc_info=True,
+ )
+ emit(CustomSelectFamilyOutcome.llm_fallback_pre_click_error)
+ return None
LOG.info(
- "Deterministic custom-select failed; falling back to LLM path",
+ "Deterministic custom-select failed after click; returning failure to avoid replaying over mutated widget",
target_value=target_value,
matched_element_id=element_id,
matched_label=matched_label,
exc_info=True,
)
- emit(
- CustomSelectFamilyOutcome.llm_fallback_post_click_unverified
- if click_attempted
- else CustomSelectFamilyOutcome.llm_fallback_pre_click_error
- )
- return None
+ emit(CustomSelectFamilyOutcome.terminal_post_click_exception)
+ return _terminal_custom_select_failure(target_value=target_value, matched_label=matched_label)
if anchor_is_combobox_input:
# Text-input comboboxes can be safely reset, so an unconfirmed read-back routes to the LLM
# mini-agent (which clears/reopens the field) instead of hard-failing the whole action.
reset_verified = await _reset_custom_select_combobox_input(readback_scope_element, page)
+ if reset_verified:
+ LOG.info(
+ "Deterministic custom-select read-back inconclusive on combobox input; routing to LLM fallback",
+ target_value=target_value,
+ matched_element_id=element_id,
+ matched_label=matched_label,
+ )
+ emit(CustomSelectFamilyOutcome.llm_fallback_reset_verified)
+ return None
LOG.info(
- "Deterministic custom-select read-back inconclusive on combobox input; routing to LLM fallback",
+ "Deterministic custom-select combobox reset failed; returning failure to avoid replaying over mutated widget",
target_value=target_value,
matched_element_id=element_id,
matched_label=matched_label,
)
- emit(
- CustomSelectFamilyOutcome.llm_fallback_reset_verified
- if reset_verified
- else CustomSelectFamilyOutcome.llm_fallback_post_click_unverified
- )
- return None
+ emit(CustomSelectFamilyOutcome.terminal_unverified_reset)
+ return _terminal_custom_select_failure(target_value=target_value, matched_label=matched_label)
if not matched_option_is_choice_input:
- # A non-toggle option (e.g. a button/div-anchored single-select listbox) can be safely
- # replayed by the LLM mini-agent. Toggle-shaped options hard-fail below instead.
LOG.info(
- "Deterministic custom-select read-back inconclusive on non-choice-input option; routing to LLM fallback",
+ "Deterministic custom-select read-back inconclusive on non-choice-input option; returning terminal failure",
target_value=target_value,
matched_element_id=element_id,
matched_label=matched_label,
)
- emit(CustomSelectFamilyOutcome.llm_fallback_post_click_unverified)
- return None
+ emit(CustomSelectFamilyOutcome.terminal_unverified_click)
+ return _terminal_custom_select_failure(target_value=target_value, matched_label=matched_label)
LOG.info(
"Deterministic custom-select read-back failed after click; returning failure to avoid replaying over mutated widget",
@@ -7849,15 +7879,8 @@ async def _select_deterministic_custom_option(
matched_element_id=element_id,
matched_label=matched_label,
)
- action_failure = ActionFailure(
- NoElementMatchedForTargetOption(
- target=target_value,
- reason="Deterministic custom-select click could not be verified by matched element read-back",
- )
- )
- action_failure.skip_remaining_actions = True
emit(CustomSelectFamilyOutcome.terminal_unverified_toggle)
- return action_failure, matched_label
+ return _terminal_custom_select_failure(target_value=target_value, matched_label=matched_label)
async def _reset_custom_select_combobox_input(element: SkyvernElement | None, page: Page) -> bool:
@@ -8005,6 +8028,7 @@ async def select_from_emerging_elements(
widget_mutated = True
deterministic_result = await _select_deterministic_custom_option(
+ execute=False,
target_value=options.target_value,
get_option_candidates=lambda: _custom_select_candidates_from_elements(shadow_candidate_elements),
field_context=options.model_dump(),
@@ -8183,6 +8207,7 @@ async def select_from_dropdown(
widget_mutated = True
deterministic_result = await _select_deterministic_custom_option(
+ execute=entry_action_type == "select_option",
target_value=target_value,
get_option_candidates=lambda: _custom_select_candidates_from_elements(trimmed_element_tree),
field_context=context.model_dump(),
@@ -8198,7 +8223,7 @@ async def select_from_dropdown(
)
if deterministic_result is not None:
action_result, matched_label = deterministic_result
- single_select_result.reasoning = "Deterministic exact/stem custom-select match"
+ single_select_result.reasoning = "Deterministic exact custom-select match"
single_select_result.value = matched_label or target_value
single_select_result.action_type = ActionType.CLICK
single_select_result.action_result = action_result
diff --git a/tests/unit/test_agent_step_characterization.py b/tests/unit/test_agent_step_characterization.py
index e135245ad..252e0bc80 100644
--- a/tests/unit/test_agent_step_characterization.py
+++ b/tests/unit/test_agent_step_characterization.py
@@ -313,6 +313,25 @@ async def test_failed_action_with_continue_flag_executes_remaining(monkeypatch:
assert [action for action, _ in output.actions_and_results] == [first, second]
[email protected](("skip", "expected_calls"), [(True, 1), (False, 2)])
+async def test_failed_action_skip_remaining_controls_duplicate_element_retry(
+ monkeypatch: pytest.MonkeyPatch, skip: bool, expected_calls: int
+) -> None:
+ first, duplicate = _click("node-1"), _click("node-1")
+ failure = ActionFailure(Exception("unverified click"))
+ failure.skip_remaining_actions = skip
+ action_handler = AsyncMock(return_value=[failure])
+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[first, duplicate], action_handler=action_handler)
+
+ step, output = await rig.run()
+
+ assert step.status == StepStatus.failed
+ assert action_handler.await_count == expected_calls
+ assert output.actions_and_results is not None
+ assert [action for action, _ in output.actions_and_results] == ([first] if skip else [first, duplicate])
+
+
@pytest.mark.asyncio
async def test_skip_remaining_actions_stops_batch_but_step_completes(monkeypatch: pytest.MonkeyPatch) -> None:
first, second = _click("node-1"), _click("node-2")
diff --git a/tests/unit/test_no_available_option.py b/tests/unit/test_no_available_option.py
index c6d5af1fe..d637e677f 100644
--- a/tests/unit/test_no_available_option.py
+++ b/tests/unit/test_no_available_option.py
@@ -504,6 +504,7 @@ class TestDeterministicCustomSelect:
get_readback_scope_element = AsyncMock()
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="United States",
get_option_candidates=lambda: [
{"label": "United States", "element_id": "us-1", "value": "US", "is_choice_input": False},
@@ -534,6 +535,7 @@ class TestDeterministicCustomSelect:
get_readback_scope_element = AsyncMock()
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="Job Board",
get_option_candidates=lambda: [
{"label": "Job Board", "element_id": "source-job-board", "value": None, "is_choice_input": False}
@@ -554,6 +556,7 @@ class TestDeterministicCustomSelect:
get_skyvern_element = AsyncMock()
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="15",
get_option_candidates=lambda: [
{"label": "15", "element_id": "day-15", "value": None, "is_choice_input": False}
@@ -583,6 +586,7 @@ class TestDeterministicCustomSelect:
selected_element = _FakeCustomElement(role="listbox")
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="Job Board",
get_option_candidates=lambda: [
{"label": "Job Board", "element_id": "source-panel", "value": None, "is_choice_input": False}
@@ -628,6 +632,7 @@ class TestDeterministicCustomSelect:
selected_element.get_locator().count = AsyncMock(return_value=1)
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="Choice",
get_option_candidates=lambda: [
{"label": "Choice", "element_id": "choice-radio", "value": None, "is_choice_input": True}
@@ -677,6 +682,7 @@ class TestDeterministicCustomSelect:
selected_element.get_locator().count = AsyncMock(return_value=1)
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="Choice",
get_option_candidates=lambda: [
{"label": "Choice", "element_id": "choice-option", "value": None, "is_choice_input": True}
@@ -718,6 +724,7 @@ class TestDeterministicCustomSelect:
selected_element.get_locator().count = AsyncMock(return_value=1)
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="WAKE",
get_option_candidates=lambda: [
{"label": "WAKE", "element_id": "county-wake", "value": "WAKE", "is_choice_input": True}
@@ -788,6 +795,7 @@ class TestDeterministicCustomSelect:
readback_scope_element = _FakeAnchorElement()
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="Job Board",
get_option_candidates=lambda: [
{"label": "Job Board", "element_id": "source-job-board", "value": None, "is_choice_input": False}
@@ -831,6 +839,7 @@ class TestDeterministicCustomSelect:
selected_element.get_locator().count = AsyncMock(return_value=1)
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="WAKE",
get_option_candidates=lambda: [
{"label": "WAKE", "element_id": "county-wake", "value": "WAKE", "is_choice_input": True}
@@ -874,6 +883,7 @@ class TestDeterministicCustomSelect:
selected_element.get_locator().count = AsyncMock(return_value=1)
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="Job Board",
get_option_candidates=lambda: [
{"label": "Job Board", "element_id": "source-job-board", "value": None, "is_choice_input": False}
@@ -918,6 +928,7 @@ class TestDeterministicCustomSelect:
readback_scope_element = _FakeAnchorElement(tag_name="button")
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="Job Board",
get_option_candidates=lambda: [
{"label": "Job Board", "element_id": "source-job-board", "value": None, "is_choice_input": True}
@@ -988,6 +999,7 @@ class TestDeterministicCustomSelect:
)
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="Alpha",
get_option_candidates=lambda: candidates,
field_context={},
@@ -1030,6 +1042,7 @@ class TestDeterministicCustomSelect:
selected_element.get_locator().count = AsyncMock(return_value=1)
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="Alpha",
get_option_candidates=lambda: [
{
@@ -1046,20 +1059,15 @@ class TestDeterministicCustomSelect:
task=_task(), # type: ignore[arg-type]
)
- if walk_is_choice_input:
- assert result is not None
- action_result, matched_label = result
- assert not action_result.success
- assert action_result.skip_remaining_actions is True
- assert matched_label == "Alpha"
- else:
- assert result is None
+ assert result is not None
+ action_result, matched_label = result
+ assert not action_result.success
+ assert action_result.skip_remaining_actions is True
+ assert matched_label == "Alpha"
selected_element.click.assert_awaited_once()
@pytest.mark.asyncio
- async def test_clicked_but_unverified_non_choice_option_soft_fails_to_llm(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
+ async def test_clicked_but_unverified_non_choice_option_is_terminal(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
handler.app,
"EXPERIMENTATION_PROVIDER",
@@ -1079,11 +1087,10 @@ class TestDeterministicCustomSelect:
)
selected_element = _FakeCustomElement()
selected_element.get_locator().count = AsyncMock(return_value=1)
- # A button/div-anchored single-select listbox (role=option, not checkbox/radio) can be
- # safely replayed by the LLM mini-agent, so an unverified click must soft-fail.
readback_scope_element = _FakeAnchorElement(tag_name="button")
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="Choice",
get_option_candidates=lambda: [
{"label": "Choice", "element_id": "choice-option", "value": None, "is_choice_input": False}
@@ -1095,7 +1102,11 @@ class TestDeterministicCustomSelect:
task=_task(), # type: ignore[arg-type]
)
- assert result is None
+ assert result is not None
+ action_result, matched_label = result
+ assert not action_result.success
+ assert action_result.skip_remaining_actions is True
+ assert matched_label == "Choice"
selected_element.click.assert_awaited_once()
@pytest.mark.asyncio
@@ -1123,8 +1134,10 @@ class TestDeterministicCustomSelect:
selected_element.get_locator().count = AsyncMock(return_value=1)
# A text-input combobox anchor is resettable, so an inconclusive read-back must NOT hard-fail.
readback_scope_element = _FakeAnchorElement(tag_name="input")
+ monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value=""))
result = await _select_deterministic_custom_option(
+ execute=True,
target_value="WAKE",
get_option_candidates=lambda: [
{"label": "WAKE", "element_id": "source-wake", "value": "WAKE", "is_choice_input": False}
diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.py
index faed9f05e..519790836 100644
--- a/tests/unit/test_select_shadow_match.py
+++ b/tests/unit/test_select_shadow_match.py
@@ -492,7 +492,8 @@ async def test_emerging_select_call_path_forwards_committed_value_to_shadow_matc
)
monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))
monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div>Committed Value</div>"))
- monkeypatch.setattr(handler, "_select_deterministic_custom_option", AsyncMock(return_value=None))
+ deterministic_select = AsyncMock(return_value=None)
+ monkeypatch.setattr(handler, "_select_deterministic_custom_option", deterministic_select)
monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
monkeypatch.setattr(
handler.LLMAPIHandlerFactory,
@@ -524,11 +525,13 @@ async def test_emerging_select_call_path_forwards_committed_value_to_shadow_matc
assert len(events) == 1
assert events[0]["prompt_name"] == "custom-select/emerging"
assert events[0]["match_agrees_with_llm"] is True
+ assert deterministic_select.await_args.kwargs["execute"] is False
@pytest.mark.asyncio
-async def test_dropdown_select_call_path_forwards_committed_value_to_shadow_match(
- monkeypatch: pytest.MonkeyPatch,
[email protected](("entry_action_type", "expected_execute"), [("select_option", True), ("input_text", False)])
+async def test_dropdown_select_call_path_forwards_committed_value_and_scopes_execution(
+ monkeypatch: pytest.MonkeyPatch, entry_action_type: str, expected_execute: bool
) -> None:
incremental_scraped = MagicMock(
get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]),
@@ -541,7 +544,8 @@ async def test_dropdown_select_call_path_forwards_committed_value_to_shadow_matc
click=AsyncMock(),
)
monkeypatch.setattr(handler, "locate_dropdown_menu", AsyncMock(return_value=None))
- monkeypatch.setattr(handler, "_select_deterministic_custom_option", AsyncMock(return_value=None))
+ deterministic_select = AsyncMock(return_value=None)
+ monkeypatch.setattr(handler, "_select_deterministic_custom_option", deterministic_select)
monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
monkeypatch.setattr(
handler.app,
@@ -564,12 +568,46 @@ async def test_dropdown_select_call_path_forwards_committed_value_to_shadow_matc
task=_shadow_task(),
force_select=True,
target_value="Committed Value",
+ entry_action_type=entry_action_type,
)
events = [log for log in logs if log.get("event") == "select_shadow_match"]
assert len(events) == 1
assert events[0]["prompt_name"] == "custom-select/dropdown"
assert events[0]["match_agrees_with_llm"] is True
+ assert deterministic_select.await_args.kwargs["execute"] is expected_execute
+
+
+async def test_dropdown_terminal_result_never_reaches_custom_select_llm(monkeypatch: pytest.MonkeyPatch) -> None:
+ incremental_scraped = MagicMock(get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]))
+ incremental_scraped.build_element_tree.return_value = "<div>Committed Value</div>"
+ failure = handler.ActionFailure(Exception("unverified deterministic click"))
+ failure.skip_remaining_actions = True
+ monkeypatch.setattr(handler, "locate_dropdown_menu", AsyncMock(return_value=None))
+ monkeypatch.setattr(
+ handler,
+ "_select_deterministic_custom_option",
+ AsyncMock(return_value=(failure, "Committed Value")),
+ )
+ llm_handler = AsyncMock()
+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
+
+ result = await handler.select_from_dropdown(
+ context=InputOrSelectContext(field="Field", is_required=True),
+ page=MagicMock(),
+ skyvern_element=MagicMock(get_id=Mock(return_value="field-id")),
+ skyvern_frame=MagicMock(),
+ incremental_scraped=incremental_scraped,
+ check_filter_funcs=[],
+ step=SimpleNamespace(step_id="step-1"),
+ task=_shadow_task(),
+ force_select=True,
+ target_value="Committed Value",
+ )
+
+ assert result.action_result is failure
+ llm_handler.assert_not_awaited()
_OUTCOME_TASK = SimpleNamespace(
@@ -590,10 +628,11 @@ async def _run_outcome_case(
*,
family_enabled: bool = True,
assigned: bool = False,
+ execute: bool = True,
entry_action_type: str = "select_option",
selection_group_id: str = "group-1",
select_depth: int = 0,
-) -> tuple[object, dict[str, object], SimpleNamespace, AsyncMock]:
+) -> tuple[object, dict[str, object] | None, SimpleNamespace, AsyncMock, AsyncMock]:
handler._COLLAPSE_XP_ASSIGNMENT_MEMO.clear()
family = (
AsyncMock(side_effect=RuntimeError("gate")) if case == "gate_error" else AsyncMock(return_value=family_enabled)
@@ -606,15 +645,23 @@ async def _run_outcome_case(
fallback_to_llm=case == "no_match",
matched_index=None if case == "no_match" else 2 if case == "bad_index" else 0,
matched_label="Choice",
- matched_tier=None if case == "no_match" else "exact",
+ matched_tier=None if case == "no_match" else "stem" if case == "stem" else "exact",
)
resolver = AsyncMock(side_effect=RuntimeError("matcher") if case == "matcher_error" else None)
resolver.return_value = resolution
+ click_error = (
+ handler.InteractWithDisabledElement(element_id="choice-1")
+ if case == "disabled_click"
+ else RuntimeError("after dispatch")
+ if case == "post_click"
+ else None
+ )
selected = SimpleNamespace(
get_attr=AsyncMock(return_value="listbox" if case == "listbox" else None),
scroll_into_view=AsyncMock(),
- click=AsyncMock(side_effect=RuntimeError("after dispatch") if case == "post_click" else None),
+ click=AsyncMock(side_effect=click_error),
)
+ get_skyvern_element = AsyncMock(return_value=selected)
monkeypatch.setattr(handler.app, "EXPERIMENTATION_PROVIDER", provider)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", SimpleNamespace(resolve_field_option=resolver))
monkeypatch.setattr(
@@ -625,41 +672,50 @@ async def _run_outcome_case(
monkeypatch.setattr(handler, "_resolve_custom_select_readback_scope_element", AsyncMock(return_value=MagicMock()))
monkeypatch.setattr(handler, "_anchor_is_combobox_input", AsyncMock(return_value=case.startswith("reset")))
monkeypatch.setattr(
- handler, "_custom_select_matched_state_confirms_pre_click", Mock(return_value=case == "precommit")
+ handler,
+ "_custom_select_matched_state_confirms_pre_click",
+ Mock(return_value=case == "precommit_matched"),
+ )
+ monkeypatch.setattr(
+ handler,
+ "_custom_select_scope_confirms_committed",
+ AsyncMock(return_value=case == "precommit_scope"),
)
- monkeypatch.setattr(handler, "_custom_select_scope_confirms_committed", AsyncMock(return_value=False))
monkeypatch.setattr(handler, "_verify_custom_select_option_with_settle", AsyncMock(return_value=case == "verified"))
monkeypatch.setattr(handler, "_reset_custom_select_combobox_input", AsyncMock(return_value=case == "reset_ok"))
candidates = (
Mock(side_effect=RuntimeError("walker"))
if case == "walker_error"
else Mock(
- return_value=[
+ return_value=[]
+ if case == "no_options"
+ else [
{
"label": "Choice",
- "element_id": None if case == "no_element_id" else "choice-1",
+ "element_id": None if case in {"no_element_id", "stem"} else "choice-1",
"value": "choice",
"is_choice_input": case == "toggle",
}
- ]
+ ],
)
)
with structlog.testing.capture_logs() as logs:
result = await handler._select_deterministic_custom_option(
- target_value="Choice",
+ target_value=None if case == "no_target" else "Choice",
get_option_candidates=candidates,
- field_context={},
+ field_context={"is_date_related": True} if case == "date_related" else {},
page=MagicMock(),
- get_skyvern_element=AsyncMock(return_value=selected),
+ get_skyvern_element=get_skyvern_element,
task=_OUTCOME_TASK,
+ execute=False if case == "execution_disabled" else execute,
step=SimpleNamespace(step_id="step"),
entry_action_type=entry_action_type,
selection_group_id=selection_group_id,
select_depth=select_depth,
)
events = [log for log in logs if log.get("event") == "custom_select_family_outcome"]
- assert len(events) == 1
- return result, events[0], provider, resolver
+ assert len(events) == (0 if case in {"no_target", "date_related", "no_options"} else 1)
+ return result, events[0] if events else None, provider, resolver, get_skyvern_element
@pytest.mark.asyncio
@@ -667,7 +723,8 @@ async def _run_outcome_case(
async def test_outcome_event_emitted_once_per_opportunity(
monkeypatch: pytest.MonkeyPatch, entry_action_type: str
) -> None:
- _, event, _, _ = await _run_outcome_case(monkeypatch, "control", entry_action_type=entry_action_type)
+ _, event, _, _, _ = await _run_outcome_case(monkeypatch, "control", entry_action_type=entry_action_type)
+ assert event is not None
assert event["outcome"] == "llm_fallback_control"
assert event["entry_action_type"] == entry_action_type
@@ -685,6 +742,7 @@ async def test_click_route_without_target_emits_nothing_and_returns_none(monkeyp
page=MagicMock(),
get_skyvern_element=AsyncMock(),
task=_OUTCOME_TASK,
+ execute=True,
entry_action_type="click",
)
@@ -845,7 +903,8 @@ async def test_outcome_event_once_per_dropdown_level_with_group_id(monkeypatch:
@pytest.mark.asyncio
async def test_outcome_event_family_off_emits_without_assignment_resolution(monkeypatch: pytest.MonkeyPatch) -> None:
- _, event, provider, resolver = await _run_outcome_case(monkeypatch, "family_off", family_enabled=False)
+ _, event, provider, resolver, _ = await _run_outcome_case(monkeypatch, "family_off", family_enabled=False)
+ assert event is not None
assert event["outcome"] == "llm_fallback_family_off"
assert (event["assigned"], event["eligible"], event["match_tier"]) == (None, True, "exact")
provider.resolve_feature_enabled_unrecorded.assert_not_awaited()
@@ -854,7 +913,8 @@ async def test_outcome_event_family_off_emits_without_assignment_resolution(monk
@pytest.mark.asyncio
async def test_outcome_event_key_allowlist_and_enum(monkeypatch: pytest.MonkeyPatch) -> None:
- _, event, _, _ = await _run_outcome_case(monkeypatch, "control")
+ _, event, _, _, _ = await _run_outcome_case(monkeypatch, "control")
+ assert event is not None
expected_keys = set(
"event log_level family workflow_run_id task_id organization_id step_id entry_action_type selection_group_id "
"select_depth family_gate_enabled assigned gate_error encountered eligible match_tier option_count attempted "
@@ -863,8 +923,9 @@ async def test_outcome_event_key_allowlist_and_enum(monkeypatch: pytest.MonkeyPa
expected_outcomes = set(
"llm_fallback_gate_error llm_fallback_eval_error llm_fallback_family_off llm_fallback_control "
"llm_fallback_no_match llm_fallback_match_unactionable llm_fallback_pre_click_error "
- "llm_fallback_reset_verified llm_fallback_post_click_unverified success_precommit success_verified "
- "terminal_post_click_exception terminal_unverified_reset terminal_unverified_click "
+ "llm_fallback_reset_verified llm_fallback_post_click_unverified llm_fallback_tier_excluded "
+ "llm_fallback_execution_disabled success_precommit success_verified terminal_post_click_exception "
+ "terminal_unverified_reset terminal_unverified_click "
"terminal_unverified_toggle".split()
)
assert event["outcome"] == "llm_fallback_control"
@@ -874,7 +935,8 @@ async def test_outcome_event_key_allowlist_and_enum(monkeypatch: pytest.MonkeyPa
@pytest.mark.asyncio
async def test_outcome_event_gate_error_fields(monkeypatch: pytest.MonkeyPatch) -> None:
- _, event, _, _ = await _run_outcome_case(monkeypatch, "gate_error")
+ _, event, _, _, _ = await _run_outcome_case(monkeypatch, "gate_error")
+ assert event is not None
assert event["outcome"] == "llm_fallback_gate_error"
assert (event["option_count"], event["eligible"], event["match_tier"]) == (None, False, None)
@@ -891,53 +953,124 @@ async def test_outcome_event_gate_error_fields(monkeypatch: pytest.MonkeyPatch)
async def test_outcome_event_eval_error_emits_on_all_arms(
monkeypatch: pytest.MonkeyPatch, case: str, family_enabled: bool, assigned: bool
) -> None:
- result, event, _, _ = await _run_outcome_case(monkeypatch, case, family_enabled=family_enabled, assigned=assigned)
+ result, event, _, _, _ = await _run_outcome_case(
+ monkeypatch, case, family_enabled=family_enabled, assigned=assigned
+ )
+ assert event is not None
assert result is None
assert event["outcome"] == "llm_fallback_eval_error"
@pytest.mark.asyncio
@pytest.mark.parametrize(
- ("case", "outcome", "attempted"),
+ ("case", "family_enabled", "assigned", "outcome", "click_attempted", "return_shape"),
[
- ("no_match", "llm_fallback_no_match", False),
- ("bad_index", "llm_fallback_match_unactionable", False),
- ("no_element_id", "llm_fallback_match_unactionable", False),
- ("listbox", "llm_fallback_match_unactionable", False),
- ("pre_click", "llm_fallback_pre_click_error", True),
- ("precommit", "success_precommit", True),
- ("verified", "success_verified", True),
- ("nonchoice", "llm_fallback_post_click_unverified", True),
- ("toggle", "terminal_unverified_toggle", True),
+ ("no_target", True, True, None, False, "none"),
+ ("date_related", True, True, None, False, "none"),
+ ("no_options", True, True, None, False, "none"),
+ ("gate_error", True, True, "llm_fallback_gate_error", False, "none"),
+ ("walker_error", True, True, "llm_fallback_eval_error", False, "none"),
+ ("matcher_error", True, True, "llm_fallback_eval_error", False, "none"),
+ ("family_off", False, True, "llm_fallback_family_off", False, "none"),
+ ("control", True, False, "llm_fallback_control", False, "none"),
+ ("no_match", True, True, "llm_fallback_no_match", False, "none"),
+ ("bad_index", True, True, "llm_fallback_match_unactionable", False, "none"),
+ ("stem", True, True, "llm_fallback_tier_excluded", False, "none"),
+ ("execution_disabled", True, True, "llm_fallback_execution_disabled", False, "none"),
+ ("no_element_id", True, True, "llm_fallback_match_unactionable", False, "none"),
+ ("listbox", True, True, "llm_fallback_match_unactionable", False, "none"),
+ ("pre_click", True, True, "llm_fallback_pre_click_error", False, "none"),
+ ("precommit_matched", True, True, "success_precommit", False, "success"),
+ ("precommit_scope", True, True, "success_precommit", False, "success"),
+ ("verified", True, True, "success_verified", True, "success"),
+ ("disabled_click", True, True, "llm_fallback_pre_click_error", True, "none"),
+ ("post_click", True, True, "terminal_post_click_exception", True, "failure"),
+ ("reset_ok", True, True, "llm_fallback_reset_verified", True, "none"),
+ ("reset_failed", True, True, "terminal_unverified_reset", True, "failure"),
+ ("nonchoice", True, True, "terminal_unverified_click", True, "failure"),
+ ("toggle", True, True, "terminal_unverified_toggle", True, "failure"),
],
)
-async def test_outcome_event_exact_for_each_treatment_exit(
- monkeypatch: pytest.MonkeyPatch, case: str, outcome: str, attempted: bool
+async def test_deterministic_custom_select_exit_matrix(
+ monkeypatch: pytest.MonkeyPatch,
+ case: str,
+ family_enabled: bool,
+ assigned: bool,
+ outcome: str | None,
+ click_attempted: bool,
+ return_shape: str,
) -> None:
- _, event, _, _ = await _run_outcome_case(monkeypatch, case, assigned=True)
- assert event["outcome"] == outcome
- assert event["attempted"] is attempted
-
+ result, event, _, _, get_skyvern_element = await _run_outcome_case(
+ monkeypatch,
+ case,
+ family_enabled=family_enabled,
+ assigned=assigned,
+ )
-async def test_click_attempted_true_when_click_dispatches_then_raises(monkeypatch: pytest.MonkeyPatch) -> None:
- result, event, _, _ = await _run_outcome_case(monkeypatch, "post_click", assigned=True)
- assert result is None
- assert event["outcome"] == "llm_fallback_post_click_unverified"
- assert event["click_attempted"] is True
+ if outcome is None:
+ assert event is None
+ else:
+ assert event is not None
+ assert event["outcome"] == outcome
+ assert event["click_attempted"] is click_attempted
+
+ if return_shape == "none":
+ assert result is None
+ else:
+ assert result is not None
+ action_result, _matched_label = result
+ expected_type = handler.ActionSuccess if return_shape == "success" else handler.ActionFailure
+ assert isinstance(action_result, expected_type)
+ if return_shape == "failure":
+ assert action_result.skip_remaining_actions is True
+
+ if event is not None and event["click_attempted"] is True and result is None:
+ assert (case, event["outcome"]) in {
+ ("disabled_click", "llm_fallback_pre_click_error"),
+ ("reset_ok", "llm_fallback_reset_verified"),
+ }
+ if case == "stem":
+ get_skyvern_element.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize(
- ("case", "outcome"),
- [("reset_ok", "llm_fallback_reset_verified"), ("reset_failed", "llm_fallback_post_click_unverified")],
+ ("case", "outcome", "attempted", "return_shape"),
+ [
+ ("execution_disabled", "llm_fallback_execution_disabled", True, "none"),
+ ("precommit_matched", "success_precommit", True, "success"),
+ ("listbox", "llm_fallback_match_unactionable", False, "none"),
+ ],
)
-async def test_reset_outcome_recorded_without_behavior_change(
- monkeypatch: pytest.MonkeyPatch, case: str, outcome: str
+async def test_execution_disabled_reads_precommit_state_without_clicking(
+ monkeypatch: pytest.MonkeyPatch,
+ case: str,
+ outcome: str,
+ attempted: bool,
+ return_shape: str,
) -> None:
- result, event, _, _ = await _run_outcome_case(monkeypatch, case, assigned=True)
- assert result is None
+ result, event, _, _, get_skyvern_element = await _run_outcome_case(
+ monkeypatch,
+ case,
+ assigned=True,
+ execute=False,
+ )
+
+ assert event is not None
assert event["outcome"] == outcome
+ assert event["attempted"] is attempted
+ assert event["click_attempted"] is False
+ get_skyvern_element.assert_awaited_once_with("choice-1")
+ selected = get_skyvern_element.return_value
+ selected.scroll_into_view.assert_not_awaited()
+ selected.click.assert_not_awaited()
+ if return_shape == "none":
+ assert result is None
+ else:
+ assert result is not None
+ action_result, matched_label = result
+ assert isinstance(action_result, handler.ActionSuccess)
+ assert matched_label == "Choice"
@pytest.mark.asyncio