Skip to content

Automate browser based workflows with AI

Execute the deterministic custom-select collapse for input_text entries with strict entry-scoped verification (#7996)

Marc Kelechavabe29a3377945
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 1da04fdbf..5fcccc69d 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -253,6 +253,7 @@ class CustomSelectFamilyOutcome(StrEnum):
llm_fallback_post_click_unverified = "llm_fallback_post_click_unverified"
success_precommit = "success_precommit"
success_verified = "success_verified"
+ terminal_llm_fallback_exception = "terminal_llm_fallback_exception"
terminal_post_click_exception = "terminal_post_click_exception"
terminal_unverified_reset = "terminal_unverified_reset"
terminal_unverified_click = "terminal_unverified_click"
@@ -4678,6 +4679,10 @@ async def handle_input_text_action(
try_to_quit_dropdown = False
return [select_result.action_result]
+ if _is_terminal_custom_select_failure(select_result.action_result):
+ auto_complete_hacky_flag = False
+ return [select_result.action_result]
+
if select_result.dropdown_menu is None:
try_to_quit_dropdown = False
@@ -5097,6 +5102,9 @@ async def handle_input_text_action(
if action.stop_batch_after_dropdown_select:
select_result.action_result.skip_remaining_actions = True
return [select_result.action_result]
+ if select_result and _is_terminal_custom_select_failure(select_result.action_result):
+ auto_complete_hacky_flag = False
+ return [select_result.action_result]
elif (
input_or_select_context is not None
and not input_or_select_context.is_search_bar
@@ -5134,6 +5142,9 @@ async def handle_input_text_action(
if action.stop_batch_after_dropdown_select:
select_result.action_result.skip_remaining_actions = True
return [select_result.action_result]
+ if select_result and _is_terminal_custom_select_failure(select_result.action_result):
+ auto_complete_hacky_flag = False
+ return [select_result.action_result]
except SkyvernPageAnalysisTimeout as inc_error:
# A page-analysis timeout after both incremental attempts previously arrived here as a
# Playwright TimeoutError (a PlaywrightError) and was re-raised; the neutral
@@ -8410,6 +8421,7 @@ _CUSTOM_SELECT_COMMITTED_STATE_JS = r"""
const expectedLabel = args.expectedLabel;
const anchorIsComboboxInput = args.anchorIsComboboxInput;
const allowAriaSelectedOptionTokens = args.allowAriaSelectedOptionTokens !== false;
+ const allowSingleValueScope = args.allowSingleValueScope === true;
const normalize = (value) => (value ?? "").replace(/\s+/g, " ").trim().toLowerCase();
const splitValues = (value) => {
const normalized = normalize(value);
@@ -8437,6 +8449,11 @@ _CUSTOM_SELECT_COMMITTED_STATE_JS = r"""
"fieldset",
".field"
];
+ const singleValueSelector = [
+ "[class*='single-value']",
+ "[class*='singleValue']",
+ "[class*='multi-value__label']"
+ ].join(",");
// Nearest matching ancestor wins; never scope to the whole form or a bare
// parent container — sibling fields showing the target label must not
// pre-confirm this one. With no recognized field wrapper, fall back to the
@@ -8444,10 +8461,32 @@ _CUSTOM_SELECT_COMMITTED_STATE_JS = r"""
const scopeCandidates = scopeSelectors
.map((selector) => anchor.closest?.(selector))
.filter(Boolean);
+ // Some combobox libraries render the committed value beside the input, invisible to
+ // input-value read-back.
+ if (allowSingleValueScope && anchorIsComboboxInput) {
+ let ancestor = anchor.parentElement;
+ for (let hops = 1; ancestor && hops <= 4; hops += 1, ancestor = ancestor.parentElement) {
+ const triggers = ancestor.querySelectorAll?.(triggerSelector);
+ if (
+ ancestor.querySelector?.(singleValueSelector)
+ && triggers?.length === 1
+ && (triggers[0] === anchor || anchor.contains?.(triggers[0]))
+ ) {
+ scopeCandidates.push(ancestor);
+ break;
+ }
+ }
+ }
const scopeRoot = (
scopeCandidates.reduce((closest, el) => (!closest || closest.contains(el) ? el : closest), null)
|| anchor
);
+ const expandedState = anchor.getAttribute?.("aria-expanded")
+ || anchor.closest?.("[aria-expanded]")?.getAttribute("aria-expanded");
+ // While the widget reports open, every reflected surface (tokens, single-value nodes,
+ // hidden inputs, trigger text) can mirror the typed filter rather than a commitment;
+ // the strict posture trusts read-back only once closed.
+ const strictReflectionClosed = !(allowSingleValueScope && anchorIsComboboxInput) || expandedState === "false";
const tokenSelectors = [
...(allowAriaSelectedOptionTokens ? ["[role='option'][aria-selected='true']"] : []),
"[data-automation-id='selectedItem']",
@@ -8455,18 +8494,34 @@ _CUSTOM_SELECT_COMMITTED_STATE_JS = r"""
".chip",
"[class*='token']"
].join(",");
- for (const token of scopeRoot.querySelectorAll(tokenSelectors)) {
- if (matchesExpected(token.textContent) || matchesExpected(token.getAttribute("aria-label"))) return true;
+ if (strictReflectionClosed) {
+ for (const token of scopeRoot.querySelectorAll(tokenSelectors)) {
+ if (matchesExpected(token.textContent) || matchesExpected(token.getAttribute("aria-label"))) {
+ return {matched: true, branch: "scope_token"};
+ }
+ }
}
- for (const hidden of scopeRoot.querySelectorAll("input[type='hidden']")) {
- if (matchesExpected(hidden.value)) return true;
+ if (strictReflectionClosed && allowSingleValueScope && anchorIsComboboxInput) {
+ for (const singleValue of scopeRoot.querySelectorAll(singleValueSelector)) {
+ if (
+ matchesExpected(singleValue.textContent)
+ || matchesExpected(singleValue.getAttribute("aria-label"))
+ ) {
+ return {matched: true, branch: "scope_single_value"};
+ }
+ }
+ }
+ if (strictReflectionClosed) {
+ for (const hidden of scopeRoot.querySelectorAll("input[type='hidden']")) {
+ if (matchesExpected(hidden.value)) return {matched: true, branch: "scope_hidden_input"};
+ }
}
const activeId = anchor.getAttribute?.("aria-activedescendant");
- if (allowAriaSelectedOptionTokens && activeId) {
+ if (strictReflectionClosed && allowAriaSelectedOptionTokens && activeId) {
const active = scopeRoot.querySelector(`#${CSS.escape(activeId)}`);
if (active && active.getAttribute("aria-selected") === "true") {
if (matchesExpected(active.textContent) || matchesExpected(active.getAttribute("aria-label"))) {
- return true;
+ return {matched: true, branch: "scope_token"};
}
}
}
@@ -8483,26 +8538,28 @@ _CUSTOM_SELECT_COMMITTED_STATE_JS = r"""
...(scopeRoot.matches?.(triggerSelector) ? [scopeRoot] : []),
...scopeRoot.querySelectorAll(triggerSelector)
];
- for (const el of triggerCandidates) {
- if (!el || seen.has(el) || !scopeRoot.contains(el)) continue;
- seen.add(el);
- if (reflectedValues(el).some(matchesExpected)) return true;
+ if (strictReflectionClosed) {
+ for (const el of triggerCandidates) {
+ if (!el || seen.has(el) || !scopeRoot.contains(el)) continue;
+ seen.add(el);
+ if (reflectedValues(el).some(matchesExpected)) {
+ return {matched: true, branch: "scope_trigger_text"};
+ }
+ }
}
// A combobox <input> may still hold the user-typed filter text; raw value equality alone is not
// a committed signal. Only trust it when the dropdown has closed (aria-expanded=false).
if (anchorIsComboboxInput) {
const valueMatches = matchesExpected(anchor.value) || matchesExpected(anchor.getAttribute("value"));
- if (valueMatches) {
- const expanded = anchor.getAttribute("aria-expanded")
- || anchor.closest?.("[aria-expanded]")?.getAttribute("aria-expanded");
- if (expanded === "false") return true;
+ if (valueMatches && expandedState === "false") {
+ return {matched: true, branch: "scope_input_value"};
}
- return false;
+ return {matched: false, branch: "none"};
}
for (const el of seen) {
- if (reflectedValues(el).some((value) => normalize(value))) return false;
+ if (reflectedValues(el).some((value) => normalize(value))) return {matched: false, branch: "none"};
}
- return false;
+ return {matched: false, branch: "none"};
}
"""
@@ -8563,9 +8620,10 @@ async def _custom_select_scope_confirms_committed(
matched_label: str | None,
expected_label: str,
allow_aria_selected_option_tokens: bool,
-) -> bool:
+ allow_single_value_scope: bool,
+) -> tuple[bool, str]:
if readback_scope_element is None:
- return False
+ return False, "none"
try:
committed = await _evaluate_element_scoped(
@@ -8575,6 +8633,7 @@ async def _custom_select_scope_confirms_committed(
"expectedLabel": expected_label,
"anchorIsComboboxInput": anchor_is_combobox_input,
"allowAriaSelectedOptionTokens": allow_aria_selected_option_tokens,
+ "allowSingleValueScope": allow_single_value_scope,
},
)
except Exception:
@@ -8584,9 +8643,11 @@ async def _custom_select_scope_confirms_committed(
matched_label=matched_label,
exc_info=True,
)
- return False
+ return False, "none"
- return committed is True
+ if not isinstance(committed, dict):
+ return False, "none"
+ return bool(committed.get("matched")), str(committed.get("branch") or "none")
async def _verify_custom_select_option(
@@ -8596,13 +8657,19 @@ async def _verify_custom_select_option(
anchor_is_combobox_input: bool,
matched_element_id: str,
matched_label: str | None,
-) -> bool:
+ use_strict_verification: bool,
+) -> tuple[bool, str]:
expected_label = _normalize_select_shadow_text(matched_label)
if not expected_label:
- return False
+ return False, "none"
- if _custom_select_matched_state_confirms(await _read_custom_select_matched_state(matched_element), expected_label):
- return True
+ matched_state_confirms = (
+ _custom_select_matched_state_confirms_pre_click
+ if use_strict_verification
+ else _custom_select_matched_state_confirms
+ )
+ if matched_state_confirms(await _read_custom_select_matched_state(matched_element), expected_label):
+ return True, "matched_state"
return await _custom_select_scope_confirms_committed(
readback_scope_element=readback_scope_element,
@@ -8610,10 +8677,14 @@ async def _verify_custom_select_option(
matched_element_id=matched_element_id,
matched_label=matched_label,
expected_label=expected_label,
- allow_aria_selected_option_tokens=True,
+ allow_aria_selected_option_tokens=not use_strict_verification,
+ allow_single_value_scope=use_strict_verification,
)
+# input_text_converted is excluded: its anchor is frequently not an <input>, so the reset path that
+# contains an unverified click does not exist for it.
+_EXECUTABLE_CUSTOM_SELECT_ENTRIES = ("select_option", "input_text")
_CUSTOM_SELECT_VERIFY_SETTLE_RETRY_DELAYS_SECONDS = (0.15, 0.15)
@@ -8624,7 +8695,8 @@ async def _verify_custom_select_option_with_settle(
anchor_is_combobox_input: bool,
matched_element_id: str,
matched_label: str | None,
-) -> bool:
+ use_strict_verification: bool,
+) -> tuple[bool, str]:
"""Retry the read-back a couple of times before giving up.
Some frameworks commit ``aria-selected``/trigger-text reflection on the next render tick
@@ -8634,15 +8706,17 @@ async def _verify_custom_select_option_with_settle(
for delay_seconds in (0.0, *_CUSTOM_SELECT_VERIFY_SETTLE_RETRY_DELAYS_SECONDS):
if delay_seconds:
await asyncio.sleep(delay_seconds)
- if await _verify_custom_select_option(
+ verified, branch = await _verify_custom_select_option(
matched_element=matched_element,
readback_scope_element=readback_scope_element,
anchor_is_combobox_input=anchor_is_combobox_input,
matched_element_id=matched_element_id,
matched_label=matched_label,
- ):
- return True
- return False
+ use_strict_verification=use_strict_verification,
+ )
+ if verified:
+ return True, branch
+ return False, "none"
async def _resolve_custom_select_readback_scope_element(
@@ -8694,9 +8768,19 @@ def _terminal_custom_select_failure(
"Deterministic custom-select click could not be verified by matched element read-back",
)
action_failure.skip_remaining_actions = True
+ action_failure.data = {"_terminal_custom_select_failure": True}
return action_failure, matched_label
+def _is_terminal_custom_select_failure(action_result: ActionResult | None) -> bool:
+ return (
+ isinstance(action_result, ActionFailure)
+ and bool(action_result.skip_remaining_actions)
+ and isinstance(action_result.data, dict)
+ and action_result.data.get("_terminal_custom_select_failure") is True
+ )
+
+
async def _select_deterministic_custom_option(
*,
target_value: str | None,
@@ -8712,6 +8796,7 @@ async def _select_deterministic_custom_option(
selection_group_id: str | None = None,
select_depth: int = 0,
on_click_attempted: Callable[[], None] | None = None,
+ on_reset_fallback: Callable[[Callable[[CustomSelectFamilyOutcome], None]], None] | None = None,
engine_selection: BrowserEngineSelection | None = UNSET_SELECTION,
) -> tuple[ActionResult, str | None] | None:
if engine_selection is UNSET_SELECTION:
@@ -8723,6 +8808,10 @@ async def _select_deterministic_custom_option(
eligible = False
match_tier: str | None = None
attempted = False
+ # emit can run before these values are otherwise assigned and swallows exceptions, so initialize
+ # both before the closure is defined.
+ anchor_is_combobox_input = False
+ verify_branch: str | None = None
click_attempted = False
script_mode_run = False
@@ -8748,10 +8837,15 @@ async def _select_deterministic_custom_option(
option_count=option_count,
attempted=attempted,
click_attempted=click_attempted,
+ anchor_is_combobox_input=anchor_is_combobox_input,
+ verify_branch=verify_branch,
verified_success=outcome
in {CustomSelectFamilyOutcome.success_precommit, CustomSelectFamilyOutcome.success_verified},
outcome=outcome.value,
- llm_fallback_requested=outcome.value.startswith("llm_fallback_"),
+ llm_fallback_requested=(
+ outcome.value.startswith("llm_fallback_")
+ or outcome == CustomSelectFamilyOutcome.terminal_llm_fallback_exception
+ ),
duration_ms=int((time.monotonic() - started_at) * 1000),
)
except Exception:
@@ -8821,7 +8915,6 @@ async def _select_deterministic_custom_option(
return None
readback_scope_element: SkyvernElement | None = None
- anchor_is_combobox_input = False
try:
selected_element = await get_skyvern_element(element_id)
if await selected_element.get_attr("role") == "listbox":
@@ -8845,20 +8938,29 @@ async def _select_deterministic_custom_option(
matched_label=matched_label,
)
anchor_is_combobox_input = await _anchor_is_combobox_input(readback_scope_element)
+ allow_single_value_scope = (
+ entry_action_type in _EXECUTABLE_CUSTOM_SELECT_ENTRIES and entry_action_type != "select_option"
+ )
+
+ if entry_action_type == "input_text" and not anchor_is_combobox_input:
+ execute = False
expected_label = _normalize_select_shadow_text(matched_label)
if expected_label:
if _custom_select_matched_state_confirms_pre_click(matched_state, expected_label):
+ verify_branch = "matched_state"
emit(CustomSelectFamilyOutcome.success_precommit)
return ActionSuccess(), matched_label
- if await _custom_select_scope_confirms_committed(
+ committed, verify_branch = await _custom_select_scope_confirms_committed(
readback_scope_element=readback_scope_element,
anchor_is_combobox_input=anchor_is_combobox_input,
matched_element_id=element_id,
matched_label=matched_label,
expected_label=expected_label,
allow_aria_selected_option_tokens=False,
- ):
+ allow_single_value_scope=allow_single_value_scope,
+ )
+ if committed:
emit(CustomSelectFamilyOutcome.success_precommit)
return ActionSuccess(), matched_label
@@ -8867,16 +8969,34 @@ async def _select_deterministic_custom_option(
return None
await selected_element.scroll_into_view()
+ # Captured before the click: the click itself can mutate the anchor, and the reset must
+ # restore the pre-click text, not the mutation.
+ pre_click_anchor_value: str | None = None
+ if allow_single_value_scope and anchor_is_combobox_input and readback_scope_element is not None:
+ pre_click_anchor_value = await get_input_value(
+ readback_scope_element.get_tag_name(),
+ readback_scope_element.get_locator(),
+ engine_selection=engine_selection,
+ )
+ if pre_click_anchor_value is None:
+ LOG.info(
+ "Anchor value unreadable before deterministic click; falling back to LLM path",
+ target_value=target_value,
+ matched_element_id=element_id,
+ )
+ emit(CustomSelectFamilyOutcome.llm_fallback_pre_click_error)
+ return None
click_attempted = True
if on_click_attempted is not None:
on_click_attempted()
await selected_element.click(page=page, engine_selection=engine_selection)
- verified = await _verify_custom_select_option_with_settle(
+ verified, verify_branch = await _verify_custom_select_option_with_settle(
matched_element=selected_element,
readback_scope_element=readback_scope_element,
anchor_is_combobox_input=anchor_is_combobox_input,
matched_element_id=element_id,
matched_label=matched_label,
+ use_strict_verification=anchor_is_combobox_input and allow_single_value_scope,
)
if verified:
emit(CustomSelectFamilyOutcome.success_verified)
@@ -8909,15 +9029,19 @@ async def _select_deterministic_custom_option(
readback_scope_element,
page,
engine_selection=engine_selection,
+ restore_value=pre_click_anchor_value if entry_action_type != "select_option" else None,
)
if reset_verified:
+ if on_reset_fallback is not None:
+ on_reset_fallback(emit)
+ else:
+ emit(CustomSelectFamilyOutcome.llm_fallback_reset_verified)
LOG.info(
"Deterministic custom-select read-back inconclusive on combobox input; routing to LLM fallback",
target_value=target_value,
matched_element_id=element_id,
matched_label=matched_label,
)
- emit(CustomSelectFamilyOutcome.llm_fallback_reset_verified)
return None
LOG.info(
"Deterministic custom-select combobox reset failed; returning failure to avoid replaying over mutated widget",
@@ -8952,6 +9076,7 @@ async def _reset_custom_select_combobox_input(
element: SkyvernElement | None,
page: Page,
engine_selection: BrowserEngineSelection | None = None,
+ restore_value: str | None = None,
) -> bool:
if element is None:
return False
@@ -8959,7 +9084,16 @@ async def _reset_custom_select_combobox_input(
locator = element.get_locator()
await locator.fill("")
await element.click(page=page, engine_selection=engine_selection)
- return await get_input_value(element.get_tag_name(), locator, engine_selection=engine_selection) == ""
+ reset_verified = await get_input_value(element.get_tag_name(), locator, engine_selection=engine_selection) == ""
+ if not reset_verified:
+ return False
+ if restore_value:
+ await locator.fill(restore_value)
+ return (
+ await get_input_value(element.get_tag_name(), locator, engine_selection=engine_selection)
+ == restore_value
+ )
+ return True
except Exception:
LOG.info(
"Failed to reset custom-select combobox input before LLM fallback",
@@ -9293,13 +9427,41 @@ async def select_from_dropdown(
html = incremental_scraped.build_element_tree(html_need_skyvern_attrs=True)
widget_mutated = False
+ post_reset_fallback = False
+ post_reset_fallback_emit: Callable[[CustomSelectFamilyOutcome], None] | None = None
+ selection_group_id = selection_group_id or str(uuid.uuid4())
def _mark_widget_mutated() -> None:
nonlocal widget_mutated
widget_mutated = True
+ def _mark_post_reset_fallback(emit: Callable[[CustomSelectFamilyOutcome], None]) -> None:
+ nonlocal post_reset_fallback, post_reset_fallback_emit
+ post_reset_fallback = True
+ post_reset_fallback_emit = emit
+
+ def _emit_post_reset_fallback_outcome(outcome: CustomSelectFamilyOutcome) -> None:
+ if post_reset_fallback_emit is not None:
+ post_reset_fallback_emit(outcome)
+
+ def _terminal_post_reset_fallback_result() -> CustomSingleSelectResult:
+ _emit_post_reset_fallback_outcome(CustomSelectFamilyOutcome.terminal_llm_fallback_exception)
+ action_failure, _ = _terminal_custom_select_failure(
+ target_value=target_value,
+ matched_label=single_select_result.value,
+ )
+ single_select_result.reasoning = "LLM fallback failed after deterministic combobox reset"
+ single_select_result.value = single_select_result.value or target_value
+ single_select_result.action_type = ActionType.CLICK
+ single_select_result.action_result = action_failure
+ return single_select_result
+
+ def _proceeded_post_reset_fallback_result() -> CustomSingleSelectResult:
+ _emit_post_reset_fallback_outcome(CustomSelectFamilyOutcome.llm_fallback_reset_verified)
+ return single_select_result
+
deterministic_result = await _select_deterministic_custom_option(
- execute=entry_action_type == "select_option",
+ execute=entry_action_type in _EXECUTABLE_CUSTOM_SELECT_ENTRIES,
target_value=target_value,
get_option_candidates=lambda: _custom_select_candidates_from_elements(trimmed_element_tree),
field_context=context.model_dump(),
@@ -9309,9 +9471,14 @@ async def select_from_dropdown(
task=task,
step=step,
entry_action_type=entry_action_type,
- selection_group_id=selection_group_id or str(uuid.uuid4()),
+ selection_group_id=selection_group_id,
select_depth=len(select_history),
on_click_attempted=_mark_widget_mutated,
+ on_reset_fallback=(
+ _mark_post_reset_fallback
+ if entry_action_type in _EXECUTABLE_CUSTOM_SELECT_ENTRIES and entry_action_type != "select_option"
+ else None
+ ),
engine_selection=skyvern_frame.engine_selection,
)
if deterministic_result is not None:
@@ -9325,178 +9492,206 @@ async def select_from_dropdown(
return single_select_result
skyvern_context = ensure_context()
- prompt = prompt_engine.load_prompt(
- "custom-select",
- is_date_related=context.is_date_related,
- field_information=context.field if not context.intention else context.intention,
- required_field=context.is_required,
- target_value=target_value,
- navigation_goal=task.navigation_goal,
- navigation_payload_str=json.dumps(task.navigation_payload),
- elements=html,
- select_history=json.dumps(build_sequential_select_history(select_history)) if select_history else "",
- local_datetime=datetime.now(skyvern_context.tz_info).isoformat(),
- )
+ try:
+ prompt = prompt_engine.load_prompt(
+ "custom-select",
+ is_date_related=context.is_date_related,
+ field_information=context.field if not context.intention else context.intention,
+ required_field=context.is_required,
+ target_value=target_value,
+ navigation_goal=task.navigation_goal,
+ navigation_payload_str=json.dumps(task.navigation_payload),
+ elements=html,
+ select_history=json.dumps(build_sequential_select_history(select_history)) if select_history else "",
+ local_datetime=datetime.now(skyvern_context.tz_info).isoformat(),
+ )
+ LOG.info("Calling LLM to find the match element", sampling=True)
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER)(
+ prompt=prompt,
+ step=step,
+ prompt_name="custom-select",
+ )
- LOG.info("Calling LLM to find the match element", sampling=True)
- json_response = await get_org_aware_secondary_llm_api_handler(default=app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER)(
- prompt=prompt, step=step, prompt_name="custom-select"
- )
- value: str | None = json_response.get("value", None)
- single_select_result.value = value
- select_reason: str | None = json_response.get("reasoning", None)
- single_select_result.reasoning = select_reason
+ if post_reset_fallback and not isinstance(json_response, dict):
+ raise TypeError("Custom-select LLM response must be a dictionary")
+ value: str | None = json_response.get("value", None)
+ single_select_result.value = value
+ select_reason: str | None = json_response.get("reasoning", None)
+ single_select_result.reasoning = select_reason
- LOG.info(
- "LLM response for the matched element",
- sampling=True,
- matched_value=value,
- response=json_response,
- )
+ LOG.info(
+ "LLM response for the matched element",
+ sampling=True,
+ matched_value=value,
+ response=json_response,
+ )
- # Check the no-match shape before ``ActionType`` coercion — coercing an empty
- # string raises ValueError and would mask the OPTION_NOT_AVAILABLE signal.
- raw_action_type: str = (json_response.get("action_type") or "").lower()
- element_id: str | None = json_response.get("id", None)
- _log_select_shadow_match(
- prompt_name="custom-select/dropdown",
- target_value=target_value,
- get_candidates=lambda: _select_shadow_candidates_from_elements(trimmed_element_tree),
- agreement=lambda candidates, matched_index: _select_shadow_agrees_with_element_choice(
- candidates,
- matched_index,
- llm_element_id=element_id,
- llm_value=value,
- ),
- )
- if not element_id or raw_action_type not in (ActionType.CLICK.value, ActionType.INPUT_TEXT.value):
- raise _no_match_exception_for_dropdown(
- reasoning=json_response.get("reasoning"),
+ # Check the no-match shape before ``ActionType`` coercion — coercing an empty
+ # string raises ValueError and would mask the OPTION_NOT_AVAILABLE signal.
+ raw_action_type: str = (json_response.get("action_type") or "").lower()
+ element_id: str | None = json_response.get("id", None)
+ _log_select_shadow_match(
+ prompt_name="custom-select/dropdown",
target_value=target_value,
- observed_options=_collect_option_texts(trimmed_element_tree),
- transient_fallback_element_id=skyvern_element.get_id(),
- widget_mutated=widget_mutated,
+ get_candidates=lambda: _select_shadow_candidates_from_elements(trimmed_element_tree),
+ agreement=lambda candidates, matched_index: _select_shadow_agrees_with_element_choice(
+ candidates,
+ matched_index,
+ llm_element_id=element_id,
+ llm_value=value,
+ ),
)
- single_select_result.action_type = ActionType(raw_action_type)
- action_type = single_select_result.action_type
+ if not element_id or raw_action_type not in (ActionType.CLICK.value, ActionType.INPUT_TEXT.value):
+ raise _no_match_exception_for_dropdown(
+ reasoning=json_response.get("reasoning"),
+ target_value=target_value,
+ observed_options=_collect_option_texts(trimmed_element_tree),
+ transient_fallback_element_id=skyvern_element.get_id(),
+ widget_mutated=widget_mutated,
+ )
+ single_select_result.action_type = ActionType(raw_action_type)
+ action_type = single_select_result.action_type
- if not force_select and target_value:
- if not json_response.get("relevant", False):
+ if not force_select and target_value and not json_response.get("relevant", False):
LOG.info(
"The selected option is not relevant to the target value",
element_id=element_id,
)
+ if post_reset_fallback:
+ return _terminal_post_reset_fallback_result()
return single_select_result
- if value is not None and action_type == ActionType.INPUT_TEXT:
- LOG.info(
- "No clickable option found, but found input element to search",
- element_id=element_id,
- )
- try:
- actual_value = get_actual_value_of_parameter_if_secret_with_task(task, value)
- is_dropdown_secret_value = actual_value != value
- is_dropdown_totp_value = _is_totp_sentinel(actual_value)
- input_element = await SkyvernElement.create_from_incremental(incremental_scraped, element_id)
- await input_element.scroll_into_view()
- await _apply_secret_visual_mask_if_needed(
- input_element,
- workflow_run_id=task.workflow_run_id,
- is_secret_value=is_dropdown_secret_value,
- is_totp_value=is_dropdown_totp_value,
- )
- current_text = await get_input_value(
- input_element.get_tag_name(),
- input_element.get_locator(),
- engine_selection=skyvern_frame.engine_selection,
+ # A value-less or empty-value input_text response would either clear the anchor and type
+ # nothing, or fall through to a click that focuses without committing — never a post-reset
+ # success.
+ if post_reset_fallback and action_type == ActionType.INPUT_TEXT and not value:
+ return _terminal_post_reset_fallback_result()
+
+ if value is not None and action_type == ActionType.INPUT_TEXT:
+ LOG.info(
+ "No clickable option found, but found input element to search",
+ element_id=element_id,
)
- if current_text == actual_value:
+ try:
+ actual_value = get_actual_value_of_parameter_if_secret_with_task(task, value)
+ is_dropdown_secret_value = actual_value != value
+ is_dropdown_totp_value = _is_totp_sentinel(actual_value)
+ input_element = await SkyvernElement.create_from_incremental(incremental_scraped, element_id)
+ await input_element.scroll_into_view()
+ await _apply_secret_visual_mask_if_needed(
+ input_element,
+ workflow_run_id=task.workflow_run_id,
+ is_secret_value=is_dropdown_secret_value,
+ is_totp_value=is_dropdown_totp_value,
+ )
+ current_text = await get_input_value(
+ input_element.get_tag_name(),
+ input_element.get_locator(),
+ engine_selection=skyvern_frame.engine_selection,
+ )
+ if current_text == actual_value and not post_reset_fallback:
+ single_select_result.action_result = ActionSuccess()
+ return single_select_result
+
+ if await input_element.is_readonly(dynamic=True):
+ LOG.warning(
+ "Try to input text on a readonly element",
+ element_id=element_id,
+ task_id=task.task_id,
+ step_id=step.step_id,
+ )
+ single_select_result.action_result = ActionFailure(InputToReadonlyElement(element_id=element_id))
+ if post_reset_fallback:
+ return _terminal_post_reset_fallback_result()
+ return single_select_result
+
+ await input_element.input_clear()
+ await input_element.input_sequentially(actual_value)
single_select_result.action_result = ActionSuccess()
+ return _proceeded_post_reset_fallback_result()
+ except Exception as e:
+ single_select_result.action_result = ActionFailure(exception=e)
+ if post_reset_fallback:
+ return _terminal_post_reset_fallback_result()
return single_select_result
- if await input_element.is_readonly(dynamic=True):
- LOG.warning(
- "Try to input text on a readonly element",
+ try:
+ selected_element = await SkyvernElement.create_from_incremental(incremental_scraped, element_id)
+ # TODO Some popup dropdowns include <select> element, we only handle the <select> element now, to prevent infinite recursion. Need to support more types of dropdowns.
+ if selected_element.get_tag_name() == InteractiveElement.SELECT and value:
+ await selected_element.scroll_into_view()
+ action = SelectOptionAction(
+ reasoning=select_reason,
element_id=element_id,
- task_id=task.task_id,
- step_id=step.step_id,
+ option=SelectOption(label=value),
+ input_or_select_context=context,
+ )
+ results = await normal_select(
+ action=action,
+ skyvern_element=selected_element,
+ task=task,
+ step=step,
+ builder=incremental_scraped,
+ engine_selection=skyvern_frame.engine_selection,
)
- single_select_result.action_result = ActionFailure(InputToReadonlyElement(element_id=element_id))
+ assert len(results) > 0
+ single_select_result.action_result = results[0]
+ if post_reset_fallback and not isinstance(results[0], ActionSuccess):
+ return _terminal_post_reset_fallback_result()
+ return _proceeded_post_reset_fallback_result()
+
+ if await selected_element.get_attr("role") == "listbox":
+ single_select_result.action_result = ActionFailure(
+ exception=InteractWithDropdownContainer(element_id=element_id)
+ )
+ if post_reset_fallback:
+ return _terminal_post_reset_fallback_result()
return single_select_result
- await input_element.input_clear()
- await input_element.input_sequentially(actual_value)
- single_select_result.action_result = ActionSuccess()
- return single_select_result
- except Exception as e:
- single_select_result.action_result = ActionFailure(exception=e)
- return single_select_result
-
- try:
- selected_element = await SkyvernElement.create_from_incremental(incremental_scraped, element_id)
- # TODO Some popup dropdowns include <select> element, we only handle the <select> element now, to prevent infinite recursion. Need to support more types of dropdowns.
- if selected_element.get_tag_name() == InteractiveElement.SELECT and value:
await selected_element.scroll_into_view()
- action = SelectOptionAction(
- reasoning=select_reason,
- element_id=element_id,
- option=SelectOption(label=value),
- input_or_select_context=context,
- )
- results = await normal_select(
- action=action,
- skyvern_element=selected_element,
- task=task,
- step=step,
- builder=incremental_scraped,
+ await selected_element.click(
+ page=page,
+ timeout=timeout,
engine_selection=skyvern_frame.engine_selection,
)
- assert len(results) > 0
- single_select_result.action_result = results[0]
+ single_select_result.action_result = ActionSuccess()
+ return _proceeded_post_reset_fallback_result()
+ except (MissingElement, MissingElementDict, MissingElementInCSSMap, MultipleElementsFound):
+ if not value:
+ raise
+
+ # sometimes we have multiple elements pointed to the same value,
+ # but only one option is clickable on the page
+ LOG.debug(
+ "Searching option with the same value in incremental elements",
+ value=value,
+ elements=incremental_scraped.element_tree,
+ )
+ locator = await incremental_scraped.select_one_element_by_value(value=value)
+ if not locator:
+ single_select_result.action_result = ActionFailure(exception=MissingElement())
+ if post_reset_fallback:
+ return _terminal_post_reset_fallback_result()
return single_select_result
- if await selected_element.get_attr("role") == "listbox":
- single_select_result.action_result = ActionFailure(
- exception=InteractWithDropdownContainer(element_id=element_id)
+ try:
+ LOG.info(
+ "Find an alternative option with the same value. Try to select the option.",
+ value=value,
)
+ await EventStrategyFactory.move_to_element(page, locator)
+ await locator.click(timeout=timeout)
+ single_select_result.action_result = ActionSuccess()
+ return _proceeded_post_reset_fallback_result()
+ except Exception as e:
+ single_select_result.action_result = ActionFailure(exception=e)
+ if post_reset_fallback:
+ return _terminal_post_reset_fallback_result()
return single_select_result
-
- await selected_element.scroll_into_view()
- await selected_element.click(
- page=page,
- timeout=timeout,
- engine_selection=skyvern_frame.engine_selection,
- )
- single_select_result.action_result = ActionSuccess()
- return single_select_result
- except (MissingElement, MissingElementDict, MissingElementInCSSMap, MultipleElementsFound):
- if not value:
- raise
-
- # sometimes we have multiple elements pointed to the same value,
- # but only one option is clickable on the page
- LOG.debug(
- "Searching option with the same value in incremental elements",
- value=value,
- elements=incremental_scraped.element_tree,
- )
- locator = await incremental_scraped.select_one_element_by_value(value=value)
- if not locator:
- single_select_result.action_result = ActionFailure(exception=MissingElement())
- return single_select_result
-
- try:
- LOG.info(
- "Find an alternative option with the same value. Try to select the option.",
- value=value,
- )
- await EventStrategyFactory.move_to_element(page, locator)
- await locator.click(timeout=timeout)
- single_select_result.action_result = ActionSuccess()
- return single_select_result
- except Exception as e:
- single_select_result.action_result = ActionFailure(exception=e)
- return single_select_result
+ except Exception:
+ if post_reset_fallback:
+ return _terminal_post_reset_fallback_result()
+ raise
def _no_element_matched_failure(value: str, reason: str) -> ActionFailure:
diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.py
index 1cd81d724..b7d4b1f79 100644
--- a/tests/unit/test_browser_effect_sink_enumeration.py
+++ b/tests/unit/test_browser_effect_sink_enumeration.py
@@ -26,7 +26,7 @@ _DISCOVERED_BROWSER_API_CALLS = {
"close": 5,
"dblclick": 2,
"evaluate": 9,
- "fill": 1,
+ "fill": 2,
"focus": 2,
"go_back": 1,
"go_forward": 1,
@@ -198,14 +198,14 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:
}
assert observed == _DISCOVERED_BROWSER_API_CALLS
- assert sum(sum(methods.values()) for methods in observed.values()) == 140
+ assert sum(sum(methods.values()) for methods in observed.values()) == 141
handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)
classified_non_browser = Counter(
{signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES}
)
assert classified_non_browser == _NON_BROWSER_CANDIDATES
assert sum(_NON_BROWSER_CANDIDATES.values()) == 5
- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 135
+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 136
def test_every_raw_evaluate_call_is_classified() -> None:
diff --git a/tests/unit/test_input_text_combobox_commit.py b/tests/unit/test_input_text_combobox_commit.py
index 4d20cfdf9..ffd061105 100644
--- a/tests/unit/test_input_text_combobox_commit.py
+++ b/tests/unit/test_input_text_combobox_commit.py
@@ -26,7 +26,7 @@ from skyvern.webeye.actions import handler
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import Action, InputOrSelectContext, InputTextAction
from skyvern.webeye.actions.handler import handle_input_text_action
-from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess
+from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
from tests.unit.conftest import make_input_element_mock
from tests.unit.helpers import make_organization, make_step, make_task
@@ -153,6 +153,10 @@ def _pressed_keys(el: MagicMock) -> list[str]:
return [call.args[0] for call in el.press_key.call_args_list if call.args]
+def _typed_values(el: MagicMock) -> list[str]:
+ return [call.args[0] if call.args else call.kwargs["text"] for call in el.input_sequentially.call_args_list]
+
+
async def _run_combobox_input(
*,
attrs: dict[str, object],
@@ -167,6 +171,9 @@ async def _run_combobox_input(
prefilter_raises: bool = False,
use_base_action: bool = False,
first_block_incremental: list[dict] | None = None,
+ terminal_failure: bool = False,
+ nonterminal_skip: bool = False,
+ nonterminal_failure_skip: bool = False,
) -> tuple[list, MagicMock, MagicMock]:
skyvern_el = make_input_element_mock(element_id="CBX", attrs=attrs)
if prefilter_raises:
@@ -211,7 +218,18 @@ async def _run_combobox_input(
)
select_result = MagicMock()
- select_result.action_result = ActionSuccess() if select_success else ActionFailure(Exception("not committed"))
+ if terminal_failure:
+ select_result.action_result, _ = handler._terminal_custom_select_failure(
+ target_value=_TARGET,
+ matched_label=_TARGET,
+ )
+ elif nonterminal_failure_skip:
+ select_result.action_result = ActionFailure(Exception("not committed"))
+ select_result.action_result.skip_remaining_actions = True
+ elif nonterminal_skip:
+ select_result.action_result = ActionResult(success=False, skip_remaining_actions=True)
+ else:
+ select_result.action_result = ActionSuccess() if select_success else ActionFailure(Exception("not committed"))
# A secret makes the resolved text differ from action.text, so is_secret_value becomes True.
action_text = "{{secret_param}}" if is_secret else _TARGET
@@ -382,6 +400,171 @@ def test_prefilter_typeahead_flag_excluded_from_serialization() -> None:
_FLAG_TYPEAHEAD_ATTRS = {"role": None, "aria-autocomplete": None, "aria-invalid": "false"}
+ ("attrs", "is_search_bar", "first_block_incremental", "expected_typed_values"),
+ [
+ pytest.param(
+ _FLAG_TYPEAHEAD_ATTRS,
+ False,
+ _listbox_with_option(_TARGET),
+ [],
+ id="autocomplete-detect",
+ ),
+ pytest.param(
+ {"role": "textbox", "aria-autocomplete": None, "aria-invalid": "false"},
+ True,
+ _listbox_with_option(_TARGET),
+ [_TARGET],
+ id="search-bar",
+ ),
+ pytest.param(_INVALID_BOTH, False, None, [_TARGET], id="invalid-combobox"),
+ ],
+)
+async def test_terminal_custom_select_failure_stops_each_input_text_caller(
+ attrs: dict[str, object],
+ is_search_bar: bool,
+ first_block_incremental: list[dict] | None,
+ expected_typed_values: list[str],
+) -> None:
+ results, el, select_mock = await _run_combobox_input(
+ attrs=attrs,
+ options=_listbox_with_option(_TARGET),
+ select_success=False,
+ stop_flag=True,
+ is_search_bar=is_search_bar,
+ first_block_incremental=first_block_incremental,
+ terminal_failure=True,
+ )
+
+ failure = select_mock.return_value.action_result
+ assert results[0] is failure
+ assert isinstance(failure, ActionFailure)
+ assert not isinstance(failure, ActionSuccess)
+ assert failure.skip_remaining_actions is True
+ assert _typed_values(el) == expected_typed_values
+ assert "Tab" not in _pressed_keys(el)
+
+
+ ("attrs", "is_search_bar", "first_block_incremental"),
+ [
+ pytest.param(_FLAG_TYPEAHEAD_ATTRS, False, _listbox_with_option(_TARGET), id="autocomplete-detect"),
+ pytest.param(
+ {"role": "textbox", "aria-autocomplete": None, "aria-invalid": "false"},
+ True,
+ _listbox_with_option(_TARGET),
+ id="search-bar",
+ ),
+ pytest.param(_INVALID_BOTH, False, None, id="invalid-combobox"),
+ ],
+)
+async def test_nonterminal_skip_carrier_falls_through_each_input_text_caller(
+ attrs: dict[str, object],
+ is_search_bar: bool,
+ first_block_incremental: list[dict] | None,
+) -> None:
+ results, el, select_mock = await _run_combobox_input(
+ attrs=attrs,
+ options=_listbox_with_option(_TARGET),
+ select_success=False,
+ stop_flag=False,
+ is_search_bar=is_search_bar,
+ first_block_incremental=first_block_incremental,
+ nonterminal_skip=True,
+ )
+
+ select_mock.assert_awaited_once()
+ assert len(results) == 1
+ assert isinstance(results[0], ActionSuccess)
+ assert results[0] is not select_mock.return_value.action_result
+ assert _typed_values(el) == [_TARGET]
+
+
+ ("attrs", "is_search_bar", "first_block_incremental"),
+ [
+ pytest.param(_FLAG_TYPEAHEAD_ATTRS, False, _listbox_with_option(_TARGET), id="autocomplete-detect"),
+ pytest.param(
+ {"role": "textbox", "aria-autocomplete": None, "aria-invalid": "false"},
+ True,
+ _listbox_with_option(_TARGET),
+ id="search-bar",
+ ),
+ pytest.param(_INVALID_BOTH, False, None, id="invalid-combobox"),
+ ],
+)
+async def test_date_related_failure_with_skip_falls_through_each_input_text_caller(
+ attrs: dict[str, object],
+ is_search_bar: bool,
+ first_block_incremental: list[dict] | None,
+) -> None:
+ """A datepicker ActionFailure+skip lacks the custom-select terminal marker and falls through."""
+ results, el, select_mock = await _run_combobox_input(
+ attrs=attrs,
+ options=_listbox_with_option(_TARGET),
+ select_success=False,
+ stop_flag=False,
+ is_search_bar=is_search_bar,
+ first_block_incremental=first_block_incremental,
+ nonterminal_failure_skip=True,
+ is_date_related=True,
+ )
+
+ select_mock.assert_awaited_once()
+ assert len(results) == 1
+ assert isinstance(results[0], ActionSuccess)
+ assert results[0] is not select_mock.return_value.action_result
+ assert _typed_values(el) == [_TARGET]
+
+
+ ("attrs", "is_search_bar", "first_block_incremental", "stop_flag", "expected_stop"),
+ [
+ pytest.param(
+ _FLAG_TYPEAHEAD_ATTRS,
+ False,
+ _listbox_with_option(_TARGET),
+ False,
+ False,
+ id="autocomplete-detect",
+ ),
+ pytest.param(
+ {"role": "textbox", "aria-autocomplete": None, "aria-invalid": "false"},
+ True,
+ _listbox_with_option(_TARGET),
+ True,
+ True,
+ id="search-bar",
+ ),
+ pytest.param(_INVALID_BOTH, False, None, True, True, id="invalid-combobox"),
+ ],
+)
+async def test_successful_custom_select_preserves_each_input_text_caller(
+ attrs: dict[str, object],
+ is_search_bar: bool,
+ first_block_incremental: list[dict] | None,
+ stop_flag: bool,
+ expected_stop: bool,
+) -> None:
+ results, _el, select_mock = await _run_combobox_input(
+ attrs=attrs,
+ options=_listbox_with_option(_TARGET),
+ select_success=True,
+ stop_flag=stop_flag,
+ is_search_bar=is_search_bar,
+ first_block_incremental=first_block_incremental,
+ )
+
+ success = select_mock.return_value.action_result
+ assert results[0] is success
+ assert isinstance(success, ActionSuccess)
+ assert bool(success.skip_remaining_actions) is expected_stop
+
+
@pytest.mark.asyncio
async def test_prefilter_flag_types_target_to_filter_before_match() -> None:
"""With prefilter_typeahead set (by Cloud Setup), Block A types the target to filter the listbox
diff --git a/tests/unit/test_no_available_option.py b/tests/unit/test_no_available_option.py
index d637e677f..def2da9b8 100644
--- a/tests/unit/test_no_available_option.py
+++ b/tests/unit/test_no_available_option.py
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from types import SimpleNamespace
-from unittest.mock import AsyncMock, MagicMock
+from unittest.mock import AsyncMock, MagicMock, call
import pytest
@@ -1134,7 +1134,7 @@ class TestDeterministicCustomSelect:
selected_element.get_locator().count = AsyncMock(return_value=1)
# A text-input combobox anchor is resettable, so an inconclusive read-back must NOT hard-fail.
readback_scope_element = _FakeAnchorElement(tag_name="input")
- monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value=""))
+ monkeypatch.setattr(handler, "get_input_value", AsyncMock(side_effect=[""]))
result = await _select_deterministic_custom_option(
execute=True,
@@ -1151,7 +1151,7 @@ class TestDeterministicCustomSelect:
assert result is None
selected_element.click.assert_awaited_once()
- readback_scope_element.get_locator().fill.assert_awaited_once_with("")
+ assert readback_scope_element.get_locator().fill.await_args_list == [call("")]
readback_scope_element.click.assert_awaited_once()
@pytest.mark.asyncio
@@ -1174,17 +1174,17 @@ class TestDeterministicCustomSelect:
)
readback_scope_element = _FakeAnchorElement()
- assert (
- await _verify_custom_select_option(
- matched_element=matched_element, # type: ignore[arg-type]
- readback_scope_element=readback_scope_element, # type: ignore[arg-type]
- anchor_is_combobox_input=False,
- matched_element_id="choice-radio",
- matched_label="Choice",
- )
- is True
+ verified, _ = await _verify_custom_select_option(
+ matched_element=matched_element, # type: ignore[arg-type]
+ readback_scope_element=readback_scope_element, # type: ignore[arg-type]
+ anchor_is_combobox_input=False,
+ matched_element_id="choice-radio",
+ matched_label="Choice",
+ use_strict_verification=False,
)
+ assert verified is True
+
@pytest.mark.asyncio
async def test_readback_accepts_scoped_trigger_reflection_without_synthetic_token(self) -> None:
matched_element = _FakeCustomElement()
@@ -1196,21 +1196,22 @@ class TestDeterministicCustomSelect:
captured["arg"] = arg
if "const nestedChoice =" in expression:
return None
- return True
+ return {"matched": True, "branch": "scope_trigger_text"}
readback_scope_element = _FakeAnchorElement()
with pytest.MonkeyPatch.context() as mp:
mp.setattr(handler.SkyvernFrame, "evaluate", AsyncMock(side_effect=evaluate))
- outcome = await _verify_custom_select_option(
+ verified, _ = await _verify_custom_select_option(
matched_element=matched_element, # type: ignore[arg-type]
readback_scope_element=readback_scope_element, # type: ignore[arg-type]
anchor_is_combobox_input=False,
matched_element_id="choice-radio",
matched_label="Choice",
+ use_strict_verification=False,
)
- assert outcome is True
+ assert verified is True
script = str(captured["expression"])
assert "document.querySelectorAll" not in script
assert "data-selected-option-id" not in script
@@ -1235,15 +1236,16 @@ class TestDeterministicCustomSelect:
"evaluate",
_stub_evaluate(matched_state=None, committed=False),
)
- outcome = await _verify_custom_select_option(
+ verified, _ = await _verify_custom_select_option(
matched_element=matched_element, # type: ignore[arg-type]
readback_scope_element=readback_scope_element, # type: ignore[arg-type]
anchor_is_combobox_input=True,
matched_element_id="source-wake",
matched_label="WAKE",
+ use_strict_verification=False,
)
- assert outcome is False
+ assert verified is False
@pytest.mark.asyncio
async def test_handle_select_option_action_does_not_value_fallback_after_deterministic_readback_failure(
@@ -1499,15 +1501,16 @@ class TestDeterministicCustomSelect:
with pytest.MonkeyPatch.context() as mp:
mp.setattr(handler.SkyvernFrame, "evaluate", AsyncMock(side_effect=evaluate))
- outcome = await _verify_custom_select_option(
+ verified, _ = await _verify_custom_select_option(
matched_element=matched_element, # type: ignore[arg-type]
readback_scope_element=readback_scope_element, # type: ignore[arg-type]
anchor_is_combobox_input=False,
matched_element_id="matched-option",
matched_label="Job Board",
+ use_strict_verification=False,
)
- assert outcome is False
+ assert verified is False
script = str(captured["expression"])
assert script.count('"output"') == 0
assert script.count("document.querySelectorAll") == 0
diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.py
index a9a379769..a64705e17 100644
--- a/tests/unit/test_select_shadow_match.py
+++ b/tests/unit/test_select_shadow_match.py
@@ -535,7 +535,10 @@ async def test_emerging_select_call_path_forwards_committed_value_to_shadow_matc
@pytest.mark.asyncio
[email protected](("entry_action_type", "expected_execute"), [("select_option", True), ("input_text", False)])
+ ("entry_action_type", "expected_execute"),
+ [("select_option", True), ("input_text", True), ("input_text_converted", False), ("click", False)],
+)
async def test_dropdown_select_call_path_forwards_committed_value_and_scopes_execution(
monkeypatch: pytest.MonkeyPatch, entry_action_type: str, expected_execute: bool
) -> None:
@@ -616,6 +619,110 @@ async def test_dropdown_terminal_result_never_reaches_custom_select_llm(monkeypa
llm_handler.assert_not_awaited()
+ ("use_strict_verification", "expected"),
+ [(False, (True, "matched_state")), (True, (False, "none"))],
+)
+async def test_custom_select_verify_strict_posture_is_entry_scoped(
+ monkeypatch: pytest.MonkeyPatch,
+ use_strict_verification: bool,
+ expected: tuple[bool, str],
+) -> None:
+ monkeypatch.setattr(
+ handler,
+ "_read_custom_select_matched_state",
+ AsyncMock(
+ return_value={
+ "label": "Choice",
+ "role": "option",
+ "inMultiselectable": False,
+ "ariaSelected": True,
+ }
+ ),
+ )
+ monkeypatch.setattr(
+ handler,
+ "_custom_select_scope_confirms_committed",
+ AsyncMock(return_value=(False, "none")),
+ )
+
+ result = await handler._verify_custom_select_option(
+ matched_element=MagicMock(),
+ readback_scope_element=MagicMock(),
+ anchor_is_combobox_input=True,
+ matched_element_id="choice-1",
+ matched_label="Choice",
+ use_strict_verification=use_strict_verification,
+ )
+
+ assert result == expected
+
+
+ ("use_strict_verification", "expected_allow_tokens", "expected_allow_single_value_scope"),
+ [(False, True, False), (True, False, True)],
+)
+async def test_custom_select_verify_strict_posture_controls_aria_selected_tokens(
+ monkeypatch: pytest.MonkeyPatch,
+ use_strict_verification: bool,
+ expected_allow_tokens: bool,
+ expected_allow_single_value_scope: bool,
+) -> None:
+ monkeypatch.setattr(handler, "_read_custom_select_matched_state", AsyncMock(return_value=None))
+ scope_confirms = AsyncMock(return_value=(False, "none"))
+ monkeypatch.setattr(handler, "_custom_select_scope_confirms_committed", scope_confirms)
+
+ await handler._verify_custom_select_option(
+ matched_element=MagicMock(),
+ readback_scope_element=MagicMock(),
+ anchor_is_combobox_input=True,
+ matched_element_id="choice-1",
+ matched_label="Choice",
+ use_strict_verification=use_strict_verification,
+ )
+
+ assert scope_confirms.await_args.kwargs["allow_aria_selected_option_tokens"] is expected_allow_tokens
+ assert scope_confirms.await_args.kwargs["allow_single_value_scope"] is expected_allow_single_value_scope
+
+
+ ("committed_state", "expected"),
+ [
+ ({"matched": True, "branch": "scope_input_value"}, (True, "scope_input_value")),
+ ({"matched": True, "branch": "scope_single_value"}, (True, "scope_single_value")),
+ ({"matched": False, "branch": "none"}, (False, "none")),
+ ],
+)
+async def test_custom_select_verify_preserves_scope_branch(
+ monkeypatch: pytest.MonkeyPatch,
+ committed_state: dict[str, object],
+ expected: tuple[bool, str],
+) -> None:
+ matched_element = MagicMock()
+ matched_element.get_locator.return_value.count = AsyncMock(return_value=1)
+
+ async def evaluate_element_scoped(_element: object, expression: str, _arg: object = None) -> object:
+ if expression == handler._CUSTOM_SELECT_MATCHED_STATE_JS:
+ return None
+ return committed_state
+
+ monkeypatch.setattr(handler, "_evaluate_element_scoped", AsyncMock(side_effect=evaluate_element_scoped))
+
+ result = await handler._verify_custom_select_option(
+ matched_element=matched_element,
+ readback_scope_element=MagicMock(),
+ anchor_is_combobox_input=True,
+ matched_element_id="choice-1",
+ matched_label="Choice",
+ use_strict_verification=True,
+ )
+
+ assert result == expected
+
+
_OUTCOME_TASK = SimpleNamespace(
workflow_run_id="wr",
task_id="task",
@@ -636,6 +743,7 @@ async def _run_outcome_case(
assigned: bool = False,
execute: bool = True,
entry_action_type: str = "select_option",
+ anchor_is_input: bool = False,
selection_group_id: str = "group-1",
select_depth: int = 0,
) -> tuple[object, dict[str, object] | None, SimpleNamespace, AsyncMock, AsyncMock]:
@@ -676,19 +784,33 @@ async def _run_outcome_case(
AsyncMock(side_effect=RuntimeError("before click") if case == "pre_click" else None, return_value=None),
)
monkeypatch.setattr(handler, "_resolve_custom_select_readback_scope_element", AsyncMock(return_value=MagicMock()))
- monkeypatch.setattr(handler, "_anchor_is_combobox_input", AsyncMock(return_value=case.startswith("reset")))
+ monkeypatch.setattr(
+ handler,
+ "_anchor_is_combobox_input",
+ AsyncMock(return_value=anchor_is_input or case.startswith("reset")),
+ )
monkeypatch.setattr(
handler,
"_custom_select_matched_state_confirms_pre_click",
Mock(return_value=case == "precommit_matched"),
)
+
+ async def scope_confirms_committed(**kwargs: object) -> tuple[bool, str]:
+ committed = case == "precommit_scope" or (
+ case == "precommit_single_value" and kwargs["allow_single_value_scope"] is True
+ )
+ return committed, "scope_single_value" if committed else "none"
+
+ monkeypatch.setattr(
+ handler, "_custom_select_scope_confirms_committed", AsyncMock(side_effect=scope_confirms_committed)
+ )
monkeypatch.setattr(
handler,
- "_custom_select_scope_confirms_committed",
- AsyncMock(return_value=case == "precommit_scope"),
+ "_verify_custom_select_option_with_settle",
+ AsyncMock(return_value=(case == "verified", "matched_state" if case == "verified" else "none")),
)
- monkeypatch.setattr(handler, "_verify_custom_select_option_with_settle", AsyncMock(return_value=case == "verified"))
monkeypatch.setattr(handler, "_reset_custom_select_combobox_input", AsyncMock(return_value=case == "reset_ok"))
+ monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value="typed"))
candidates = (
Mock(side_effect=RuntimeError("walker"))
if case == "walker_error"
@@ -735,6 +857,140 @@ async def test_outcome_event_emitted_once_per_opportunity(
assert event["entry_action_type"] == entry_action_type
+ (
+ "entry_action_type",
+ "anchor_is_input",
+ "expected_use_strict_verification",
+ "expected_allow_single_value_scope",
+ ),
+ [
+ ("select_option", True, False, False),
+ ("select_option", False, False, False),
+ ("input_text", True, True, True),
+ ("input_text_converted", True, False, False),
+ ],
+)
+async def test_deterministic_custom_select_verification_is_entry_scoped(
+ monkeypatch: pytest.MonkeyPatch,
+ entry_action_type: str,
+ anchor_is_input: bool,
+ expected_use_strict_verification: bool,
+ expected_allow_single_value_scope: bool,
+) -> None:
+ await _run_outcome_case(
+ monkeypatch,
+ "verified",
+ assigned=True,
+ entry_action_type=entry_action_type,
+ anchor_is_input=anchor_is_input,
+ )
+
+ verify = handler._verify_custom_select_option_with_settle
+ assert isinstance(verify, AsyncMock)
+ assert verify.await_args.kwargs["use_strict_verification"] is expected_use_strict_verification
+ scope_confirms = handler._custom_select_scope_confirms_committed
+ assert isinstance(scope_confirms, AsyncMock)
+ assert scope_confirms.await_args.kwargs["allow_single_value_scope"] is expected_allow_single_value_scope
+
+
+async def test_executable_entry_constant_disables_input_text_single_value_scope(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(handler, "_EXECUTABLE_CUSTOM_SELECT_ENTRIES", ("select_option",))
+
+ await _run_outcome_case(
+ monkeypatch,
+ "verified",
+ assigned=True,
+ entry_action_type="input_text",
+ anchor_is_input=True,
+ )
+
+ scope_confirms = handler._custom_select_scope_confirms_committed
+ assert isinstance(scope_confirms, AsyncMock)
+ assert scope_confirms.await_args.kwargs["allow_single_value_scope"] is False
+ verify = handler._verify_custom_select_option_with_settle
+ assert isinstance(verify, AsyncMock)
+ assert verify.await_args.kwargs["use_strict_verification"] is False
+
+
+ ("case", "expected_outcome"),
+ [
+ ("precommit_matched", "success_precommit"),
+ ("precommit_scope", "success_precommit"),
+ ("verified", "llm_fallback_execution_disabled"),
+ ],
+)
+async def test_input_text_non_input_anchor_keeps_precommit_but_never_clicks(
+ monkeypatch: pytest.MonkeyPatch,
+ case: str,
+ expected_outcome: str,
+) -> None:
+ result, event, _, _, get_skyvern_element = await _run_outcome_case(
+ monkeypatch,
+ case,
+ assigned=True,
+ entry_action_type="input_text",
+ anchor_is_input=False,
+ )
+
+ if expected_outcome == "success_precommit":
+ assert result is not None
+ assert isinstance(result[0], handler.ActionSuccess)
+ else:
+ assert result is None
+ assert event is not None
+ assert event["outcome"] == expected_outcome
+ get_skyvern_element.return_value.click.assert_not_awaited()
+
+
[email protected]("entry_action_type", ["click", "input_text_converted"])
+async def test_nonexecutable_entry_does_not_single_value_preconfirm(
+ monkeypatch: pytest.MonkeyPatch,
+ entry_action_type: str,
+) -> None:
+ result, event, _, _, get_skyvern_element = await _run_outcome_case(
+ monkeypatch,
+ "precommit_single_value",
+ assigned=True,
+ execute=False,
+ entry_action_type=entry_action_type,
+ anchor_is_input=True,
+ )
+
+ assert result is None
+ assert event is not None
+ assert event["outcome"] == "llm_fallback_execution_disabled"
+ get_skyvern_element.return_value.click.assert_not_awaited()
+
+
+async def test_executable_entry_constant_disables_input_text_single_value_preconfirm(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(handler, "_EXECUTABLE_CUSTOM_SELECT_ENTRIES", ("select_option",))
+
+ result, event, _, _, get_skyvern_element = await _run_outcome_case(
+ monkeypatch,
+ "precommit_single_value",
+ assigned=True,
+ execute=False,
+ entry_action_type="input_text",
+ anchor_is_input=True,
+ )
+
+ assert result is None
+ assert event is not None
+ assert event["outcome"] == "llm_fallback_execution_disabled"
+ get_skyvern_element.return_value.click.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_click_route_without_target_emits_nothing_and_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
gate = AsyncMock()
@@ -907,6 +1163,217 @@ async def test_outcome_event_once_per_dropdown_level_with_group_id(monkeypatch:
assert [event["select_depth"] for event in events] == [0, 1, 2]
+async def _run_post_reset_fallback_case(
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ llm_response: object,
+ entry_action_type: str = "input_text",
+ input_value_reads: list[str | None] | None = None,
+) -> tuple[object, list[dict[str, object]], MagicMock, MagicMock, BaseException | None, AsyncMock]:
+ handler._COLLAPSE_XP_ASSIGNMENT_MEMO.clear()
+ monkeypatch.setattr(
+ handler.app,
+ "EXPERIMENTATION_PROVIDER",
+ SimpleNamespace(
+ is_feature_enabled_cached=AsyncMock(return_value=True),
+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),
+ ),
+ )
+ monkeypatch.setattr(
+ handler.app,
+ "AGENT_FUNCTION",
+ SimpleNamespace(
+ resolve_field_option=AsyncMock(
+ return_value=SimpleNamespace(
+ fallback_to_llm=False,
+ matched_index=0,
+ matched_label="Committed Value",
+ matched_tier="exact",
+ )
+ )
+ ),
+ )
+ llm_handler = (
+ AsyncMock(side_effect=llm_response)
+ if isinstance(llm_response, Exception)
+ else AsyncMock(return_value=llm_response)
+ )
+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
+ monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
+ monkeypatch.setattr(handler, "_read_custom_select_matched_state", AsyncMock(return_value=None))
+ monkeypatch.setattr(handler, "_custom_select_scope_confirms_committed", AsyncMock(return_value=(False, "none")))
+ monkeypatch.setattr(handler, "_verify_custom_select_option_with_settle", AsyncMock(return_value=(False, "none")))
+ if input_value_reads is None:
+ input_value_reads = (
+ [""]
+ if entry_action_type == "select_option"
+ else ["Committed Value", "", "Committed Value", "Committed Value"]
+ )
+ get_input_value_mock = AsyncMock(side_effect=input_value_reads)
+ monkeypatch.setattr(handler, "get_input_value", get_input_value_mock)
+ monkeypatch.setattr(
+ handler, "get_actual_value_of_parameter_if_secret_with_task", Mock(side_effect=lambda _, value: value)
+ )
+
+ reset_locator = MagicMock(fill=AsyncMock())
+ anchor = MagicMock(
+ get_id=Mock(return_value="field-id"),
+ get_tag_name=Mock(return_value="input"),
+ get_locator=Mock(return_value=reset_locator),
+ click=AsyncMock(),
+ )
+ matched_element = MagicMock(
+ get_attr=AsyncMock(return_value=None),
+ scroll_into_view=AsyncMock(),
+ click=AsyncMock(),
+ )
+ input_element = MagicMock(
+ get_tag_name=Mock(return_value="input"),
+ get_locator=Mock(return_value=MagicMock()),
+ scroll_into_view=AsyncMock(),
+ is_readonly=AsyncMock(return_value=False),
+ input_clear=AsyncMock(),
+ input_sequentially=AsyncMock(),
+ )
+ monkeypatch.setattr(
+ handler.SkyvernElement,
+ "create_from_incremental",
+ AsyncMock(side_effect=[matched_element, input_element]),
+ )
+
+ dropdown = MagicMock(get_element_handler=AsyncMock(return_value=MagicMock()))
+ skyvern_frame = MagicMock(get_element_scrollable=AsyncMock(return_value=False), engine_selection=object())
+ incremental_scraped = MagicMock(get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]))
+ incremental_scraped.build_element_tree.return_value = "<div>Committed Value</div>"
+ monkeypatch.setattr(handler, "try_to_find_potential_scrollable_element", AsyncMock(return_value=dropdown))
+
+ result: object = None
+ raised: BaseException | None = None
+ with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
+ with structlog.testing.capture_logs() as logs:
+ try:
+ result = await handler.select_from_dropdown(
+ context=InputOrSelectContext(field="Field"),
+ page=MagicMock(),
+ skyvern_element=anchor,
+ skyvern_frame=skyvern_frame,
+ incremental_scraped=incremental_scraped,
+ check_filter_funcs=[],
+ step=SimpleNamespace(step_id="step"),
+ task=_OUTCOME_TASK,
+ dropdown_menu_element=dropdown,
+ force_select=True,
+ target_value="Committed Value",
+ entry_action_type=entry_action_type,
+ selection_group_id="group-1",
+ )
+ except Exception as exc:
+ raised = exc
+
+ events = [log for log in logs if log.get("event") == "custom_select_family_outcome"]
+ return result, events, input_element, reset_locator, raised, get_input_value_mock
+
+
+ "llm_response",
+ [
+ RuntimeError("fallback failed"),
+ {"id": "", "action_type": "", "reasoning": "no match"},
+ {"id": "matched-id", "value": "Committed Value", "action_type": {"x": 1}},
+ {"id": "matched-id", "action_type": "input_text", "reasoning": "no value"},
+ {"id": "matched-id", "value": "", "action_type": "input_text", "reasoning": "empty value"},
+ ],
+)
+async def test_post_reset_llm_failure_emits_single_terminal_outcome(
+ monkeypatch: pytest.MonkeyPatch,
+ llm_response: object,
+) -> None:
+ result, events, _, _, raised, _ = await _run_post_reset_fallback_case(monkeypatch, llm_response=llm_response)
+
+ assert raised is None
+ assert isinstance(result.action_result, handler.ActionFailure)
+ assert result.action_result.skip_remaining_actions is True
+ assert [event["outcome"] for event in events] == ["terminal_llm_fallback_exception"]
+ assert events[0]["entry_action_type"] == "input_text"
+ assert events[0]["selection_group_id"] == "group-1"
+ assert events[0]["select_depth"] == 0
+ assert events[0]["family_gate_enabled"] is True
+ assert events[0]["assigned"] is True
+ assert events[0]["llm_fallback_requested"] is True
+
+
+async def test_post_reset_restored_text_is_retyped_and_emits_single_proceeded_outcome(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ result, events, input_element, reset_locator, raised, _ = await _run_post_reset_fallback_case(
+ monkeypatch,
+ llm_response={"id": "matched-id", "value": "Committed Value", "action_type": "input_text"},
+ )
+
+ assert raised is None
+ assert isinstance(result.action_result, handler.ActionSuccess)
+ input_element.input_clear.assert_awaited_once()
+ input_element.input_sequentially.assert_awaited_once_with("Committed Value")
+ assert [c.args for c in reset_locator.fill.await_args_list] == [("",), ("Committed Value",)]
+ assert [event["outcome"] for event in events] == ["llm_fallback_reset_verified"]
+ assert events[0]["entry_action_type"] == "input_text"
+ assert events[0]["selection_group_id"] == "group-1"
+ assert events[0]["select_depth"] == 0
+ assert events[0]["family_gate_enabled"] is True
+ assert events[0]["assigned"] is True
+ assert events[0]["llm_fallback_requested"] is True
+
+
+async def test_select_option_reset_path_matches_main(monkeypatch: pytest.MonkeyPatch) -> None:
+ llm_error = RuntimeError("fallback failed")
+ result, events, _, reset_locator, raised, get_input_value_mock = await _run_post_reset_fallback_case(
+ monkeypatch,
+ llm_response=llm_error,
+ entry_action_type="select_option",
+ )
+
+ assert raised is llm_error
+ assert result is None
+ assert [event["outcome"] for event in events] == ["llm_fallback_reset_verified"]
+ assert events[0]["entry_action_type"] == "select_option"
+ assert [c.args for c in reset_locator.fill.await_args_list] == [("",)]
+ assert get_input_value_mock.await_count == 1
+
+
+async def test_failed_restore_readback_is_terminal_not_reset_verified(monkeypatch: pytest.MonkeyPatch) -> None:
+ llm_handler_never_called = RuntimeError("llm should not run")
+ result, events, _, reset_locator, raised, _ = await _run_post_reset_fallback_case(
+ monkeypatch,
+ llm_response=llm_handler_never_called,
+ input_value_reads=["Committed Value", "", "typed over"],
+ )
+
+ assert raised is None
+ assert isinstance(result.action_result, handler.ActionFailure)
+ assert result.action_result.skip_remaining_actions is True
+ assert [event["outcome"] for event in events] == ["terminal_unverified_reset"]
+ assert [c.args for c in reset_locator.fill.await_args_list] == [("",), ("Committed Value",)]
+
+
+async def test_unreadable_anchor_before_click_falls_back_without_clicking(monkeypatch: pytest.MonkeyPatch) -> None:
+ llm_error = RuntimeError("llm path")
+ result, events, _, reset_locator, raised, _ = await _run_post_reset_fallback_case(
+ monkeypatch,
+ llm_response=llm_error,
+ input_value_reads=[None],
+ )
+
+ assert raised is llm_error
+ assert result is None
+ assert [event["outcome"] for event in events] == ["llm_fallback_pre_click_error"]
+ reset_locator.fill.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_outcome_event_family_off_emits_without_assignment_resolution(monkeypatch: pytest.MonkeyPatch) -> None:
_, event, provider, resolver, _ = await _run_outcome_case(monkeypatch, "family_off", family_enabled=False)
@@ -924,17 +1391,20 @@ async def test_outcome_event_key_allowlist_and_enum(monkeypatch: pytest.MonkeyPa
expected_keys = set(
"event log_level family workflow_run_id task_id organization_id step_id entry_action_type selection_group_id "
"select_depth script_mode family_gate_enabled assigned gate_error encountered eligible match_tier "
- "option_count attempted click_attempted verified_success outcome llm_fallback_requested duration_ms".split()
+ "option_count attempted click_attempted anchor_is_combobox_input verify_branch verified_success outcome "
+ "llm_fallback_requested duration_ms".split()
)
expected_outcomes = set(
"llm_fallback_gate_error llm_fallback_eval_error llm_fallback_family_off llm_fallback_control "
"llm_fallback_no_match llm_fallback_match_unactionable llm_fallback_pre_click_error "
"llm_fallback_reset_verified llm_fallback_post_click_unverified llm_fallback_tier_excluded "
"llm_fallback_execution_disabled success_precommit success_verified terminal_post_click_exception "
+ "terminal_llm_fallback_exception "
"terminal_unverified_reset terminal_unverified_click "
"terminal_unverified_toggle".split()
)
assert event["outcome"] == "llm_fallback_control"
+ assert (event["anchor_is_combobox_input"], event["verify_branch"]) == (False, None)
assert set(event) == expected_keys
assert {outcome.value for outcome in handler.CustomSelectFamilyOutcome} == expected_outcomes