Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Execute the deterministic custom-select collapse for input_text entries with strict entry-scoped verification (#7996)
Marc Kelechavabe29a3377945
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 1da04fdbf..5fcccc69d 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -253,6 +253,7 @@ class CustomSelectFamilyOutcome(StrEnum):llm_fallback_post_click_unverified = "llm_fallback_post_click_unverified"success_precommit = "success_precommit"success_verified = "success_verified"+ terminal_llm_fallback_exception = "terminal_llm_fallback_exception"terminal_post_click_exception = "terminal_post_click_exception"terminal_unverified_reset = "terminal_unverified_reset"terminal_unverified_click = "terminal_unverified_click"@@ -4678,6 +4679,10 @@ async def handle_input_text_action(try_to_quit_dropdown = Falsereturn [select_result.action_result]+ if _is_terminal_custom_select_failure(select_result.action_result):+ auto_complete_hacky_flag = False+ return [select_result.action_result]+if select_result.dropdown_menu is None:try_to_quit_dropdown = False@@ -5097,6 +5102,9 @@ async def handle_input_text_action(if action.stop_batch_after_dropdown_select:select_result.action_result.skip_remaining_actions = Truereturn [select_result.action_result]+ if select_result and _is_terminal_custom_select_failure(select_result.action_result):+ auto_complete_hacky_flag = False+ return [select_result.action_result]elif (input_or_select_context is not Noneand not input_or_select_context.is_search_bar@@ -5134,6 +5142,9 @@ async def handle_input_text_action(if action.stop_batch_after_dropdown_select:select_result.action_result.skip_remaining_actions = Truereturn [select_result.action_result]+ if select_result and _is_terminal_custom_select_failure(select_result.action_result):+ auto_complete_hacky_flag = False+ return [select_result.action_result]except SkyvernPageAnalysisTimeout as inc_error:# A page-analysis timeout after both incremental attempts previously arrived here as a# Playwright TimeoutError (a PlaywrightError) and was re-raised; the neutral@@ -8410,6 +8421,7 @@ _CUSTOM_SELECT_COMMITTED_STATE_JS = r"""const expectedLabel = args.expectedLabel;const anchorIsComboboxInput = args.anchorIsComboboxInput;const allowAriaSelectedOptionTokens = args.allowAriaSelectedOptionTokens !== false;+ const allowSingleValueScope = args.allowSingleValueScope === true;const normalize = (value) => (value ?? "").replace(/\s+/g, " ").trim().toLowerCase();const splitValues = (value) => {const normalized = normalize(value);@@ -8437,6 +8449,11 @@ _CUSTOM_SELECT_COMMITTED_STATE_JS = r""""fieldset",".field"];+ const singleValueSelector = [+ "[class*='single-value']",+ "[class*='singleValue']",+ "[class*='multi-value__label']"+ ].join(",");// Nearest matching ancestor wins; never scope to the whole form or a bare// parent container — sibling fields showing the target label must not// pre-confirm this one. With no recognized field wrapper, fall back to the@@ -8444,10 +8461,32 @@ _CUSTOM_SELECT_COMMITTED_STATE_JS = r"""const scopeCandidates = scopeSelectors.map((selector) => anchor.closest?.(selector)).filter(Boolean);+ // Some combobox libraries render the committed value beside the input, invisible to+ // input-value read-back.+ if (allowSingleValueScope && anchorIsComboboxInput) {+ let ancestor = anchor.parentElement;+ for (let hops = 1; ancestor && hops <= 4; hops += 1, ancestor = ancestor.parentElement) {+ const triggers = ancestor.querySelectorAll?.(triggerSelector);+ if (+ ancestor.querySelector?.(singleValueSelector)+ && triggers?.length === 1+ && (triggers[0] === anchor || anchor.contains?.(triggers[0]))+ ) {+ scopeCandidates.push(ancestor);+ break;+ }+ }+ }const scopeRoot = (scopeCandidates.reduce((closest, el) => (!closest || closest.contains(el) ? el : closest), null)|| anchor);+ const expandedState = anchor.getAttribute?.("aria-expanded")+ || anchor.closest?.("[aria-expanded]")?.getAttribute("aria-expanded");+ // While the widget reports open, every reflected surface (tokens, single-value nodes,+ // hidden inputs, trigger text) can mirror the typed filter rather than a commitment;+ // the strict posture trusts read-back only once closed.+ const strictReflectionClosed = !(allowSingleValueScope && anchorIsComboboxInput) || expandedState === "false";const tokenSelectors = [...(allowAriaSelectedOptionTokens ? ["[role='option'][aria-selected='true']"] : []),"[data-automation-id='selectedItem']",@@ -8455,18 +8494,34 @@ _CUSTOM_SELECT_COMMITTED_STATE_JS = r"""".chip","[class*='token']"].join(",");- for (const token of scopeRoot.querySelectorAll(tokenSelectors)) {- if (matchesExpected(token.textContent) || matchesExpected(token.getAttribute("aria-label"))) return true;+ if (strictReflectionClosed) {+ for (const token of scopeRoot.querySelectorAll(tokenSelectors)) {+ if (matchesExpected(token.textContent) || matchesExpected(token.getAttribute("aria-label"))) {+ return {matched: true, branch: "scope_token"};+ }+ }}- for (const hidden of scopeRoot.querySelectorAll("input[type='hidden']")) {- if (matchesExpected(hidden.value)) return true;+ if (strictReflectionClosed && allowSingleValueScope && anchorIsComboboxInput) {+ for (const singleValue of scopeRoot.querySelectorAll(singleValueSelector)) {+ if (+ matchesExpected(singleValue.textContent)+ || matchesExpected(singleValue.getAttribute("aria-label"))+ ) {+ return {matched: true, branch: "scope_single_value"};+ }+ }+ }+ if (strictReflectionClosed) {+ for (const hidden of scopeRoot.querySelectorAll("input[type='hidden']")) {+ if (matchesExpected(hidden.value)) return {matched: true, branch: "scope_hidden_input"};+ }}const activeId = anchor.getAttribute?.("aria-activedescendant");- if (allowAriaSelectedOptionTokens && activeId) {+ if (strictReflectionClosed && allowAriaSelectedOptionTokens && activeId) {const active = scopeRoot.querySelector(`#${CSS.escape(activeId)}`);if (active && active.getAttribute("aria-selected") === "true") {if (matchesExpected(active.textContent) || matchesExpected(active.getAttribute("aria-label"))) {- return true;+ return {matched: true, branch: "scope_token"};}}}@@ -8483,26 +8538,28 @@ _CUSTOM_SELECT_COMMITTED_STATE_JS = r"""...(scopeRoot.matches?.(triggerSelector) ? [scopeRoot] : []),...scopeRoot.querySelectorAll(triggerSelector)];- for (const el of triggerCandidates) {- if (!el || seen.has(el) || !scopeRoot.contains(el)) continue;- seen.add(el);- if (reflectedValues(el).some(matchesExpected)) return true;+ if (strictReflectionClosed) {+ for (const el of triggerCandidates) {+ if (!el || seen.has(el) || !scopeRoot.contains(el)) continue;+ seen.add(el);+ if (reflectedValues(el).some(matchesExpected)) {+ return {matched: true, branch: "scope_trigger_text"};+ }+ }}// A combobox <input> may still hold the user-typed filter text; raw value equality alone is not// a committed signal. Only trust it when the dropdown has closed (aria-expanded=false).if (anchorIsComboboxInput) {const valueMatches = matchesExpected(anchor.value) || matchesExpected(anchor.getAttribute("value"));- if (valueMatches) {- const expanded = anchor.getAttribute("aria-expanded")- || anchor.closest?.("[aria-expanded]")?.getAttribute("aria-expanded");- if (expanded === "false") return true;+ if (valueMatches && expandedState === "false") {+ return {matched: true, branch: "scope_input_value"};}- return false;+ return {matched: false, branch: "none"};}for (const el of seen) {- if (reflectedValues(el).some((value) => normalize(value))) return false;+ if (reflectedValues(el).some((value) => normalize(value))) return {matched: false, branch: "none"};}- return false;+ return {matched: false, branch: "none"};}"""@@ -8563,9 +8620,10 @@ async def _custom_select_scope_confirms_committed(matched_label: str | None,expected_label: str,allow_aria_selected_option_tokens: bool,-) -> bool:+ allow_single_value_scope: bool,+) -> tuple[bool, str]:if readback_scope_element is None:- return False+ return False, "none"try:committed = await _evaluate_element_scoped(@@ -8575,6 +8633,7 @@ async def _custom_select_scope_confirms_committed("expectedLabel": expected_label,"anchorIsComboboxInput": anchor_is_combobox_input,"allowAriaSelectedOptionTokens": allow_aria_selected_option_tokens,+ "allowSingleValueScope": allow_single_value_scope,},)except Exception:@@ -8584,9 +8643,11 @@ async def _custom_select_scope_confirms_committed(matched_label=matched_label,exc_info=True,)- return False+ return False, "none"- return committed is True+ if not isinstance(committed, dict):+ return False, "none"+ return bool(committed.get("matched")), str(committed.get("branch") or "none")async def _verify_custom_select_option(@@ -8596,13 +8657,19 @@ async def _verify_custom_select_option(anchor_is_combobox_input: bool,matched_element_id: str,matched_label: str | None,-) -> bool:+ use_strict_verification: bool,+) -> tuple[bool, str]:expected_label = _normalize_select_shadow_text(matched_label)if not expected_label:- return False+ return False, "none"- if _custom_select_matched_state_confirms(await _read_custom_select_matched_state(matched_element), expected_label):- return True+ matched_state_confirms = (+ _custom_select_matched_state_confirms_pre_click+ if use_strict_verification+ else _custom_select_matched_state_confirms+ )+ if matched_state_confirms(await _read_custom_select_matched_state(matched_element), expected_label):+ return True, "matched_state"return await _custom_select_scope_confirms_committed(readback_scope_element=readback_scope_element,@@ -8610,10 +8677,14 @@ async def _verify_custom_select_option(matched_element_id=matched_element_id,matched_label=matched_label,expected_label=expected_label,- allow_aria_selected_option_tokens=True,+ allow_aria_selected_option_tokens=not use_strict_verification,+ allow_single_value_scope=use_strict_verification,)+# input_text_converted is excluded: its anchor is frequently not an <input>, so the reset path that+# contains an unverified click does not exist for it.+_EXECUTABLE_CUSTOM_SELECT_ENTRIES = ("select_option", "input_text")_CUSTOM_SELECT_VERIFY_SETTLE_RETRY_DELAYS_SECONDS = (0.15, 0.15)@@ -8624,7 +8695,8 @@ async def _verify_custom_select_option_with_settle(anchor_is_combobox_input: bool,matched_element_id: str,matched_label: str | None,-) -> bool:+ use_strict_verification: bool,+) -> tuple[bool, str]:"""Retry the read-back a couple of times before giving up.Some frameworks commit ``aria-selected``/trigger-text reflection on the next render tick@@ -8634,15 +8706,17 @@ async def _verify_custom_select_option_with_settle(for delay_seconds in (0.0, *_CUSTOM_SELECT_VERIFY_SETTLE_RETRY_DELAYS_SECONDS):if delay_seconds:await asyncio.sleep(delay_seconds)- if await _verify_custom_select_option(+ verified, branch = await _verify_custom_select_option(matched_element=matched_element,readback_scope_element=readback_scope_element,anchor_is_combobox_input=anchor_is_combobox_input,matched_element_id=matched_element_id,matched_label=matched_label,- ):- return True- return False+ use_strict_verification=use_strict_verification,+ )+ if verified:+ return True, branch+ return False, "none"async def _resolve_custom_select_readback_scope_element(@@ -8694,9 +8768,19 @@ def _terminal_custom_select_failure("Deterministic custom-select click could not be verified by matched element read-back",)action_failure.skip_remaining_actions = True+ action_failure.data = {"_terminal_custom_select_failure": True}return action_failure, matched_label+def _is_terminal_custom_select_failure(action_result: ActionResult | None) -> bool:+ return (+ isinstance(action_result, ActionFailure)+ and bool(action_result.skip_remaining_actions)+ and isinstance(action_result.data, dict)+ and action_result.data.get("_terminal_custom_select_failure") is True+ )++async def _select_deterministic_custom_option(*,target_value: str | None,@@ -8712,6 +8796,7 @@ async def _select_deterministic_custom_option(selection_group_id: str | None = None,select_depth: int = 0,on_click_attempted: Callable[[], None] | None = None,+ on_reset_fallback: Callable[[Callable[[CustomSelectFamilyOutcome], None]], None] | None = None,engine_selection: BrowserEngineSelection | None = UNSET_SELECTION,) -> tuple[ActionResult, str | None] | None:if engine_selection is UNSET_SELECTION:@@ -8723,6 +8808,10 @@ async def _select_deterministic_custom_option(eligible = Falsematch_tier: str | None = Noneattempted = False+ # emit can run before these values are otherwise assigned and swallows exceptions, so initialize+ # both before the closure is defined.+ anchor_is_combobox_input = False+ verify_branch: str | None = Noneclick_attempted = Falsescript_mode_run = False@@ -8748,10 +8837,15 @@ async def _select_deterministic_custom_option(option_count=option_count,attempted=attempted,click_attempted=click_attempted,+ anchor_is_combobox_input=anchor_is_combobox_input,+ verify_branch=verify_branch,verified_success=outcomein {CustomSelectFamilyOutcome.success_precommit, CustomSelectFamilyOutcome.success_verified},outcome=outcome.value,- llm_fallback_requested=outcome.value.startswith("llm_fallback_"),+ llm_fallback_requested=(+ outcome.value.startswith("llm_fallback_")+ or outcome == CustomSelectFamilyOutcome.terminal_llm_fallback_exception+ ),duration_ms=int((time.monotonic() - started_at) * 1000),)except Exception:@@ -8821,7 +8915,6 @@ async def _select_deterministic_custom_option(return Nonereadback_scope_element: SkyvernElement | None = None- anchor_is_combobox_input = Falsetry:selected_element = await get_skyvern_element(element_id)if await selected_element.get_attr("role") == "listbox":@@ -8845,20 +8938,29 @@ async def _select_deterministic_custom_option(matched_label=matched_label,)anchor_is_combobox_input = await _anchor_is_combobox_input(readback_scope_element)+ allow_single_value_scope = (+ entry_action_type in _EXECUTABLE_CUSTOM_SELECT_ENTRIES and entry_action_type != "select_option"+ )++ if entry_action_type == "input_text" and not anchor_is_combobox_input:+ execute = Falseexpected_label = _normalize_select_shadow_text(matched_label)if expected_label:if _custom_select_matched_state_confirms_pre_click(matched_state, expected_label):+ verify_branch = "matched_state"emit(CustomSelectFamilyOutcome.success_precommit)return ActionSuccess(), matched_label- if await _custom_select_scope_confirms_committed(+ committed, verify_branch = await _custom_select_scope_confirms_committed(readback_scope_element=readback_scope_element,anchor_is_combobox_input=anchor_is_combobox_input,matched_element_id=element_id,matched_label=matched_label,expected_label=expected_label,allow_aria_selected_option_tokens=False,- ):+ allow_single_value_scope=allow_single_value_scope,+ )+ if committed:emit(CustomSelectFamilyOutcome.success_precommit)return ActionSuccess(), matched_label@@ -8867,16 +8969,34 @@ async def _select_deterministic_custom_option(return Noneawait selected_element.scroll_into_view()+ # Captured before the click: the click itself can mutate the anchor, and the reset must+ # restore the pre-click text, not the mutation.+ pre_click_anchor_value: str | None = None+ if allow_single_value_scope and anchor_is_combobox_input and readback_scope_element is not None:+ pre_click_anchor_value = await get_input_value(+ readback_scope_element.get_tag_name(),+ readback_scope_element.get_locator(),+ engine_selection=engine_selection,+ )+ if pre_click_anchor_value is None:+ LOG.info(+ "Anchor value unreadable before deterministic click; falling back to LLM path",+ target_value=target_value,+ matched_element_id=element_id,+ )+ emit(CustomSelectFamilyOutcome.llm_fallback_pre_click_error)+ return Noneclick_attempted = Trueif on_click_attempted is not None:on_click_attempted()await selected_element.click(page=page, engine_selection=engine_selection)- verified = await _verify_custom_select_option_with_settle(+ verified, verify_branch = await _verify_custom_select_option_with_settle(matched_element=selected_element,readback_scope_element=readback_scope_element,anchor_is_combobox_input=anchor_is_combobox_input,matched_element_id=element_id,matched_label=matched_label,+ use_strict_verification=anchor_is_combobox_input and allow_single_value_scope,)if verified:emit(CustomSelectFamilyOutcome.success_verified)@@ -8909,15 +9029,19 @@ async def _select_deterministic_custom_option(readback_scope_element,page,engine_selection=engine_selection,+ restore_value=pre_click_anchor_value if entry_action_type != "select_option" else None,)if reset_verified:+ if on_reset_fallback is not None:+ on_reset_fallback(emit)+ else:+ emit(CustomSelectFamilyOutcome.llm_fallback_reset_verified)LOG.info("Deterministic custom-select read-back inconclusive on combobox input; routing to LLM fallback",target_value=target_value,matched_element_id=element_id,matched_label=matched_label,)- emit(CustomSelectFamilyOutcome.llm_fallback_reset_verified)return NoneLOG.info("Deterministic custom-select combobox reset failed; returning failure to avoid replaying over mutated widget",@@ -8952,6 +9076,7 @@ async def _reset_custom_select_combobox_input(element: SkyvernElement | None,page: Page,engine_selection: BrowserEngineSelection | None = None,+ restore_value: str | None = None,) -> bool:if element is None:return False@@ -8959,7 +9084,16 @@ async def _reset_custom_select_combobox_input(locator = element.get_locator()await locator.fill("")await element.click(page=page, engine_selection=engine_selection)- return await get_input_value(element.get_tag_name(), locator, engine_selection=engine_selection) == ""+ reset_verified = await get_input_value(element.get_tag_name(), locator, engine_selection=engine_selection) == ""+ if not reset_verified:+ return False+ if restore_value:+ await locator.fill(restore_value)+ return (+ await get_input_value(element.get_tag_name(), locator, engine_selection=engine_selection)+ == restore_value+ )+ return Trueexcept Exception:LOG.info("Failed to reset custom-select combobox input before LLM fallback",@@ -9293,13 +9427,41 @@ async def select_from_dropdown(html = incremental_scraped.build_element_tree(html_need_skyvern_attrs=True)widget_mutated = False+ post_reset_fallback = False+ post_reset_fallback_emit: Callable[[CustomSelectFamilyOutcome], None] | None = None+ selection_group_id = selection_group_id or str(uuid.uuid4())def _mark_widget_mutated() -> None:nonlocal widget_mutatedwidget_mutated = True+ def _mark_post_reset_fallback(emit: Callable[[CustomSelectFamilyOutcome], None]) -> None:+ nonlocal post_reset_fallback, post_reset_fallback_emit+ post_reset_fallback = True+ post_reset_fallback_emit = emit++ def _emit_post_reset_fallback_outcome(outcome: CustomSelectFamilyOutcome) -> None:+ if post_reset_fallback_emit is not None:+ post_reset_fallback_emit(outcome)++ def _terminal_post_reset_fallback_result() -> CustomSingleSelectResult:+ _emit_post_reset_fallback_outcome(CustomSelectFamilyOutcome.terminal_llm_fallback_exception)+ action_failure, _ = _terminal_custom_select_failure(+ target_value=target_value,+ matched_label=single_select_result.value,+ )+ single_select_result.reasoning = "LLM fallback failed after deterministic combobox reset"+ single_select_result.value = single_select_result.value or target_value+ single_select_result.action_type = ActionType.CLICK+ single_select_result.action_result = action_failure+ return single_select_result++ def _proceeded_post_reset_fallback_result() -> CustomSingleSelectResult:+ _emit_post_reset_fallback_outcome(CustomSelectFamilyOutcome.llm_fallback_reset_verified)+ return single_select_result+deterministic_result = await _select_deterministic_custom_option(- execute=entry_action_type == "select_option",+ execute=entry_action_type in _EXECUTABLE_CUSTOM_SELECT_ENTRIES,target_value=target_value,get_option_candidates=lambda: _custom_select_candidates_from_elements(trimmed_element_tree),field_context=context.model_dump(),@@ -9309,9 +9471,14 @@ async def select_from_dropdown(task=task,step=step,entry_action_type=entry_action_type,- selection_group_id=selection_group_id or str(uuid.uuid4()),+ selection_group_id=selection_group_id,select_depth=len(select_history),on_click_attempted=_mark_widget_mutated,+ on_reset_fallback=(+ _mark_post_reset_fallback+ if entry_action_type in _EXECUTABLE_CUSTOM_SELECT_ENTRIES and entry_action_type != "select_option"+ else None+ ),engine_selection=skyvern_frame.engine_selection,)if deterministic_result is not None:@@ -9325,178 +9492,206 @@ async def select_from_dropdown(return single_select_resultskyvern_context = ensure_context()- prompt = prompt_engine.load_prompt(- "custom-select",- is_date_related=context.is_date_related,- field_information=context.field if not context.intention else context.intention,- required_field=context.is_required,- target_value=target_value,- navigation_goal=task.navigation_goal,- navigation_payload_str=json.dumps(task.navigation_payload),- elements=html,- select_history=json.dumps(build_sequential_select_history(select_history)) if select_history else "",- local_datetime=datetime.now(skyvern_context.tz_info).isoformat(),- )+ try:+ prompt = prompt_engine.load_prompt(+ "custom-select",+ is_date_related=context.is_date_related,+ field_information=context.field if not context.intention else context.intention,+ required_field=context.is_required,+ target_value=target_value,+ navigation_goal=task.navigation_goal,+ navigation_payload_str=json.dumps(task.navigation_payload),+ elements=html,+ select_history=json.dumps(build_sequential_select_history(select_history)) if select_history else "",+ local_datetime=datetime.now(skyvern_context.tz_info).isoformat(),+ )+ LOG.info("Calling LLM to find the match element", sampling=True)+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER)(+ prompt=prompt,+ step=step,+ prompt_name="custom-select",+ )- LOG.info("Calling LLM to find the match element", sampling=True)- json_response = await get_org_aware_secondary_llm_api_handler(default=app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER)(- prompt=prompt, step=step, prompt_name="custom-select"- )- value: str | None = json_response.get("value", None)- single_select_result.value = value- select_reason: str | None = json_response.get("reasoning", None)- single_select_result.reasoning = select_reason+ if post_reset_fallback and not isinstance(json_response, dict):+ raise TypeError("Custom-select LLM response must be a dictionary")+ value: str | None = json_response.get("value", None)+ single_select_result.value = value+ select_reason: str | None = json_response.get("reasoning", None)+ single_select_result.reasoning = select_reason- LOG.info(- "LLM response for the matched element",- sampling=True,- matched_value=value,- response=json_response,- )+ LOG.info(+ "LLM response for the matched element",+ sampling=True,+ matched_value=value,+ response=json_response,+ )- # Check the no-match shape before ``ActionType`` coercion — coercing an empty- # string raises ValueError and would mask the OPTION_NOT_AVAILABLE signal.- raw_action_type: str = (json_response.get("action_type") or "").lower()- element_id: str | None = json_response.get("id", None)- _log_select_shadow_match(- prompt_name="custom-select/dropdown",- target_value=target_value,- get_candidates=lambda: _select_shadow_candidates_from_elements(trimmed_element_tree),- agreement=lambda candidates, matched_index: _select_shadow_agrees_with_element_choice(- candidates,- matched_index,- llm_element_id=element_id,- llm_value=value,- ),- )- if not element_id or raw_action_type not in (ActionType.CLICK.value, ActionType.INPUT_TEXT.value):- raise _no_match_exception_for_dropdown(- reasoning=json_response.get("reasoning"),+ # Check the no-match shape before ``ActionType`` coercion — coercing an empty+ # string raises ValueError and would mask the OPTION_NOT_AVAILABLE signal.+ raw_action_type: str = (json_response.get("action_type") or "").lower()+ element_id: str | None = json_response.get("id", None)+ _log_select_shadow_match(+ prompt_name="custom-select/dropdown",target_value=target_value,- observed_options=_collect_option_texts(trimmed_element_tree),- transient_fallback_element_id=skyvern_element.get_id(),- widget_mutated=widget_mutated,+ get_candidates=lambda: _select_shadow_candidates_from_elements(trimmed_element_tree),+ agreement=lambda candidates, matched_index: _select_shadow_agrees_with_element_choice(+ candidates,+ matched_index,+ llm_element_id=element_id,+ llm_value=value,+ ),)- single_select_result.action_type = ActionType(raw_action_type)- action_type = single_select_result.action_type+ if not element_id or raw_action_type not in (ActionType.CLICK.value, ActionType.INPUT_TEXT.value):+ raise _no_match_exception_for_dropdown(+ reasoning=json_response.get("reasoning"),+ target_value=target_value,+ observed_options=_collect_option_texts(trimmed_element_tree),+ transient_fallback_element_id=skyvern_element.get_id(),+ widget_mutated=widget_mutated,+ )+ single_select_result.action_type = ActionType(raw_action_type)+ action_type = single_select_result.action_type- if not force_select and target_value:- if not json_response.get("relevant", False):+ if not force_select and target_value and not json_response.get("relevant", False):LOG.info("The selected option is not relevant to the target value",element_id=element_id,)+ if post_reset_fallback:+ return _terminal_post_reset_fallback_result()return single_select_result- if value is not None and action_type == ActionType.INPUT_TEXT:- LOG.info(- "No clickable option found, but found input element to search",- element_id=element_id,- )- try:- actual_value = get_actual_value_of_parameter_if_secret_with_task(task, value)- is_dropdown_secret_value = actual_value != value- is_dropdown_totp_value = _is_totp_sentinel(actual_value)- input_element = await SkyvernElement.create_from_incremental(incremental_scraped, element_id)- await input_element.scroll_into_view()- await _apply_secret_visual_mask_if_needed(- input_element,- workflow_run_id=task.workflow_run_id,- is_secret_value=is_dropdown_secret_value,- is_totp_value=is_dropdown_totp_value,- )- current_text = await get_input_value(- input_element.get_tag_name(),- input_element.get_locator(),- engine_selection=skyvern_frame.engine_selection,+ # A value-less or empty-value input_text response would either clear the anchor and type+ # nothing, or fall through to a click that focuses without committing — never a post-reset+ # success.+ if post_reset_fallback and action_type == ActionType.INPUT_TEXT and not value:+ return _terminal_post_reset_fallback_result()++ if value is not None and action_type == ActionType.INPUT_TEXT:+ LOG.info(+ "No clickable option found, but found input element to search",+ element_id=element_id,)- if current_text == actual_value:+ try:+ actual_value = get_actual_value_of_parameter_if_secret_with_task(task, value)+ is_dropdown_secret_value = actual_value != value+ is_dropdown_totp_value = _is_totp_sentinel(actual_value)+ input_element = await SkyvernElement.create_from_incremental(incremental_scraped, element_id)+ await input_element.scroll_into_view()+ await _apply_secret_visual_mask_if_needed(+ input_element,+ workflow_run_id=task.workflow_run_id,+ is_secret_value=is_dropdown_secret_value,+ is_totp_value=is_dropdown_totp_value,+ )+ current_text = await get_input_value(+ input_element.get_tag_name(),+ input_element.get_locator(),+ engine_selection=skyvern_frame.engine_selection,+ )+ if current_text == actual_value and not post_reset_fallback:+ single_select_result.action_result = ActionSuccess()+ return single_select_result++ if await input_element.is_readonly(dynamic=True):+ LOG.warning(+ "Try to input text on a readonly element",+ element_id=element_id,+ task_id=task.task_id,+ step_id=step.step_id,+ )+ single_select_result.action_result = ActionFailure(InputToReadonlyElement(element_id=element_id))+ if post_reset_fallback:+ return _terminal_post_reset_fallback_result()+ return single_select_result++ await input_element.input_clear()+ await input_element.input_sequentially(actual_value)single_select_result.action_result = ActionSuccess()+ return _proceeded_post_reset_fallback_result()+ except Exception as e:+ single_select_result.action_result = ActionFailure(exception=e)+ if post_reset_fallback:+ return _terminal_post_reset_fallback_result()return single_select_result- if await input_element.is_readonly(dynamic=True):- LOG.warning(- "Try to input text on a readonly element",+ try:+ selected_element = await SkyvernElement.create_from_incremental(incremental_scraped, element_id)+ # TODO Some popup dropdowns include <select> element, we only handle the <select> element now, to prevent infinite recursion. Need to support more types of dropdowns.+ if selected_element.get_tag_name() == InteractiveElement.SELECT and value:+ await selected_element.scroll_into_view()+ action = SelectOptionAction(+ reasoning=select_reason,element_id=element_id,- task_id=task.task_id,- step_id=step.step_id,+ option=SelectOption(label=value),+ input_or_select_context=context,+ )+ results = await normal_select(+ action=action,+ skyvern_element=selected_element,+ task=task,+ step=step,+ builder=incremental_scraped,+ engine_selection=skyvern_frame.engine_selection,)- single_select_result.action_result = ActionFailure(InputToReadonlyElement(element_id=element_id))+ assert len(results) > 0+ single_select_result.action_result = results[0]+ if post_reset_fallback and not isinstance(results[0], ActionSuccess):+ return _terminal_post_reset_fallback_result()+ return _proceeded_post_reset_fallback_result()++ if await selected_element.get_attr("role") == "listbox":+ single_select_result.action_result = ActionFailure(+ exception=InteractWithDropdownContainer(element_id=element_id)+ )+ if post_reset_fallback:+ return _terminal_post_reset_fallback_result()return single_select_result- await input_element.input_clear()- await input_element.input_sequentially(actual_value)- single_select_result.action_result = ActionSuccess()- return single_select_result- except Exception as e:- single_select_result.action_result = ActionFailure(exception=e)- return single_select_result-- try:- selected_element = await SkyvernElement.create_from_incremental(incremental_scraped, element_id)- # TODO Some popup dropdowns include <select> element, we only handle the <select> element now, to prevent infinite recursion. Need to support more types of dropdowns.- if selected_element.get_tag_name() == InteractiveElement.SELECT and value:await selected_element.scroll_into_view()- action = SelectOptionAction(- reasoning=select_reason,- element_id=element_id,- option=SelectOption(label=value),- input_or_select_context=context,- )- results = await normal_select(- action=action,- skyvern_element=selected_element,- task=task,- step=step,- builder=incremental_scraped,+ await selected_element.click(+ page=page,+ timeout=timeout,engine_selection=skyvern_frame.engine_selection,)- assert len(results) > 0- single_select_result.action_result = results[0]+ single_select_result.action_result = ActionSuccess()+ return _proceeded_post_reset_fallback_result()+ except (MissingElement, MissingElementDict, MissingElementInCSSMap, MultipleElementsFound):+ if not value:+ raise++ # sometimes we have multiple elements pointed to the same value,+ # but only one option is clickable on the page+ LOG.debug(+ "Searching option with the same value in incremental elements",+ value=value,+ elements=incremental_scraped.element_tree,+ )+ locator = await incremental_scraped.select_one_element_by_value(value=value)+ if not locator:+ single_select_result.action_result = ActionFailure(exception=MissingElement())+ if post_reset_fallback:+ return _terminal_post_reset_fallback_result()return single_select_result- if await selected_element.get_attr("role") == "listbox":- single_select_result.action_result = ActionFailure(- exception=InteractWithDropdownContainer(element_id=element_id)+ try:+ LOG.info(+ "Find an alternative option with the same value. Try to select the option.",+ value=value,)+ await EventStrategyFactory.move_to_element(page, locator)+ await locator.click(timeout=timeout)+ single_select_result.action_result = ActionSuccess()+ return _proceeded_post_reset_fallback_result()+ except Exception as e:+ single_select_result.action_result = ActionFailure(exception=e)+ if post_reset_fallback:+ return _terminal_post_reset_fallback_result()return single_select_result-- await selected_element.scroll_into_view()- await selected_element.click(- page=page,- timeout=timeout,- engine_selection=skyvern_frame.engine_selection,- )- single_select_result.action_result = ActionSuccess()- return single_select_result- except (MissingElement, MissingElementDict, MissingElementInCSSMap, MultipleElementsFound):- if not value:- raise-- # sometimes we have multiple elements pointed to the same value,- # but only one option is clickable on the page- LOG.debug(- "Searching option with the same value in incremental elements",- value=value,- elements=incremental_scraped.element_tree,- )- locator = await incremental_scraped.select_one_element_by_value(value=value)- if not locator:- single_select_result.action_result = ActionFailure(exception=MissingElement())- return single_select_result-- try:- LOG.info(- "Find an alternative option with the same value. Try to select the option.",- value=value,- )- await EventStrategyFactory.move_to_element(page, locator)- await locator.click(timeout=timeout)- single_select_result.action_result = ActionSuccess()- return single_select_result- except Exception as e:- single_select_result.action_result = ActionFailure(exception=e)- return single_select_result+ except Exception:+ if post_reset_fallback:+ return _terminal_post_reset_fallback_result()+ raisedef _no_element_matched_failure(value: str, reason: str) -> ActionFailure:diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.pyindex 1cd81d724..b7d4b1f79 100644--- a/tests/unit/test_browser_effect_sink_enumeration.py+++ b/tests/unit/test_browser_effect_sink_enumeration.py@@ -26,7 +26,7 @@ _DISCOVERED_BROWSER_API_CALLS = {"close": 5,"dblclick": 2,"evaluate": 9,- "fill": 1,+ "fill": 2,"focus": 2,"go_back": 1,"go_forward": 1,@@ -198,14 +198,14 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:}assert observed == _DISCOVERED_BROWSER_API_CALLS- assert sum(sum(methods.values()) for methods in observed.values()) == 140+ assert sum(sum(methods.values()) for methods in observed.values()) == 141handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)classified_non_browser = Counter({signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES})assert classified_non_browser == _NON_BROWSER_CANDIDATESassert sum(_NON_BROWSER_CANDIDATES.values()) == 5- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 135+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 136def test_every_raw_evaluate_call_is_classified() -> None:diff --git a/tests/unit/test_input_text_combobox_commit.py b/tests/unit/test_input_text_combobox_commit.pyindex 4d20cfdf9..ffd061105 100644--- a/tests/unit/test_input_text_combobox_commit.py+++ b/tests/unit/test_input_text_combobox_commit.py@@ -26,7 +26,7 @@ from skyvern.webeye.actions import handlerfrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import Action, InputOrSelectContext, InputTextActionfrom skyvern.webeye.actions.handler import handle_input_text_action-from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess+from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccessfrom tests.unit.conftest import make_input_element_mockfrom tests.unit.helpers import make_organization, make_step, make_task@@ -153,6 +153,10 @@ def _pressed_keys(el: MagicMock) -> list[str]:return [call.args[0] for call in el.press_key.call_args_list if call.args]+def _typed_values(el: MagicMock) -> list[str]:+ return [call.args[0] if call.args else call.kwargs["text"] for call in el.input_sequentially.call_args_list]++async def _run_combobox_input(*,attrs: dict[str, object],@@ -167,6 +171,9 @@ async def _run_combobox_input(prefilter_raises: bool = False,use_base_action: bool = False,first_block_incremental: list[dict] | None = None,+ terminal_failure: bool = False,+ nonterminal_skip: bool = False,+ nonterminal_failure_skip: bool = False,) -> tuple[list, MagicMock, MagicMock]:skyvern_el = make_input_element_mock(element_id="CBX", attrs=attrs)if prefilter_raises:@@ -211,7 +218,18 @@ async def _run_combobox_input()select_result = MagicMock()- select_result.action_result = ActionSuccess() if select_success else ActionFailure(Exception("not committed"))+ if terminal_failure:+ select_result.action_result, _ = handler._terminal_custom_select_failure(+ target_value=_TARGET,+ matched_label=_TARGET,+ )+ elif nonterminal_failure_skip:+ select_result.action_result = ActionFailure(Exception("not committed"))+ select_result.action_result.skip_remaining_actions = True+ elif nonterminal_skip:+ select_result.action_result = ActionResult(success=False, skip_remaining_actions=True)+ else:+ select_result.action_result = ActionSuccess() if select_success else ActionFailure(Exception("not committed"))# A secret makes the resolved text differ from action.text, so is_secret_value becomes True.action_text = "{{secret_param}}" if is_secret else _TARGET@@ -382,6 +400,171 @@ def test_prefilter_typeahead_flag_excluded_from_serialization() -> None:_FLAG_TYPEAHEAD_ATTRS = {"role": None, "aria-autocomplete": None, "aria-invalid": "false"}+ ("attrs", "is_search_bar", "first_block_incremental", "expected_typed_values"),+ [+ pytest.param(+ _FLAG_TYPEAHEAD_ATTRS,+ False,+ _listbox_with_option(_TARGET),+ [],+ id="autocomplete-detect",+ ),+ pytest.param(+ {"role": "textbox", "aria-autocomplete": None, "aria-invalid": "false"},+ True,+ _listbox_with_option(_TARGET),+ [_TARGET],+ id="search-bar",+ ),+ pytest.param(_INVALID_BOTH, False, None, [_TARGET], id="invalid-combobox"),+ ],+)+async def test_terminal_custom_select_failure_stops_each_input_text_caller(+ attrs: dict[str, object],+ is_search_bar: bool,+ first_block_incremental: list[dict] | None,+ expected_typed_values: list[str],+) -> None:+ results, el, select_mock = await _run_combobox_input(+ attrs=attrs,+ options=_listbox_with_option(_TARGET),+ select_success=False,+ stop_flag=True,+ is_search_bar=is_search_bar,+ first_block_incremental=first_block_incremental,+ terminal_failure=True,+ )++ failure = select_mock.return_value.action_result+ assert results[0] is failure+ assert isinstance(failure, ActionFailure)+ assert not isinstance(failure, ActionSuccess)+ assert failure.skip_remaining_actions is True+ assert _typed_values(el) == expected_typed_values+ assert "Tab" not in _pressed_keys(el)+++ ("attrs", "is_search_bar", "first_block_incremental"),+ [+ pytest.param(_FLAG_TYPEAHEAD_ATTRS, False, _listbox_with_option(_TARGET), id="autocomplete-detect"),+ pytest.param(+ {"role": "textbox", "aria-autocomplete": None, "aria-invalid": "false"},+ True,+ _listbox_with_option(_TARGET),+ id="search-bar",+ ),+ pytest.param(_INVALID_BOTH, False, None, id="invalid-combobox"),+ ],+)+async def test_nonterminal_skip_carrier_falls_through_each_input_text_caller(+ attrs: dict[str, object],+ is_search_bar: bool,+ first_block_incremental: list[dict] | None,+) -> None:+ results, el, select_mock = await _run_combobox_input(+ attrs=attrs,+ options=_listbox_with_option(_TARGET),+ select_success=False,+ stop_flag=False,+ is_search_bar=is_search_bar,+ first_block_incremental=first_block_incremental,+ nonterminal_skip=True,+ )++ select_mock.assert_awaited_once()+ assert len(results) == 1+ assert isinstance(results[0], ActionSuccess)+ assert results[0] is not select_mock.return_value.action_result+ assert _typed_values(el) == [_TARGET]+++ ("attrs", "is_search_bar", "first_block_incremental"),+ [+ pytest.param(_FLAG_TYPEAHEAD_ATTRS, False, _listbox_with_option(_TARGET), id="autocomplete-detect"),+ pytest.param(+ {"role": "textbox", "aria-autocomplete": None, "aria-invalid": "false"},+ True,+ _listbox_with_option(_TARGET),+ id="search-bar",+ ),+ pytest.param(_INVALID_BOTH, False, None, id="invalid-combobox"),+ ],+)+async def test_date_related_failure_with_skip_falls_through_each_input_text_caller(+ attrs: dict[str, object],+ is_search_bar: bool,+ first_block_incremental: list[dict] | None,+) -> None:+ """A datepicker ActionFailure+skip lacks the custom-select terminal marker and falls through."""+ results, el, select_mock = await _run_combobox_input(+ attrs=attrs,+ options=_listbox_with_option(_TARGET),+ select_success=False,+ stop_flag=False,+ is_search_bar=is_search_bar,+ first_block_incremental=first_block_incremental,+ nonterminal_failure_skip=True,+ is_date_related=True,+ )++ select_mock.assert_awaited_once()+ assert len(results) == 1+ assert isinstance(results[0], ActionSuccess)+ assert results[0] is not select_mock.return_value.action_result+ assert _typed_values(el) == [_TARGET]+++ ("attrs", "is_search_bar", "first_block_incremental", "stop_flag", "expected_stop"),+ [+ pytest.param(+ _FLAG_TYPEAHEAD_ATTRS,+ False,+ _listbox_with_option(_TARGET),+ False,+ False,+ id="autocomplete-detect",+ ),+ pytest.param(+ {"role": "textbox", "aria-autocomplete": None, "aria-invalid": "false"},+ True,+ _listbox_with_option(_TARGET),+ True,+ True,+ id="search-bar",+ ),+ pytest.param(_INVALID_BOTH, False, None, True, True, id="invalid-combobox"),+ ],+)+async def test_successful_custom_select_preserves_each_input_text_caller(+ attrs: dict[str, object],+ is_search_bar: bool,+ first_block_incremental: list[dict] | None,+ stop_flag: bool,+ expected_stop: bool,+) -> None:+ results, _el, select_mock = await _run_combobox_input(+ attrs=attrs,+ options=_listbox_with_option(_TARGET),+ select_success=True,+ stop_flag=stop_flag,+ is_search_bar=is_search_bar,+ first_block_incremental=first_block_incremental,+ )++ success = select_mock.return_value.action_result+ assert results[0] is success+ assert isinstance(success, ActionSuccess)+ assert bool(success.skip_remaining_actions) is expected_stop++@pytest.mark.asyncioasync def test_prefilter_flag_types_target_to_filter_before_match() -> None:"""With prefilter_typeahead set (by Cloud Setup), Block A types the target to filter the listboxdiff --git a/tests/unit/test_no_available_option.py b/tests/unit/test_no_available_option.pyindex d637e677f..def2da9b8 100644--- a/tests/unit/test_no_available_option.py+++ b/tests/unit/test_no_available_option.py@@ -4,7 +4,7 @@ from __future__ import annotationsfrom datetime import UTC, datetimefrom types import SimpleNamespace-from unittest.mock import AsyncMock, MagicMock+from unittest.mock import AsyncMock, MagicMock, callimport pytest@@ -1134,7 +1134,7 @@ class TestDeterministicCustomSelect:selected_element.get_locator().count = AsyncMock(return_value=1)# A text-input combobox anchor is resettable, so an inconclusive read-back must NOT hard-fail.readback_scope_element = _FakeAnchorElement(tag_name="input")- monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value=""))+ monkeypatch.setattr(handler, "get_input_value", AsyncMock(side_effect=[""]))result = await _select_deterministic_custom_option(execute=True,@@ -1151,7 +1151,7 @@ class TestDeterministicCustomSelect:assert result is Noneselected_element.click.assert_awaited_once()- readback_scope_element.get_locator().fill.assert_awaited_once_with("")+ assert readback_scope_element.get_locator().fill.await_args_list == [call("")]readback_scope_element.click.assert_awaited_once()@pytest.mark.asyncio@@ -1174,17 +1174,17 @@ class TestDeterministicCustomSelect:)readback_scope_element = _FakeAnchorElement()- assert (- await _verify_custom_select_option(- matched_element=matched_element, # type: ignore[arg-type]- readback_scope_element=readback_scope_element, # type: ignore[arg-type]- anchor_is_combobox_input=False,- matched_element_id="choice-radio",- matched_label="Choice",- )- is True+ verified, _ = await _verify_custom_select_option(+ matched_element=matched_element, # type: ignore[arg-type]+ readback_scope_element=readback_scope_element, # type: ignore[arg-type]+ anchor_is_combobox_input=False,+ matched_element_id="choice-radio",+ matched_label="Choice",+ use_strict_verification=False,)+ assert verified is True+@pytest.mark.asyncioasync def test_readback_accepts_scoped_trigger_reflection_without_synthetic_token(self) -> None:matched_element = _FakeCustomElement()@@ -1196,21 +1196,22 @@ class TestDeterministicCustomSelect:captured["arg"] = argif "const nestedChoice =" in expression:return None- return True+ return {"matched": True, "branch": "scope_trigger_text"}readback_scope_element = _FakeAnchorElement()with pytest.MonkeyPatch.context() as mp:mp.setattr(handler.SkyvernFrame, "evaluate", AsyncMock(side_effect=evaluate))- outcome = await _verify_custom_select_option(+ verified, _ = await _verify_custom_select_option(matched_element=matched_element, # type: ignore[arg-type]readback_scope_element=readback_scope_element, # type: ignore[arg-type]anchor_is_combobox_input=False,matched_element_id="choice-radio",matched_label="Choice",+ use_strict_verification=False,)- assert outcome is True+ assert verified is Truescript = str(captured["expression"])assert "document.querySelectorAll" not in scriptassert "data-selected-option-id" not in script@@ -1235,15 +1236,16 @@ class TestDeterministicCustomSelect:"evaluate",_stub_evaluate(matched_state=None, committed=False),)- outcome = await _verify_custom_select_option(+ verified, _ = await _verify_custom_select_option(matched_element=matched_element, # type: ignore[arg-type]readback_scope_element=readback_scope_element, # type: ignore[arg-type]anchor_is_combobox_input=True,matched_element_id="source-wake",matched_label="WAKE",+ use_strict_verification=False,)- assert outcome is False+ assert verified is False@pytest.mark.asyncioasync def test_handle_select_option_action_does_not_value_fallback_after_deterministic_readback_failure(@@ -1499,15 +1501,16 @@ class TestDeterministicCustomSelect:with pytest.MonkeyPatch.context() as mp:mp.setattr(handler.SkyvernFrame, "evaluate", AsyncMock(side_effect=evaluate))- outcome = await _verify_custom_select_option(+ verified, _ = await _verify_custom_select_option(matched_element=matched_element, # type: ignore[arg-type]readback_scope_element=readback_scope_element, # type: ignore[arg-type]anchor_is_combobox_input=False,matched_element_id="matched-option",matched_label="Job Board",+ use_strict_verification=False,)- assert outcome is False+ assert verified is Falsescript = str(captured["expression"])assert script.count('"output"') == 0assert script.count("document.querySelectorAll") == 0diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.pyindex a9a379769..a64705e17 100644--- a/tests/unit/test_select_shadow_match.py+++ b/tests/unit/test_select_shadow_match.py@@ -535,7 +535,10 @@ async def test_emerging_select_call_path_forwards_committed_value_to_shadow_matc@pytest.mark.asyncio[email protected](("entry_action_type", "expected_execute"), [("select_option", True), ("input_text", False)])+ ("entry_action_type", "expected_execute"),+ [("select_option", True), ("input_text", True), ("input_text_converted", False), ("click", False)],+)async def test_dropdown_select_call_path_forwards_committed_value_and_scopes_execution(monkeypatch: pytest.MonkeyPatch, entry_action_type: str, expected_execute: bool) -> None:@@ -616,6 +619,110 @@ async def test_dropdown_terminal_result_never_reaches_custom_select_llm(monkeypallm_handler.assert_not_awaited()+ ("use_strict_verification", "expected"),+ [(False, (True, "matched_state")), (True, (False, "none"))],+)+async def test_custom_select_verify_strict_posture_is_entry_scoped(+ monkeypatch: pytest.MonkeyPatch,+ use_strict_verification: bool,+ expected: tuple[bool, str],+) -> None:+ monkeypatch.setattr(+ handler,+ "_read_custom_select_matched_state",+ AsyncMock(+ return_value={+ "label": "Choice",+ "role": "option",+ "inMultiselectable": False,+ "ariaSelected": True,+ }+ ),+ )+ monkeypatch.setattr(+ handler,+ "_custom_select_scope_confirms_committed",+ AsyncMock(return_value=(False, "none")),+ )++ result = await handler._verify_custom_select_option(+ matched_element=MagicMock(),+ readback_scope_element=MagicMock(),+ anchor_is_combobox_input=True,+ matched_element_id="choice-1",+ matched_label="Choice",+ use_strict_verification=use_strict_verification,+ )++ assert result == expected+++ ("use_strict_verification", "expected_allow_tokens", "expected_allow_single_value_scope"),+ [(False, True, False), (True, False, True)],+)+async def test_custom_select_verify_strict_posture_controls_aria_selected_tokens(+ monkeypatch: pytest.MonkeyPatch,+ use_strict_verification: bool,+ expected_allow_tokens: bool,+ expected_allow_single_value_scope: bool,+) -> None:+ monkeypatch.setattr(handler, "_read_custom_select_matched_state", AsyncMock(return_value=None))+ scope_confirms = AsyncMock(return_value=(False, "none"))+ monkeypatch.setattr(handler, "_custom_select_scope_confirms_committed", scope_confirms)++ await handler._verify_custom_select_option(+ matched_element=MagicMock(),+ readback_scope_element=MagicMock(),+ anchor_is_combobox_input=True,+ matched_element_id="choice-1",+ matched_label="Choice",+ use_strict_verification=use_strict_verification,+ )++ assert scope_confirms.await_args.kwargs["allow_aria_selected_option_tokens"] is expected_allow_tokens+ assert scope_confirms.await_args.kwargs["allow_single_value_scope"] is expected_allow_single_value_scope+++ ("committed_state", "expected"),+ [+ ({"matched": True, "branch": "scope_input_value"}, (True, "scope_input_value")),+ ({"matched": True, "branch": "scope_single_value"}, (True, "scope_single_value")),+ ({"matched": False, "branch": "none"}, (False, "none")),+ ],+)+async def test_custom_select_verify_preserves_scope_branch(+ monkeypatch: pytest.MonkeyPatch,+ committed_state: dict[str, object],+ expected: tuple[bool, str],+) -> None:+ matched_element = MagicMock()+ matched_element.get_locator.return_value.count = AsyncMock(return_value=1)++ async def evaluate_element_scoped(_element: object, expression: str, _arg: object = None) -> object:+ if expression == handler._CUSTOM_SELECT_MATCHED_STATE_JS:+ return None+ return committed_state++ monkeypatch.setattr(handler, "_evaluate_element_scoped", AsyncMock(side_effect=evaluate_element_scoped))++ result = await handler._verify_custom_select_option(+ matched_element=matched_element,+ readback_scope_element=MagicMock(),+ anchor_is_combobox_input=True,+ matched_element_id="choice-1",+ matched_label="Choice",+ use_strict_verification=True,+ )++ assert result == expected++_OUTCOME_TASK = SimpleNamespace(workflow_run_id="wr",task_id="task",@@ -636,6 +743,7 @@ async def _run_outcome_case(assigned: bool = False,execute: bool = True,entry_action_type: str = "select_option",+ anchor_is_input: bool = False,selection_group_id: str = "group-1",select_depth: int = 0,) -> tuple[object, dict[str, object] | None, SimpleNamespace, AsyncMock, AsyncMock]:@@ -676,19 +784,33 @@ async def _run_outcome_case(AsyncMock(side_effect=RuntimeError("before click") if case == "pre_click" else None, return_value=None),)monkeypatch.setattr(handler, "_resolve_custom_select_readback_scope_element", AsyncMock(return_value=MagicMock()))- monkeypatch.setattr(handler, "_anchor_is_combobox_input", AsyncMock(return_value=case.startswith("reset")))+ monkeypatch.setattr(+ handler,+ "_anchor_is_combobox_input",+ AsyncMock(return_value=anchor_is_input or case.startswith("reset")),+ )monkeypatch.setattr(handler,"_custom_select_matched_state_confirms_pre_click",Mock(return_value=case == "precommit_matched"),)++ async def scope_confirms_committed(**kwargs: object) -> tuple[bool, str]:+ committed = case == "precommit_scope" or (+ case == "precommit_single_value" and kwargs["allow_single_value_scope"] is True+ )+ return committed, "scope_single_value" if committed else "none"++ monkeypatch.setattr(+ handler, "_custom_select_scope_confirms_committed", AsyncMock(side_effect=scope_confirms_committed)+ )monkeypatch.setattr(handler,- "_custom_select_scope_confirms_committed",- AsyncMock(return_value=case == "precommit_scope"),+ "_verify_custom_select_option_with_settle",+ AsyncMock(return_value=(case == "verified", "matched_state" if case == "verified" else "none")),)- monkeypatch.setattr(handler, "_verify_custom_select_option_with_settle", AsyncMock(return_value=case == "verified"))monkeypatch.setattr(handler, "_reset_custom_select_combobox_input", AsyncMock(return_value=case == "reset_ok"))+ monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value="typed"))candidates = (Mock(side_effect=RuntimeError("walker"))if case == "walker_error"@@ -735,6 +857,140 @@ async def test_outcome_event_emitted_once_per_opportunity(assert event["entry_action_type"] == entry_action_type+ (+ "entry_action_type",+ "anchor_is_input",+ "expected_use_strict_verification",+ "expected_allow_single_value_scope",+ ),+ [+ ("select_option", True, False, False),+ ("select_option", False, False, False),+ ("input_text", True, True, True),+ ("input_text_converted", True, False, False),+ ],+)+async def test_deterministic_custom_select_verification_is_entry_scoped(+ monkeypatch: pytest.MonkeyPatch,+ entry_action_type: str,+ anchor_is_input: bool,+ expected_use_strict_verification: bool,+ expected_allow_single_value_scope: bool,+) -> None:+ await _run_outcome_case(+ monkeypatch,+ "verified",+ assigned=True,+ entry_action_type=entry_action_type,+ anchor_is_input=anchor_is_input,+ )++ verify = handler._verify_custom_select_option_with_settle+ assert isinstance(verify, AsyncMock)+ assert verify.await_args.kwargs["use_strict_verification"] is expected_use_strict_verification+ scope_confirms = handler._custom_select_scope_confirms_committed+ assert isinstance(scope_confirms, AsyncMock)+ assert scope_confirms.await_args.kwargs["allow_single_value_scope"] is expected_allow_single_value_scope+++async def test_executable_entry_constant_disables_input_text_single_value_scope(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(handler, "_EXECUTABLE_CUSTOM_SELECT_ENTRIES", ("select_option",))++ await _run_outcome_case(+ monkeypatch,+ "verified",+ assigned=True,+ entry_action_type="input_text",+ anchor_is_input=True,+ )++ scope_confirms = handler._custom_select_scope_confirms_committed+ assert isinstance(scope_confirms, AsyncMock)+ assert scope_confirms.await_args.kwargs["allow_single_value_scope"] is False+ verify = handler._verify_custom_select_option_with_settle+ assert isinstance(verify, AsyncMock)+ assert verify.await_args.kwargs["use_strict_verification"] is False+++ ("case", "expected_outcome"),+ [+ ("precommit_matched", "success_precommit"),+ ("precommit_scope", "success_precommit"),+ ("verified", "llm_fallback_execution_disabled"),+ ],+)+async def test_input_text_non_input_anchor_keeps_precommit_but_never_clicks(+ monkeypatch: pytest.MonkeyPatch,+ case: str,+ expected_outcome: str,+) -> None:+ result, event, _, _, get_skyvern_element = await _run_outcome_case(+ monkeypatch,+ case,+ assigned=True,+ entry_action_type="input_text",+ anchor_is_input=False,+ )++ if expected_outcome == "success_precommit":+ assert result is not None+ assert isinstance(result[0], handler.ActionSuccess)+ else:+ assert result is None+ assert event is not None+ assert event["outcome"] == expected_outcome+ get_skyvern_element.return_value.click.assert_not_awaited()++[email protected]("entry_action_type", ["click", "input_text_converted"])+async def test_nonexecutable_entry_does_not_single_value_preconfirm(+ monkeypatch: pytest.MonkeyPatch,+ entry_action_type: str,+) -> None:+ result, event, _, _, get_skyvern_element = await _run_outcome_case(+ monkeypatch,+ "precommit_single_value",+ assigned=True,+ execute=False,+ entry_action_type=entry_action_type,+ anchor_is_input=True,+ )++ assert result is None+ assert event is not None+ assert event["outcome"] == "llm_fallback_execution_disabled"+ get_skyvern_element.return_value.click.assert_not_awaited()+++async def test_executable_entry_constant_disables_input_text_single_value_preconfirm(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(handler, "_EXECUTABLE_CUSTOM_SELECT_ENTRIES", ("select_option",))++ result, event, _, _, get_skyvern_element = await _run_outcome_case(+ monkeypatch,+ "precommit_single_value",+ assigned=True,+ execute=False,+ entry_action_type="input_text",+ anchor_is_input=True,+ )++ assert result is None+ assert event is not None+ assert event["outcome"] == "llm_fallback_execution_disabled"+ get_skyvern_element.return_value.click.assert_not_awaited()++@pytest.mark.asyncioasync def test_click_route_without_target_emits_nothing_and_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:gate = AsyncMock()@@ -907,6 +1163,217 @@ async def test_outcome_event_once_per_dropdown_level_with_group_id(monkeypatch:assert [event["select_depth"] for event in events] == [0, 1, 2]+async def _run_post_reset_fallback_case(+ monkeypatch: pytest.MonkeyPatch,+ *,+ llm_response: object,+ entry_action_type: str = "input_text",+ input_value_reads: list[str | None] | None = None,+) -> tuple[object, list[dict[str, object]], MagicMock, MagicMock, BaseException | None, AsyncMock]:+ handler._COLLAPSE_XP_ASSIGNMENT_MEMO.clear()+ monkeypatch.setattr(+ handler.app,+ "EXPERIMENTATION_PROVIDER",+ SimpleNamespace(+ is_feature_enabled_cached=AsyncMock(return_value=True),+ resolve_feature_enabled_unrecorded=AsyncMock(return_value=True),+ ),+ )+ monkeypatch.setattr(+ handler.app,+ "AGENT_FUNCTION",+ SimpleNamespace(+ resolve_field_option=AsyncMock(+ return_value=SimpleNamespace(+ fallback_to_llm=False,+ matched_index=0,+ matched_label="Committed Value",+ matched_tier="exact",+ )+ )+ ),+ )+ llm_handler = (+ AsyncMock(side_effect=llm_response)+ if isinstance(llm_response, Exception)+ else AsyncMock(return_value=llm_response)+ )+ monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)+ monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))+ monkeypatch.setattr(handler, "_read_custom_select_matched_state", AsyncMock(return_value=None))+ monkeypatch.setattr(handler, "_custom_select_scope_confirms_committed", AsyncMock(return_value=(False, "none")))+ monkeypatch.setattr(handler, "_verify_custom_select_option_with_settle", AsyncMock(return_value=(False, "none")))+ if input_value_reads is None:+ input_value_reads = (+ [""]+ if entry_action_type == "select_option"+ else ["Committed Value", "", "Committed Value", "Committed Value"]+ )+ get_input_value_mock = AsyncMock(side_effect=input_value_reads)+ monkeypatch.setattr(handler, "get_input_value", get_input_value_mock)+ monkeypatch.setattr(+ handler, "get_actual_value_of_parameter_if_secret_with_task", Mock(side_effect=lambda _, value: value)+ )++ reset_locator = MagicMock(fill=AsyncMock())+ anchor = MagicMock(+ get_id=Mock(return_value="field-id"),+ get_tag_name=Mock(return_value="input"),+ get_locator=Mock(return_value=reset_locator),+ click=AsyncMock(),+ )+ matched_element = MagicMock(+ get_attr=AsyncMock(return_value=None),+ scroll_into_view=AsyncMock(),+ click=AsyncMock(),+ )+ input_element = MagicMock(+ get_tag_name=Mock(return_value="input"),+ get_locator=Mock(return_value=MagicMock()),+ scroll_into_view=AsyncMock(),+ is_readonly=AsyncMock(return_value=False),+ input_clear=AsyncMock(),+ input_sequentially=AsyncMock(),+ )+ monkeypatch.setattr(+ handler.SkyvernElement,+ "create_from_incremental",+ AsyncMock(side_effect=[matched_element, input_element]),+ )++ dropdown = MagicMock(get_element_handler=AsyncMock(return_value=MagicMock()))+ skyvern_frame = MagicMock(get_element_scrollable=AsyncMock(return_value=False), engine_selection=object())+ incremental_scraped = MagicMock(get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]))+ incremental_scraped.build_element_tree.return_value = "<div>Committed Value</div>"+ monkeypatch.setattr(handler, "try_to_find_potential_scrollable_element", AsyncMock(return_value=dropdown))++ result: object = None+ raised: BaseException | None = None+ with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):+ with structlog.testing.capture_logs() as logs:+ try:+ result = await handler.select_from_dropdown(+ context=InputOrSelectContext(field="Field"),+ page=MagicMock(),+ skyvern_element=anchor,+ skyvern_frame=skyvern_frame,+ incremental_scraped=incremental_scraped,+ check_filter_funcs=[],+ step=SimpleNamespace(step_id="step"),+ task=_OUTCOME_TASK,+ dropdown_menu_element=dropdown,+ force_select=True,+ target_value="Committed Value",+ entry_action_type=entry_action_type,+ selection_group_id="group-1",+ )+ except Exception as exc:+ raised = exc++ events = [log for log in logs if log.get("event") == "custom_select_family_outcome"]+ return result, events, input_element, reset_locator, raised, get_input_value_mock+++ "llm_response",+ [+ RuntimeError("fallback failed"),+ {"id": "", "action_type": "", "reasoning": "no match"},+ {"id": "matched-id", "value": "Committed Value", "action_type": {"x": 1}},+ {"id": "matched-id", "action_type": "input_text", "reasoning": "no value"},+ {"id": "matched-id", "value": "", "action_type": "input_text", "reasoning": "empty value"},+ ],+)+async def test_post_reset_llm_failure_emits_single_terminal_outcome(+ monkeypatch: pytest.MonkeyPatch,+ llm_response: object,+) -> None:+ result, events, _, _, raised, _ = await _run_post_reset_fallback_case(monkeypatch, llm_response=llm_response)++ assert raised is None+ assert isinstance(result.action_result, handler.ActionFailure)+ assert result.action_result.skip_remaining_actions is True+ assert [event["outcome"] for event in events] == ["terminal_llm_fallback_exception"]+ assert events[0]["entry_action_type"] == "input_text"+ assert events[0]["selection_group_id"] == "group-1"+ assert events[0]["select_depth"] == 0+ assert events[0]["family_gate_enabled"] is True+ assert events[0]["assigned"] is True+ assert events[0]["llm_fallback_requested"] is True+++async def test_post_reset_restored_text_is_retyped_and_emits_single_proceeded_outcome(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ result, events, input_element, reset_locator, raised, _ = await _run_post_reset_fallback_case(+ monkeypatch,+ llm_response={"id": "matched-id", "value": "Committed Value", "action_type": "input_text"},+ )++ assert raised is None+ assert isinstance(result.action_result, handler.ActionSuccess)+ input_element.input_clear.assert_awaited_once()+ input_element.input_sequentially.assert_awaited_once_with("Committed Value")+ assert [c.args for c in reset_locator.fill.await_args_list] == [("",), ("Committed Value",)]+ assert [event["outcome"] for event in events] == ["llm_fallback_reset_verified"]+ assert events[0]["entry_action_type"] == "input_text"+ assert events[0]["selection_group_id"] == "group-1"+ assert events[0]["select_depth"] == 0+ assert events[0]["family_gate_enabled"] is True+ assert events[0]["assigned"] is True+ assert events[0]["llm_fallback_requested"] is True+++async def test_select_option_reset_path_matches_main(monkeypatch: pytest.MonkeyPatch) -> None:+ llm_error = RuntimeError("fallback failed")+ result, events, _, reset_locator, raised, get_input_value_mock = await _run_post_reset_fallback_case(+ monkeypatch,+ llm_response=llm_error,+ entry_action_type="select_option",+ )++ assert raised is llm_error+ assert result is None+ assert [event["outcome"] for event in events] == ["llm_fallback_reset_verified"]+ assert events[0]["entry_action_type"] == "select_option"+ assert [c.args for c in reset_locator.fill.await_args_list] == [("",)]+ assert get_input_value_mock.await_count == 1+++async def test_failed_restore_readback_is_terminal_not_reset_verified(monkeypatch: pytest.MonkeyPatch) -> None:+ llm_handler_never_called = RuntimeError("llm should not run")+ result, events, _, reset_locator, raised, _ = await _run_post_reset_fallback_case(+ monkeypatch,+ llm_response=llm_handler_never_called,+ input_value_reads=["Committed Value", "", "typed over"],+ )++ assert raised is None+ assert isinstance(result.action_result, handler.ActionFailure)+ assert result.action_result.skip_remaining_actions is True+ assert [event["outcome"] for event in events] == ["terminal_unverified_reset"]+ assert [c.args for c in reset_locator.fill.await_args_list] == [("",), ("Committed Value",)]+++async def test_unreadable_anchor_before_click_falls_back_without_clicking(monkeypatch: pytest.MonkeyPatch) -> None:+ llm_error = RuntimeError("llm path")+ result, events, _, reset_locator, raised, _ = await _run_post_reset_fallback_case(+ monkeypatch,+ llm_response=llm_error,+ input_value_reads=[None],+ )++ assert raised is llm_error+ assert result is None+ assert [event["outcome"] for event in events] == ["llm_fallback_pre_click_error"]+ reset_locator.fill.assert_not_awaited()++@pytest.mark.asyncioasync def test_outcome_event_family_off_emits_without_assignment_resolution(monkeypatch: pytest.MonkeyPatch) -> None:_, event, provider, resolver, _ = await _run_outcome_case(monkeypatch, "family_off", family_enabled=False)@@ -924,17 +1391,20 @@ async def test_outcome_event_key_allowlist_and_enum(monkeypatch: pytest.MonkeyPaexpected_keys = set("event log_level family workflow_run_id task_id organization_id step_id entry_action_type selection_group_id ""select_depth script_mode family_gate_enabled assigned gate_error encountered eligible match_tier "- "option_count attempted click_attempted verified_success outcome llm_fallback_requested duration_ms".split()+ "option_count attempted click_attempted anchor_is_combobox_input verify_branch verified_success outcome "+ "llm_fallback_requested duration_ms".split())expected_outcomes = set("llm_fallback_gate_error llm_fallback_eval_error llm_fallback_family_off llm_fallback_control ""llm_fallback_no_match llm_fallback_match_unactionable llm_fallback_pre_click_error ""llm_fallback_reset_verified llm_fallback_post_click_unverified llm_fallback_tier_excluded ""llm_fallback_execution_disabled success_precommit success_verified terminal_post_click_exception "+ "terminal_llm_fallback_exception ""terminal_unverified_reset terminal_unverified_click ""terminal_unverified_toggle".split())assert event["outcome"] == "llm_fallback_control"+ assert (event["anchor_is_combobox_input"], event["verify_branch"]) == (False, None)assert set(event) == expected_keysassert {outcome.value for outcome in handler.CustomSelectFamilyOutcome} == expected_outcomes