Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Revert the six flag-gated agent-handler verification PRs for SKY-12398/12399/12400 (#7590)
Shuchang Zheng584eb7896599
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex a00e5ffe8..1f98176a4 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -207,38 +207,6 @@ class CardNumberInputMismatch(SkyvernException):)-class MaskedInputReadbackMismatch(SkyvernException):- # A masked field (postal code, SSN, etc.) accepted the keystrokes but committed nothing usable —- # it reads back empty or still shows mask placeholders. Counts only; the raw value may be a secret.- def __init__(self, *, expected_char_count: int, committed_char_count: int):- self.expected_char_count = expected_char_count- self.committed_char_count = committed_char_count- super().__init__(- "Masked input read-back mismatch: "- f"expected {expected_char_count} characters, committed {committed_char_count}."- )---class InputTextCommitMismatch(SkyvernException):- def __init__(self, *, expected_length: int, actual_length: int):- self.expected_length = expected_length- self.actual_length = actual_length- super().__init__(- f"Input text commit mismatch: expected length {expected_length}, found actual length {actual_length}."- )---class RepeatedActionFailure(SkyvernException):- def __init__(self, *, element_id: str, attempt_count: int, failure_description: str):- self.element_id = element_id- self.attempt_count = attempt_count- self.failure_description = failure_description- super().__init__(- f"Action on element {element_id} failed identically {attempt_count} consecutive times: "- f"{failure_description}."- )--class SecretInputMismatch(SkyvernException):def __init__(self) -> None:# No secret material in the message: not the value, its length, or its character classes.@@ -1071,17 +1039,9 @@ class NoElementBoudingBox(SkyvernException):super().__init__(f"Element does not have a bounding box. element_id={element_id}")-class AutoCompletionCommitFailure(SkyvernException):- def __init__(self, stage: str, attempt_trail: list[str] | None = None) -> None:- trail = attempt_trail or [stage]- super().__init__(f"Autocomplete selection failed. stage={stage} attempt_trail={','.join(trail)}")- self.stage = stage- self.attempt_trail = trail---# Deprecated: superseded by AutoCompletionCommitFailure, which carries the failing stage.-# Kept for one release so downstream `except` clauses keep working.-NoIncrementalElementFoundForAutoCompletion = AutoCompletionCommitFailure+class NoIncrementalElementFoundForAutoCompletion(SkyvernException):+ def __init__(self, element_id: str, text: str) -> None:+ super().__init__(f"No auto completion shown up after fill in [{text}]. element_id={element_id}")class NoSuitableAutoCompleteOption(SkyvernException):diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 60fc1dd7e..8520f2683 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -57,7 +57,6 @@ from skyvern.exceptions import (MissingExtractActionsResponse,NoTOTPVerificationCodeFound,PDFEmbedBase64DecodeError,- RepeatedActionFailure,ScrapingFailed,SkyvernException,StepTerminationError,@@ -169,7 +168,7 @@ from skyvern.webeye.actions.parse_actions import (parse_cua_actions,parse_ui_tars_actions,)-from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess+from skyvern.webeye.actions.responses import ActionResult, ActionSuccessfrom skyvern.webeye.browser_state import BrowserStatefrom skyvern.webeye.cdp_download_interceptor import download_filename_from_suffixfrom skyvern.webeye.scraper.scraped_page import ElementTreeFormat, ScrapedPage@@ -181,103 +180,6 @@ RECOVERABLE_BLANK_WORKFLOW_TASK_URLS = {":"}EXTRACT_ACTION_TEMPLATE = "extract-action"DECISIVE_CRITERION_VALIDATE_TEMPLATE = "decisive-criterion-validate"-REPEATED_ACTION_FAILURE_LIMIT = 3-REPEATED_ACTION_CIRCUIT_BREAKER_FLAG = "REPEATED_ACTION_CIRCUIT_BREAKER"-_ACTION_REPEAT_PAYLOAD_FIELDS = (- "button",- "download",- "duration",- "file_name",- "file_url",- "hold",- "hold_seconds",- "is_checked",- "is_magic_link",- "keys",- "option",- "path",- "repeat",- "start_x",- "start_y",- "text",- "url",- "verification_code",- "x",- "y",-)-_PERSISTED_ACTION_REPEAT_PAYLOAD_FIELDS = tuple(- field for field in _ACTION_REPEAT_PAYLOAD_FIELDS if field in Action.model_fields-)---def _repeated_action_identity(action: Action) -> tuple[ActionType, str, str] | None:- if not action.action_type.is_web_action() or not action.element_id:- return None- payload = {field: getattr(action, field) for field in _PERSISTED_ACTION_REPEAT_PAYLOAD_FIELDS}- payload_digest = hashlib.sha256(- json.dumps(payload, default=str, sort_keys=True, separators=(",", ":")).encode()- ).hexdigest()- return action.action_type, action.element_id, payload_digest---def _get_repeated_action_failure(- attempts: list[tuple[Action, list[ActionResult]]],-) -> RepeatedActionFailure | None:- recent_attempts = attempts[-REPEATED_ACTION_FAILURE_LIMIT:]- if len(recent_attempts) < REPEATED_ACTION_FAILURE_LIMIT:- return None- identities = [_repeated_action_identity(action) for action, _ in recent_attempts]- if identities[0] is None or any(identity != identities[0] for identity in identities[1:]):- return None- if any(not results or any(result.success for result in results) for _, results in recent_attempts):- return None- _, element_id, _ = identities[0]- final_result = recent_attempts[-1][1][-1]- failure_description = final_result.exception_type or "UnknownFailure"- return RepeatedActionFailure(- element_id=element_id,- attempt_count=REPEATED_ACTION_FAILURE_LIMIT,- failure_description=failure_description,- )---async def _is_repeated_action_breaker_enabled(task: Task) -> bool:- try:- distinct_id = task.workflow_run_id if task.workflow_run_id else task.task_id- return bool(- await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(- REPEATED_ACTION_CIRCUIT_BREAKER_FLAG,- distinct_id,- properties={"organization_id": task.organization_id},- )- )- except Exception:- LOG.warning(- "Failed to evaluate repeated-action circuit breaker flag; defaulting to disabled",- task_id=task.task_id,- exc_info=True,- )- return False---async def _get_previous_action_attempts(task: Task, current_step_id: str) -> list[tuple[Action, list[ActionResult]]]:- try:- steps = await app.DATABASE.tasks.get_task_steps(- task_id=task.task_id,- organization_id=task.organization_id,- )- except Exception:- LOG.warning("Failed to load action history for repeated-action detection", task_id=task.task_id, exc_info=True)- return []- if not isinstance(steps, list):- return []- attempts: list[tuple[Action, list[ActionResult]]] = []- for previous_step in sorted(steps, key=lambda item: (item.order, item.retry_index)):- if previous_step.step_id == current_step_id:- continue- if previous_step.output and previous_step.output.actions_and_results:- attempts.extend(previous_step.output.actions_and_results)- return attemptsasync def resolve_inherited_workflow_task_page(browser_state: BrowserState, workflow_run_id: str) -> Page:@@ -1875,7 +1777,6 @@ class ForgeAgent:element_id_to_action_index[action.element_id] = action_idxelement_id_to_last_action: dict[str, int] = dict()- previous_step_attempts: list[tuple[Action, list[ActionResult]]] | None = Nonetry:wait_config = await get_or_create_wait_config(task.task_id, task.workflow_run_id, task.organization_id)base_delay = get_wait_time(wait_config, "inter_action_delay", default=0.5)@@ -1990,42 +1891,6 @@ class ForgeAgent:action,results,)- if (- results- and not any(result.success for result in results)- and await _is_repeated_action_breaker_enabled(task)- ):- current_step_attempts = detailed_agent_step_output.actions_and_results[: action_idx + 1]- repeated_failure = _get_repeated_action_failure(current_step_attempts)- current_identity = _repeated_action_identity(action)- current_attempts_can_follow_history = current_identity is not None and all(- _repeated_action_identity(attempt_action) == current_identity- and attempt_results- and not any(result.success for result in attempt_results)- for attempt_action, attempt_results in current_step_attempts- )- if (- repeated_failure is None- and len(current_step_attempts) < REPEATED_ACTION_FAILURE_LIMIT- and current_attempts_can_follow_history- ):- if previous_step_attempts is None:- previous_step_attempts = await _get_previous_action_attempts(task, step.step_id)- repeated_failure = _get_repeated_action_failure(previous_step_attempts + current_step_attempts)- if repeated_failure is not None:- final_result = results[-1]- repeated_result = ActionFailure(- repeated_failure,- stop_execution_on_failure=final_result.stop_execution_on_failure,- download_triggered=final_result.download_triggered,- interacted_with_sibling=bool(final_result.interacted_with_sibling),- interacted_with_parent=bool(final_result.interacted_with_parent),- )- repeated_result.skip_remaining_actions = final_result.skip_remaining_actions- repeated_result.needs_followup = final_result.needs_followup- repeated_result.followup_message = final_result.followup_message- results = [repeated_result]- detailed_agent_step_output.actions_and_results[action_idx] = (action, results)# Determine wait time between actionswait_time = random.uniform(base_delay, base_delay * 2) if base_delay > 0 else 0.0diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 999d8b2ce..2f90646c4 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -33,7 +33,6 @@ from skyvern.constants import (from skyvern.core.script_generations.fuzzy_matcher import match_option_exact_or_stemfrom skyvern.errors.errors import TOTPExpiredError, UserDefinedError, filter_to_user_defined_codesfrom skyvern.exceptions import (- AutoCompletionCommitFailure,CaptchaSolveError,CardNumberInputMismatch,EmptySelect,@@ -48,13 +47,11 @@ from skyvern.exceptions import (IllegitComplete,ImaginaryFileUrl,ImaginarySecretValue,- InputTextCommitMismatch,InputToInvisibleElement,InputToReadonlyElement,InteractWithDisabledElement,InteractWithDropdownContainer,InvalidElementForTextInput,- MaskedInputReadbackMismatch,MissingElement,MissingElementDict,MissingElementInCSSMap,@@ -63,6 +60,7 @@ from skyvern.exceptions import (NoAutoCompleteOptionMeetCondition,NoAvailableOptionFoundForCustomSelection,NoElementMatchedForTargetOption,+ NoIncrementalElementFoundForAutoCompletion,NoIncrementalElementFoundForCustomSelection,NoSuitableAutoCompleteOption,NoTOTPSecretFound,@@ -177,10 +175,6 @@ LOG = structlog.get_logger()UPLOAD_PENDING_FOLLOWUP_MESSAGE = "Upload is not complete yet. Continue the upload flow."FIX_TEL_INPUT_DIGIT_DROP_FLAG = "FIX_TEL_INPUT_DIGIT_DROP"-VERIFY_EMERGING_SELECT_PICK_FLAG = "VERIFY_EMERGING_SELECT_PICK"-FIX_MASKED_INPUT_READBACK_FLAG = "FIX_MASKED_INPUT_READBACK"-SURFACE_BLOCKED_FORM_ADVANCE_FLAG = "SURFACE_BLOCKED_FORM_ADVANCE"-VERIFY_INPUT_TEXT_COMMIT_FLAG = "VERIFY_INPUT_TEXT_COMMIT"COLLAPSE_SELECT_FANOUT_FLAG = "COLLAPSE_SELECT_FANOUT"COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG = "COLLAPSE_CUSTOM_SELECT_FANOUT"COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG = "COLLAPSE_AUTOCOMPLETE_FANOUT"@@ -218,31 +212,6 @@ DOWNLOAD_EVENT_ACTIVE_DIR_GRACE_SECONDS = 60DOWNLOAD_DUPLICATE_STEM_SUFFIX_RE = re.compile(r"(?:\s+\(\d{1,3}\)|_\d{1,3})$")SELECT_SHADOW_MATCH_APOSTROPHE_RE = re.compile(r"['`‘’]")SELECT_SHADOW_MATCH_WORD_RE = re.compile(r"\w+")-AUTOCOMPLETE_COMMIT_REQUIRED_FLAG = "AUTOCOMPLETE_COMMIT_REQUIRED"-AUTOCOMPLETE_COMMIT_WORD_RE = re.compile(r"[^\W_]+")-AUTOCOMPLETE_POLL_INTERVAL_SECONDS = 0.5-AUTOCOMPLETE_POLL_MAX_ATTEMPTS = 16-AUTOCOMPLETE_POLL_DEFAULT_BUDGET_SECONDS = 4.0-AUTOCOMPLETE_TOTAL_DEADLINE_SECONDS = 8.0-AUTOCOMPLETE_READBACK_TIMEOUT_SECONDS = 2.0-AUTOCOMPLETE_TEARDOWN_ERROR_MARKERS = (- "context was destroyed",- "navigation",- "target closed",- "has been closed",- "frame was detached",-)-AUTOCOMPLETE_DETACHED_ERROR_MARKERS = (- "not attached",- "detached",-)-AUTOCOMPLETE_RERESOLVE_FAILURE_MARKERS = (- "timeout",- "timed out",- "not found",- "no element",- "failed to find",-)def _select_shadow_match_enabled() -> bool:@@ -469,16 +438,6 @@ def _resolve_autocomplete_candidate(return matched_index, candidates[matched_index]-def _autocomplete_candidate_for_element_id(- elements: list[dict], element_id: str-) -> tuple[int, dict[str, str | None]] | None:- candidates = _autocomplete_candidates_from_elements(elements)- for index, candidate in enumerate(candidates):- if candidate.get("element_id") == element_id:- return index, candidate- return None--async def _read_autocomplete_option_identity(*,skyvern_frame: SkyvernFrame,@@ -515,202 +474,39 @@ async def _verify_autocomplete_option_identity(LOG.info("Autocomplete option index differed from deterministic candidate; accepting label match",expected_index=matched_index,- expected_label_length=len(matched_label),+ expected_label=matched_label,actual_index=actual_index,- actual_label_length=len(str(actual_label or "")),+ actual_label=actual_label,)return TrueLOG.info("Autocomplete option identity did not match deterministic candidate",expected_index=matched_index,- expected_label_length=len(matched_label),+ expected_label=matched_label,actual_index=actual_index,- actual_label_length=len(str(actual_label or "")),+ actual_label=actual_label,)return False-class AutocompleteCommitOutcome(BaseModel):- committed: bool = False- side_effects_observed: bool = False---def _autocomplete_commit_tokens(value: str | None) -> list[str]:- return AUTOCOMPLETE_COMMIT_WORD_RE.findall(str(value or "").casefold())---async def _read_autocomplete_committed_value(skyvern_element: SkyvernElement) -> tuple[str | None, bool]:- detached_error_observed = False- for attempt in range(2):- try:- async with asyncio.timeout(AUTOCOMPLETE_READBACK_TIMEOUT_SECONDS):- value = await get_input_value(skyvern_element.get_tag_name(), skyvern_element.get_locator())- return str(value or ""), False- except Exception as exc:- message = str(exc).casefold()- if any(marker in message for marker in AUTOCOMPLETE_TEARDOWN_ERROR_MARKERS):- return None, True- detached_error = any(marker in message for marker in AUTOCOMPLETE_DETACHED_ERROR_MARKERS)- if attempt == 0 and detached_error:- detached_error_observed = True- continue- reresolution_failed = (- detached_error- or isinstance(exc, TimeoutError)- or any(marker in message for marker in AUTOCOMPLETE_RERESOLVE_FAILURE_MARKERS)- )- if detached_error_observed and reresolution_failed:- LOG.info(- "Autocomplete read-back element stayed unreachable after detach, treating as implicit commit",- element_id=skyvern_element.get_id(),- error=type(exc).__name__,- )- return None, True- LOG.info(- "Autocomplete read-back could not read the input value",- element_id=skyvern_element.get_id(),- error=type(exc).__name__,- )- return None, False- return None, False---async def _autocomplete_suggestions_closed(locator: Locator | None) -> bool | None:- if locator is None:- return None- try:- async with asyncio.timeout(AUTOCOMPLETE_READBACK_TIMEOUT_SECONDS):- if await locator.count() == 0:- return True- return not await locator.is_visible()- except Exception:- return None---async def _autocomplete_selection_state(locator: Locator | None) -> bool | None:- if locator is None:- return None- try:- async with asyncio.timeout(AUTOCOMPLETE_READBACK_TIMEOUT_SECONDS):- aria_selected = await locator.get_attribute("aria-selected")- except Exception:- return None- if aria_selected is None:- return None- return str(aria_selected).casefold() == "true"---async def _autocomplete_input_reports_closed(skyvern_element: SkyvernElement) -> bool | None:- try:- async with asyncio.timeout(AUTOCOMPLETE_READBACK_TIMEOUT_SECONDS):- # Must be a live read: get_attr defaults to static-first, and an input scraped before the- # dropdown opened carries a cached aria-expanded="false" that would read as commit evidence- # no matter what the keypress actually did.- aria_expanded = await skyvern_element.get_attr("aria-expanded", mode="dynamic")- except Exception:- return None- if aria_expanded is None:- return None- normalized = str(aria_expanded).casefold()- if normalized not in ("true", "false"):- return None- return normalized == "false"--async def _verify_autocomplete_input_readback(*,skyvern_element: SkyvernElement,+ matched_index: int,matched_label: str,- typed_prefix: str,- pre_action_value: str | None = None,- suggestions_closed: bool | None = None,- selection_state_present: bool | None = None,- allow_unchanged_value: bool = False,- commit_required: bool = False,-) -> AutocompleteCommitOutcome:- actual_value, teardown_observed = await _read_autocomplete_committed_value(skyvern_element)- if teardown_observed:- return AutocompleteCommitOutcome(committed=True, side_effects_observed=True)- if actual_value is None:- return AutocompleteCommitOutcome(committed=False, side_effects_observed=bool(suggestions_closed))-- actual_tokens = _autocomplete_commit_tokens(actual_value)- matched_tokens = _autocomplete_commit_tokens(matched_label)- baseline = typed_prefix if pre_action_value is None else pre_action_value- baseline_tokens = _autocomplete_commit_tokens(baseline)- actual_normalized = " ".join(actual_tokens)- matched_normalized = " ".join(matched_tokens)- baseline_normalized = " ".join(baseline_tokens)- prefix_normalized = " ".join(_autocomplete_commit_tokens(typed_prefix))-- value_changed = actual_normalized != baseline_normalized- side_effects_observed = value_changed or bool(suggestions_closed)- actual_set = set(actual_tokens)- matched_set = set(matched_tokens)- # Field shows the whole label (plus any prefix like "Selected:"), or a shorter read-back that still- # covers at least half the label - a lone token of a multi-word label does not identify the selection.- token_superset = matched_set <= actual_set- token_subset = actual_set <= matched_set and len(actual_set) * 2 >= len(matched_set)- token_match = bool(actual_tokens) and bool(matched_tokens) and (token_superset or token_subset)-- committed = False- if allow_unchanged_value:- committed = token_match or (bool(actual_tokens) and actual_normalized == prefix_normalized)- elif not actual_tokens:- # Chip/token widgets clear the input after committing, so cleared + closed listbox is a commit.- # Under commit-required a dismissal click that clears the field looks identical, so an- # affirmative selection signal is needed on top of the closed listbox.- committed = bool(suggestions_closed) and (not commit_required or bool(selection_state_present))- elif token_match:- label_distinguishable = matched_normalized != baseline_normalized- if label_distinguishable:- committed = value_changed- else:- committed = value_changed or bool(suggestions_closed) or bool(selection_state_present)-- if not committed:- LOG.info(- "Autocomplete read-back did not confirm selected suggestion",- element_id=skyvern_element.get_id(),- expected_length=len(matched_normalized),- actual_length=len(actual_normalized),- value_changed=value_changed,- suggestions_closed=suggestions_closed,- )- return AutocompleteCommitOutcome(committed=committed, side_effects_observed=side_effects_observed)-+) -> bool:+ actual_value = await get_input_value(skyvern_element.get_tag_name(), skyvern_element.get_locator())+ if _normalize_select_shadow_text(actual_value) == _normalize_select_shadow_text(matched_label):+ return True-async def _poll_autocomplete_incremental_elements(- *,- incremental_scraped: IncrementalScrapePage,- cleanup_factory: CleanupElementTreeFunc,- deadline: float | None = None,-) -> list[dict]:- loop = asyncio.get_running_loop()- if deadline is None:- deadline = loop.time() + AUTOCOMPLETE_POLL_DEFAULT_BUDGET_SECONDS- extracted_elements: list[dict] = []- for _ in range(AUTOCOMPLETE_POLL_MAX_ATTEMPTS):- try:- async with asyncio.timeout(AUTOCOMPLETE_POLL_INTERVAL_SECONDS):- probe_count = int(await incremental_scraped.get_incremental_elements_num())- except Exception:- probe_count = 0- # Re-extract on every positive probe rather than only on a count change: a listbox can swap a- # loading placeholder for an option in place, leaving the incremental count constant. We return- # as soon as an option candidate renders, so the common case still extracts at most once.- if probe_count > 0:- incremental_elements = await incremental_scraped.get_incremental_element_tree(cleanup_factory)- if incremental_elements:- extracted_elements = incremental_elements- if _autocomplete_candidates_from_elements(incremental_elements):- return extracted_elements- remaining = deadline - loop.time()- if remaining <= 0:- break- await asyncio.sleep(min(AUTOCOMPLETE_POLL_INTERVAL_SECONDS, remaining))- return extracted_elements+ LOG.info(+ "Autocomplete read-back did not match deterministic option",+ expected_index=matched_index,+ expected_label=matched_label,+ actual_value=actual_value,+ )+ return Falseasync def _reset_autocomplete_for_llm_fallback(@@ -724,7 +520,6 @@ async def _reset_autocomplete_for_llm_fallback(text: str,task: Task,step: Step,- poll_deadline: float | None = None,) -> tuple[IncrementalScrapePage, list[dict], list[dict], str, list[str]]:await current_incremental_scraped.stop_listen_dom_increment()await skyvern_element.input_clear()@@ -732,14 +527,13 @@ async def _reset_autocomplete_for_llm_fallback(incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())await skyvern_element.press_fill(text)- incremental_element = await _poll_autocomplete_incremental_elements(- incremental_scraped=incremental_scraped,- cleanup_factory=clean_and_remove_element_tree_factory(+ await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1, caller="autocomplete.fallback_refill")+ incremental_element = await incremental_scraped.get_incremental_element_tree(+ clean_and_remove_element_tree_factory(task=task,step=step,check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],),- deadline=poll_deadline,)if len(incremental_element) > 0:@@ -757,7 +551,7 @@ async def _reset_autocomplete_for_llm_fallback(if (await dom_after_open.get_skyvern_element_by_id(element_id)).is_interactable()]if len(new_interactable_element_ids) == 0:- raise AutoCompletionCommitFailure(stage="suggestions_never_rendered")+ raise NoIncrementalElementFoundForAutoCompletion(element_id=skyvern_element.get_id(), text=text)LOG.info("New elements detected after resetting autocomplete fallback input",@@ -1388,7 +1182,7 @@ def _plan_tel_text(*, is_tel: bool, is_secret: bool, value: str, pattern: str |return value, False, is_tel and not is_secret-async def _is_org_flag_enabled(task: Task, flag: str, log_label: str) -> bool:+async def _is_tel_digit_fix_enabled(task: Task) -> bool:organization_id = task.organization_idif not organization_id:return False@@ -1399,44 +1193,20 @@ async def _is_org_flag_enabled(task: Task, flag: str, log_label: str) -> bool:# Bucket by org (not per-run) for a stable, monitorable ramp and clean rollback.return bool(await experimentation_provider.is_feature_enabled_cached(- flag,+ FIX_TEL_INPUT_DIGIT_DROP_FLAG,organization_id,properties={"organization_id": organization_id},))except Exception:LOG.warning(- f"Failed to evaluate {log_label} flag; defaulting to disabled",+ "Failed to evaluate tel-digit-fix flag; defaulting to disabled",organization_id=organization_id,exc_info=True,)return False-async def _is_tel_digit_fix_enabled(task: Task) -> bool:- return await _is_org_flag_enabled(task, FIX_TEL_INPUT_DIGIT_DROP_FLAG, "tel-digit-fix")---async def _is_verify_emerging_select_pick_enabled(task: Task) -> bool:- return await _is_org_flag_enabled(task, VERIFY_EMERGING_SELECT_PICK_FLAG, "verify-emerging-select-pick")---async def _is_masked_input_readback_fix_enabled(task: Task) -> bool:- return await _is_org_flag_enabled(task, FIX_MASKED_INPUT_READBACK_FLAG, "masked-input-readback")---async def _is_surface_blocked_form_advance_enabled(task: Task) -> bool:- return await _is_org_flag_enabled(task, SURFACE_BLOCKED_FORM_ADVANCE_FLAG, "surface-blocked-form-advance")---async def _is_autocomplete_commit_required_enabled(task: Task) -> bool:- return await _is_org_flag_enabled(task, AUTOCOMPLETE_COMMIT_REQUIRED_FLAG, "autocomplete-commit-required")---async def _is_input_text_commit_verification_enabled(task: Task) -> bool:- return await _is_org_flag_enabled(task, VERIFY_INPUT_TEXT_COMMIT_FLAG, "input-text-commit")--async def _resolve_collapse_xp_assignment(experimentation_provider: BaseExperimentationProvider,task: Task,@@ -1647,274 +1417,6 @@ async def _fill_nanp_tel_with_readback(return None-_INPUT_TEXT_MASK_FORMAT_RE = re.compile(r"[\s\-_().]")-_NON_TEXT_INPUT_TYPES = {- "button",- "checkbox",- "color",- "date",- "datetime-local",- "file",- "hidden",- "image",- "month",- "password",- "radio",- "range",- "reset",- "submit",- "tel",- "time",- "week",-}-_INPUT_COMMIT_NAVIGATION_ERRORS = (- "execution context was destroyed",- "navigation",- "target closed",-)-_INPUT_COMMIT_DETACHED_ERRORS = ("detached", "not attached")-_MASK_NORMALIZED_INPUT_TYPES = {"search", "text", "textarea"}-_SET_NATIVE_INPUT_VALUE_JS = """-(element, value) => {- const prototype = element instanceof HTMLTextAreaElement- ? HTMLTextAreaElement.prototype- : HTMLInputElement.prototype;- const setter = Object.getOwnPropertyDescriptor(prototype, "value").set;- setter.call(element, value);- element.dispatchEvent(new Event("input", { bubbles: true }));- element.dispatchEvent(new Event("change", { bubbles: true }));- element.blur();-}-"""---def _normalize_input_text_value(value: str | None) -> str:- return _INPUT_TEXT_MASK_FORMAT_RE.sub("", value or "")---def input_text_values_match(expected_value: str, actual_value: str | None, *, input_type: str) -> bool:- """Decide whether a field committed what was typed.-- A field has committed when its value derives from the typed text. That covers an exact match, a- reformat that only rearranges separators ("1234567890" -> "(123) 456-7890"), and an enrichment- that extends the typed value ("12345" -> "12345-6789"). Only three things are failures: an empty- field, a field still showing unfilled mask placeholders, and a genuinely different value.-- Deriving is read as a prefix rather than a substring: extending what was typed is a commit, but a- value that merely happens to contain it is not.- """- expected_trimmed = expected_value.strip()- actual_trimmed = (actual_value or "").strip()- if not expected_trimmed:- return True- if not actual_trimmed:- return False- if expected_trimmed == actual_trimmed:- return True- if input_type not in _MASK_NORMALIZED_INPUT_TYPES:- return False- expected_normalized = _normalize_input_text_value(expected_trimmed).casefold()- actual_normalized = _normalize_input_text_value(actual_trimmed).casefold()- if not expected_normalized:- # Nothing but separators was typed, so only separators may come back - anything else is a- # different value, not a reformat of this one.- return not actual_normalized- # An all-placeholder commit ("_____") normalizes away to nothing: the mask took the keystrokes- # and kept none of them.- if not actual_normalized:- return False- return actual_normalized.startswith(expected_normalized)---def _is_input_commit_navigation_error(exc: PlaywrightError) -> bool:- return any(message in str(exc).lower() for message in _INPUT_COMMIT_NAVIGATION_ERRORS)---def _is_input_commit_detached_error(exc: PlaywrightError) -> bool:- return any(message in str(exc).lower() for message in _INPUT_COMMIT_DETACHED_ERRORS)---def _is_plausible_autocomplete_rewrite(expected_value: str, actual_value: str | None) -> bool:- # A suggestion rewrite contains or extends the typed text; the reverse containment- # (actual is a prefix/fragment of expected) is a dropped-input failure, not a rewrite.- expected_trimmed = expected_value.strip().casefold()- actual_trimmed = (actual_value or "").strip().casefold()- return bool(expected_trimmed and actual_trimmed) and expected_trimmed in actual_trimmed---_OPTION_UI_ROLES = {"listbox", "option"}---def _incremental_elements_contain_option_ui(elements: list[dict]) -> bool:- stack: list[dict] = [element for element in elements if isinstance(element, dict)]- while stack:- element = stack.pop()- if str(element.get("tagName") or "").lower() == "li":- return True- attributes = element.get("attributes")- if isinstance(attributes, dict) and str(attributes.get("role") or "").lower() in _OPTION_UI_ROLES:- return True- children = element.get("children")- if isinstance(children, list):- stack.extend(child for child in children if isinstance(child, dict))- return False---async def _read_generic_input_commit(- *,- skyvern_element: SkyvernElement,- dom: DomUtil,- tag_name: str,- input_type: str,- expected_value: str,- allow_autocomplete_rewrite: bool,- allow_reresolve: bool,-) -> tuple[bool, str | None, SkyvernElement]:- try:- actual_value = await get_input_value(tag_name=tag_name, locator=skyvern_element.get_locator())- except PlaywrightError as exc:- if _is_input_commit_navigation_error(exc):- return True, None, skyvern_element- if not allow_reresolve or not _is_input_commit_detached_error(exc):- raise- skyvern_element = await dom.get_skyvern_element_by_id(skyvern_element.get_id())- actual_value = await get_input_value(tag_name=tag_name, locator=skyvern_element.get_locator())- committed = input_text_values_match(expected_value, actual_value, input_type=input_type)- if not committed and allow_autocomplete_rewrite:- committed = _is_plausible_autocomplete_rewrite(expected_value, actual_value)- return committed, actual_value, skyvern_element---async def _verify_generic_input_commit_with_escalation(- *,- skyvern_element: SkyvernElement,- skyvern_frame: SkyvernFrame,- dom: DomUtil,- tag_name: str,- input_type: str,- expected_value: str,- allow_autocomplete_rewrite: bool,-) -> ActionFailure | None:- # React-style inputs re-render nodes on every value change, so each escalation stage- # gets its own re-resolve allowance instead of one shared across the ladder.- committed, actual_value, skyvern_element = await _read_generic_input_commit(- skyvern_element=skyvern_element,- dom=dom,- tag_name=tag_name,- input_type=input_type,- expected_value=expected_value,- allow_autocomplete_rewrite=allow_autocomplete_rewrite,- allow_reresolve=True,- )- if committed:- return None-- # A masked or debounced field can land its value a beat after the keystrokes settle. Let it- # before escalating, or the clear+retype below destroys a value that was about to commit.- await skyvern_frame.safe_wait_for_animation_end(caller="input_text.commit_readback")- committed, actual_value, skyvern_element = await _read_generic_input_commit(- skyvern_element=skyvern_element,- dom=dom,- tag_name=tag_name,- input_type=input_type,- expected_value=expected_value,- allow_autocomplete_rewrite=allow_autocomplete_rewrite,- allow_reresolve=True,- )- if committed:- return None-- await skyvern_element.input_clear()- await skyvern_element.input_sequentially(text=expected_value)- committed, actual_value, skyvern_element = await _read_generic_input_commit(- skyvern_element=skyvern_element,- dom=dom,- tag_name=tag_name,- input_type=input_type,- expected_value=expected_value,- allow_autocomplete_rewrite=allow_autocomplete_rewrite,- allow_reresolve=True,- )- if committed:- return None-- await skyvern_element.input_fill(text=expected_value)- committed, actual_value, skyvern_element = await _read_generic_input_commit(- skyvern_element=skyvern_element,- dom=dom,- tag_name=tag_name,- input_type=input_type,- expected_value=expected_value,- allow_autocomplete_rewrite=allow_autocomplete_rewrite,- allow_reresolve=True,- )- if committed:- return None-- await skyvern_element.get_locator().evaluate(_SET_NATIVE_INPUT_VALUE_JS, expected_value)- committed, actual_value, skyvern_element = await _read_generic_input_commit(- skyvern_element=skyvern_element,- dom=dom,- tag_name=tag_name,- input_type=input_type,- expected_value=expected_value,- allow_autocomplete_rewrite=allow_autocomplete_rewrite,- allow_reresolve=True,- )- if committed:- return None-- expected_length = len(_normalize_input_text_value(expected_value))- actual_length = len(_normalize_input_text_value(actual_value))- LOG.warning(- "Input text commit mismatch after fallback ladder",- element_id=skyvern_element.get_id(),- expected_length=expected_length,- actual_length=actual_length,- )- return ActionFailure(InputTextCommitMismatch(expected_length=expected_length, actual_length=actual_length))---async def _verify_generic_input_commit(- *,- skyvern_element: SkyvernElement,- skyvern_frame: SkyvernFrame,- dom: DomUtil,- tag_name: str,- input_type: str,- expected_value: str,- allow_autocomplete_rewrite: bool = False,-) -> ActionFailure | None:- try:- return await _verify_generic_input_commit_with_escalation(- skyvern_element=skyvern_element,- skyvern_frame=skyvern_frame,- dom=dom,- tag_name=tag_name,- input_type=input_type,- expected_value=expected_value,- allow_autocomplete_rewrite=allow_autocomplete_rewrite,- )- except PlaywrightError as exc:- if _is_input_commit_navigation_error(exc):- return None- if _is_input_commit_detached_error(exc):- LOG.info(- "Element detached during input commit verification; skipping verification",- element_id=skyvern_element.get_id(),- )- return None- raise---def _is_generic_input_commit_candidate(*, tag_name: str, live_input_type: str) -> str | None:- if tag_name == "textarea":- return "textarea"- if tag_name != InteractiveElement.INPUT:- return None- return live_input_type if live_input_type not in _NON_TEXT_INPUT_TYPES else None--async def _log_tel_fallback_fill_digit_counts(*, skyvern_element: SkyvernElement, tag_name: str, expected_value: str, task_id: str | None, step_id: str | None) -> None:@@ -2058,66 +1560,6 @@ async def _fill_card_number_with_readback()-# Character an input-mask library leaves in unfilled slots (jQuery Inputmask, Cleave, and component-library-# masked inputs generally).-_MASK_PLACEHOLDER_CHAR = "_"---def _alnum(value: str | None) -> str:- return "".join(ch for ch in (value or "") if ch.isalnum())---def _has_input_mask_evidence(*, pattern: str | None, placeholder: str | None) -> bool:- # Only treat a field as masked when it advertises a constraint: an HTML `pattern`, or a placeholder- # rendered from mask slots (e.g. "_____", "___-__-____"). Keeps unrelated inputs untouched.- if pattern:- return True- return placeholder is not None and _MASK_PLACEHOLDER_CHAR in placeholder---def _committed_value_is_masked_incomplete(expected_value: str, committed_value: str | None) -> bool:- # The mask accepted keystrokes but did not commit them: reads back empty (only separators), or still- # shows unfilled placeholder slots while holding fewer characters than were typed. Compares character- # counts only — never the raw value, which may be a secret. A placeholder char inside an otherwise- # complete value (e.g. "foo_bar") is not treated as incomplete.- expected_alnum = _alnum(expected_value)- if not expected_alnum:- return False- committed_alnum = _alnum(committed_value)- if not committed_alnum:- return True- return (- committed_value is not None- and _MASK_PLACEHOLDER_CHAR in committed_value- and len(committed_alnum) < len(expected_alnum)- )---async def _verify_masked_input_after_fill(- *,- skyvern_element: SkyvernElement,- skyvern_frame: SkyvernFrame,- tag_name: str,- expected_value: str,- known_masked: bool,-) -> None:- committed_value = await get_input_value(tag_name, skyvern_element.get_locator())- triggered = known_masked or (committed_value is not None and _MASK_PLACEHOLDER_CHAR in committed_value)- if not triggered or not _committed_value_is_masked_incomplete(expected_value, committed_value):- return-- # A masked value can land a beat after the keystrokes settle; let the field commit before verdict.- await skyvern_frame.safe_wait_for_animation_end(caller="input_text.masked_readback")- committed_value = await get_input_value(tag_name, skyvern_element.get_locator())- if not _committed_value_is_masked_incomplete(expected_value, committed_value):- return-- raise MaskedInputReadbackMismatch(- expected_char_count=len(_alnum(expected_value)),- committed_char_count=len(_alnum(committed_value)),- )--# Native input types whose DOM .value round-trips typed text exactly, so an exact read-back is meaningful.# password/text/email/search/url and an untyped input default to a text field; tel/number/date-like inputs# normalize or auto-format their value and are excluded, as are textarea/contenteditable/select (non-input@@ -2507,12 +1949,7 @@ async def check_date_format(class AutoCompletionResult(BaseModel):auto_completion_attempt: bool = Falseincremental_elements: list[dict] = []- action_result: ActionResult | None = None- stage_outcome: str | None = None- # A click landed and mutated the field but the commit could not be verified. Once this is- # set the field is already dirty, so the caller must stop trying further selections instead- # of clicking more options on top of it.- committed_side_effects: bool = False+ action_result: ActionResult = ActionSuccess()class ScopedXhrDownloadCapture:@@ -3300,91 +2737,6 @@ async def _retarget_disabled_element_for_click(return child_element-_SUBMIT_LIKE_TEXT = re.compile(- r"\b(next|submit|continue|proceed|confirm|finish|save|apply|update|complete|register|sign\s?up|log\s?in)\b",- re.IGNORECASE,-)--_BLOCKED_FORM_ADVANCE_MESSAGE = (- "Clicked '{label}' but the page did not advance and no new elements appeared. The form is likely "- "blocked by a validation error — inspect the visible fields for invalid or required inputs and any "- "error messages, resolve them, and only then click to advance again instead of repeating this click."-)---def _submit_like_label(element_dict: dict) -> str:- attributes = element_dict.get("attributes") or {}- for candidate in (- element_dict.get("text"),- attributes.get("value"),- attributes.get("aria-label"),- attributes.get("title"),- ):- text = str(candidate or "").strip()- if text:- return text- return "the button"---def _click_is_submit_like(element_dict: dict) -> bool:- attributes = element_dict.get("attributes") or {}- tag = str(element_dict.get("tagName") or "").lower()- input_type = str(attributes.get("type") or "").lower()- if tag == "input" and input_type in {"submit", "image"}:- return True- role = str(attributes.get("role") or attributes.get("aria-role") or "").lower()- # A bare <a> is a navigation link (a real one changes the URL and is excluded by the- # caller's URL-change check); only an <a role="button"> counts as a submit control.- is_button_like = tag == "button" or role == "button" or (tag == "input" and input_type == "button")- if not is_button_like:- return False- label = " ".join(- str(part)- for part in (- element_dict.get("text"),- attributes.get("value"),- attributes.get("aria-label"),- attributes.get("title"),- )- if part- )- return bool(_SUBMIT_LIKE_TEXT.search(label))---async def _maybe_flag_blocked_form_advance(- *,- results: list[ActionResult],- skyvern_element: SkyvernElement,- page: Page,- original_url: str,- incremental_scraped: IncrementalScrapePage,- task: Task,-) -> None:- """Surface a diagnostic when a submit-like click neither navigated nor revealed new elements.-- The signal flows to the next step's planner through ActionResult.followup_message (see- get_action_history), directing the agent to resolve the blocking validation instead of- re-clicking a button that is not advancing the form.- """- if not results or not isinstance(results[-1], ActionSuccess):- return- element_dict = skyvern_element.get_element_dict()- if not _click_is_submit_like(element_dict):- return- if page.url != original_url:- return- if not await _is_surface_blocked_form_advance_enabled(task):- return- if await incremental_scraped.get_incremental_elements_num() > 0:- return- results[-1].needs_followup = True- results[-1].followup_message = _BLOCKED_FORM_ADVANCE_MESSAGE.format(label=_submit_like_label(element_dict))- LOG.info(- "Surfacing blocked-form-advance diagnostic after a non-advancing submit-like click",- element_id=skyvern_element.get_id(),- )--@traced(name="skyvern.agent.action.click")async def handle_click_action(action: actions.ClickAction,@@ -3543,17 +2895,6 @@ async def handle_click_action(results.append(sequential_click_result)return results- # No navigation and no dropdown surfaced: flag a non-advancing submit-like click so the- # next step's planner resolves the blocking validation instead of re-clicking.- await _maybe_flag_blocked_form_advance(- results=results,- skyvern_element=skyvern_element,- page=page,- original_url=original_url,- incremental_scraped=incremental_scraped,- task=task,- )-except Exception:LOG.warning("Failed to do sequential logic for the click action, skipping",@@ -4100,7 +3441,6 @@ async def handle_input_text_action(incremental_element: list[dict] = []auto_complete_hacky_flag: bool = False- autocomplete_option_signal: bool = Falseinput_or_select_context = await _get_input_or_select_context(action=action,@@ -4268,8 +3608,7 @@ async def handle_input_text_action()return [ActionFailure(InputToReadonlyElement(element_id=skyvern_element.get_id()))]- live_input_type = ((await skyvern_element.get_attr("type")) or "text").lower()- is_tel = live_input_type == "tel"+ is_tel = await skyvern_element.get_attr("type") == "tel"candidate_card_digits = _card_number_digits(text)is_card_number_input = _is_probable_card_number(candidate_card_digits) and await _is_card_number_field(skyvern_element@@ -4375,11 +3714,8 @@ async def handle_input_text_action("Find a blocking element to the current element, going to input on the blocking element",)if await blocking_element.is_editable():- blocking_tag_name = blocking_element.get_tag_name()- blocking_input_type = ((await blocking_element.get_attr("type")) or "text").lower()skyvern_element = blocking_element- tag_name = blocking_tag_name- live_input_type = blocking_input_type+ tag_name = blocking_element.get_tag_name()if used_bare_nanp:# The tel plan read constraints from the original element; re-derive them from# the element actually being filled.@@ -4438,7 +3774,7 @@ async def handle_input_text_action(if len(text) == 0:return [ActionSuccess()]- if tag_name == InteractiveElement.INPUT and live_input_type == "date":+ if tag_name == InteractiveElement.INPUT and await skyvern_element.get_attr("type") == "date":try:action.set_has_mini_agent()text = await check_date_format(@@ -4512,32 +3848,7 @@ async def handle_input_text_action(and secret_input_type in _EXACT_VALUE_INPUT_TYPES)secret_maxlength = await skyvern_element.get_attr("maxlength") if verify_secret_input else None-- # Masked inputs (postal code, SSN) can report a successful fill while the mask committed nothing.- # Read the value back for masked fields and, on an empty/placeholder commit, retype once. Scoped to- # non-credential fields: a credential value is handled by the exact secret read-back above.- verify_masked_input_after_fill = False- mask_attr_evidence = False- if not is_tel and not verify_secret_input and await _is_masked_input_readback_fix_enabled(task):- mask_attr_evidence = _has_input_mask_evidence(- pattern=await skyvern_element.get_attr("pattern"),- placeholder=await skyvern_element.get_attr("placeholder"),- )- verify_masked_input_after_fill = True-- generic_commit_input_type = None- if (- not is_tel- and not card_expected_digits- and not verify_secret_input- and await _is_input_text_commit_verification_enabled(task)- ):- generic_commit_input_type = _is_generic_input_commit_candidate(- tag_name=tag_name,- live_input_type=live_input_type,- )await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())- input_context_destroyed = Falsetry:# Read-back/recover priority: card > tel > secret > masked > plain fill (each gate is mutually exclusive).@@ -4577,41 +3888,6 @@ async def handle_input_text_action()if secret_failure is not None:return [secret_failure]- elif verify_masked_input_after_fill:- await skyvern_element.input_sequentially(text=text)- # Read the committed value back; on an empty/placeholder commit, refocus, clear and retype- # once. A second failure fails the action with a diagnostic rather than a silent empty fill.- try:- await _verify_masked_input_after_fill(- skyvern_element=skyvern_element,- skyvern_frame=skyvern_frame,- tag_name=tag_name,- expected_value=text,- known_masked=mask_attr_evidence,- )- except MaskedInputReadbackMismatch:- await skyvern_element.get_locator().focus(timeout=timeout)- await skyvern_element.input_clear()- await skyvern_element.input_sequentially(text=text)- try:- # Attempt 1 only raises once the field is proven masked, so the retry keeps that- # verdict: a mask that swallows the retype reads back clean of placeholder chars.- await _verify_masked_input_after_fill(- skyvern_element=skyvern_element,- skyvern_frame=skyvern_frame,- tag_name=tag_name,- expected_value=text,- known_masked=True,- )- except MaskedInputReadbackMismatch as mismatch:- LOG.warning(- "Masked input read-back mismatch after retry",- action=action,- element_id=skyvern_element.get_id(),- expected_char_count=mismatch.expected_char_count,- committed_char_count=mismatch.committed_char_count,- )- return [ActionFailure(mismatch)]else:await skyvern_element.input_sequentially(text=text)if log_tel_fallback_readback:@@ -4632,7 +3908,6 @@ async def handle_input_text_action()if len(incremental_element) > 0:auto_complete_hacky_flag = True- autocomplete_option_signal = _incremental_elements_contain_option_ui(incremental_element)if (input_or_select_contextand input_or_select_context.is_search_bar@@ -4641,6 +3916,7 @@ async def handle_input_text_action(LOG.info("Detected target-matching dropdown after search-bar input; attempting custom selection",element_id=skyvern_element.get_id(),+ target_value=text,)action.set_has_mini_agent()select_result = await sequentially_select_from_dropdown(@@ -4710,7 +3986,6 @@ async def handle_input_text_action(or "navigation" in error_messageor "target closed" in error_message):- input_context_destroyed = True# These are expected during page navigation/auto-submit, silently continueLOG.debug("Playwright error during incremental element processing (likely page navigation)",@@ -4735,19 +4010,6 @@ async def handle_input_text_action(# Always stop listeningawait incremental_scraped.stop_listen_dom_increment()- if generic_commit_input_type is not None and not input_context_destroyed:- commit_failure = await _verify_generic_input_commit(- skyvern_element=skyvern_element,- skyvern_frame=skyvern_frame,- dom=dom,- tag_name=tag_name,- input_type=generic_commit_input_type,- expected_value=text,- allow_autocomplete_rewrite=autocomplete_option_signal,- )- if commit_failure is not None:- return [commit_failure]-return [ActionSuccess()]except Exception as e:# Handle any other unexpected errors during text input@@ -6464,11 +5726,8 @@ async def choose_auto_completion_dropdown(preserved_elements: list[dict] | None = None,relevance_threshold: float = 0.8,is_location_input: bool = False,- commit_required: bool = False,- verify_commit: bool = True,collapse_autocomplete_fanout_enabled: bool = False,action: InputTextAction | None = None,- poll_deadline: float | None = None,) -> AutoCompletionResult:preserved_elements = preserved_elements or []clear_input = True@@ -6481,12 +5740,12 @@ async def choose_auto_completion_dropdown(try:await skyvern_element.press_fill(text)- incremental_element = await _poll_autocomplete_incremental_elements(- incremental_scraped=incremental_scraped,- cleanup_factory=clean_and_remove_element_tree_factory(+ # wait for new elemnts to load+ await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1, caller="autocomplete.fill")+ incremental_element = await incremental_scraped.get_incremental_element_tree(+ clean_and_remove_element_tree_factory(task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]),- deadline=poll_deadline,)# check if elements in preserve list are still on the page@@ -6548,7 +5807,7 @@ async def choose_auto_completion_dropdown("Autocomplete deterministic option identity failed, resetting input before LLM fallback",element_id=matched_element_id,matched_index=matched_index,- matched_label_length=len(matched_label),+ matched_label=matched_label,)(incremental_scraped,@@ -6566,7 +5825,6 @@ async def choose_auto_completion_dropdown(text=text,task=task,step=step,- poll_deadline=poll_deadline,)result.incremental_elements = copy.deepcopy(fallback_incremental_elements)cleaned_incremental_element = shadow_candidate_elements@@ -6574,45 +5832,32 @@ async def choose_auto_completion_dropdown(LOG.info("Autocomplete deterministic fast path: exact/stem option found, skipping LLM",element_id=matched_element_id,- input_length=len(text),+ input_value=text,matched_index=matched_index,- matched_label_length=len(matched_label),+ matched_label=matched_label,)try:await matched_locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)- # This deterministic read-back predates the commit-verification feature- # (it is the one verify site that exists on main), so it stays on- # regardless of verify_commit - gating it would reintroduce the silent- # deterministic-click success this whole change exists to stop.- commit_outcome = await _verify_autocomplete_input_readback(+ if await _verify_autocomplete_input_readback(skyvern_element=skyvern_element,+ matched_index=matched_index,matched_label=matched_label,- typed_prefix=text,- pre_action_value=text,- suggestions_closed=await _autocomplete_suggestions_closed(matched_locator),- selection_state_present=await _autocomplete_selection_state(matched_locator),- commit_required=commit_required,- )- if commit_outcome.committed:+ ):clear_input = Falseresult.action_result = ActionSuccess()return result- result.stage_outcome = "clicked_but_not_committed"- if commit_outcome.side_effects_observed:- result.committed_side_effects = True- raise AutoCompletionCommitFailure(stage="clicked_but_not_committed")LOG.info("Autocomplete deterministic read-back failed, resetting input before LLM fallback",element_id=matched_element_id,matched_index=matched_index,+ matched_label=matched_label,)- except AutoCompletionCommitFailure:- raiseexcept Exception:LOG.info("Autocomplete deterministic fast-path click/read-back failed, falling through to LLM",element_id=matched_element_id,matched_index=matched_index,+ matched_label=matched_label,exc_info=True,)(@@ -6631,7 +5876,6 @@ async def choose_auto_completion_dropdown(text=text,task=task,step=step,- poll_deadline=poll_deadline,)result.incremental_elements = copy.deepcopy(fallback_incremental_elements)cleaned_incremental_element = shadow_candidate_elements@@ -6643,7 +5887,7 @@ async def choose_auto_completion_dropdown("Autocomplete deterministic option detached before click, resetting input before LLM fallback",element_id=matched_element_id,matched_index=matched_index,- matched_label_length=len(matched_label),+ matched_label=matched_label,)(incremental_scraped,@@ -6661,7 +5905,6 @@ async def choose_auto_completion_dropdown(text=text,task=task,step=step,- poll_deadline=poll_deadline,)result.incremental_elements = copy.deepcopy(fallback_incremental_elements)cleaned_incremental_element = shadow_candidate_elements@@ -6681,62 +5924,18 @@ async def choose_auto_completion_dropdown(LOG.info("Location auto-completion fast path: single option found, skipping LLM",element_id=fast_path_element_id,- input_length=len(text),+ input_value=text,)try:await fast_path_locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)- if verify_commit:- commit_outcome = await _verify_autocomplete_input_readback(- skyvern_element=skyvern_element,- matched_label=only_element.get("text") or "",- typed_prefix=text,- pre_action_value=text,- suggestions_closed=await _autocomplete_suggestions_closed(fast_path_locator),- selection_state_present=await _autocomplete_selection_state(fast_path_locator),- commit_required=commit_required,- )- else:- commit_outcome = AutocompleteCommitOutcome(committed=True)- if commit_outcome.committed:- clear_input = False- result.action_result = ActionSuccess()- return result- result.stage_outcome = "clicked_but_not_committed"- if commit_outcome.side_effects_observed:- result.committed_side_effects = True- raise AutoCompletionCommitFailure(stage="clicked_but_not_committed")- LOG.info(- "Location fast-path read-back failed, resetting input before LLM fallback",- element_id=fast_path_element_id,- )- except AutoCompletionCommitFailure:- raise+ clear_input = False+ result.action_result = ActionSuccess()+ return resultexcept Exception:LOG.info("Location fast-path click failed, falling through to LLM",element_id=fast_path_element_id,- exc_info=True,)- (- incremental_scraped,- fallback_incremental_elements,- shadow_candidate_elements,- html,- new_interactable_element_ids,- ) = await _reset_autocomplete_for_llm_fallback(- current_incremental_scraped=incremental_scraped,- skyvern_frame=skyvern_frame,- skyvern_element=skyvern_element,- page=page,- scraped_page=scraped_page,- dom=dom,- text=text,- task=task,- step=step,- poll_deadline=poll_deadline,- )- result.incremental_elements = copy.deepcopy(fallback_incremental_elements)- cleaned_incremental_element = shadow_candidate_elementsif not html:html = incremental_scraped.build_html_tree(cleaned_incremental_element)@@ -6753,7 +5952,7 @@ async def choose_auto_completion_dropdown(if (await dom_after_open.get_skyvern_element_by_id(element_id)).is_interactable()]if len(new_interactable_element_ids) == 0:- raise AutoCompletionCommitFailure(stage="suggestions_never_rendered")+ raise NoIncrementalElementFoundForAutoCompletion(element_id=skyvern_element.get_id(), text=text)LOG.info("New elements detected after the input",new_elements_ids=new_interactable_element_ids,@@ -6800,31 +5999,11 @@ async def choose_auto_completion_dropdown(if json_response.get("direct_searching", False):LOG.info("Decided to directly search with the current value",- element_id=skyvern_element.get_id(),- input_length=len(text),+ value=text,)await skyvern_element.press_key("Enter")- matched_label = str(json_response.get("value") or "")- # Searching with the typed text leaves the value unchanged by design, so an unchanged- # read-back is a commit here - but not on a field that must commit a real suggestion,- # where it is indistinguishable from Enter doing nothing at all.- if verify_commit:- commit_outcome = await _verify_autocomplete_input_readback(- skyvern_element=skyvern_element,- matched_label=matched_label or text,- typed_prefix=text,- pre_action_value=text,- allow_unchanged_value=not commit_required,- commit_required=commit_required,- )- else:- commit_outcome = AutocompleteCommitOutcome(committed=True)- if commit_outcome.committed:- clear_input = False- result.action_result = ActionSuccess()- return result- result.committed_side_effects = commit_outcome.side_effects_observed- raise AutoCompletionCommitFailure(stage="clicked_but_not_committed")+ clear_input = False+ return resultif not element_id:reasoning = json_response.get("reasoning")@@ -6853,15 +6032,6 @@ async def choose_auto_completion_dropdown(if await locator.count() == 0:raise MissingElement(element_id=element_id)- candidate_match = _autocomplete_candidate_for_element_id(shadow_candidate_elements, element_id)- if candidate_match is None:- raise NoSuitableAutoCompleteOption(- reasoning="Selected element was not a rendered suggestion",- target_value=text,- )- _matched_index, candidate = candidate_match- matched_label = candidate.get("label") or ""-# Use SkyvernElement.click() so we get the full fallback chain# (Playwright click → coordinate click → JavaScript click). Plain# locator.click() can fail when the item or one of its ancestors has@@ -6873,36 +6043,16 @@ async def choose_auto_completion_dropdown()await selected_element.scroll_into_view()await selected_element.click(page=page)- if verify_commit:- commit_outcome = await _verify_autocomplete_input_readback(- skyvern_element=skyvern_element,- matched_label=matched_label,- typed_prefix=text,- pre_action_value=text,- suggestions_closed=await _autocomplete_suggestions_closed(locator),- selection_state_present=await _autocomplete_selection_state(locator),- commit_required=commit_required,- )- else:- commit_outcome = AutocompleteCommitOutcome(committed=True)- if commit_outcome.committed:- clear_input = False- result.action_result = ActionSuccess()- return result- result.committed_side_effects = commit_outcome.side_effects_observed- raise AutoCompletionCommitFailure(stage="clicked_but_not_committed")+ clear_input = False+ return resultexcept Exception as e:- stage = e.stage if isinstance(e, AutoCompletionCommitFailure) else "suggestion_not_matched"LOG.info("Failed to choose the auto completion dropdown",sampling=True,- element_id=skyvern_element.get_id(),- input_length=len(text),- stage=stage,exc_info=True,+ input_value=text,)- result.stage_outcome = stageresult.action_result = ActionFailure(exception=e)return resultfinally:@@ -6923,21 +6073,6 @@ def remove_duplicated_HTML_element(elements: list[dict]) -> list[dict]:return new_elements-def _autocomplete_side_effect_terminal(- *, commit_required: bool, attempt_trail: list[str], element_id: str-) -> ActionResult | None:- LOG.info(- "Autocomplete click mutated the field without a verified commit; stopping to avoid clicking further options on a dirty field",- element_id=element_id,- attempt_trail=attempt_trail,- )- if commit_required:- return ActionFailure(- AutoCompletionCommitFailure(stage="clicked_but_not_committed", attempt_trail=attempt_trail)- )- return None--async def input_or_auto_complete_input(input_or_select_context: InputOrSelectContext,scraped_page: ScrapedPage,@@ -6965,16 +6100,6 @@ async def input_or_auto_complete_input(current_attemp = 0current_value = textresult = AutoCompletionResult()- attempt_trail: list[str] = []- is_location = input_or_select_context.is_location_input or False- # Commit verification (and everything downstream of a failed read-back) only runs behind the- # flag, so flag-off orgs keep the pre-verification behavior: a selection click is assumed- # committed, with no read-back cost and no extra LLM fall-through.- verify_commit = await _is_autocomplete_commit_required_enabled(task)- # A location-search widget can be both. Search bars keep their documented fall-through, so they- # never enter strict commit mode - otherwise the search-bar branch below returns a hard failure.- commit_required = is_location and not input_or_select_context.is_search_bar and verify_commit- poll_deadline = asyncio.get_running_loop().time() + AUTOCOMPLETE_TOTAL_DEADLINE_SECONDSwhile current_attemp < MAX_AUTO_COMPLETE_ATTEMP:current_attemp += 1@@ -6984,9 +6109,9 @@ async def input_or_auto_complete_input(LOG.info("Try the potential value for auto completion",sampling=True,- attempt=current_attemp,- input_length=len(current_value),+ input_value=current_value,)+ is_location = input_or_select_context.is_location_input or Falseresult = await choose_auto_completion_dropdown(context=input_or_select_context,page=page,@@ -6998,30 +6123,19 @@ async def input_or_auto_complete_input(step=step,task=task,is_location_input=is_location,- commit_required=commit_required,- verify_commit=verify_commit,collapse_autocomplete_fanout_enabled=collapse_autocomplete_fanout_enabled,action=action,- poll_deadline=poll_deadline,)if isinstance(result.action_result, ActionSuccess):return ActionSuccess()- attempt_trail.append(f"attempt_{len(attempt_trail) + 1}:{result.stage_outcome or 'suggestion_not_matched'}")if input_or_select_context.is_search_bar:- # commit_required is derived with `and not is_search_bar`, so it is always False here;- # search bars keep their documented fall-through.LOG.info("Stop generating potential values for the auto-completion since it's a search bar",context=input_or_select_context,)return None- if result.committed_side_effects:- return _autocomplete_side_effect_terminal(- commit_required=commit_required, attempt_trail=attempt_trail, element_id=skyvern_element.get_id()- )-tried_values.append(current_value)whole_new_elements.extend(result.incremental_elements)@@ -7043,7 +6157,7 @@ async def input_or_auto_complete_input(LOG.info("Ask LLM to give potential values based on the current value",- input_length=len(current_value),+ current_value=current_value,potential_value_count=AUTO_COMPLETION_POTENTIAL_VALUES_COUNT,)if collapse_autocomplete_fanout_enabled and action is not None:@@ -7058,14 +6172,13 @@ async def input_or_auto_complete_input(if not value:LOG.info("Empty potential value, skip this attempt",- attempt=current_attemp,+ value=each_value,)continueLOG.info("Try the potential value for auto completion",sampling=True,- attempt=current_attemp,- input_length=len(value),+ input_value=value,)result = await choose_auto_completion_dropdown(context=input_or_select_context,@@ -7078,20 +6191,11 @@ async def input_or_auto_complete_input(step=step,task=task,is_location_input=is_location,- commit_required=commit_required,- verify_commit=verify_commit,collapse_autocomplete_fanout_enabled=collapse_autocomplete_fanout_enabled,action=action,- poll_deadline=poll_deadline,)if isinstance(result.action_result, ActionSuccess):return ActionSuccess()- attempt_trail.append(f"attempt_{len(attempt_trail) + 1}:{result.stage_outcome or 'suggestion_not_matched'}")-- if result.committed_side_effects:- return _autocomplete_side_effect_terminal(- commit_required=commit_required, attempt_trail=attempt_trail, element_id=skyvern_element.get_id()- )tried_values.append(value)whole_new_elements.extend(result.incremental_elements)@@ -7100,8 +6204,8 @@ async def input_or_auto_complete_input(if current_attemp < MAX_AUTO_COMPLETE_ATTEMP:LOG.info("Ask LLM to tweak the current value based on tried input values",+ current_value=current_value,current_attemp=current_attemp,- input_length=len(current_value),)cleaned_new_elements = remove_duplicated_HTML_element(whole_new_elements)prompt = prompt_engine.load_prompt(@@ -7120,15 +6224,12 @@ async def input_or_auto_complete_input(context_reasoning = json_respone.get("reasoning")new_current_value = json_respone.get("tweaked_value", "")if not new_current_value:- if commit_required:- stage = result.stage_outcome or "suggestion_not_matched"- return ActionFailure(AutoCompletionCommitFailure(stage=stage, attempt_trail=attempt_trail))return ActionFailure(ErrEmptyTweakValue(reasoning=context_reasoning, current_value=current_value))LOG.info("Ask LLM tweaked the current value with a new value",field_information=input_or_select_context.field,- previous_length=len(current_value),- new_length=len(new_current_value),+ current_value=current_value,+ new_value=new_current_value,)current_value = new_current_value@@ -7137,38 +6238,26 @@ async def input_or_auto_complete_input(LOG.info("Auto completion attempts exhausted, trying discover-all-options fallback",element_id=skyvern_element.get_id(),- input_length=len(text),+ original_text=text,)- try:- fallback_result = await discover_and_select_from_full_dropdown(- context=input_or_select_context,- page=page,- scraped_page=scraped_page,- dom=dom,- original_text=text,- skyvern_element=skyvern_element,- step=step,- task=task,- commit_required=commit_required,- verify_commit=verify_commit,- )- except AutoCompletionCommitFailure as fallback_failure:- fallback_stage = fallback_failure.stage- else:- if isinstance(fallback_result, ActionSuccess):- return fallback_result- fallback_stage = "suggestion_not_matched"- attempt_trail.append(f"attempt_{len(attempt_trail) + 1}:{fallback_stage}")+ fallback_result = await discover_and_select_from_full_dropdown(+ context=input_or_select_context,+ page=page,+ scraped_page=scraped_page,+ dom=dom,+ original_text=text,+ skyvern_element=skyvern_element,+ step=step,+ task=task,+ )+ if fallback_result is not None:+ return fallback_resultLOG.info("Auto completion didn't finish, this might leave the input value to be empty.",sampling=True,context=input_or_select_context,- attempt_trail=attempt_trail,)- if commit_required:- final_stage = attempt_trail[-1].split(":", 1)[1] if attempt_trail else "suggestion_not_matched"- return ActionFailure(AutoCompletionCommitFailure(stage=final_stage, attempt_trail=attempt_trail))return None@@ -7183,8 +6272,6 @@ async def discover_and_select_from_full_dropdown(step: Step,task: Task,relevance_threshold: float = 0.6,- commit_required: bool = False,- verify_commit: bool = True,) -> ActionResult | None:"""Fallback for auto-completion: clear input, click/ArrowDown to reveal all options,then ask LLM to pick the best semantic match from actual dropdown values."""@@ -7293,7 +6380,7 @@ async def discover_and_select_from_full_dropdown(LOG.info("Discover fallback: asking LLM to pick from actual options",element_id=skyvern_element.get_id(),- input_length=len(original_text),+ original_text=original_text,)json_response = await app.AUTO_COMPLETION_LLM_API_HANDLER(prompt=prompt, step=step, prompt_name="auto-completion-choose-option"@@ -7316,7 +6403,7 @@ async def discover_and_select_from_full_dropdown("Discover fallback: found suitable option, typing discovered value to trigger auto-completion",element_id=element_id,relevance_float=relevance_float,- discovered_value_length=len(discovered_value),+ discovered_value=discovered_value,)if not discovered_value:@@ -7337,48 +6424,24 @@ async def discover_and_select_from_full_dropdown(try:await skyvern_element.press_key("ArrowDown")await skyvern_element.press_key("Enter")- # This path fills the field with the option text itself, so a widget that commits that text- # verbatim reads back identical to the fill and the value alone proves nothing. aria-expanded- # is the tiebreak: closed means Enter took, open means it did not. A widget that exposes no- # expanded state at all - the shadow-DOM case this fallback exists for - leaves the verbatim- # read-back as the only evidence there is, so accept it rather than fail a committed field.- if verify_commit:- suggestions_closed = await _autocomplete_input_reports_closed(skyvern_element)- commit_outcome = await _verify_autocomplete_input_readback(- skyvern_element=skyvern_element,- matched_label=discovered_value,- typed_prefix=discovered_value,- pre_action_value=discovered_value,- suggestions_closed=suggestions_closed,- allow_unchanged_value=suggestions_closed is None,- commit_required=commit_required,- )- else:- commit_outcome = AutocompleteCommitOutcome(committed=True)- if commit_outcome.committed:- LOG.info(- "Discover fallback: selected option via keyboard",- element_id=skyvern_element.get_id(),- )- return ActionSuccess()- raise AutoCompletionCommitFailure(stage="clicked_but_not_committed")- except AutoCompletionCommitFailure:- raise+ LOG.info(+ "Discover fallback: selected option via keyboard",+ discovered_value=discovered_value,+ )+ return ActionSuccess()except Exception:LOG.info("Discover fallback: keyboard selection failed",- discovered_value_length=len(discovered_value),exc_info=True,+ discovered_value=discovered_value,)return None- except AutoCompletionCommitFailure:- raiseexcept Exception:LOG.warning("Discover fallback failed",- input_length=len(original_text),exc_info=True,+ original_text=original_text,)return Nonefinally:@@ -8356,74 +7419,6 @@ def _collect_new_roots(element: dict, new_ids: set[str], out: list[dict]) -> Non_collect_new_roots(child, new_ids, out)-def _custom_select_fallback_subtrees(elements: list[dict], current_element_id: str) -> list[dict]:- anchor: dict | None = None- anchor_is_open = False- listboxes: list[dict] = []- anchor_descendant_listboxes: list[dict] = []- all_nodes: list[dict] = []-- queue: deque[tuple[dict, bool, bool]] = deque((element, False, False) for element in elements)- while queue:- node, in_anchor, in_expanded_combobox = queue.popleft()- if not isinstance(node, dict):- continue- attrs = node.get("attributes") or {}- all_nodes.append(node)- role = str(attrs.get("role") or "").lower()- is_anchor = node.get("id") == current_element_id- child_in_anchor = in_anchor or is_anchor- expanded = str(attrs.get("aria-expanded") or "").lower() == "true"- child_in_expanded_combobox = in_expanded_combobox or (role == "combobox" and expanded)-- if is_anchor:- anchor = node- anchor_is_open = expanded or in_expanded_combobox- if role == "listbox":- listboxes.append(node)- if child_in_anchor:- anchor_descendant_listboxes.append(node)-- for child in node.get("children") or []:- queue.append((child, child_in_anchor, child_in_expanded_combobox))-- if anchor is None:- return []-- anchor_attrs = anchor.get("attributes") or {}- controlled_dom_ids = {- reference- for attribute_name in ("aria-controls", "aria-owns")- for reference in str(anchor_attrs.get(attribute_name) or "").split()- if reference- }- anchor_dom_id = str(anchor_attrs.get("id") or "")- controlled_subtrees = [- node- for node in all_nodes- if str((node.get("attributes") or {}).get("id") or "") in controlled_dom_ids- and _custom_select_candidates_from_elements([node])- ]- if controlled_subtrees:- return controlled_subtrees-- linked_listboxes = []- for listbox in listboxes:- attrs = listbox.get("attributes") or {}- listbox_dom_id = str(attrs.get("id") or "")- labelled_by = set(str(attrs.get("aria-labelledby") or "").split())- if listbox_dom_id in controlled_dom_ids or (anchor_dom_id and anchor_dom_id in labelled_by):- linked_listboxes.append(listbox)-- if linked_listboxes:- return linked_listboxes- if anchor_descendant_listboxes:- return anchor_descendant_listboxes- if len(listboxes) == 1 and anchor_is_open:- return listboxes- return []--@traced(name="skyvern.agent.dropdown.select_emerging")async def select_from_emerging_elements(current_element_id: str,@@ -8453,44 +7448,12 @@ async def select_from_emerging_elements(if (await dom_after_open.get_skyvern_element_by_id(element_id)).is_interactable()]- fallback_element_subtrees: list[dict] = []- if len(new_interactable_element_ids) == 0:- # Resolve ownership from the untrimmed tree: trim_element_tree drops aria-controls /- # aria-owns / aria-labelledby / the DOM id, which are exactly the linkage attributes this- # fallback keys off, so resolving against the trimmed tree leaves it inert. Re-trim the- # small resolved subtree set below so the prompt stays trimmed (no page-wide token cost).- fallback_element_subtrees = _custom_select_fallback_subtrees(- scraped_page_after_open.element_tree,- current_element_id,- )- fallback_element_subtrees = trim_element_tree(copy.deepcopy(fallback_element_subtrees))- fallback_candidates = _custom_select_candidates_from_elements(fallback_element_subtrees)- fallback_candidate_ids: list[str] = []- for candidate in fallback_candidates:- fallback_id = candidate.get("element_id")- if fallback_id is not None:- fallback_candidate_ids.append(fallback_id)- new_interactable_element_ids = [- element_id- for element_id in fallback_candidate_ids- if (await dom_after_open.get_skyvern_element_by_id(element_id)).is_interactable()- ]- if new_interactable_element_ids:- LOG.info(- "Found current-page custom-select options after incremental detection returned no elements",- current_element_id=current_element_id,- fallback_option_count=len(new_interactable_element_ids),- )-if len(new_interactable_element_ids) == 0:raise NoIncrementalElementFoundForCustomSelection(element_id=current_element_id)- interactable_element_ids = set(new_interactable_element_ids)# Extract minimal subtrees rooted at new elements — avoids sending the full page DOM# which gets truncated on large pages, losing portal-rendered dropdown items.- new_element_subtrees = fallback_element_subtrees or _extract_new_subtrees(- scraped_page_after_open.element_tree_trimmed, new_element_ids- )+ new_element_subtrees = _extract_new_subtrees(scraped_page_after_open.element_tree_trimmed, new_element_ids)shadow_candidate_elements: list[dict] = []_ctx = skyvern_context.current()lean_enabled = bool(_ctx and _ctx.enable_lean_element_tree)@@ -8533,20 +7496,9 @@ async def select_from_emerging_elements(async def get_readback_scope_element() -> SkyvernElement | None:return await dom_after_open.get_skyvern_element_by_id(current_element_id)- verify_pick_enabled = await _is_verify_emerging_select_pick_enabled(task)- deterministic_candidates = _custom_select_candidates_from_elements(shadow_candidate_elements)- if verify_pick_enabled:- # Restrict deterministic matching to the elements the dropdown actually surfaced. Gated:- # custom-widget option nodes are often not flagged interactable, so flag off keeps main's- # behavior (all candidates) and this ramps with the off-list rejection below.- deterministic_candidates = [- candidate- for candidate in deterministic_candidates- if candidate.get("element_id") in interactable_element_ids- ]deterministic_result = await _select_deterministic_custom_option(target_value=options.target_value,- get_option_candidates=lambda: deterministic_candidates,+ get_option_candidates=lambda: _custom_select_candidates_from_elements(shadow_candidate_elements),field_context=options.model_dump(),page=page,get_skyvern_element=dom_after_open.get_skyvern_element_by_id,@@ -8599,20 +7551,7 @@ async def select_from_emerging_elements(llm_value=value,),)- anchor_input_pick = raw_action_type == ActionType.INPUT_TEXT.value and element_id == current_element_id- # A pick outside the elements the dropdown actually surfaced is a wrong-field click that used to- # proceed on a warning. Rejecting it turns those silent successes into loud failures, so it ramps- # behind a flag rather than shipping to every org at once.- off_list = element_id not in interactable_element_ids and not anchor_input_pick- if off_list and not verify_pick_enabled:- # Keep the pre-flag warning so the reject rate is measurable before ramping the flag on.- LOG.warning(- "Emerging-select pick is outside the surfaced interactable elements; proceeding because the verify flag is off",- element_id=element_id,- current_element_id=current_element_id,- )- off_list_pick = off_list and verify_pick_enabled- if not element_id or raw_action_type not in (ActionType.CLICK.value, ActionType.INPUT_TEXT.value) or off_list_pick:+ if not element_id or raw_action_type not in (ActionType.CLICK.value, ActionType.INPUT_TEXT.value):raise _no_match_exception_for_dropdown(reasoning=json_response.get("reasoning"),target_value=options.target_value,@@ -8621,6 +7560,14 @@ async def select_from_emerging_elements()action_type = ActionType(raw_action_type)+ new_ids_set = set(new_interactable_element_ids)+ if element_id not in new_ids_set:+ LOG.warning(+ "custom-select returned element outside new_interactable_element_ids",+ selected_element_id=element_id,+ new_interactable_count=len(new_ids_set),+ )+if value is not None and action_type == ActionType.INPUT_TEXT:actual_value = get_actual_value_of_parameter_if_secret_with_task(task, value)LOG.info(diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex 764f7da91..a3b3dbe1c 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -2875,160 +2875,6 @@ function asyncSleepFor(ms) {return new Promise((resolve) => setTimeout(resolve, ms));}-const incrementalObserverAttributeFilter = [- "class",- "style",- "hidden",- "aria-expanded",- "aria-hidden",-];-const maxIncrementalShadowRootDepth = 10;-const maxIncrementalShadowRootCount = 200;-const maxIncrementalShadowDiscoveryVisits = 4000;-const incrementalShadowDiscoveryBudgetMs = 25;--function getIncrementalObserverOptions() {- return {- attributes: true,- attributeOldValue: true,- childList: true,- subtree: true,- characterData: true,- };-}--function getIncrementalOptionLikeElements(- root,- recurseObservedShadowRoots = false,-) {- const selector =- '[role="listbox"], [role="option"], [role="listbox"] li, [role="combobox"] li';- const elements = [];- const pending = [root];- const visited = new Set();- while (pending.length > 0) {- const currentRoot = pending.pop();- if (!currentRoot || visited.has(currentRoot)) continue;- visited.add(currentRoot);- if (currentRoot instanceof Element && currentRoot.matches(selector)) {- elements.push(currentRoot);- }- if (currentRoot.querySelectorAll) {- elements.push(...currentRoot.querySelectorAll(selector));- }- if (!recurseObservedShadowRoots) continue;- const possibleHosts =- currentRoot instanceof Element- ? [currentRoot, ...currentRoot.querySelectorAll("*")]- : [...currentRoot.querySelectorAll("*")];- for (const possibleHost of possibleHosts) {- if (- possibleHost.shadowRoot &&- window.globalObservedShadowRoots.has(possibleHost.shadowRoot)- ) {- pending.push(possibleHost.shadowRoot);- }- }- }- return elements;-}--function isIncrementalOptionSubtreeVisible(element) {- return (- !element.hidden &&- element.getAttribute("aria-hidden") !== "true" &&- isElementVisible(element)- );-}--function trackIncrementalOptionVisibility(root) {- for (const optionLikeElement of getIncrementalOptionLikeElements(root)) {- let currentElement = optionLikeElement;- while (currentElement instanceof Element) {- if (window.globalIncrementalVisibilityMap.has(currentElement)) break;- window.globalIncrementalVisibilityMap.set(- currentElement,- isIncrementalOptionSubtreeVisible(currentElement),- );- if (currentElement.parentElement) {- currentElement = currentElement.parentElement;- continue;- }- const currentRoot = currentElement.getRootNode();- currentElement =- currentRoot instanceof ShadowRoot ? currentRoot.host : null;- }- }-}--function observeIncrementalTarget(target) {- window.globalObserverForDOMIncrement.observe(- target,- getIncrementalObserverOptions(),- );- trackIncrementalOptionVisibility(target);-}--function observeIncrementalShadowRoot(shadowRoot, countTowardBudget = true) {- if (window.globalObservedShadowRoots.has(shadowRoot)) return;- window.globalObservedShadowRoots.add(shadowRoot);- if (countTowardBudget) {- window.globalObservedShadowRootCount++;- }- observeIncrementalTarget(shadowRoot);-}--function discoverAndObserveOpenShadowRoots(- startNode,- initialDepth = 0,- deadline = Number.POSITIVE_INFINITY,- countTowardBudget = true,-) {- const pending = [{ node: startNode, depth: initialDepth }];- let currentIndex = 0;- while (- currentIndex < pending.length &&- currentIndex < maxIncrementalShadowDiscoveryVisits &&- (!countTowardBudget ||- window.globalObservedShadowRootCount < maxIncrementalShadowRootCount) &&- performance.now() < deadline- ) {- const { node, depth } = pending[currentIndex++];- if (!node || depth > maxIncrementalShadowRootDepth) continue;-- if (- node instanceof Element &&- node.shadowRoot &&- depth < maxIncrementalShadowRootDepth- ) {- const shadowRoot = node.shadowRoot;- if (!window.globalObservedShadowRoots.has(shadowRoot)) {- observeIncrementalShadowRoot(shadowRoot, countTowardBudget);- }- for (const child of shadowRoot.children) {- pending.push({ node: child, depth: depth + 1 });- }- }-- if (node.children) {- for (const child of node.children) {- pending.push({ node: child, depth });- }- }- }-}--function isNewlyVisibleIncrementalOptionSubtree(node) {- const wasVisible = window.globalIncrementalVisibilityMap.get(node) ?? false;- const isVisible = isIncrementalOptionSubtreeVisible(node);- window.globalIncrementalVisibilityMap.set(node, isVisible);- return (- !wasVisible &&- isVisible &&- getIncrementalOptionLikeElements(node, true).length > 0- );-}-async function addIncrementalNodeToMap(parentNode, childrenNode) {const maxParsedElement = 3000;const maxElementToWait = 100;@@ -3045,7 +2891,10 @@ async function addIncrementalNodeToMap(parentNode, childrenNode) {if (window.globalListnerFlag) {// calculate the depth of targetNode element for sortingconst depth = getElementDomDepth(parentNode);- const parsedNodeTrees = [];+ let newNodesTreeList = [];+ if (window.globalDomDepthMap.has(depth)) {+ newNodesTreeList = window.globalDomDepthMap.get(depth);+ }try {for (const child of childrenNode) {@@ -3063,17 +2912,13 @@ async function addIncrementalNodeToMap(parentNode, childrenNode) {window.globalHoverStylesMap,);if (newNodeTree.length > 0) {- parsedNodeTrees.push(...newNodeTree);+ newNodesTreeList.push(...newNodeTree);}}} catch (error) {_jsConsoleError("Error building incremental element node:", error);}- const currentNodeTrees = window.globalDomDepthMap.get(depth) || [];- window.globalDomDepthMap.set(depth, [- ...currentNodeTrees,- ...parsedNodeTrees,- ]);+ window.globalDomDepthMap.set(depth, newNodesTreeList);}await window.globalParsedElementCounter.add();}@@ -3083,29 +2928,12 @@ if (window.globalObserverForDOMIncrement === undefined) {mutationsList,observer,) {- const shadowDiscoveryDeadline =- performance.now() + incrementalShadowDiscoveryBudgetMs;// TODO: how to detect duplicated recreate element?for (const mutation of mutationsList) {- const node =- mutation.target instanceof ShadowRoot- ? mutation.target.host- : mutation.target;+ const node = mutation.target;if (node.nodeType === Node.TEXT_NODE) continue;const tagName = node.tagName?.toLowerCase();- if (mutation.type === "childList") {- for (const addedNode of mutation.addedNodes) {- if (!(addedNode instanceof Element)) continue;- trackIncrementalOptionVisibility(addedNode);- discoverAndObserveOpenShadowRoots(- addedNode,- 0,- shadowDiscoveryDeadline,- );- }- }-// ignore unique_id change to avoid infinite loop about DOM changesif (mutation.attributeName === "unique_id") continue;@@ -3127,19 +2955,6 @@ if (window.globalObserverForDOMIncrement === undefined) {// we detect the element based on the following rulesswitch (mutation.type) {case "attributes": {- if (- incrementalObserverAttributeFilter.includes(- mutation.attributeName,- ) &&- isNewlyVisibleIncrementalOptionSubtree(node)- ) {- window.globalOneTimeIncrementElements.push({- targetNode: node,- newNodes: [node],- });- await addIncrementalNodeToMap(node, [node]);- break;- }switch (mutation.attributeName) {case "hidden": {if (!node.hidden) {@@ -3235,48 +3050,25 @@ async function startGlobalIncrementalObserver(element = null) {window.globalOneTimeIncrementElements = [];await getHoverStylesMap();window.globalParsedElementCounter = new SafeCounter();- window.globalIncrementalVisibilityMap = new WeakMap();- window.globalObservedShadowRoots = new WeakSet();- window.globalObservedShadowRootCount = 0;- window.globalIncrementalObserverGeneration =- (window.globalIncrementalObserverGeneration || 0) + 1;window.globalObserverForDOMIncrement.takeRecords(); // cleanup the older data+ window.globalObserverForDOMIncrement.observe(document.body, {+ attributes: true,+ attributeOldValue: true,+ childList: true,+ subtree: true,+ characterData: true,+ });- let composedTreeElement = element;- let composedTreeDepth = 0;- const composedTreeRoots = [];- while (- composedTreeElement instanceof Element &&- composedTreeDepth < maxIncrementalShadowRootDepth- ) {- const root = composedTreeElement.getRootNode();- if (!(root instanceof ShadowRoot)) break;- observeIncrementalShadowRoot(root, false);- composedTreeRoots.push({ root, depth: composedTreeDepth });- composedTreeElement = root.host;- composedTreeDepth++;- }-- const anchorDiscoveryDeadline = performance.now() + 50;- for (const { root, depth } of composedTreeRoots) {- discoverAndObserveOpenShadowRoots(root, depth, anchorDiscoveryDeadline);+ // if the element is in shadow DOM, we need to observe the shadow DOM as well+ if (element && element.getRootNode() instanceof ShadowRoot) {+ window.globalObserverForDOMIncrement.observe(element.getRootNode(), {+ attributes: true,+ attributeOldValue: true,+ childList: true,+ subtree: true,+ characterData: true,+ });}-- observeIncrementalTarget(document.body);- const observerGeneration = window.globalIncrementalObserverGeneration;- setTimeout(() => {- if (- !window.globalListnerFlag ||- observerGeneration !== window.globalIncrementalObserverGeneration- ) {- return;- }- discoverAndObserveOpenShadowRoots(- document.documentElement,- 0,- performance.now() + 50,- );- }, 0);}async function stopGlobalIncrementalObserver() {diff --git a/tests/unit/test_agent_step_characterization.py b/tests/unit/test_agent_step_characterization.pyindex 14a170b3a..cad0ad6d7 100644--- a/tests/unit/test_agent_step_characterization.py+++ b/tests/unit/test_agent_step_characterization.py@@ -9,14 +9,13 @@ from zoneinfo import ZoneInfoimport pytest-from skyvern.exceptions import NoTOTPVerificationCodeFound, RepeatedActionFailure-from skyvern.forge.agent import ForgeAgent, StepPromptResult, _get_repeated_action_failure+from skyvern.exceptions import NoTOTPVerificationCodeFound+from skyvern.forge.agent import ForgeAgent, StepPromptResultfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.models import Step, StepStatusfrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.tasks import Task-from skyvern.schemas.steps import AgentStepOutputfrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import (Action,@@ -24,7 +23,6 @@ from skyvern.webeye.actions.actions import (CompleteAction,DownloadFileAction,ExtractAction,- InputTextAction,WaitAction,)from skyvern.webeye.actions.models import DetailedAgentStepOutput@@ -79,7 +77,6 @@ def make_agent_step_rig(injected_actions: list[Action] | None = None,task_overrides: dict[str, Any] | None = None,disable_user_goal_check: bool = True,- repeated_action_breaker: bool = False,) -> AgentStepRig:agent = ForgeAgent()now = datetime.now(UTC)@@ -151,8 +148,6 @@ def make_agent_step_rig(async def _flag(flag_name: str, *_args, **_kwargs) -> bool:if flag_name == "DISABLE_USER_GOAL_CHECK":return disable_user_goal_check- if flag_name == "REPEATED_ACTION_CIRCUIT_BREAKER":- return repeated_action_breakerreturn Falsemonkeypatch.setattr(@@ -289,137 +284,6 @@ async def test_failed_action_marks_step_failed_and_skips_remaining(monkeypatch:assert output.actions_and_results[0][1][0].success is False-async def test_third_identical_failed_persisted_input_surfaces_repeated_action_failure(- monkeypatch: pytest.MonkeyPatch,-) -> None:- action = InputTextAction(element_id="framework-managed-input", text="private payload")- handler = AsyncMock(- return_value=[ActionFailure(Exception("value did not commit"), stop_execution_on_failure=False)]- )- rig = make_agent_step_rig(- monkeypatch, parsed_actions=[action], action_handler=handler, repeated_action_breaker=True- )- previous_steps = []- for order in range(2):- previous_action = action.model_copy(update={"step_id": f"step-{order}", "step_order": order})- persisted_output = AgentStepOutput.model_validate(- AgentStepOutput(- action_results=[ActionFailure(Exception("value did not commit"), stop_execution_on_failure=False)],- actions_and_results=[- (- previous_action,- [ActionFailure(Exception("value did not commit"), stop_execution_on_failure=False)],- ),- ],- errors=[],- ).model_dump(mode="json")- )- assert persisted_output.actions_and_results is not None- persisted_action = persisted_output.actions_and_results[0][0]- assert type(persisted_action) is Action- assert persisted_action.action_type == ActionType.INPUT_TEXT- assert persisted_action.element_id == "framework-managed-input"- assert persisted_action.text == "private payload"- previous_step = make_step(- datetime.now(UTC),- rig.task,- step_id=f"step-{order}",- status=StepStatus.failed,- order=order,- output=persisted_output,- )- previous_steps.append(previous_step)- monkeypatch.setattr(- "skyvern.forge.agent.app.DATABASE.tasks.get_task_steps",- AsyncMock(return_value=previous_steps),- )-- step, output = await rig.run()-- assert step.status == StepStatus.completed- assert handler.await_count == 1- assert output.actions_and_results is not None- repeated_result = output.actions_and_results[-1][1][-1]- assert repeated_result.exception_type == RepeatedActionFailure.__name__- assert "framework-managed-input" in (repeated_result.exception_message or "")- assert "3" in (repeated_result.exception_message or "")- assert "Exception" in (repeated_result.exception_message or "")- assert "private payload" not in (repeated_result.exception_message or "")- assert repeated_result.stop_execution_on_failure is False---async def test_repeat_ineligible_failure_does_not_load_task_history(monkeypatch: pytest.MonkeyPatch) -> None:- action = WaitAction(seconds=1)- history = AsyncMock()- monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task_steps", history)- rig = make_agent_step_rig(- monkeypatch,- parsed_actions=[action],- action_handler=AsyncMock(return_value=[ActionFailure(Exception("failed"))]),- )-- await rig.run()-- history.assert_not_awaited()---async def test_repeated_action_breaker_disabled_adds_no_db_read_and_no_failure(- monkeypatch: pytest.MonkeyPatch,-) -> None:- """Flag off is today's behavior: the failure surfaces as itself and the task-steps history is never- loaded, so a flag-off org pays nothing for the detector on the failing-action hot path."""- action = InputTextAction(element_id="framework-managed-input", text="private payload")- history = AsyncMock(return_value=[])- monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task_steps", history)- rig = make_agent_step_rig(- monkeypatch,- parsed_actions=[action, action.model_copy(), action.model_copy()],- action_handler=AsyncMock(- return_value=[ActionFailure(Exception("value did not commit"), stop_execution_on_failure=False)]- ),- repeated_action_breaker=False,- )-- _, output = await rig.run()-- history.assert_not_awaited()- for _action, results in output.actions_and_results:- assert results[-1].exception_type == "Exception"- assert results[-1].exception_type != RepeatedActionFailure.__name__---async def test_repeated_action_history_loaded_at_most_once_per_step(monkeypatch: pytest.MonkeyPatch) -> None:- action = InputTextAction(element_id="framework-managed-input", text="private payload")- history = AsyncMock(return_value=[])- monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task_steps", history)- rig = make_agent_step_rig(- monkeypatch,- parsed_actions=[action, action.model_copy()],- action_handler=AsyncMock(- return_value=[ActionFailure(Exception("value did not commit"), stop_execution_on_failure=False)]- ),- repeated_action_breaker=True,- )-- await rig.run()-- assert rig.action_handler.await_count == 2- assert history.await_count == 1---def test_repeated_action_failure_requires_matching_unsuccessful_attempts() -> None:- first = InputTextAction(element_id="field", text="first")- second = InputTextAction(element_id="field", text="second")- failure = [ActionFailure(Exception("failed"))]-- assert _get_repeated_action_failure([(first, failure), (first, [ActionSuccess()]), (first, failure)]) is None- assert _get_repeated_action_failure([(first, failure), (second, failure), (first, failure)]) is None--@pytest.mark.asyncioasync def test_failed_action_with_continue_flag_executes_remaining(monkeypatch: pytest.MonkeyPatch) -> None:# Mirror of the skip case: a failure result that opts out of stopping the batchdiff --git a/tests/unit/test_auto_completion_discover_fallback.py b/tests/unit/test_auto_completion_discover_fallback.pyindex 3126078f6..bc7a8bc1e 100644--- a/tests/unit/test_auto_completion_discover_fallback.py+++ b/tests/unit/test_auto_completion_discover_fallback.py@@ -13,7 +13,6 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest-from skyvern.exceptions import AutoCompletionCommitFailurefrom skyvern.forge.sdk.models import StepStatusfrom skyvern.webeye.actions.actions import InputOrSelectContextfrom skyvern.webeye.actions.handler import (@@ -71,10 +70,7 @@ def _mock_skyvern_element(frame: MagicMock | None = None) -> MagicMock:# get_locator().click() needs to be async for the discover fallback's click callmock_locator = MagicMock()mock_locator.click = AsyncMock()- mock_locator.input_value = AsyncMock(return_value="I do not wish to disclose")el.get_locator.return_value = mock_locator- el.get_tag_name.return_value = "input"- el.get_attr = AsyncMock(return_value="false")return el@@ -99,7 +95,6 @@ def _mock_incremental_scrape(elements: list[dict]) -> MagicMock:inc = MagicMock()inc.start_listen_dom_increment = AsyncMock()inc.stop_listen_dom_increment = AsyncMock()- inc.get_incremental_elements_num = AsyncMock(return_value=len(elements))inc.get_incremental_element_tree = AsyncMock(return_value=copy.deepcopy(elements))inc.build_html_tree.return_value = "<div>mocked options</div>"inc.id_to_element_dict = {e["id"]: e for e in elements}@@ -170,260 +165,6 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:inc_scrape.stop_listen_dom_increment.assert_awaited()-async def test_discover_fallback_noop_keyboard_selection_is_not_a_commit() -> None:- """The input still holding the fallback's own fill with the listbox open is not commit evidence."""- frame = _mock_frame(locator_count=1)- skyvern_el = _mock_skyvern_element(frame)- skyvern_el.get_attr = AsyncMock(return_value="true")- inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)-- llm_response = {- "auto_completion_attempt": True,- "relevance_float": 0.9,- "id": "OPT3",- "value": "I do not wish to disclose",- "reasoning": "'I do not wish to disclose' matches 'Decline to Self Identify'",- }-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),- ),- patch(- "skyvern.webeye.actions.handler.IncrementalScrapePage",- return_value=inc_scrape,- ),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- ):- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- with pytest.raises(AutoCompletionCommitFailure) as exc_info:- await discover_and_select_from_full_dropdown(- context=_make_context(),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- original_text="Decline to Self Identify",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- )-- assert exc_info.value.stage == "clicked_but_not_committed"- inc_scrape.stop_listen_dom_increment.assert_awaited()---async def test_discover_fallback_assumes_commit_when_verification_disabled() -> None:- """Flag off: the keyboard selection is assumed committed - same scenario the noop test above- rejects with verification enabled."""- frame = _mock_frame(locator_count=1)- skyvern_el = _mock_skyvern_element(frame)- skyvern_el.get_attr = AsyncMock(return_value="true")- inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)-- llm_response = {- "auto_completion_attempt": True,- "relevance_float": 0.9,- "id": "OPT3",- "value": "I do not wish to disclose",- "reasoning": "'I do not wish to disclose' matches 'Decline to Self Identify'",- }-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),- ),- patch(- "skyvern.webeye.actions.handler.IncrementalScrapePage",- return_value=inc_scrape,- ),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- ):- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- result = await discover_and_select_from_full_dropdown(- context=_make_context(),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- original_text="Decline to Self Identify",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- verify_commit=False,- )-- assert isinstance(result, ActionSuccess)- inc_scrape.stop_listen_dom_increment.assert_awaited()---async def test_discover_fallback_canonicalized_commit_succeeds() -> None:- """A widget that expands the committed value beyond the fallback's own fill counts as a commit."""- frame = _mock_frame(locator_count=1)- skyvern_el = _mock_skyvern_element(frame)- skyvern_el.get_attr = AsyncMock(return_value=None)- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value="I do not wish to disclose (private)")- inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)-- llm_response = {- "auto_completion_attempt": True,- "relevance_float": 0.9,- "id": "OPT3",- "value": "I do not wish to disclose",- "reasoning": "Best semantic match",- }-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),- ),- patch(- "skyvern.webeye.actions.handler.IncrementalScrapePage",- return_value=inc_scrape,- ),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- ):- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- result = await discover_and_select_from_full_dropdown(- context=_make_context(),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- original_text="Decline to Self Identify",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- )-- assert isinstance(result, ActionSuccess)---async def test_discover_fallback_exact_echo_without_expanded_state_is_a_commit() -> None:- """A widget that commits the option text verbatim and exposes no expanded state is a commit, not a failure."""- frame = _mock_frame(locator_count=1)- skyvern_el = _mock_skyvern_element(frame)- # No aria-expanded at all: the widget gives no open/closed signal, and the committed value is the- # option text verbatim, so it is indistinguishable from the fallback's own fill by value alone.- skyvern_el.get_attr = AsyncMock(return_value=None)- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value="I do not wish to disclose")- inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)-- llm_response = {- "auto_completion_attempt": True,- "relevance_float": 0.9,- "id": "OPT3",- "value": "I do not wish to disclose",- "reasoning": "Best semantic match",- }-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),- ),- patch(- "skyvern.webeye.actions.handler.IncrementalScrapePage",- return_value=inc_scrape,- ),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- ):- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- result = await discover_and_select_from_full_dropdown(- context=_make_context(),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- original_text="Decline to Self Identify",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- )-- assert isinstance(result, ActionSuccess)---async def test_discover_fallback_ignores_stale_cached_expanded_state() -> None:- """A no-op Enter must not read as committed just because the element was scraped before the- dropdown opened. get_attr is static-first, so the cached aria-expanded="false" from that scrape- would otherwise stand in as commit evidence for an unchanged value."""- frame = _mock_frame(locator_count=1)- skyvern_el = _mock_skyvern_element(frame)- # Behaves like the real SkyvernElement.get_attr: static cache says "false" (scraped while closed),- # the live DOM says "true" (the listbox is still open, so Enter did nothing).- static_attrs = {"aria-expanded": "false"}- live_attrs = {"aria-expanded": "true"}-- async def _get_attr(name: str, mode: str = "auto", **_kwargs: object) -> object:- if mode == "dynamic":- return live_attrs.get(name)- cached = static_attrs.get(name)- return cached if cached is not None or mode == "static" else live_attrs.get(name)-- skyvern_el.get_attr = AsyncMock(side_effect=_get_attr)- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value="I do not wish to disclose")- inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)-- llm_response = {- "auto_completion_attempt": True,- "relevance_float": 0.9,- "id": "OPT3",- "value": "I do not wish to disclose",- "reasoning": "Best semantic match",- }-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),- ),- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- ):- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- with pytest.raises(AutoCompletionCommitFailure):- await discover_and_select_from_full_dropdown(- context=_make_context(),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- original_text="Decline to Self Identify",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- )--@pytest.mark.asyncioasync def test_discover_fallback_returns_none_when_no_options() -> None:"""ArrowDown produces no incremental elements → returns None."""@@ -871,7 +612,7 @@ async def test_input_or_auto_complete_skips_discover_for_search_bar() -> None:mock_prompt.load_prompt.return_value = "mocked prompt"mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)- # Search bars fall through to plain typing (None) instead of hard-failing,+ # Note: search bars bail out early at L2989 (before potential_values),# so discover_fallback is never reached. This test verifies that.result = await input_or_auto_complete_input(input_or_select_context=_make_context(is_search_bar=True),@@ -884,225 +625,6 @@ async def test_input_or_auto_complete_skips_discover_for_search_bar() -> None:task=_TASK,)+ # Search bar returns None early, never reaching discover fallbackassert result is Nonemock_discover.assert_not_called()---async def test_input_or_auto_complete_failure_includes_attempt_trail() -> None:- """A location field with the commit-required flag on returns a typed failure with the attempt trail."""- skyvern_el = _mock_skyvern_element()- choose_result = AutoCompletionResult(- action_result=ActionFailure(AutoCompletionCommitFailure(stage="suggestions_never_rendered")),- stage_outcome="suggestions_never_rendered",- )-- with (- patch(- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",- new=AsyncMock(return_value=choose_result),- ),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- patch(- "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",- new=AsyncMock(return_value=None),- ),- ):- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=True)- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- result = await input_or_auto_complete_input(- input_or_select_context=_make_context(is_location_input=True),- scraped_page=MagicMock(),- page=MagicMock(),- dom=MagicMock(),- text="private value",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- )-- assert isinstance(result, ActionFailure)- assert result.exception_type == AutoCompletionCommitFailure.__name__- assert result.exception_message is not None- assert "attempt_1:suggestions_never_rendered" in result.exception_message- assert "attempt_2:suggestion_not_matched" in result.exception_message- assert "private value" not in result.exception_message---def _side_effect_choose_result() -> AutoCompletionResult:- return AutoCompletionResult(- action_result=ActionFailure(AutoCompletionCommitFailure(stage="clicked_but_not_committed")),- stage_outcome="clicked_but_not_committed",- committed_side_effects=True,- )---async def test_side_effectful_click_stops_retrying_with_flag_on() -> None:- """Once a click has mutated the field but could not be verified, the orchestration loop must- stop instead of generating more values and clicking additional options on the dirty field."""- skyvern_el = _mock_skyvern_element()- choose_mock = AsyncMock(return_value=_side_effect_choose_result())- discover_mock = AsyncMock(return_value=ActionSuccess())-- with (- patch("skyvern.webeye.actions.handler.choose_auto_completion_dropdown", new=choose_mock),- patch("skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown", new=discover_mock),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- ):- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=True)- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- result = await input_or_auto_complete_input(- input_or_select_context=_make_context(is_location_input=True),- scraped_page=MagicMock(),- page=MagicMock(),- dom=MagicMock(),- text="private value",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- )-- assert choose_mock.call_count == 1- discover_mock.assert_not_called()- mock_app.SECONDARY_LLM_API_HANDLER.assert_not_called()- assert isinstance(result, ActionFailure)- assert result.exception_type == AutoCompletionCommitFailure.__name__- assert "clicked_but_not_committed" in (result.exception_message or "")---async def test_side_effectful_click_stops_retrying_with_flag_off() -> None:- """With the flag off the loop still stops after a side-effectful click (returning None to fall- through to plain typing) rather than clicking more options on the mutated field."""- skyvern_el = _mock_skyvern_element()- choose_mock = AsyncMock(return_value=_side_effect_choose_result())- discover_mock = AsyncMock(return_value=ActionSuccess())-- with (- patch("skyvern.webeye.actions.handler.choose_auto_completion_dropdown", new=choose_mock),- patch("skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown", new=discover_mock),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- ):- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- result = await input_or_auto_complete_input(- input_or_select_context=_make_context(is_location_input=True),- scraped_page=MagicMock(),- page=MagicMock(),- dom=MagicMock(),- text="private value",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- )-- assert choose_mock.call_count == 1- discover_mock.assert_not_called()- assert result is None---async def test_input_or_auto_complete_falls_through_when_flag_disabled() -> None:- """With the commit-required flag off, an exhausted location autocomplete falls through to plain typing."""- skyvern_el = _mock_skyvern_element()- choose_result = AutoCompletionResult(- action_result=ActionFailure(AutoCompletionCommitFailure(stage="suggestions_never_rendered")),- stage_outcome="suggestions_never_rendered",- )-- with (- patch(- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",- new=AsyncMock(return_value=choose_result),- ),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- patch(- "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",- new=AsyncMock(return_value=None),- ),- ):- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- result = await input_or_auto_complete_input(- input_or_select_context=_make_context(is_location_input=True),- scraped_page=MagicMock(),- page=MagicMock(),- dom=MagicMock(),- text="private value",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- )-- assert result is None---async def test_fallback_stage_is_read_structurally_not_from_message_text() -> None:- """The fallback stage comes from the exception's stage attribute, even when the- message's attempt trail embeds every other stage name ahead of it."""- skyvern_el = _mock_skyvern_element()- choose_result = AutoCompletionResult(- action_result=ActionFailure(AutoCompletionCommitFailure(stage="suggestions_never_rendered")),- stage_outcome="suggestions_never_rendered",- )- fallback_failure = AutoCompletionCommitFailure(- stage="clicked_but_not_committed",- attempt_trail=["attempt_1:suggestions_never_rendered", "attempt_2:suggestion_not_matched"],- )- assert "suggestions_never_rendered" in str(fallback_failure)-- with (- patch(- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",- new=AsyncMock(return_value=choose_result),- ),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- patch(- "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",- new=AsyncMock(side_effect=fallback_failure),- ),- ):- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=True)- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- result = await input_or_auto_complete_input(- input_or_select_context=_make_context(is_location_input=True),- scraped_page=MagicMock(),- page=MagicMock(),- dom=MagicMock(),- text="private value",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- )-- assert isinstance(result, ActionFailure)- assert result.exception_type == AutoCompletionCommitFailure.__name__- assert "attempt_2:clicked_but_not_committed" in (result.exception_message or "")- assert "stage=clicked_but_not_committed" in (result.exception_message or "")diff --git a/tests/unit/test_auto_completion_location.py b/tests/unit/test_auto_completion_location.pyindex 778594980..f33020687 100644--- a/tests/unit/test_auto_completion_location.py+++ b/tests/unit/test_auto_completion_location.py@@ -6,27 +6,23 @@ suggestion appears, we skip the LLM call and click the suggestion directly.from __future__ import annotations-import asyncioimport copyfrom datetime import UTC, datetimefrom unittest.mock import AsyncMock, MagicMock, patchimport pytest-import skyvern.webeye.actions.handler as handler_modulefrom skyvern.constants import SKYVERN_ID_ATTR-from skyvern.exceptions import AutoCompletionCommitFailure, MissingElement, NoIncrementalElementFoundForAutoCompletion+from skyvern.exceptions import NoIncrementalElementFoundForAutoCompletionfrom skyvern.forge.sdk.models import StepStatusfrom skyvern.webeye.actions.actions import InputOrSelectContextfrom skyvern.webeye.actions.handler import (AutoCompletionResult,- _poll_autocomplete_incremental_elements,_reset_autocomplete_for_llm_fallback,- _verify_autocomplete_input_readback,choose_auto_completion_dropdown,input_or_auto_complete_input,)-from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess+from skyvern.webeye.actions.responses import ActionSuccessfrom tests.unit.helpers import make_organization, make_step, make_task# ---------------------------------------------------------------------------@@ -38,15 +34,11 @@ _ORG = make_organization(_NOW)_TASK = make_task(_NOW, _ORG, navigation_payload={"address": "123 Main St"})_STEP = make_step(_NOW, _TASK, step_id="stp-1", status=StepStatus.created, order=0, output=None)-SINGLE_ELEMENT = [- {"id": "AAAA", "tagName": "li", "attributes": {"role": "option"}, "text": "123 Main St, Springfield, IL"}-]+SINGLE_ELEMENT = [{"id": "AAAA", "tag": "div", "text": "123 Main St, Springfield, IL"}]MULTI_ELEMENTS = [- {"id": "AAAA", "tagName": "li", "attributes": {"role": "option"}, "text": "123 Main St, Springfield, IL"},- {"id": "AAAB", "tagName": "li", "attributes": {"role": "option"}, "text": "123 Main St, Springfield, MO"},+ {"id": "AAAA", "tag": "div", "text": "123 Main St, Springfield, IL"},+ {"id": "AAAB", "tag": "div", "text": "123 Main St, Springfield, MO"},]-LOADING_ELEMENT = {"id": "LOAD", "tagName": "div", "text": "Loading..."}-OPTION_ELEMENT = {"id": "OPT", "tagName": "li", "attributes": {"role": "option"}, "text": "123 Main St, Springfield"}def _make_location_context(**overrides: object) -> InputOrSelectContext:@@ -76,767 +68,79 @@ def _mock_skyvern_element(frame: MagicMock | None = None) -> MagicMock:el.get_frame.return_value = frame or _mock_frame()el.get_frame_id.return_value = "frame-1"el.is_interactable.return_value = True- el.press_fill = AsyncMock()- el.input_clear = AsyncMock()- el.is_visible = AsyncMock(return_value=True)- el.get_element_handler = AsyncMock(return_value=MagicMock())- el.get_tag_name.return_value = "input"- el.press_key = AsyncMock()- el.get_attr = AsyncMock(return_value=None)- input_locator = MagicMock()- input_locator.input_value = AsyncMock(return_value="123 Main St, Springfield, IL")- el.get_locator.return_value = input_locator- return el---def _mock_frame(locator_count: int = 1) -> MagicMock:- """Return a mock Playwright Frame with a configurable locator."""- frame = MagicMock()- locator = MagicMock()- locator.count = AsyncMock(return_value=locator_count)- locator.click = AsyncMock()- locator.element_handle = AsyncMock(return_value=MagicMock())- locator.is_visible = AsyncMock(return_value=True)- locator.get_attribute = AsyncMock(return_value=None)- frame.locator.return_value = locator- return frame---def _mock_incremental_scrape(elements: list[dict]) -> MagicMock:- """Return a mock IncrementalScrapePage that yields *elements*."""- inc = MagicMock()- inc.start_listen_dom_increment = AsyncMock()- inc.stop_listen_dom_increment = AsyncMock()- inc.get_incremental_elements_num = AsyncMock(return_value=len(elements))- inc.get_incremental_element_tree = AsyncMock(return_value=copy.deepcopy(elements))- inc.build_html_tree.return_value = "<div>mocked</div>"- return inc---class _FakeSuggestionList:- """Stateful listbox fake: each patched sleep advances the rendered frame."""-- def __init__(self, frames: list[list[dict]]) -> None:- self._frames = frames- self._tick = 0-- def _current(self) -> list[dict]:- return self._frames[min(self._tick, len(self._frames) - 1)]-- async def advance(self, _delay: float) -> None:- self._tick += 1-- def scrape(self) -> MagicMock:- inc = MagicMock()- inc.start_listen_dom_increment = AsyncMock()- inc.stop_listen_dom_increment = AsyncMock()- inc.get_incremental_elements_num = AsyncMock(side_effect=lambda: len(self._current()))- inc.get_incremental_element_tree = AsyncMock(side_effect=lambda _cleanup: copy.deepcopy(self._current()))- inc.build_html_tree.return_value = "<div>mocked</div>"- return inc---def test_auto_completion_result_does_not_default_to_success() -> None:- assert AutoCompletionResult().action_result is None---def test_removed_autocomplete_exception_name_still_catchable() -> None:- """The deprecated OSS alias must keep downstream `except` clauses working for one release."""- with pytest.raises(NoIncrementalElementFoundForAutoCompletion):- raise AutoCompletionCommitFailure(stage="suggestion_not_matched")--- ("actual_value", "matched_label", "typed_prefix", "expected"),- [- ("123 MAIN ST., Springfield!", "123 Main St Springfield", "123 Main", True),- ("Selected: 123 Main St, Springfield", "123 Main St Springfield", "123 Main", True),- ("123 Main", "123 Main St Springfield", "123 Main", False),- ("scala 5", "ca", "sc", False),- ("123 Main St Springfield", "123 Main St, Springfield, USA", "123 Main", True),- ],-)-async def test_autocomplete_readback_matches_on_token_boundaries(- actual_value: str,- matched_label: str,- typed_prefix: str,- expected: bool,-) -> None:- skyvern_el = _mock_skyvern_element()- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value=actual_value)-- outcome = await _verify_autocomplete_input_readback(- skyvern_element=skyvern_el,- matched_label=matched_label,- typed_prefix=typed_prefix,- )- assert outcome.committed is expected---async def test_autocomplete_readback_rejects_lone_token_of_multiword_label() -> None:- # A single token of a multi-word label (here "Springfield" of "123 Main St Springfield") no longer- # identifies the selection, even though the value changed - the field must still cover a substantial- # share of the label. Substantial truncation ("123 Main St" of the same label) stays a commit.- lone_token = _mock_skyvern_element()- lone_token.get_locator.return_value.input_value = AsyncMock(return_value="Springfield")- weak = await _verify_autocomplete_input_readback(- skyvern_element=lone_token,- matched_label="123 Main St Springfield",- typed_prefix="123 Main",- )- assert weak.committed is False-- substantial = _mock_skyvern_element()- substantial.get_locator.return_value.input_value = AsyncMock(return_value="123 Main St")- strong = await _verify_autocomplete_input_readback(- skyvern_element=substantial,- matched_label="123 Main St Springfield",- typed_prefix="123",- )- assert strong.committed is True---async def test_autocomplete_readback_requires_secondary_signal_when_label_equals_typed_text() -> None:- skyvern_el = _mock_skyvern_element()- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value="Oakland, California")-- no_signal = await _verify_autocomplete_input_readback(- skyvern_element=skyvern_el,- matched_label="Oakland, California",- typed_prefix="Oakland, California",- )- assert no_signal.committed is False- assert no_signal.side_effects_observed is False-- list_closed = await _verify_autocomplete_input_readback(- skyvern_element=skyvern_el,- matched_label="Oakland, California",- typed_prefix="Oakland, California",- suggestions_closed=True,- )- assert list_closed.committed is True-- aria_selected = await _verify_autocomplete_input_readback(- skyvern_element=skyvern_el,- matched_label="Oakland, California",- typed_prefix="Oakland, California",- selection_state_present=True,- )- assert aria_selected.committed is True---async def test_autocomplete_readback_accepts_cleared_input_only_when_list_closed() -> None:- skyvern_el = _mock_skyvern_element()- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value="")-- chip_commit = await _verify_autocomplete_input_readback(- skyvern_element=skyvern_el,- matched_label="Oakland, California",- typed_prefix="Oakland",- suggestions_closed=True,- )- assert chip_commit.committed is True-- still_open = await _verify_autocomplete_input_readback(- skyvern_element=skyvern_el,- matched_label="Oakland, California",- typed_prefix="Oakland",- )- assert still_open.committed is False---async def test_autocomplete_readback_empty_value_needs_selection_signal_when_commit_required() -> None:- # A click that dismisses the suggestions while clearing the field reads back identical to a chip- # commit; under commit-required only an affirmative selection signal separates the two.- skyvern_el = _mock_skyvern_element()- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value="")-- dismissal = await _verify_autocomplete_input_readback(- skyvern_element=skyvern_el,- matched_label="Oakland, California",- typed_prefix="Oakland",- suggestions_closed=True,- commit_required=True,- )- assert dismissal.committed is False-- affirmed = await _verify_autocomplete_input_readback(- skyvern_element=skyvern_el,- matched_label="Oakland, California",- typed_prefix="Oakland",- suggestions_closed=True,- selection_state_present=True,- commit_required=True,- )- assert affirmed.committed is True---async def test_autocomplete_poll_returns_immediately_when_suggestions_exist() -> None:- fake_list = _FakeSuggestionList([SINGLE_ELEMENT])- incremental_scrape = fake_list.scrape()-- with patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock(side_effect=fake_list.advance)) as sleep:- result = await _poll_autocomplete_incremental_elements(- incremental_scraped=incremental_scrape,- cleanup_factory=MagicMock(),- )-- assert result == SINGLE_ELEMENT- sleep.assert_not_awaited()---async def test_autocomplete_poll_waits_until_suggestions_render() -> None:- fake_list = _FakeSuggestionList([[], [], SINGLE_ELEMENT])- incremental_scrape = fake_list.scrape()-- with patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock(side_effect=fake_list.advance)) as sleep:- result = await _poll_autocomplete_incremental_elements(- incremental_scraped=incremental_scrape,- cleanup_factory=MagicMock(),- )-- assert result == SINGLE_ELEMENT- assert sleep.await_count == 2---async def test_autocomplete_poll_keeps_waiting_past_loading_nodes_until_options_render() -> None:- fake_list = _FakeSuggestionList([[LOADING_ELEMENT], [LOADING_ELEMENT], [LOADING_ELEMENT, OPTION_ELEMENT]])- incremental_scrape = fake_list.scrape()-- with patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock(side_effect=fake_list.advance)):- result = await _poll_autocomplete_incremental_elements(- incremental_scraped=incremental_scrape,- cleanup_factory=MagicMock(),- )-- assert OPTION_ELEMENT in result---async def test_autocomplete_poll_reextracts_when_placeholder_replaced_in_place() -> None:- # A listbox can swap a loading placeholder for an option in the same node, so the incremental- # count stays constant (1 -> 1). A count-delta guard would extract the placeholder once and never- # re-extract, missing the rendered option.- fake_list = _FakeSuggestionList([[LOADING_ELEMENT], [OPTION_ELEMENT]])- incremental_scrape = fake_list.scrape()-- with patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock(side_effect=fake_list.advance)):- result = await _poll_autocomplete_incremental_elements(- incremental_scraped=incremental_scrape,- cleanup_factory=MagicMock(),- )-- assert OPTION_ELEMENT in result---async def test_autocomplete_poll_is_bounded_when_suggestions_never_render() -> None:- fake_list = _FakeSuggestionList([[]])- incremental_scrape = fake_list.scrape()-- with patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock(side_effect=fake_list.advance)) as sleep:- result = await _poll_autocomplete_incremental_elements(- incremental_scraped=incremental_scrape,- cleanup_factory=MagicMock(),- )-- assert result == []- incremental_scrape.get_incremental_element_tree.assert_not_awaited()- assert sleep.await_count <= handler_module.AUTOCOMPLETE_POLL_MAX_ATTEMPTS---async def test_autocomplete_poll_does_not_cancel_slow_extraction(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(handler_module, "AUTOCOMPLETE_POLL_INTERVAL_SECONDS", 0.05)- incremental_scrape = MagicMock()- incremental_scrape.get_incremental_elements_num = AsyncMock(return_value=1)-- async def _slow_extract(_cleanup: object) -> list[dict]:- await asyncio.sleep(0.2)- return copy.deepcopy(SINGLE_ELEMENT)-- incremental_scrape.get_incremental_element_tree = AsyncMock(side_effect=_slow_extract)-- result = await _poll_autocomplete_incremental_elements(- incremental_scraped=incremental_scrape,- cleanup_factory=MagicMock(),- )-- assert result == SINGLE_ELEMENT- assert incremental_scrape.get_incremental_element_tree.await_count == 1---async def test_autocomplete_poll_bounds_a_hanging_probe(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(handler_module, "AUTOCOMPLETE_POLL_INTERVAL_SECONDS", 0.02)- monkeypatch.setattr(handler_module, "AUTOCOMPLETE_POLL_MAX_ATTEMPTS", 2)- incremental_scrape = MagicMock()-- async def _hanging_probe() -> int:- await asyncio.sleep(10)- return 1-- incremental_scrape.get_incremental_elements_num = AsyncMock(side_effect=_hanging_probe)- incremental_scrape.get_incremental_element_tree = AsyncMock(return_value=copy.deepcopy(SINGLE_ELEMENT))-- result = await _poll_autocomplete_incremental_elements(- incremental_scraped=incremental_scrape,- cleanup_factory=MagicMock(),- )-- assert result == []- incremental_scrape.get_incremental_element_tree.assert_not_awaited()---async def test_autocomplete_timeout_reports_suggestions_never_rendered() -> None:- skyvern_el = _mock_skyvern_element()- incremental_scrape = _mock_incremental_scrape([])- scraped_page = MagicMock(id_to_css_dict={})- scraped_after_open = MagicMock(id_to_css_dict={})- scraped_page.generate_scraped_page_without_screenshots = AsyncMock(return_value=scraped_after_open)-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=MagicMock()),- ),- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=incremental_scrape),- patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock()),- ):- result = await choose_auto_completion_dropdown(- context=_make_location_context(),- page=MagicMock(),- scraped_page=scraped_page,- dom=MagicMock(),- text="123 Main",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- is_location_input=True,- )-- assert isinstance(result.action_result, ActionFailure)- assert result.action_result.exception_type == AutoCompletionCommitFailure.__name__- assert "suggestions_never_rendered" in (result.action_result.exception_message or "")---# ----------------------------------------------------------------------------# Tests for choose_auto_completion_dropdown-# ------------------------------------------------------------------------------async def test_location_single_option_skips_llm() -> None:- """When is_location_input=True and exactly 1 option appears, the LLM must NOT be called."""- frame = _mock_frame(locator_count=1)- skyvern_el = _mock_skyvern_element(frame)- inc_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),- ),- patch(- "skyvern.webeye.actions.handler.IncrementalScrapePage",- return_value=inc_scrape,- ),- patch("skyvern.webeye.actions.handler.app") as mock_app,- ):- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock()-- result = await choose_auto_completion_dropdown(- context=_make_location_context(),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- text="123 Main St",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- is_location_input=True,- )-- # The LLM should never have been called- mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_not_called()-- # The locator should have been clicked- frame.locator.assert_called_with(f'[{SKYVERN_ID_ATTR}="AAAA"]')- frame.locator.return_value.click.assert_awaited_once()-- # Result should indicate success- assert isinstance(result.action_result, ActionSuccess)---async def test_location_single_option_click_with_side_effects_does_not_refire() -> None:- """A click that changed the value (but did not match) must hard-fail instead of re-firing."""- frame, option_locator, skyvern_el, skyvern_frame = _mock_autocomplete_input(- selected_value="Wrong city",- click_closes_listbox=False,- )- inc_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=skyvern_frame),- ),- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),- patch("skyvern.webeye.actions.handler.app") as mock_app,- ):- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock()-- result = await choose_auto_completion_dropdown(- context=_make_location_context(),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- text="123 Main",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- is_location_input=True,- )-- option_locator.click.assert_awaited_once()- skyvern_el.press_fill.assert_awaited_once()- mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_not_called()- assert isinstance(result.action_result, ActionFailure)- assert result.action_result.exception_type == AutoCompletionCommitFailure.__name__- assert "clicked_but_not_committed" in (result.action_result.exception_message or "")---async def test_location_single_option_side_effect_free_noncommit_falls_back_to_llm() -> None:- """A no-op click (value unchanged, listbox still open) resets and runs the LLM chooser in the same call."""- _frame, option_locator, skyvern_el, skyvern_frame = _mock_autocomplete_input(- selected_value="123 Main",- click_closes_listbox=False,- )- events: list[str] = []- skyvern_el.input_clear = AsyncMock(side_effect=lambda: events.append("clear"))- skyvern_el.press_fill = AsyncMock(side_effect=lambda value: events.append(f"fill:{value}"))- skyvern_el.press_key = AsyncMock()- initial_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)- initial_scrape.build_html_tree.return_value = "<div>stale options</div>"- fallback_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)- fallback_scrape.build_html_tree.return_value = "<div>fresh options</div>"-- llm_response = {- "auto_completion_attempt": False,- "relevance_float": 0.0,- "id": "",- "direct_searching": True,- "value": "123 Main",- "reasoning": "Read-back mismatch",- }-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=skyvern_frame),- ),- patch(- "skyvern.webeye.actions.handler.IncrementalScrapePage",- side_effect=[initial_scrape, fallback_scrape],- ) as scrape_factory,- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- ):-- async def _llm_handler(**_: object) -> dict[str, object]:- events.append("llm")- return llm_response-- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(side_effect=_llm_handler)- mock_app.AGENT_FUNCTION = MagicMock()- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- result = await choose_auto_completion_dropdown(- context=_make_location_context(),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- text="123 Main",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- is_location_input=True,- )-- option_locator.click.assert_awaited_once()- assert scrape_factory.call_count == 2- skyvern_el.input_clear.assert_awaited_once()- assert skyvern_el.press_fill.await_count == 2- assert events == ["fill:123 Main", "clear", "fill:123 Main", "llm"]- assert mock_prompt.load_prompt.call_args.kwargs["elements"] == "<div>fresh options</div>"- mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_awaited_once()- skyvern_el.press_key.assert_awaited_once_with("Enter")- assert isinstance(result.action_result, ActionSuccess)---async def test_direct_searching_unchanged_value_fails_when_commit_required() -> None:- """Searching with the typed text leaves the value unchanged, which is indistinguishable from Enter- doing nothing. On a field that must commit a real suggestion that is a failure, not a success."""- _frame, _option_locator, skyvern_el, skyvern_frame = _mock_autocomplete_input(- selected_value="123 Main",- click_closes_listbox=False,- )- skyvern_el.input_clear = AsyncMock()- skyvern_el.press_fill = AsyncMock()- skyvern_el.press_key = AsyncMock()- initial_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)- initial_scrape.build_html_tree.return_value = "<div>stale options</div>"- fallback_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)- fallback_scrape.build_html_tree.return_value = "<div>fresh options</div>"-- llm_response = {- "auto_completion_attempt": False,- "relevance_float": 0.0,- "id": "",- "direct_searching": True,- "value": "123 Main",- "reasoning": "Read-back mismatch",- }-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=skyvern_frame),- ),- patch(- "skyvern.webeye.actions.handler.IncrementalScrapePage",- side_effect=[initial_scrape, fallback_scrape],- ),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- ):- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)- mock_app.AGENT_FUNCTION = MagicMock()- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- result = await choose_auto_completion_dropdown(- context=_make_location_context(),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- text="123 Main",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- is_location_input=True,- commit_required=True,- )-- assert isinstance(result.action_result, ActionFailure)- assert result.action_result.exception_type == AutoCompletionCommitFailure.__name__- assert "clicked_but_not_committed" in (result.action_result.exception_message or "")---async def test_unexpected_exception_surfaces_as_itself_not_commit_failure() -> None:- """A genuine bug-shaped error keeps its identity in the ActionFailure instead of being- rewrapped as an expected no-match commit failure."""- skyvern_el = _mock_skyvern_element()- skyvern_el.press_fill = AsyncMock(side_effect=MissingElement(element_id="AAAA"))- inc_scrape = _mock_incremental_scrape([])-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=MagicMock()),- ),- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),- ):- result = await choose_auto_completion_dropdown(- context=_make_location_context(),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- text="123 Main",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- is_location_input=True,- )-- assert isinstance(result.action_result, ActionFailure)- assert result.action_result.exception_type == MissingElement.__name__- assert result.stage_outcome == "suggestion_not_matched"---async def test_commit_failure_keeps_its_stage_in_action_failure() -> None:- """An AutoCompletionCommitFailure raised inside the dropdown flow keeps its own stage."""- skyvern_el = _mock_skyvern_element()- skyvern_el.press_fill = AsyncMock(side_effect=AutoCompletionCommitFailure(stage="suggestions_never_rendered"))- inc_scrape = _mock_incremental_scrape([])-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=MagicMock()),- ),- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),- ):- result = await choose_auto_completion_dropdown(- context=_make_location_context(),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- text="123 Main",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- is_location_input=True,- )+ el.press_fill = AsyncMock()+ el.input_clear = AsyncMock()+ el.is_visible = AsyncMock(return_value=True)+ el.get_element_handler = AsyncMock(return_value=MagicMock())+ return el- assert isinstance(result.action_result, ActionFailure)- assert result.action_result.exception_type == AutoCompletionCommitFailure.__name__- assert result.stage_outcome == "suggestions_never_rendered"+def _mock_frame(locator_count: int = 1) -> MagicMock:+ """Return a mock Playwright Frame with a configurable locator."""+ frame = MagicMock()+ locator = MagicMock()+ locator.count = AsyncMock(return_value=locator_count)+ locator.click = AsyncMock()+ frame.locator.return_value = locator+ return frame-async def test_unexpected_exception_still_hard_fails_location_field_when_flag_on() -> None:- """A preserved unexpected exception from the dropdown flow still produces the staged- location failure when the commit-required flag is on."""- skyvern_el = _mock_skyvern_element()- choose_result = AutoCompletionResult(- action_result=ActionFailure(MissingElement(element_id="AAAA")),- stage_outcome="suggestion_not_matched",- )- with (- patch(- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",- new=AsyncMock(return_value=choose_result),- ),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- patch(- "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",- new=AsyncMock(return_value=None),- ),- ):- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=True)- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)+def _mock_incremental_scrape(elements: list[dict]) -> MagicMock:+ """Return a mock IncrementalScrapePage that yields *elements*."""+ inc = MagicMock()+ inc.start_listen_dom_increment = AsyncMock()+ inc.stop_listen_dom_increment = AsyncMock()+ inc.get_incremental_element_tree = AsyncMock(return_value=copy.deepcopy(elements))+ inc.build_html_tree.return_value = "<div>mocked</div>"+ return inc- result = await input_or_auto_complete_input(- input_or_select_context=_make_location_context(),- scraped_page=MagicMock(),- page=MagicMock(),- dom=MagicMock(),- text="123 Main St",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- )- assert isinstance(result, ActionFailure)- assert result.exception_type == AutoCompletionCommitFailure.__name__- assert "attempt_1:suggestion_not_matched" in (result.exception_message or "")+# ---------------------------------------------------------------------------+# Tests for choose_auto_completion_dropdown+# ---------------------------------------------------------------------------@pytest.mark.asyncio-async def test_location_search_bar_keeps_fallthrough_even_with_flag_on() -> None:- """A location-search widget is both a location input and a search bar. Search bars keep their- documented fall-through, so strict commit mode must not claim one and hard-fail it."""- skyvern_el = _mock_skyvern_element()- choose_result = AutoCompletionResult(- action_result=ActionFailure(MissingElement(element_id="AAAA")),- stage_outcome="suggestion_not_matched",- )+async def test_location_single_option_skips_llm() -> None:+ """When is_location_input=True and exactly 1 option appears, the LLM must NOT be called."""+ frame = _mock_frame(locator_count=1)+ skyvern_el = _mock_skyvern_element(frame)+ inc_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)with (patch(- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",- new=AsyncMock(return_value=choose_result),+ "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",+ new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,patch(- "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",- new=AsyncMock(return_value=None),+ "skyvern.webeye.actions.handler.IncrementalScrapePage",+ return_value=inc_scrape,),+ patch("skyvern.webeye.actions.handler.app") as mock_app,):- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=True)- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)+ mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock()- result = await input_or_auto_complete_input(- input_or_select_context=_make_location_context(is_search_bar=True),- scraped_page=MagicMock(),+ result = await choose_auto_completion_dropdown(+ context=_make_location_context(),page=MagicMock(),+ scraped_page=MagicMock(),dom=MagicMock(),text="123 Main St",skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_location_input=True,)- assert not isinstance(result, ActionFailure)---async def test_unexpected_exception_falls_through_to_plain_typing_when_flag_off() -> None:- """With the commit-required flag off, a preserved unexpected exception still lets the- location field fall through to plain typing."""- skyvern_el = _mock_skyvern_element()- choose_result = AutoCompletionResult(- action_result=ActionFailure(MissingElement(element_id="AAAA")),- stage_outcome="suggestion_not_matched",- )-- with (- patch(- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",- new=AsyncMock(return_value=choose_result),- ),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- patch(- "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",- new=AsyncMock(return_value=None),- ),- ):- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)+ # The LLM should never have been called+ mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_not_called()- result = await input_or_auto_complete_input(- input_or_select_context=_make_location_context(),- scraped_page=MagicMock(),- page=MagicMock(),- dom=MagicMock(),- text="123 Main St",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- )+ # The locator should have been clicked+ frame.locator.assert_called_with(f'[{SKYVERN_ID_ATTR}="AAAA"]')+ frame.locator.return_value.click.assert_awaited_once()- assert result is None+ # Result should indicate success+ assert isinstance(result.action_result, ActionSuccess)@pytest.mark.asyncio@@ -891,7 +195,6 @@ async def test_location_multiple_options_calls_llm() -> None:"direct_searching": False,"reasoning": "First option matches",}- selected_element = MagicMock(scroll_into_view=AsyncMock(), click=AsyncMock())with (patch(@@ -905,14 +208,13 @@ async def test_location_multiple_options_calls_llm() -> None:patch("skyvern.webeye.actions.handler.app") as mock_app,patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- patch("skyvern.webeye.actions.handler.SkyvernElement", return_value=selected_element),):mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)mock_app.AGENT_FUNCTION = MagicMock()mock_prompt.load_prompt.return_value = "mocked prompt"mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)- result = await choose_auto_completion_dropdown(+ await choose_auto_completion_dropdown(context=_make_location_context(),page=MagicMock(),scraped_page=MagicMock(),@@ -926,8 +228,6 @@ async def test_location_multiple_options_calls_llm() -> None:# LLM should have been called because there are 2 optionsmock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_awaited_once()- selected_element.click.assert_awaited_once()- assert isinstance(result.action_result, ActionSuccess)@pytest.mark.asyncio@@ -1120,44 +420,12 @@ async def test_input_or_auto_complete_passes_false_for_non_location() -> None:assert call_kwargs["is_location_input"] is False[email protected]("flag_enabled", [True, False])-async def test_input_or_auto_complete_gates_verification_on_the_flag(flag_enabled: bool) -> None:- """Commit verification runs only behind the flag; flag-off keeps the pre-verification behavior."""- context = _make_location_context()-- with (- patch(- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",- new=AsyncMock(return_value=AutoCompletionResult(action_result=ActionSuccess())),- ) as mock_choose,- patch("skyvern.webeye.actions.handler.app") as mock_app,- ):- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=flag_enabled)-- result = await input_or_auto_complete_input(- input_or_select_context=context,- scraped_page=MagicMock(),- page=MagicMock(),- dom=MagicMock(),- text="123 Main St",- skyvern_element=_mock_skyvern_element(),- step=_STEP,- task=_TASK,- )-- assert isinstance(result, ActionSuccess)- call_kwargs = mock_choose.call_args.kwargs- assert call_kwargs["verify_commit"] is flag_enabled- assert call_kwargs["commit_required"] is flag_enabled--# ---------------------------------------------------------------------------# Integration tests: options that don't contain the input fall through to LLM# ----------------------------------------------------------------------------NO_RESULT_ELEMENTS = [{"id": "AAAA", "tagName": "div", "interactable": True, "text": "No results"}]-UNRELATED_ELEMENTS = [{"id": "AAAA", "tagName": "div", "interactable": True, "text": "Something completely different"}]+NO_RESULT_ELEMENTS = [{"id": "AAAA", "tag": "div", "text": "No results"}]+UNRELATED_ELEMENTS = [{"id": "AAAA", "tag": "div", "text": "Something completely different"}]DETERMINISTIC_ELEMENTS = [{"id": "AAAA", "tagName": "li", "attributes": {"role": "option"}, "text": "San Francisco, California"},{"id": "AAAB", "tagName": "li", "attributes": {"role": "option"}, "text": "Oakland, California"},@@ -1172,20 +440,12 @@ def _mock_autocomplete_input(*,selected_value: str,option_identity: dict[str, object] | None = None,- click_closes_listbox: bool = True,) -> tuple[MagicMock, MagicMock, MagicMock, MagicMock]:frame = MagicMock()option_locator = MagicMock()option_locator.count = AsyncMock(return_value=1)option_locator.element_handle = AsyncMock(return_value=MagicMock())- option_locator.is_visible = AsyncMock(return_value=True)- option_locator.get_attribute = AsyncMock(return_value=None)-- async def _click(**_: object) -> None:- if click_closes_listbox:- option_locator.count = AsyncMock(return_value=0)-- option_locator.click = AsyncMock(side_effect=_click)+ option_locator.click = AsyncMock()frame.locator.return_value = option_locatorinput_locator = MagicMock()@@ -1215,7 +475,6 @@ async def test_location_no_results_option_falls_through_to_llm() -> None:"relevance_float": 0.0,"id": "","direct_searching": True,- "value": "123 Main St, Springfield, IL","reasoning": "No results shown",}@@ -1392,7 +651,6 @@ async def test_collapse_autocomplete_search_bar_exact_match_uses_llm() -> None:"relevance_float": 0.0,"id": "","direct_searching": True,- "value": "Oakland, California","reasoning": "Search bars should direct search",}@@ -1411,7 +669,7 @@ async def test_collapse_autocomplete_search_bar_exact_match_uses_llm() -> None:mock_prompt.load_prompt.return_value = "mocked prompt"mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)- result = await choose_auto_completion_dropdown(+ await choose_auto_completion_dropdown(context=_make_non_location_context(field="Search", is_search_bar=True),page=MagicMock(),scraped_page=MagicMock(),@@ -1426,7 +684,6 @@ async def test_collapse_autocomplete_search_bar_exact_match_uses_llm() -> None:option_locator.click.assert_not_called()mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_awaited_once()skyvern_el.press_key.assert_awaited_once_with("Enter")- assert isinstance(result.action_result, ActionSuccess)@pytest.mark.asyncio@@ -1443,7 +700,6 @@ async def test_collapse_autocomplete_ambiguous_exact_match_falls_back_to_llm() -"relevance_float": 0.0,"id": "","direct_searching": True,- "value": "Oakland, California","reasoning": "Ambiguous duplicate labels",}@@ -1502,7 +758,6 @@ async def test_collapse_autocomplete_identity_mismatch_resets_before_llm() -> No"relevance_float": 0.0,"id": "","direct_searching": True,- "value": "Oakland, California","reasoning": "Option rerendered",}@@ -1554,11 +809,10 @@ async def test_collapse_autocomplete_identity_mismatch_resets_before_llm() -> No@pytest.mark.asyncioasync def test_collapse_autocomplete_readback_mismatch_falls_back_to_llm() -> None:- """A no-op deterministic click (value unchanged, listbox open) is not accepted and resets before the LLM."""+ """A deterministic click is not accepted when the input read-back is not the matched label."""_frame, option_locator, skyvern_el, skyvern_frame_mock = _mock_autocomplete_input(- selected_value="Oakland, California",+ selected_value="Wrong city",option_identity={"index": 1, "label": "Oakland, California"},- click_closes_listbox=False,)events: list[str] = []skyvern_el.input_clear = AsyncMock(side_effect=lambda: events.append("clear"))@@ -1576,7 +830,6 @@ async def test_collapse_autocomplete_readback_mismatch_falls_back_to_llm() -> No"relevance_float": 0.0,"id": "","direct_searching": True,- "value": "Oakland, California","reasoning": "Read-back mismatch",}@@ -1625,149 +878,6 @@ async def test_collapse_autocomplete_readback_mismatch_falls_back_to_llm() -> Noskyvern_el.press_key.assert_awaited_once_with("Enter")-async def test_collapse_autocomplete_deterministic_path_verifies_even_when_flag_off() -> None:- # The deterministic read-back is the one verify site that exists on main, so it stays on even at- # flag-off: a click that leaves the value unchanged with the listbox open is not a commit and must- # still reset and fall back to the LLM chooser, never be reported as a silent success.- _frame, option_locator, skyvern_el, skyvern_frame_mock = _mock_autocomplete_input(- selected_value="Oakland, California",- option_identity={"index": 1, "label": "Oakland, California"},- click_closes_listbox=False,- )- events: list[str] = []- skyvern_el.input_clear = AsyncMock(side_effect=lambda: events.append("clear"))- skyvern_el.press_fill = AsyncMock(side_effect=lambda value: events.append(f"fill:{value}"))- skyvern_el.press_key = AsyncMock()- initial_scrape = _mock_incremental_scrape(DETERMINISTIC_ELEMENTS)- fallback_scrape = _mock_incremental_scrape(- [{"id": "AAAB", "tagName": "li", "attributes": {"role": "option"}, "text": "Oakland, California"}]- )- llm_response = {- "auto_completion_attempt": False,- "relevance_float": 0.0,- "id": "",- "direct_searching": True,- "value": "Oakland, California",- "reasoning": "Read-back mismatch",- }-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=skyvern_frame_mock),- ),- patch(- "skyvern.webeye.actions.handler.IncrementalScrapePage",- side_effect=[initial_scrape, fallback_scrape],- ),- patch("skyvern.webeye.actions.handler.app") as mock_app,- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- ):-- async def _llm_handler(**_: object) -> dict[str, object]:- events.append("llm")- return llm_response-- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(side_effect=_llm_handler)- mock_app.AGENT_FUNCTION = MagicMock()- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)-- await choose_auto_completion_dropdown(- context=_make_non_location_context(field="Current location"),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- text="Oakland, California",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- verify_commit=False,- collapse_autocomplete_fanout_enabled=True,- )-- option_locator.click.assert_awaited_once()- skyvern_el.get_locator.return_value.input_value.assert_awaited()- mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_awaited_once()---async def test_collapse_autocomplete_chip_widget_commit_succeeds() -> None:- """A chip/token widget clears the input after committing; cleared value + closed listbox is a commit."""- _frame, option_locator, skyvern_el, skyvern_frame_mock = _mock_autocomplete_input(- selected_value="",- option_identity={"index": 1, "label": "Oakland, California"},- )- inc_scrape = _mock_incremental_scrape(DETERMINISTIC_ELEMENTS)-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=skyvern_frame_mock),- ),- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),- patch("skyvern.webeye.actions.handler.app") as mock_app,- ):- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock()-- result = await choose_auto_completion_dropdown(- context=_make_non_location_context(field="Current location"),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- text="Oakland, California",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- collapse_autocomplete_fanout_enabled=True,- )-- option_locator.click.assert_awaited_once()- mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_not_called()- assert isinstance(result.action_result, ActionSuccess)---async def test_collapse_autocomplete_click_with_side_effects_does_not_refire() -> None:- """A click that changed the value (but did not match) must not be re-fired via the LLM fallback."""- _frame, option_locator, skyvern_el, skyvern_frame_mock = _mock_autocomplete_input(- selected_value="Wrong city",- option_identity={"index": 1, "label": "Oakland, California"},- click_closes_listbox=False,- )- inc_scrape = _mock_incremental_scrape(DETERMINISTIC_ELEMENTS)-- with (- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=skyvern_frame_mock),- ),- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),- patch("skyvern.webeye.actions.handler.app") as mock_app,- ):- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock()-- result = await choose_auto_completion_dropdown(- context=_make_non_location_context(field="Current location"),- page=MagicMock(),- scraped_page=MagicMock(),- dom=MagicMock(),- text="Oakland, California",- skyvern_element=skyvern_el,- step=_STEP,- task=_TASK,- collapse_autocomplete_fanout_enabled=True,- )-- option_locator.click.assert_awaited_once()- skyvern_el.press_fill.assert_awaited_once()- mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_not_called()- assert isinstance(result.action_result, ActionFailure)- assert "clicked_but_not_committed" in (result.action_result.exception_message or "")--@pytest.mark.asyncioasync def test_collapse_autocomplete_detached_option_resets_before_llm() -> None:"""A deterministic candidate that detaches before the click must refresh before the LLM chooser."""@@ -1792,7 +902,6 @@ async def test_collapse_autocomplete_detached_option_resets_before_llm() -> None"relevance_float": 0.0,"id": "","direct_searching": True,- "value": "Oakland, California","reasoning": "Option detached",}@@ -1871,7 +980,6 @@ async def test_reset_autocomplete_empty_incremental_rescrapes_page_elements() ->with (patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=fallback_scrape) as scrape_factory,patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_after_open),- patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock()),):(returned_scrape,@@ -1894,9 +1002,9 @@ async def test_reset_autocomplete_empty_incremental_rescrapes_page_elements() ->current_scrape.stop_listen_dom_increment.assert_awaited_once()skyvern_el.input_clear.assert_awaited_once()skyvern_el.press_fill.assert_awaited_once_with("Oakland")+ skyvern_frame.safe_wait_for_animation_end.assert_awaited_once()scrape_factory.assert_called_once_with(skyvern_frame=skyvern_frame)- fallback_scrape.get_incremental_elements_num.assert_awaited()- fallback_scrape.get_incremental_element_tree.assert_not_awaited()+ fallback_scrape.get_incremental_element_tree.assert_awaited_once()scraped_page.generate_scraped_page_without_screenshots.assert_awaited_once()dom_after_open.get_skyvern_element_by_id.assert_awaited_once_with("FRESH")assert returned_scrape is fallback_scrape@@ -1932,8 +1040,7 @@ async def test_reset_autocomplete_empty_rescrape_without_interactable_elements_rwith (patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=fallback_scrape),patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_after_open),- patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock()),- pytest.raises(AutoCompletionCommitFailure),+ pytest.raises(NoIncrementalElementFoundForAutoCompletion),):await _reset_autocomplete_for_llm_fallback(current_incremental_scraped=current_scrape,diff --git a/tests/unit/test_blocked_form_advance_followup.py b/tests/unit/test_blocked_form_advance_followup.pydeleted file mode 100644index 8cee8faba..000000000--- a/tests/unit/test_blocked_form_advance_followup.py+++ /dev/null@@ -1,134 +0,0 @@-from types import SimpleNamespace-from unittest.mock import AsyncMock, Mock--import pytest--from skyvern.webeye.actions import handler-from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess---def _element(tag: str, *, attributes: dict | None = None, text: str | None = None) -> dict:- return {"tagName": tag, "attributes": attributes or {}, "text": text}--- ("element", "expected"),- [- (_element("button", text="Next"), True),- (_element("button", text="Continue"), True),- (_element("button", text="Save and proceed"), True),- (_element("input", attributes={"type": "submit", "value": "Go"}), True),- (_element("input", attributes={"type": "image"}), True),- (_element("a", attributes={"role": "button"}, text="Confirm"), True),- (_element("button", attributes={"aria-label": "Submit application"}), True),- (_element("button", text="Add another item"), False),- (_element("input", attributes={"type": "checkbox"}), False),- (_element("div", text="Next"), False),- (_element("a", text="Next"), False),- (_element("button", text=""), False),- ],-)-def test_click_is_submit_like(element: dict, expected: bool) -> None:- assert handler._click_is_submit_like(element) is expected---def _submit_element() -> object:- return SimpleNamespace(- get_element_dict=Mock(return_value=_element("button", text="Next")),- get_id=Mock(return_value="next-btn"),- )---async def test_blocked_form_advance_sets_followup_on_non_advancing_submit(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(handler, "_is_surface_blocked_form_advance_enabled", AsyncMock(return_value=True))- results: list[object] = [ActionSuccess()]- await handler._maybe_flag_blocked_form_advance(- results=results,- skyvern_element=_submit_element(),- page=SimpleNamespace(url="https://example.com/step"),- original_url="https://example.com/step",- incremental_scraped=SimpleNamespace(get_incremental_elements_num=AsyncMock(return_value=0)),- task=SimpleNamespace(organization_id=None),- )- assert results[-1].needs_followup is True- assert results[-1].followup_message- assert "Next" in results[-1].followup_message---async def test_blocked_form_advance_noop_when_url_changed(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(handler, "_is_surface_blocked_form_advance_enabled", AsyncMock(return_value=True))- results: list[object] = [ActionSuccess()]- await handler._maybe_flag_blocked_form_advance(- results=results,- skyvern_element=_submit_element(),- page=SimpleNamespace(url="https://example.com/next-step"),- original_url="https://example.com/step",- incremental_scraped=SimpleNamespace(get_incremental_elements_num=AsyncMock(return_value=0)),- task=SimpleNamespace(organization_id=None),- )- assert results[-1].needs_followup is None---async def test_blocked_form_advance_noop_when_new_elements_appeared(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(handler, "_is_surface_blocked_form_advance_enabled", AsyncMock(return_value=True))- results: list[object] = [ActionSuccess()]- await handler._maybe_flag_blocked_form_advance(- results=results,- skyvern_element=_submit_element(),- page=SimpleNamespace(url="https://example.com/step"),- original_url="https://example.com/step",- incremental_scraped=SimpleNamespace(get_incremental_elements_num=AsyncMock(return_value=5)),- task=SimpleNamespace(organization_id=None),- )- assert results[-1].needs_followup is None---async def test_blocked_form_advance_noop_when_not_submit_like(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(handler, "_is_surface_blocked_form_advance_enabled", AsyncMock(return_value=True))- results: list[object] = [ActionSuccess()]- await handler._maybe_flag_blocked_form_advance(- results=results,- skyvern_element=SimpleNamespace(- get_element_dict=Mock(return_value=_element("input", attributes={"type": "checkbox"})),- get_id=Mock(return_value="cb"),- ),- page=SimpleNamespace(url="https://example.com/step"),- original_url="https://example.com/step",- incremental_scraped=SimpleNamespace(get_incremental_elements_num=AsyncMock(return_value=0)),- task=SimpleNamespace(organization_id=None),- )- assert results[-1].needs_followup is None---async def test_blocked_form_advance_noop_when_flag_disabled(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(handler, "_is_surface_blocked_form_advance_enabled", AsyncMock(return_value=False))- results: list[object] = [ActionSuccess()]- await handler._maybe_flag_blocked_form_advance(- results=results,- skyvern_element=_submit_element(),- page=SimpleNamespace(url="https://example.com/step"),- original_url="https://example.com/step",- incremental_scraped=SimpleNamespace(get_incremental_elements_num=AsyncMock(return_value=0)),- task=SimpleNamespace(organization_id=None),- )- assert results[-1].needs_followup is None---async def test_blocked_form_advance_noop_when_last_result_not_success(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(handler, "_is_surface_blocked_form_advance_enabled", AsyncMock(return_value=True))- results: list[object] = [ActionFailure(Exception("boom"))]- await handler._maybe_flag_blocked_form_advance(- results=results,- skyvern_element=_submit_element(),- page=SimpleNamespace(url="https://example.com/step"),- original_url="https://example.com/step",- incremental_scraped=SimpleNamespace(get_incremental_elements_num=AsyncMock(return_value=0)),- task=SimpleNamespace(organization_id=None),- )- assert results[-1].needs_followup is Nonediff --git a/tests/unit/test_input_text_combobox_commit.py b/tests/unit/test_input_text_combobox_commit.pyindex a8984d5ce..a927b8592 100644--- a/tests/unit/test_input_text_combobox_commit.py+++ b/tests/unit/test_input_text_combobox_commit.py@@ -206,10 +206,6 @@ async def _run_combobox_input(),patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),patch("skyvern.webeye.actions.handler.sequentially_select_from_dropdown", new=select_mock),- patch(- "skyvern.webeye.actions.handler._is_input_text_commit_verification_enabled",- new=AsyncMock(return_value=False),- ),):results = await handle_input_text_action(action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEPdiff --git a/tests/unit/test_input_text_masked_readback.py b/tests/unit/test_input_text_masked_readback.pydeleted file mode 100644index 16313e34c..000000000--- a/tests/unit/test_input_text_masked_readback.py+++ /dev/null@@ -1,190 +0,0 @@-from __future__ import annotations--from datetime import UTC, datetime-from unittest.mock import AsyncMock, MagicMock, patch--import pytest--from skyvern.exceptions import MaskedInputReadbackMismatch-from skyvern.forge.sdk.models import StepStatus-from skyvern.webeye.actions.actions import InputTextAction-from skyvern.webeye.actions.handler import (- _committed_value_is_masked_incomplete,- _has_input_mask_evidence,- _verify_masked_input_after_fill,- handle_input_text_action,-)-from skyvern.webeye.actions.responses import ActionFailure-from tests.unit.conftest import make_input_element_mock-from tests.unit.helpers import make_organization, make_step, make_task--_NOW = datetime.now(UTC)-_ORG = make_organization(_NOW)-_TASK = make_task(_NOW, _ORG, navigation_payload={}, navigation_goal="Fill the postal code field")-_STEP = make_step(_NOW, _TASK, step_id="stp-masked-readback", status=StepStatus.created, order=0, output=None)---def test_mask_evidence_from_pattern() -> None:- assert _has_input_mask_evidence(pattern=r"\d{5}(-\d{4})?", placeholder=None) is True---def test_mask_evidence_from_placeholder_slots() -> None:- assert _has_input_mask_evidence(pattern=None, placeholder="_____") is True- assert _has_input_mask_evidence(pattern=None, placeholder="___-__-____") is True---def test_no_mask_evidence_for_plain_field() -> None:- assert _has_input_mask_evidence(pattern=None, placeholder="Enter ZIP") is False- assert _has_input_mask_evidence(pattern=None, placeholder=None) is False---def test_incomplete_when_commit_is_empty() -> None:- assert _committed_value_is_masked_incomplete("12345", "") is True- assert _committed_value_is_masked_incomplete("12345", None) is True---def test_incomplete_when_only_placeholders_remain() -> None:- assert _committed_value_is_masked_incomplete("12345", "_____") is True- assert _committed_value_is_masked_incomplete("12345", "123__") is True---def test_complete_when_value_committed() -> None:- assert _committed_value_is_masked_incomplete("12345", "12345") is False- # A ZIP+4 field that reformats the committed value is complete, not incomplete.- assert _committed_value_is_masked_incomplete("12345", "12345-6789") is False---def test_placeholder_char_inside_complete_value_is_not_incomplete() -> None:- # An underscore that is part of a fully-typed value must not read as an unfilled mask slot.- assert _committed_value_is_masked_incomplete("foo_bar", "foo_bar") is False---def test_nothing_expected_is_never_incomplete() -> None:- assert _committed_value_is_masked_incomplete("", "") is False- assert _committed_value_is_masked_incomplete("---", "_____") is False---def _mock_element(input_values: list[str | None]) -> MagicMock:- locator = MagicMock()- locator.input_value = AsyncMock(side_effect=input_values)- element = MagicMock()- element.get_locator = MagicMock(return_value=locator)- return element---def _mock_frame() -> MagicMock:- frame = MagicMock()- frame.safe_wait_for_animation_end = AsyncMock()- return frame---async def test_masked_readback_raises_on_empty_commit() -> None:- element = _mock_element(["", ""])- with pytest.raises(MaskedInputReadbackMismatch) as exc:- await _verify_masked_input_after_fill(- skyvern_element=element,- skyvern_frame=_mock_frame(),- tag_name="input",- expected_value="12345",- known_masked=True,- )- assert exc.value.expected_char_count == 5- assert exc.value.committed_char_count == 0---async def test_masked_readback_accepts_committed_value() -> None:- element = _mock_element(["12345"])- await _verify_masked_input_after_fill(- skyvern_element=element,- skyvern_frame=_mock_frame(),- tag_name="input",- expected_value="12345",- known_masked=True,- )---async def test_masked_readback_settles_before_declaring_failure() -> None:- # First read is still empty, the value lands after the settle wait — no failure.- element = _mock_element(["", "12345"])- await _verify_masked_input_after_fill(- skyvern_element=element,- skyvern_frame=_mock_frame(),- tag_name="input",- expected_value="12345",- known_masked=True,- )---async def test_masked_readback_skips_field_without_mask_signal() -> None:- # No attribute evidence and no placeholder chars in the commit: leave the field alone even if empty.- element = _mock_element([""])- await _verify_masked_input_after_fill(- skyvern_element=element,- skyvern_frame=_mock_frame(),- tag_name="input",- expected_value="12345",- known_masked=False,- )---async def test_masked_readback_triggers_on_placeholder_commit_without_attr_evidence() -> None:- element = _mock_element(["_____", "_____"])- with pytest.raises(MaskedInputReadbackMismatch):- await _verify_masked_input_after_fill(- skyvern_element=element,- skyvern_frame=_mock_frame(),- tag_name="input",- expected_value="12345",- known_masked=False,- )---async def _run_masked_input_text(text: str, read_backs: list[str | None]) -> list:- element = make_input_element_mock()- dom_instance = MagicMock()- dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=element)-- inc = MagicMock()- inc.start_listen_dom_increment = AsyncMock()- inc.stop_listen_dom_increment = AsyncMock()- inc.get_incremental_element_tree = AsyncMock(return_value=[])-- skyvern_frame = MagicMock()- skyvern_frame.safe_wait_for_animation_end = AsyncMock()-- scraped_page = MagicMock()- scraped_page.id_to_element_dict = {"AADC": {"tagName": "input"}}-- with (- patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),- patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance", new=AsyncMock(return_value=skyvern_frame)),- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),- patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(side_effect=read_backs)),- patch("skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task", return_value=text),- patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=None)),- patch("skyvern.webeye.actions.handler._is_masked_input_readback_fix_enabled", new=AsyncMock(return_value=True)),- ):- return await handle_input_text_action(- action=InputTextAction(element_id="AADC", text=text, reasoning="fill postal code"),- page=MagicMock(),- scraped_page=scraped_page,- task=_TASK,- step=_STEP,- )---async def test_masked_retype_swallowed_by_mask_fails_loudly() -> None:- # The field advertises no mask attrs, so the only mask evidence is the placeholder chars seen on the- # first read-back. The retype is then swallowed whole and reads back empty: without carrying that- # evidence into the retry, the empty commit looks like an unmasked field and passes silently.- results = await _run_masked_input_text("12345", ["", "12_45", "12_45", "", ""])-- assert len(results) == 1- assert isinstance(results[0], ActionFailure)- assert results[0].exception_type == MaskedInputReadbackMismatch.__name__diff --git a/tests/unit/test_input_text_phone_readback.py b/tests/unit/test_input_text_phone_readback.pyindex 1e88a6b72..9ec6afc57 100644--- a/tests/unit/test_input_text_phone_readback.py+++ b/tests/unit/test_input_text_phone_readback.py@@ -3,17 +3,13 @@ from __future__ import annotationsfrom unittest.mock import AsyncMock, MagicMockimport pytest-from playwright._impl._errors import Error as PlaywrightError-from skyvern.exceptions import InputTextCommitMismatch, PhoneNumberInputMismatch+from skyvern.exceptions import PhoneNumberInputMismatchfrom skyvern.webeye.actions.handler import (_fill_nanp_tel_with_readback,_log_tel_fallback_fill_digit_counts,- _verify_generic_input_commit,- input_text_values_match,verify_phone_input_digits,)-from skyvern.webeye.actions.responses import ActionFailureNANP_DIGITS = "2245550199"NANP_E164 = f"+1{NANP_DIGITS}"@@ -214,328 +210,3 @@ async def test_tel_fallback_digit_count_log_swallows_read_errors() -> None:task_id="tsk_1",step_id="stp_1",)--- ("expected", "actual", "input_type"),- [- ("5551234567", "(555) 123-4567", "text"),- ("12345", "12345-____", "text"),- ("ab12 cd", "AB12-CD", "search"),- ("postal code", "POSTAL_CODE", "textarea"),- ],-)-def test_input_text_values_match_ignores_plain_text_mask_formatting(- expected: str, actual: str, input_type: str-) -> None:- assert input_text_values_match(expected, actual, input_type=input_type)--- ("expected", "actual", "input_type"),- [- ("[email protected]", "user@exampletest", "email"),- ("https://example.test/path", "https://exampletest/path", "url"),- ("-12.5", "125", "number"),- ],-)-def test_input_text_values_match_preserves_semantic_punctuation(expected: str, actual: str, input_type: str) -> None:- assert not input_text_values_match(expected, actual, input_type=input_type)---def test_input_text_values_match_trims_semantic_input_whitespace() -> None:- assert input_text_values_match(" [email protected] ", "[email protected]", input_type="email")--- ("expected", "actual", "committed", "why"),- [- # A field has committed when its value derives from what was typed.- ("12345", "12345", True, "exact"),- ("1234567890", "(123) 456-7890", True, "reformat only rearranges separators"),- ("12345", "12345-6789", True, "widget enriches the typed value"),- ("foo_bar", "foo_bar", True, "placeholder char inside a fully typed value"),- # Only these are failures.- ("12345", "", False, "empty"),- ("12345", "_____", False, "mask kept none of the keystrokes"),- ("12345", "123__", False, "mask kept only some of them"),- ("12345", "99999", False, "genuinely different value"),- ("12345", "999123459", False, "contains the typed value but does not derive from it"),- ("---", "___", True, "separators typed, separators back: a reformat of nothing"),- ("---", "xyz", False, "separators typed, a real value back: not a reformat of it"),- ],-)-def test_input_text_values_match_treats_derived_values_as_committed(- expected: str, actual: str, committed: bool, why: str-) -> None:- assert input_text_values_match(expected, actual, input_type="text") is committed, why---def test_input_text_values_match_rejects_empty_commit() -> None:- assert not input_text_values_match("-", "", input_type="text")---def _make_commit_element(readbacks: list[str]) -> MagicMock:- locator = MagicMock()- locator.input_value = AsyncMock(side_effect=readbacks)- locator.evaluate = AsyncMock()- element = _make_element(locator)- element.input_clear = AsyncMock()- element.input_sequentially = AsyncMock()- element.input_fill = AsyncMock()- return element---def _make_commit_frame() -> MagicMock:- frame = MagicMock()- frame.safe_wait_for_animation_end = AsyncMock()- return frame---async def test_generic_commit_verification_accepts_initial_readback() -> None:- element = _make_commit_element(["committed"])- dom = MagicMock()-- result = await _verify_generic_input_commit(- skyvern_element=element,- skyvern_frame=_make_commit_frame(),- dom=dom,- tag_name="input",- input_type="text",- expected_value="committed",- )-- assert result is None- element.input_clear.assert_not_awaited()- element.input_fill.assert_not_awaited()- element.get_locator().evaluate.assert_not_awaited()---async def test_generic_commit_verification_settles_before_destroying_a_slow_value() -> None:- # A masked or debounced field can land its value a beat after the keystrokes settle. Escalating on- # the first read-back would clear and retype a field that was about to commit correctly.- element = _make_commit_element(["", "committed"])- frame = _make_commit_frame()- dom = MagicMock()-- result = await _verify_generic_input_commit(- skyvern_element=element,- skyvern_frame=frame,- dom=dom,- tag_name="input",- input_type="text",- expected_value="committed",- )-- assert result is None- frame.safe_wait_for_animation_end.assert_awaited_once()- element.input_clear.assert_not_awaited()- element.input_sequentially.assert_not_awaited()---async def test_generic_commit_verification_escalates_to_fill() -> None:- element = _make_commit_element(["", "", "", "committed"])- dom = MagicMock()-- result = await _verify_generic_input_commit(- skyvern_element=element,- skyvern_frame=_make_commit_frame(),- dom=dom,- tag_name="input",- input_type="text",- expected_value="committed",- )-- assert result is None- element.input_clear.assert_awaited_once()- element.input_sequentially.assert_awaited_once_with(text="committed")- element.input_fill.assert_awaited_once_with(text="committed")- element.get_locator().evaluate.assert_not_awaited()---async def test_generic_commit_verification_escalates_to_native_setter() -> None:- element = _make_commit_element(["", "", "", "", "committed"])- dom = MagicMock()-- result = await _verify_generic_input_commit(- skyvern_element=element,- skyvern_frame=_make_commit_frame(),- dom=dom,- tag_name="textarea",- input_type="textarea",- expected_value="committed",- )-- assert result is None- element.input_fill.assert_awaited_once_with(text="committed")- element.get_locator().evaluate.assert_awaited_once()- assert element.get_locator().evaluate.await_args.args[1] == "committed"---async def test_generic_commit_verification_returns_typed_failure_without_value() -> None:- element = _make_commit_element(["", "", "", "", ""])- dom = MagicMock()-- result = await _verify_generic_input_commit(- skyvern_element=element,- skyvern_frame=_make_commit_frame(),- dom=dom,- tag_name="input",- input_type="text",- expected_value="sensitive-value",- )-- assert isinstance(result, ActionFailure)- assert result.exception_type == InputTextCommitMismatch.__name__- assert "expected length 14" in (result.exception_message or "")- assert "actual length 0" in (result.exception_message or "")- assert "sensitive-value" not in (result.exception_message or "")---async def test_generic_commit_verification_skips_navigation_during_escalation() -> None:- element = _make_commit_element(["", ""])- dom = MagicMock()- element.input_clear.side_effect = PlaywrightError("Execution context was destroyed during navigation")-- result = await _verify_generic_input_commit(- skyvern_element=element,- skyvern_frame=_make_commit_frame(),- dom=dom,- tag_name="input",- input_type="text",- expected_value="committed",- )-- assert result is None---async def test_generic_commit_verification_reresolves_detached_element() -> None:- detached_element = _make_commit_element([])- detached_element.get_locator().input_value.side_effect = PlaywrightError("Element is not attached to the DOM")- replacement_element = _make_commit_element(["committed"])- dom = MagicMock()- dom.get_skyvern_element_by_id = AsyncMock(return_value=replacement_element)-- result = await _verify_generic_input_commit(- skyvern_element=detached_element,- skyvern_frame=_make_commit_frame(),- dom=dom,- tag_name="input",- input_type="text",- expected_value="committed",- )-- assert result is None- dom.get_skyvern_element_by_id.assert_awaited_once_with(detached_element.get_id())---async def test_generic_commit_verification_accepts_plausible_autocomplete_rewrite() -> None:- element = _make_commit_element(["committed option"])-- result = await _verify_generic_input_commit(- skyvern_element=element,- skyvern_frame=_make_commit_frame(),- dom=MagicMock(),- tag_name="input",- input_type="text",- expected_value="committed",- allow_autocomplete_rewrite=True,- )-- assert result is None- element.input_clear.assert_not_awaited()---async def test_generic_commit_verification_rejects_truncated_value_as_autocomplete_rewrite() -> None:- element = _make_commit_element(["123", "123", "123", "123", "123"])-- result = await _verify_generic_input_commit(- skyvern_element=element,- skyvern_frame=_make_commit_frame(),- dom=MagicMock(),- tag_name="input",- input_type="text",- expected_value="12345",- allow_autocomplete_rewrite=True,- )-- assert isinstance(result, ActionFailure)- assert result.exception_type == InputTextCommitMismatch.__name__- element.input_clear.assert_awaited_once()- element.input_fill.assert_awaited_once_with(text="12345")---async def test_generic_commit_verification_escalates_garbled_readback_without_rewrite_allowance() -> None:- element = _make_commit_element(["123", "123", "12345"])-- result = await _verify_generic_input_commit(- skyvern_element=element,- skyvern_frame=_make_commit_frame(),- dom=MagicMock(),- tag_name="input",- input_type="text",- expected_value="12345",- )-- assert result is None- element.input_clear.assert_awaited_once()- element.input_sequentially.assert_awaited_once_with(text="12345")---async def test_generic_commit_verification_reresolves_once_per_escalation_stage() -> None:- first_element = _make_commit_element([])- first_element.get_locator().input_value.side_effect = PlaywrightError("Element is not attached to the DOM")- second_element = _make_commit_element([])- second_element.get_locator().input_value.side_effect = [- "",- "",- PlaywrightError("Element is not attached to the DOM"),- ]- third_element = _make_commit_element(["committed"])- dom = MagicMock()- dom.get_skyvern_element_by_id = AsyncMock(side_effect=[second_element, third_element])-- result = await _verify_generic_input_commit(- skyvern_element=first_element,- skyvern_frame=_make_commit_frame(),- dom=dom,- tag_name="input",- input_type="text",- expected_value="committed",- )-- assert result is None- assert dom.get_skyvern_element_by_id.await_count == 2- second_element.input_sequentially.assert_awaited_once_with(text="committed")---async def test_generic_commit_verification_soft_skips_detach_during_fill_stage() -> None:- detached_element = _make_commit_element([])- detached_element.get_locator().input_value.side_effect = PlaywrightError("Element is not attached to the DOM")- replacement_element = _make_commit_element(["", ""])- replacement_element.input_clear.side_effect = PlaywrightError("Element is not attached to the DOM")- dom = MagicMock()- dom.get_skyvern_element_by_id = AsyncMock(return_value=replacement_element)-- result = await _verify_generic_input_commit(- skyvern_element=detached_element,- skyvern_frame=_make_commit_frame(),- dom=dom,- tag_name="input",- input_type="text",- expected_value="committed",- )-- assert result is Nonediff --git a/tests/unit/test_input_text_search_bar_skip.py b/tests/unit/test_input_text_search_bar_skip.pyindex edb9edb90..c666663d9 100644--- a/tests/unit/test_input_text_search_bar_skip.py+++ b/tests/unit/test_input_text_search_bar_skip.py@@ -9,19 +9,16 @@ and non-clobbering follow-ups (Tab/Escape/Arrow, different-element actions) arefrom __future__ import annotations-import copyfrom datetime import UTC, datetimefrom unittest.mock import AsyncMock, MagicMock, patchimport pytest-from playwright.async_api import Error as PlaywrightError-from skyvern.exceptions import AutoCompletionCommitFailurefrom skyvern.forge.sdk.models import StepStatusfrom skyvern.webeye.actions.actions import InputOrSelectContext, InputTextAction, KeypressAction-from skyvern.webeye.actions.handler import AUTOCOMPLETE_COMMIT_REQUIRED_FLAG, handle_input_text_action+from skyvern.webeye.actions.handler import handle_input_text_actionfrom skyvern.webeye.actions.handler_utils import keys_include_enter, should_stop_batch_after_dropdown_select-from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess+from skyvern.webeye.actions.responses import ActionSuccessfrom tests.unit.conftest import make_input_element_mockfrom tests.unit.helpers import make_organization, make_step, make_task@@ -122,94 +119,12 @@ async def _run_search_bar_input(stop_flag: bool, incremental: list[dict]) -> lis"skyvern.webeye.actions.handler.sequentially_select_from_dropdown",new=AsyncMock(return_value=select_result),),- patch(- "skyvern.webeye.actions.handler._is_input_text_commit_verification_enabled",- new=AsyncMock(return_value=False),- ),):return await handle_input_text_action(action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP)-def _fake_experimentation_provider(enabled_flags: set[str]) -> MagicMock:- provider = MagicMock()-- async def _is_enabled(flag: str, distinct_id: str, properties: dict | None = None) -> bool:- return flag in enabled_flags-- provider.is_feature_enabled_cached = AsyncMock(side_effect=_is_enabled)- provider.resolve_feature_enabled_unrecorded = AsyncMock(return_value=False)- return provider---async def _run_autocomplete_input_without_suggestions(- *,- is_search_bar: bool,- is_location_input: bool,- commit_flag_enabled: bool,-) -> tuple[list, MagicMock]:- """Drive handle_input_text_action through the REAL autocomplete chain with a widget- that never renders suggestions."""- skyvern_el = make_input_element_mock(element_id="AADC")- skyvern_el.is_auto_completion_input = AsyncMock(return_value=True)- skyvern_el.press_fill = AsyncMock()- skyvern_el.has_attr = AsyncMock(return_value=False)- skyvern_el.get_locator.return_value.click = AsyncMock()-- dom_instance = MagicMock()- dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=skyvern_el)-- inc = MagicMock()- inc.start_listen_dom_increment = AsyncMock()- inc.stop_listen_dom_increment = AsyncMock()- inc.get_incremental_elements_num = AsyncMock(return_value=0)- inc.get_incremental_element_tree = AsyncMock(return_value=[])- inc.build_html_tree.return_value = "<div></div>"- inc.id_to_element_dict = {}-- skyvern_frame = MagicMock()- skyvern_frame.safe_wait_for_animation_end = AsyncMock()-- scraped_page = MagicMock()- scraped_page.id_to_element_dict = {"AADC": {"tagName": "input"}}- scraped_page.id_to_css_dict = {"AADC": "#aadc"}- scraped_after = MagicMock()- scraped_after.id_to_css_dict = {"AADC": "#aadc"}- scraped_page.generate_scraped_page_without_screenshots = AsyncMock(return_value=scraped_after)-- context = InputOrSelectContext(field="Account", is_search_bar=is_search_bar, is_location_input=is_location_input)- enabled_flags = {AUTOCOMPLETE_COMMIT_REQUIRED_FLAG} if commit_flag_enabled else set()-- action = _input()-- with (- patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),- patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance", new=AsyncMock(return_value=skyvern_frame)),- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),- patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(return_value="")),- patch(- "skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task",- return_value="123456",- ),- patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),- patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock()),- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- patch("skyvern.webeye.actions.handler.app") as mock_app,- ):- mock_app.EXPERIMENTATION_PROVIDER = _fake_experimentation_provider(enabled_flags)- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value={})- mock_app.AGENT_FUNCTION = MagicMock()- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)- results = await handle_input_text_action(- action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP- )- return results, skyvern_el--@pytest.mark.asyncioasync def test_selection_with_clobbering_next_stops_batch() -> None:"""Selection committed + a clobbering next action (flag set) -> skip_remaining_actions=True."""@@ -232,128 +147,3 @@ async def test_plain_search_input_does_not_stop_batch() -> None:results = await _run_search_bar_input(stop_flag=True, incremental=[])assert len(results) == 1 and isinstance(results[0], ActionSuccess)assert not results[0].skip_remaining_actions---async def test_search_bar_without_suggestions_falls_through_to_plain_typing() -> None:- """A search bar whose autocomplete never renders suggestions must still be typed into (flag off)."""- results, skyvern_el = await _run_autocomplete_input_without_suggestions(- is_search_bar=True,- is_location_input=False,- commit_flag_enabled=False,- )- assert len(results) == 1 and isinstance(results[0], ActionSuccess)- skyvern_el.input_sequentially.assert_awaited_once()---async def test_location_field_without_suggestions_falls_through_when_flag_off() -> None:- results, skyvern_el = await _run_autocomplete_input_without_suggestions(- is_search_bar=False,- is_location_input=True,- commit_flag_enabled=False,- )- assert len(results) == 1 and isinstance(results[0], ActionSuccess)- skyvern_el.input_sequentially.assert_awaited_once()---async def test_search_bar_direct_search_enter_navigation_is_implicit_commit() -> None:- """direct_searching presses Enter, the page navigates, and every post-Enter read-back hits a stale- element; the action must succeed as an implicit commit without falling through to plain typing."""- skyvern_el = make_input_element_mock(element_id="AADC")- skyvern_el.is_auto_completion_input = AsyncMock(return_value=True)- skyvern_el.press_fill = AsyncMock()- skyvern_el.has_attr = AsyncMock(return_value=False)-- enter_pressed = False-- async def _press_key(key: str, **_: object) -> None:- nonlocal enter_pressed- if key == "Enter":- enter_pressed = True-- skyvern_el.press_key = AsyncMock(side_effect=_press_key)-- async def _read_input_value(tag_name: str, locator: object) -> str:- if enter_pressed:- raise PlaywrightError("Element is not attached to the DOM")- return ""-- suggestions = [- {"id": "OPT1", "tagName": "li", "attributes": {"role": "option"}, "text": "Target 123456"},- {"id": "OPT2", "tagName": "li", "attributes": {"role": "option"}, "text": "Target 654321"},- ]-- dom_instance = MagicMock()- dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=skyvern_el)-- inc = MagicMock()- inc.start_listen_dom_increment = AsyncMock()- inc.stop_listen_dom_increment = AsyncMock()- inc.get_incremental_elements_num = AsyncMock(return_value=len(suggestions))- inc.get_incremental_element_tree = AsyncMock(return_value=copy.deepcopy(suggestions))- inc.build_html_tree.return_value = "<div>options</div>"- inc.id_to_element_dict = {}-- scraped_page = MagicMock()- scraped_page.id_to_element_dict = {"AADC": {"tagName": "input"}}- scraped_page.id_to_css_dict = {"AADC": "#aadc"}-- context = InputOrSelectContext(field="Search", is_search_bar=True, is_location_input=False)- llm_response = {- "auto_completion_attempt": False,- "relevance_float": 0.0,- "id": "",- "direct_searching": True,- "value": "123456",- "reasoning": "Search directly",- }-- action = _input()-- with (- patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),- patch(- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),- ),- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),- patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(side_effect=_read_input_value)),- patch(- "skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task",- return_value="123456",- ),- patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),- patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock()),- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- patch("skyvern.webeye.actions.handler.app") as mock_app,- ):- mock_app.EXPERIMENTATION_PROVIDER = _fake_experimentation_provider(set())- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})- mock_app.AGENT_FUNCTION = MagicMock()- mock_prompt.load_prompt.return_value = "mocked prompt"- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)- results = await handle_input_text_action(- action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP- )-- assert len(results) == 1 and isinstance(results[0], ActionSuccess)- skyvern_el.press_key.assert_awaited_once_with("Enter")- skyvern_el.input_sequentially.assert_not_awaited()---async def test_location_field_without_commit_fails_when_flag_on() -> None:- """With the commit-required flag on, a location field never falls through to plain typing."""- results, skyvern_el = await _run_autocomplete_input_without_suggestions(- is_search_bar=False,- is_location_input=True,- commit_flag_enabled=True,- )- assert len(results) == 1 and isinstance(results[0], ActionFailure)- assert results[0].exception_type == AutoCompletionCommitFailure.__name__- assert "suggestions_never_rendered" in (results[0].exception_message or "")- skyvern_el.input_sequentially.assert_not_awaited()diff --git a/tests/unit/test_input_text_tel_card_routing.py b/tests/unit/test_input_text_tel_card_routing.pyindex c431a0552..085e333af 100644--- a/tests/unit/test_input_text_tel_card_routing.py+++ b/tests/unit/test_input_text_tel_card_routing.py@@ -1,7 +1,7 @@from __future__ import annotationsfrom datetime import UTC, datetime-from unittest.mock import ANY, AsyncMock, MagicMock, patch+from unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -55,11 +55,9 @@ async def _run_input_text(resolved: str | None = None,tel_fix_enabled: bool = True,tel_verify_side_effect: list[Exception | None] | None = None,- generic_flag: bool = True,- incremental_elements: list[dict] | None = None,tag_name: str = "input",blocker: MagicMock | None = None,-) -> tuple[list, AsyncMock, AsyncMock, AsyncMock, MagicMock, AsyncMock, AsyncMock]:+) -> tuple[list, AsyncMock, AsyncMock, AsyncMock, MagicMock, AsyncMock]:dom_instance = MagicMock()dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=el)if blocker is not None:@@ -69,7 +67,7 @@ async def _run_input_text(inc = MagicMock()inc.start_listen_dom_increment = AsyncMock()inc.stop_listen_dom_increment = AsyncMock()- inc.get_incremental_element_tree = AsyncMock(return_value=incremental_elements or [])+ inc.get_incremental_element_tree = AsyncMock(return_value=[])skyvern_frame = MagicMock()skyvern_frame.safe_wait_for_animation_end = AsyncMock()@@ -79,7 +77,6 @@ async def _run_input_text(card_readback = AsyncMock(return_value=None)tel_verify = AsyncMock(side_effect=tel_verify_side_effect)- generic_verify = AsyncMock(return_value=None)phone_format = AsyncMock(return_value=text)warning_log = MagicMock()secret_readback = AsyncMock(return_value=None)@@ -97,16 +94,10 @@ async def _run_input_text(),patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=None)),patch("skyvern.webeye.actions.handler._is_tel_digit_fix_enabled", new=AsyncMock(return_value=tel_fix_enabled)),- patch(- "skyvern.webeye.actions.handler._is_input_text_commit_verification_enabled",- new=AsyncMock(return_value=generic_flag),- ),patch("skyvern.webeye.actions.handler.check_phone_number_format", new=phone_format),patch("skyvern.webeye.actions.handler._fill_card_number_with_readback", new=card_readback),patch("skyvern.webeye.actions.handler._fill_secret_with_readback", new=secret_readback),patch("skyvern.webeye.actions.handler._verify_tel_input_after_fill", new=tel_verify),- patch("skyvern.webeye.actions.handler._verify_generic_input_commit", new=generic_verify),- patch("skyvern.webeye.actions.handler.check_date_format", new=AsyncMock(return_value=text)),patch("skyvern.webeye.actions.handler.LOG.warning", new=warning_log),):results = await handle_input_text_action(@@ -117,7 +108,7 @@ async def _run_input_text(step=_STEP,)- return results, card_readback, tel_verify, phone_format, warning_log, generic_verify, secret_readback+ return results, card_readback, tel_verify, phone_format, warning_log, secret_readback@pytest.mark.asyncio@@ -131,9 +122,7 @@ async def _run_input_text(async def test_tel_card_number_field_uses_card_readback_not_phone_format(attrs: dict[str, str | None]) -> None:el = _mock_input(attrs)- results, card_readback, tel_verify, phone_format, _, generic_verify, secret_readback = await _run_input_text(- el, VISA_16- )+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, VISA_16)assert len(results) == 1 and isinstance(results[0], ActionSuccess)card_readback.assert_awaited_once_with(@@ -144,7 +133,6 @@ async def test_tel_card_number_field_uses_card_readback_not_phone_format(attrs:)phone_format.assert_not_awaited()tel_verify.assert_not_awaited()- generic_verify.assert_not_awaited()secret_readback.assert_not_awaited()el.input_sequentially.assert_not_awaited()@@ -153,9 +141,7 @@ async def test_tel_card_number_field_uses_card_readback_not_phone_format(attrs:async def test_ten_digit_tel_phone_uses_tel_readback_not_card_readback() -> None:el = _mock_input({"type": "tel", "autocomplete": None, "name": "phone"})- results, card_readback, tel_verify, phone_format, _, generic_verify, secret_readback = await _run_input_text(- el, "224-555-0199"- )+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, "224-555-0199")assert len(results) == 1 and isinstance(results[0], ActionSuccess)el.input_sequentially.assert_awaited_once_with(text="2245550199")@@ -169,7 +155,6 @@ async def test_ten_digit_tel_phone_uses_tel_readback_not_card_readback() -> None)card_readback.assert_not_awaited()phone_format.assert_not_awaited()- generic_verify.assert_not_awaited()secret_readback.assert_not_awaited()@@ -177,7 +162,7 @@ async def test_ten_digit_tel_phone_uses_tel_readback_not_card_readback() -> Noneasync def test_tel_flag_off_preserves_legacy_format_and_sequential_fill() -> None:el = _mock_input({"type": "tel", "autocomplete": None, "name": "phone"})- results, _, tel_verify, phone_format, _, _, _ = await _run_input_text(+ results, _, tel_verify, phone_format, _, _ = await _run_input_text(el,"224-555-0199",tel_fix_enabled=False,@@ -199,7 +184,7 @@ async def test_separator_only_tel_never_forces_nanp_country_code() -> None:PhoneNumberInputMismatch(expected_digit_count=10, actual_digit_count=12),]- results, _, tel_verify, _, warning_log, _, _ = await _run_input_text(+ results, _, tel_verify, _, warning_log, _ = await _run_input_text(el,"224-555-0199",tel_verify_side_effect=mismatches,@@ -227,7 +212,7 @@ async def test_explicit_nanp_tel_keeps_constraint_safe_e164_fallback() -> None:None,]- results, _, tel_verify, _, _, _, _ = await _run_input_text(+ results, _, tel_verify, _, _, _ = await _run_input_text(el,"+1 (224) 555-0199",tel_verify_side_effect=mismatches_then_success,@@ -243,43 +228,6 @@ async def test_explicit_nanp_tel_keeps_constraint_safe_e164_fallback() -> None:]-async def test_generic_text_input_runs_commit_verification() -> None:- el = _mock_input({"type": "text", "autocomplete": None, "name": "postal_code"})-- results, card_readback, tel_verify, phone_format, _, generic_verify, _ = await _run_input_text(el, "12345")-- assert len(results) == 1 and isinstance(results[0], ActionSuccess)- el.input_sequentially.assert_awaited_once_with(text="12345")- generic_verify.assert_awaited_once_with(- skyvern_element=el,- skyvern_frame=ANY,- dom=ANY,- tag_name="input",- input_type="text",- expected_value="12345",- allow_autocomplete_rewrite=False,- )- card_readback.assert_not_awaited()- tel_verify.assert_not_awaited()- phone_format.assert_not_awaited()---async def test_blocking_date_input_refreshes_type_before_routing() -> None:- original = _mock_input({"type": "text", "autocomplete": None, "name": "start_date"})- blocking = _mock_input({"type": "date", "autocomplete": None, "name": "start_date"})- blocking.get_id.return_value = "BLOCKING"- original.find_blocking_element = AsyncMock(return_value=(blocking, True))-- results, _, _, _, _, generic_verify, _ = await _run_input_text(original, "2026-07-17")-- assert len(results) == 1 and isinstance(results[0], ActionSuccess)- blocking.input_fill.assert_awaited_once_with(text="2026-07-17")- blocking.input_sequentially.assert_not_awaited()- generic_verify.assert_not_awaited()--@pytest.mark.asyncioasync def test_blocking_tel_input_rechecks_constraints_before_readback() -> None:original = _mock_input({"type": "tel", "autocomplete": None, "name": "phone"})@@ -288,7 +236,7 @@ async def test_blocking_tel_input_rechecks_constraints_before_readback() -> None)blocking.get_id.return_value = "BLOCKING"- results, _, tel_verify, _, _, _, _ = await _run_input_text(original, "+1 (224) 555-0199", blocker=blocking)+ results, _, tel_verify, _, _, _ = await _run_input_text(original, "+1 (224) 555-0199", blocker=blocking)assert len(results) == 1 and isinstance(results[0], ActionSuccess)blocking.input_sequentially.assert_awaited_once_with(text="2245550199")@@ -303,67 +251,6 @@ async def test_blocking_tel_input_rechecks_constraints_before_readback() -> None)-async def test_generic_text_input_skips_commit_verification_when_flag_disabled() -> None:- el = _mock_input({"type": "text", "autocomplete": None, "name": "postal_code"})-- results, _, _, _, _, generic_verify, _ = await _run_input_text(el, "12345", generic_flag=False)-- assert len(results) == 1 and isinstance(results[0], ActionSuccess)- el.input_sequentially.assert_awaited_once_with(text="12345")- generic_verify.assert_not_awaited()- # The credential-scope classification and the baseline live_input_type each read the live type,- # independent of the commit-verification flag; a disabled flag still queries type exactly twice.- type_queries = [call for call in el.get_attr.await_args_list if call.args == ("type",)]- assert len(type_queries) == 2---async def test_generic_text_input_verifies_after_unrelated_dom_mutation() -> None:- el = _mock_input({"type": "text", "autocomplete": None, "name": "postal_code"})-- results, _, _, _, _, generic_verify, _ = await _run_input_text(- el,- "12345",- incremental_elements=[{"tagName": "div", "attributes": {"class": "field-error"}, "children": []}],- )-- assert len(results) == 1 and isinstance(results[0], ActionSuccess)- generic_verify.assert_awaited_once()- assert generic_verify.await_args.kwargs["allow_autocomplete_rewrite"] is False---async def test_generic_text_input_allows_autocomplete_rewrite_for_option_mutation() -> None:- el = _mock_input({"type": "text", "autocomplete": None, "name": "postal_code"})-- results, _, _, _, _, generic_verify, _ = await _run_input_text(- el,- "12345",- incremental_elements=[- {- "tagName": "ul",- "attributes": {"role": "listbox"},- "children": [{"tagName": "li", "attributes": {"role": "option"}, "children": []}],- }- ],- )-- assert len(results) == 1 and isinstance(results[0], ActionSuccess)- generic_verify.assert_awaited_once()- assert generic_verify.await_args.kwargs["allow_autocomplete_rewrite"] is True---async def test_password_input_skips_commit_verification() -> None:- el = _mock_input({"type": "password", "autocomplete": None, "name": "credential"})-- results, _, _, _, _, generic_verify, _ = await _run_input_text(el, "private payload")-- assert len(results) == 1 and isinstance(results[0], ActionSuccess)- generic_verify.assert_not_awaited()--@pytest.mark.asyncioasync def test_secret_tel_value_uses_tel_verifier_not_secret_readback() -> None:# A resolved secret that is a NANP phone number must keep the digit-normalized tel verification@@ -371,7 +258,7 @@ async def test_secret_tel_value_uses_tel_verifier_not_secret_readback() -> None:# the bare digits against the formatted value and fail a correct fill.el = _mock_input({"type": "tel", "autocomplete": None, "name": "phone"})- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, "{{ phone }}", resolved="224-555-0199")@@ -395,7 +282,7 @@ async def test_single_character_secret_skips_readback() -> None:# (e.g. a multi-field TOTP digit routed into a masked box: is_secret_value True, is_totp_value False).el = _mock_input({"type": "password", "autocomplete": None, "name": "otp-digit"})- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, "{{ digit }}", resolved="5")@@ -413,7 +300,7 @@ async def test_secret_in_non_input_element_skips_readback() -> None:el = _mock_input({"type": None, "autocomplete": None, "name": "note"})el.get_tag_name.return_value = "div"- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, "{{ sec }}", resolved="mysecretvalue", tag_name="div")@@ -429,7 +316,7 @@ async def test_secret_in_exact_value_input_uses_readback(input_type: str | None)# its .value exactly, so the credential read-back verifier runs and is told the live type.el = _mock_input({"type": input_type, "autocomplete": None, "name": "credential"})- results, card_readback, tel_verify, phone_format, _, generic_verify, secret_readback = await _run_input_text(+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, "{{ sec }}", resolved="mysecretvalue", tag_name="input")@@ -442,8 +329,6 @@ async def test_secret_in_exact_value_input_uses_readback(input_type: str | None)maxlength=None,)el.input_sequentially.assert_not_awaited()- # A credential takes only the exact secret read-back; the generic commit verifier must not also run.- generic_verify.assert_not_awaited()@pytest.mark.asyncio@@ -454,7 +339,7 @@ async def test_secret_in_non_exact_value_input_skips_readback(input_type: str) -# path earlier and never reaches this gate.)el = _mock_input({"type": input_type, "autocomplete": None, "name": "field"})- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, "{{ sec }}", resolved="mysecretvalue", tag_name="input")@@ -471,7 +356,7 @@ async def test_secret_readback_skips_when_retargeted_to_out_of_scope_blocker() -el = _mock_input({"type": "text", "autocomplete": None, "name": "credential"})blocker = _mock_input({"type": "number", "autocomplete": None, "name": "overlay"})- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, "{{ sec }}", resolved="mysecretvalue", tag_name="input", blocker=blocker)@@ -488,7 +373,7 @@ async def test_secret_readback_runs_on_retargeted_element_type() -> None:el = _mock_input({"type": "text", "autocomplete": None, "name": "overlay"})blocker = _mock_input({"type": "password", "autocomplete": None, "name": "password"})- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, "{{ sec }}", resolved="mysecretvalue", tag_name="input", blocker=blocker)@@ -508,7 +393,7 @@ async def test_non_secret_exact_value_input_skips_readback() -> None:# only secrets are read back.el = _mock_input({"type": "text", "autocomplete": None, "name": "search"})- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, "not a secret value")@@ -525,7 +410,7 @@ async def test_totp_value_short_circuits_before_secret_readback() -> None:# `not is_totp_value` conjunct is a defensive backstop for this invariant.el = _mock_input({"type": "password", "autocomplete": None, "name": "otp"})- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, "{{ totp }}", resolved=str(BitwardenConstants.TOTP))diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.pyindex 2a644fd91..faed9f05e 100644--- a/tests/unit/test_select_shadow_match.py+++ b/tests/unit/test_select_shadow_match.py@@ -1,4 +1,3 @@-import copyfrom types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, Mockfrom zoneinfo import ZoneInfo@@ -427,7 +426,6 @@ async def test_auto_completion_call_path_forwards_committed_value_to_shadow_matcincremental_scraped = MagicMock(start_listen_dom_increment=AsyncMock(),stop_listen_dom_increment=AsyncMock(),- get_incremental_elements_num=AsyncMock(return_value=1),get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]),)incremental_scraped.build_html_tree.return_value = "<div>Committed Value</div>"@@ -489,7 +487,9 @@ async def test_emerging_select_call_path_forwards_committed_value_to_shadow_matcclick=AsyncMock(),)dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=selected_element))- llm_handler = AsyncMock(return_value={"id": "matched-id", "value": " committed value ", "action_type": "click"})+ llm_handler = AsyncMock(+ return_value={"id": "different-id", "value": " committed value ", "action_type": "click"}+ )monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div>Committed Value</div>"))monkeypatch.setattr(handler, "_select_deterministic_custom_option", AsyncMock(return_value=None))@@ -526,622 +526,6 @@ async def test_emerging_select_call_path_forwards_committed_value_to_shadow_matcassert events[0]["match_agrees_with_llm"] is True-async def test_emerging_select_falls_back_to_visible_controlled_listbox_when_no_ids_are_new(- monkeypatch: pytest.MonkeyPatch,-) -> None:- anchor = {- "id": "field-id",- "tagName": "button",- "attributes": {- "aria-controls": "menu-dom-id",- "aria-expanded": "true",- "role": "combobox",- },- "interactable": True,- "children": [],- }- listbox = {- "id": "menu-id",- "tagName": "div",- "attributes": {"id": "menu-dom-id", "role": "listbox"},- "children": [- {- "id": "option-a",- "tagName": "button",- "attributes": {"role": "option"},- "text": "Alpha",- "interactable": True,- "children": [],- },- {- "id": "option-hidden",- "tagName": "button",- "attributes": {"role": "option"},- "text": "Beta",- "interactable": False,- "children": [],- },- {- "id": "option-b",- "tagName": "button",- "attributes": {"role": "option"},- "text": "Beta",- "interactable": True,- "children": [],- },- ],- }- tree = [anchor, listbox]- ids = {- "field-id": "[unique_id=field-id]",- "menu-id": "[unique_id=menu-id]",- "option-a": "[unique_id=option-a]",- "option-hidden": "[unique_id=option-hidden]",- "option-b": "[unique_id=option-b]",- }-- def element_for_id(element_id: str) -> MagicMock:- return MagicMock(- is_interactable=Mock(return_value=element_id.startswith("option-") and element_id != "option-hidden")- )-- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(side_effect=element_for_id))-- async def select_deterministically(**kwargs: object) -> tuple[handler.ActionSuccess, str]:- get_option_candidates = kwargs["get_option_candidates"]- assert callable(get_option_candidates)- assert [candidate["label"] for candidate in get_option_candidates()] == ["Alpha", "Beta"]- return handler.ActionSuccess(), "Beta"-- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))- deterministic_select = AsyncMock(side_effect=select_deterministically)- monkeypatch.setattr(handler, "_select_deterministic_custom_option", deterministic_select)- llm_handler = AsyncMock()- monkeypatch.setattr(- handler.LLMAPIHandlerFactory,- "get_override_llm_api_handler",- Mock(return_value=llm_handler),- )- monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)-- scraped_page = SimpleNamespace(id_to_css_dict=ids)- scraped_page_after_open = SimpleNamespace(id_to_css_dict=ids, element_tree=tree, element_tree_trimmed=tree)- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):- result = await handler.select_from_emerging_elements(- current_element_id="field-id",- options=handler.CustomSelectPromptOptions(target_value="Beta"),- page=MagicMock(),- scraped_page=scraped_page,- scraped_page_after_open=scraped_page_after_open,- step=SimpleNamespace(step_id="step-1"),- task=_shadow_task(),- )-- assert isinstance(result, handler.ActionSuccess)- deterministic_select.assert_awaited_once()- llm_handler.assert_not_awaited()---async def test_emerging_select_fallback_resolves_owned_listbox_when_trim_strips_ownership_attrs(- monkeypatch: pytest.MonkeyPatch,-) -> None:- # A custom combobox whose listbox is linked by aria-controls. trim_element_tree drops- # aria-controls / aria-owns / the DOM id from the trimmed tree, so ownership must be resolved- # from the untrimmed element_tree — resolving it from the trimmed tree leaves the fallback inert.- anchor = {- "id": "field-id",- "tagName": "button",- "attributes": {"aria-controls": "menu-dom-id", "aria-expanded": "true", "role": "combobox"},- "interactable": True,- "children": [],- }- listbox = {- "id": "menu-id",- "tagName": "div",- "attributes": {"id": "menu-dom-id", "role": "listbox"},- "children": [- {- "id": "option-a",- "tagName": "button",- "attributes": {"role": "option"},- "text": "Alpha",- "interactable": True,- "children": [],- },- {- "id": "option-b",- "tagName": "button",- "attributes": {"role": "option"},- "text": "Beta",- "interactable": True,- "children": [],- },- ],- }- element_tree = [anchor, listbox]- element_tree_trimmed = handler.trim_element_tree(copy.deepcopy(element_tree))- # Guard the premise: trim really removes the ownership signal the fallback keys off.- assert "aria-controls" not in (element_tree_trimmed[0].get("attributes") or {})- assert "id" not in (element_tree_trimmed[1].get("attributes") or {})-- ids = {- "field-id": "[unique_id=field-id]",- "menu-id": "[unique_id=menu-id]",- "option-a": "[unique_id=option-a]",- "option-b": "[unique_id=option-b]",- }-- def element_for_id(element_id: str) -> MagicMock:- return MagicMock(is_interactable=Mock(return_value=element_id.startswith("option-")))-- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(side_effect=element_for_id))-- async def select_deterministically(**kwargs: object) -> tuple[handler.ActionSuccess, str]:- get_option_candidates = kwargs["get_option_candidates"]- assert callable(get_option_candidates)- assert [candidate["label"] for candidate in get_option_candidates()] == ["Alpha", "Beta"]- return handler.ActionSuccess(), "Beta"-- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))- deterministic_select = AsyncMock(side_effect=select_deterministically)- monkeypatch.setattr(handler, "_select_deterministic_custom_option", deterministic_select)- monkeypatch.setattr(- handler.LLMAPIHandlerFactory,- "get_override_llm_api_handler",- Mock(return_value=AsyncMock()),- )-- scraped_page = SimpleNamespace(id_to_css_dict=ids)- scraped_page_after_open = SimpleNamespace(- id_to_css_dict=ids,- element_tree=element_tree,- element_tree_trimmed=element_tree_trimmed,- )- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):- result = await handler.select_from_emerging_elements(- current_element_id="field-id",- options=handler.CustomSelectPromptOptions(target_value="Beta"),- page=MagicMock(),- scraped_page=scraped_page,- scraped_page_after_open=scraped_page_after_open,- step=SimpleNamespace(step_id="step-1"),- task=_shadow_task(),- )-- assert isinstance(result, handler.ActionSuccess)- deterministic_select.assert_awaited_once()---async def test_emerging_select_keeps_off_list_candidate_for_deterministic_match_when_flag_off(- monkeypatch: pytest.MonkeyPatch,-) -> None:- # Option nodes in custom widgets are often not flagged interactable. With the verify flag off we- # must not pre-filter deterministic candidates to the interactable set (main's behavior) — dropping- # them there would regress selections that used to succeed.- tree = [- {- "id": "option-x",- "tagName": "div",- "attributes": {"role": "option"},- "text": "Alpha",- "interactable": True,- "children": [],- },- {- "id": "option-y",- "tagName": "div",- "attributes": {"role": "option"},- "text": "Beta",- "children": [],- },- ]- monkeypatch.setattr(handler, "_is_verify_emerging_select_pick_enabled", AsyncMock(return_value=False))- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock())- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))- monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div role='option'>option</div>"))- monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))- llm_handler = AsyncMock()- monkeypatch.setattr(- handler.LLMAPIHandlerFactory,- "get_override_llm_api_handler",- Mock(return_value=llm_handler),- )- monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)-- async def select_deterministically(**kwargs: object) -> tuple[handler.ActionSuccess, str]:- get_option_candidates = kwargs["get_option_candidates"]- assert callable(get_option_candidates)- assert [candidate["element_id"] for candidate in get_option_candidates()] == ["option-x", "option-y"]- return handler.ActionSuccess(), "Beta"-- deterministic_select = AsyncMock(side_effect=select_deterministically)- monkeypatch.setattr(handler, "_select_deterministic_custom_option", deterministic_select)-- scraped_page = SimpleNamespace(id_to_css_dict={})- scraped_page_after_open = SimpleNamespace(- id_to_css_dict={"option-x": "[unique_id=option-x]", "option-y": "[unique_id=option-y]"},- element_tree_trimmed=tree,- )- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):- result = await handler.select_from_emerging_elements(- current_element_id="field-id",- options=handler.CustomSelectPromptOptions(target_value="Beta"),- page=MagicMock(),- scraped_page=scraped_page,- scraped_page_after_open=scraped_page_after_open,- new_interactable_element_ids=["option-x"],- step=SimpleNamespace(step_id="step-1"),- task=_shadow_task(),- )-- assert isinstance(result, handler.ActionSuccess)- deterministic_select.assert_awaited_once()- llm_handler.assert_not_awaited()--[email protected](("anchor_expanded", "expected"), [(False, []), (True, ["listbox-id"])])-def test_custom_select_fallback_requires_open_anchor_for_sole_listbox(- anchor_expanded: bool,- expected: list[str],-) -> None:- tree = [- {- "id": "field-id",- "tagName": "button",- "attributes": {"aria-expanded": str(anchor_expanded).lower()},- "children": [],- },- {- "id": "listbox-id",- "tagName": "div",- "attributes": {"role": "listbox"},- "children": [- {- "id": "option-id",- "tagName": "button",- "attributes": {"role": "option"},- "text": "Alpha",- "children": [],- }- ],- },- ]-- result = handler._custom_select_fallback_subtrees(tree, "field-id")-- assert [element["id"] for element in result] == expected---def test_custom_select_fallback_accepts_sole_listbox_for_expanded_combobox_ancestor() -> None:- tree = [- {- "id": "combobox-id",- "tagName": "div",- "attributes": {"aria-expanded": "true", "role": "combobox"},- "children": [- {- "id": "field-id",- "tagName": "input",- "attributes": {},- "children": [],- }- ],- },- {- "id": "listbox-id",- "tagName": "div",- "attributes": {"role": "listbox"},- "children": [- {- "id": "option-id",- "tagName": "button",- "attributes": {"role": "option"},- "text": "Alpha",- "children": [],- }- ],- },- ]-- result = handler._custom_select_fallback_subtrees(tree, "field-id")-- assert [element["id"] for element in result] == ["listbox-id"]---async def test_emerging_select_rejects_page_global_options_without_owned_listbox(- monkeypatch: pytest.MonkeyPatch,-) -> None:- tree = [- {- "id": "field-id",- "tagName": "button",- "attributes": {"aria-expanded": "true", "role": "combobox"},- "children": [],- },- {- "id": "unrelated-option",- "tagName": "button",- "attributes": {"role": "option"},- "text": "Wrong field",- "children": [],- },- ]- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock())- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))- scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})- scraped_page_after_open = SimpleNamespace(- id_to_css_dict={"field-id": "[unique_id=field-id]"},- element_tree=tree,- element_tree_trimmed=tree,- )-- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):- with pytest.raises(handler.NoIncrementalElementFoundForCustomSelection):- await handler.select_from_emerging_elements(- current_element_id="field-id",- options=handler.CustomSelectPromptOptions(target_value="Wrong field"),- page=MagicMock(),- scraped_page=scraped_page,- scraped_page_after_open=scraped_page_after_open,- step=SimpleNamespace(step_id="step-1"),- task=_shadow_task(),- )---async def test_emerging_select_rejects_hidden_candidate_from_deterministic_and_llm_paths(- monkeypatch: pytest.MonkeyPatch,-) -> None:- tree = [- {- "id": "hidden-id",- "tagName": "button",- "attributes": {"role": "option"},- "text": "Exact target",- "children": [],- },- {- "id": "visible-id",- "tagName": "button",- "attributes": {"role": "option"},- "text": "Visible alternative",- "children": [],- },- ]- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock())- monkeypatch.setattr(handler, "_is_verify_emerging_select_pick_enabled", AsyncMock(return_value=True))- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))- monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<button role='option'>option</button>"))- monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))- llm_handler = AsyncMock(return_value={"id": "hidden-id", "value": "Exact target", "action_type": "click"})- monkeypatch.setattr(- handler.LLMAPIHandlerFactory,- "get_override_llm_api_handler",- Mock(return_value=llm_handler),- )- monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)-- async def select_deterministically(**kwargs: object) -> None:- get_option_candidates = kwargs["get_option_candidates"]- assert callable(get_option_candidates)- assert [candidate["element_id"] for candidate in get_option_candidates()] == ["visible-id"]- return None-- monkeypatch.setattr(- handler,- "_select_deterministic_custom_option",- AsyncMock(side_effect=select_deterministically),- )- scraped_page = SimpleNamespace(id_to_css_dict={})- scraped_page_after_open = SimpleNamespace(- id_to_css_dict={- "hidden-id": "[unique_id=hidden-id]",- "visible-id": "[unique_id=visible-id]",- },- element_tree_trimmed=tree,- )-- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):- with pytest.raises(handler.NoAvailableOptionFoundForCustomSelection):- await handler.select_from_emerging_elements(- current_element_id="field-id",- options=handler.CustomSelectPromptOptions(target_value="Exact target"),- page=MagicMock(),- scraped_page=scraped_page,- scraped_page_after_open=scraped_page_after_open,- new_interactable_element_ids=["visible-id"],- step=SimpleNamespace(step_id="step-1"),- task=_shadow_task(),- )-- dom_after_open.get_skyvern_element_by_id.assert_not_awaited()---async def test_emerging_select_allows_input_text_on_anchor_element(- monkeypatch: pytest.MonkeyPatch,-) -> None:- anchor_input = MagicMock(- scroll_into_view=AsyncMock(),- get_tag_name=Mock(return_value="input"),- get_locator=Mock(return_value=MagicMock()),- is_readonly=AsyncMock(return_value=False),- input_clear=AsyncMock(),- input_sequentially=AsyncMock(),- )- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=anchor_input))- llm_handler = AsyncMock(return_value={"id": "field-id", "value": "filter text", "action_type": "input_text"})- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))- monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div>Option</div>"))- monkeypatch.setattr(handler, "_select_deterministic_custom_option", AsyncMock(return_value=None))- monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value=""))- monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))- monkeypatch.setattr(- handler.LLMAPIHandlerFactory,- "get_override_llm_api_handler",- Mock(return_value=llm_handler),- )- monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)-- scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})- scraped_page_after_open = SimpleNamespace(- id_to_css_dict={- "field-id": "[unique_id=field-id]",- "option-x": "[unique_id=option-x]",- },- element_tree_trimmed=[- {- "id": "option-x",- "tagName": "div",- "attributes": {"role": "option"},- "text": "Other Option",- "interactable": True,- }- ],- )- task = SimpleNamespace(- navigation_goal="goal",- navigation_payload={},- llm_key=None,- workflow_run_id=None,- organization_id=None,- workflow_permanent_id=None,- )- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):- result = await handler.select_from_emerging_elements(- current_element_id="field-id",- options=handler.CustomSelectPromptOptions(target_value="filter text"),- page=MagicMock(),- scraped_page=scraped_page,- scraped_page_after_open=scraped_page_after_open,- new_interactable_element_ids=["option-x"],- step=SimpleNamespace(step_id="step-1"),- task=task,- )-- assert isinstance(result, handler.ActionSuccess)- anchor_input.input_clear.assert_awaited_once()- anchor_input.input_sequentially.assert_awaited_once_with("filter text")---async def test_emerging_select_rejects_input_text_on_element_outside_anchor_and_new_ids(- monkeypatch: pytest.MonkeyPatch,-) -> None:- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock())- llm_handler = AsyncMock(return_value={"id": "stranger-id", "value": "filter text", "action_type": "input_text"})- monkeypatch.setattr(handler, "_is_verify_emerging_select_pick_enabled", AsyncMock(return_value=True))- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))- monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div>Option</div>"))- monkeypatch.setattr(handler, "_select_deterministic_custom_option", AsyncMock(return_value=None))- monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))- monkeypatch.setattr(- handler.LLMAPIHandlerFactory,- "get_override_llm_api_handler",- Mock(return_value=llm_handler),- )- monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)-- scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})- scraped_page_after_open = SimpleNamespace(- id_to_css_dict={- "field-id": "[unique_id=field-id]",- "option-x": "[unique_id=option-x]",- },- element_tree_trimmed=[- {- "id": "option-x",- "tagName": "div",- "attributes": {"role": "option"},- "text": "Other Option",- "interactable": True,- }- ],- )- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):- with pytest.raises(handler.NoAvailableOptionFoundForCustomSelection):- await handler.select_from_emerging_elements(- current_element_id="field-id",- options=handler.CustomSelectPromptOptions(target_value="filter text"),- page=MagicMock(),- scraped_page=scraped_page,- scraped_page_after_open=scraped_page_after_open,- new_interactable_element_ids=["option-x"],- step=SimpleNamespace(step_id="step-1"),- task=_shadow_task(),- )-- dom_after_open.get_skyvern_element_by_id.assert_not_awaited()---async def test_emerging_select_off_list_pick_still_proceeds_when_flag_disabled(- monkeypatch: pytest.MonkeyPatch,-) -> None:- # Flag off keeps today's behavior: an off-list pick is still acted on rather than rejected, so- # merging this cannot change any org's outcomes until the flag is ramped.- selected_element = MagicMock(- get_tag_name=Mock(return_value="div"),- get_attr=AsyncMock(return_value=None),- scroll_into_view=AsyncMock(),- click=AsyncMock(),- )- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=selected_element))- llm_handler = AsyncMock(return_value={"id": "stranger-id", "value": None, "action_type": "click"})- monkeypatch.setattr(handler, "_is_verify_emerging_select_pick_enabled", AsyncMock(return_value=False))- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))- monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div>Option</div>"))- monkeypatch.setattr(handler, "_select_deterministic_custom_option", AsyncMock(return_value=None))- monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))- monkeypatch.setattr(- handler.LLMAPIHandlerFactory,- "get_override_llm_api_handler",- Mock(return_value=llm_handler),- )- monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)-- scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})- scraped_page_after_open = SimpleNamespace(- id_to_css_dict={- "field-id": "[unique_id=field-id]",- "option-x": "[unique_id=option-x]",- },- element_tree_trimmed=[- {- "id": "option-x",- "tagName": "div",- "attributes": {"role": "option"},- "text": "Other Option",- "interactable": True,- }- ],- )- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):- with structlog.testing.capture_logs() as logs:- result = await handler.select_from_emerging_elements(- current_element_id="field-id",- options=handler.CustomSelectPromptOptions(target_value="filter text"),- page=MagicMock(),- scraped_page=scraped_page,- scraped_page_after_open=scraped_page_after_open,- new_interactable_element_ids=["option-x"],- step=SimpleNamespace(step_id="step-1"),- task=_shadow_task(),- )-- assert isinstance(result, handler.ActionSuccess)- selected_element.click.assert_awaited()- # Flag off still emits telemetry so the reject rate is measurable before ramping the flag on.- off_list_warnings = [- log for log in logs if log.get("log_level") == "warning" and log.get("element_id") == "stranger-id"- ]- assert off_list_warnings--@pytest.mark.asyncioasync def test_dropdown_select_call_path_forwards_committed_value_to_shadow_match(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/test_tel_input_bare_digits.py b/tests/unit/test_tel_input_bare_digits.pyindex bac4d4da1..463800368 100644--- a/tests/unit/test_tel_input_bare_digits.py+++ b/tests/unit/test_tel_input_bare_digits.py@@ -8,7 +8,6 @@ import pytestfrom skyvern.webeye.actions import handler_utilsfrom skyvern.webeye.actions.handler import (- _is_org_flag_enabled,_is_tel_digit_fix_enabled,_nanp_e164_fallback,_nanp_national_digits,@@ -187,29 +186,3 @@ async def test_is_tel_digit_fix_enabled_uses_org_keyed_rollout(monkeypatch: pyteorganization.organization_id,properties={"organization_id": organization.organization_id},)---async def test_is_org_flag_enabled_buckets_any_flag_by_org(monkeypatch: pytest.MonkeyPatch) -> None:- now = datetime.now(UTC)- organization = make_organization(now)- task = make_task(now, organization)- provider = SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True))- monkeypatch.setattr("skyvern.webeye.actions.handler.app.EXPERIMENTATION_PROVIDER", provider)-- assert await _is_org_flag_enabled(task, "SOME_OTHER_FLAG", "some-other") is True- provider.is_feature_enabled_cached.assert_awaited_once_with(- "SOME_OTHER_FLAG",- organization.organization_id,- properties={"organization_id": organization.organization_id},- )---async def test_is_org_flag_enabled_fails_closed_when_provider_raises(monkeypatch: pytest.MonkeyPatch) -> None:- now = datetime.now(UTC)- task = make_task(now, make_organization(now))- provider = SimpleNamespace(is_feature_enabled_cached=AsyncMock(side_effect=RuntimeError("posthog down")))- monkeypatch.setattr("skyvern.webeye.actions.handler.app.EXPERIMENTATION_PROVIDER", provider)-- assert await _is_org_flag_enabled(task, "SOME_OTHER_FLAG", "some-other") is False